From 6de9cdb26b801489a007756ad9eb8d99f4262f07 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 9 Sep 2026 10:37:36 +0200 Subject: [PATCH 01/65] mtmd: propagate video ID to bitmap (#28601) --- tools/mtmd/mtmd-helper.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index dc2ab414e742..bdf8bf6fe450 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -371,6 +371,7 @@ static bool is_webp_file(const unsigned char * buf, size_t len) { #ifdef MTMD_VIDEO static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder, const mtmd_helper_video_init_params & params); +static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string & id); #endif mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder, @@ -436,6 +437,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * LOG_ERR("%s: failed to decode buffer as either image/audio/video\n", __func__); return {nullptr, nullptr}; } + mtmd_helper_video_set_id(video_ctx, id); // propagate the hash to the frames result = mtmd_bitmap_init_lazy(ctx, id.empty() ? nullptr : id.c_str(), video_ctx, @@ -527,6 +529,7 @@ struct mtmd_helper_video { std::string ffprobe_bin; float fps_target = 0.0f; mtmd_helper_video_info info = {}; + std::string id; // hash of the input video // RAII wrapper for managing subprocess struct subprocess_handle { @@ -785,9 +788,14 @@ struct mtmd_helper_video { } LOG_DBG("%s: frame %d read OK\n", __func__, current_frame); - current_frame++; mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data()); mtmd_bitmap_set_mergeable(frame, true); + if (!id.empty()) { + // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache + std::string frame_id = id + "+" + std::to_string(current_frame); + mtmd_bitmap_set_id(frame, frame_id.c_str()); + } + current_frame++; return frame; } @@ -886,6 +894,10 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) { } #ifdef MTMD_VIDEO +static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string & id) { + vctx->id = id; +} + static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder, const mtmd_helper_video_init_params & params) { mtmd_helper_video vctx; From e2d2c0d6aa9b996d5d3a3c1d5e24c8c19728bb3d Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 9 Sep 2026 16:46:22 +0800 Subject: [PATCH 02/65] model: fix granite3 moe unknown parameter count (#28632) Signed-off-by: Aaron Teo --- src/llama-model.cpp | 1 + src/llama-model.h | 1 + src/models/granite-moe.cpp | 1 + 3 files changed, 3 insertions(+) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 0adc07449be0..54009b3696a5 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -935,6 +935,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_17B_16E: return "17Bx16E (Scout)"; case LLM_TYPE_17B_128E: return "17Bx128E (Maverick)"; case LLM_TYPE_A13B: return "A13B"; + case LLM_TYPE_1B_A400M: return "1B.A400M"; case LLM_TYPE_7B_A1B: return "7B.A1B"; case LLM_TYPE_8B_A1B: return "8B.A1B"; case LLM_TYPE_7_9B_A1_3B: return "7.9B.A1.3B"; diff --git a/src/llama-model.h b/src/llama-model.h index 4c4a30e018bc..c0cc4065567a 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -116,6 +116,7 @@ enum llm_type { LLM_TYPE_17B_16E, // llama4 Scout LLM_TYPE_17B_128E, // llama4 Maverick LLM_TYPE_A13B, + LLM_TYPE_1B_A400M, // Granite3 MoE LLM_TYPE_7B_A1B, LLM_TYPE_8B_A1B, // lfm2moe LLM_TYPE_7_9B_A1_3B, // Ling-3.0-tiny diff --git a/src/models/granite-moe.cpp b/src/models/granite-moe.cpp index 09be49393e30..156553edfd04 100644 --- a/src/models/granite-moe.cpp +++ b/src/models/granite-moe.cpp @@ -8,6 +8,7 @@ void llama_model_granite_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false); switch (hparams.n_layer()) { + case 24: type = LLM_TYPE_1B_A400M; break; case 32: type = LLM_TYPE_3B; break; case 40: type = LLM_TYPE_3B; break; // Add additional layer/vocab/etc checks here for other model sizes From 14a9d09f75683c94c2c4f229efe54670d4209089 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 9 Sep 2026 18:36:27 +0800 Subject: [PATCH 03/65] args: officially deprecate --mmap|mlock|dio (#28334) Signed-off-by: Aaron Teo --- common/arg.cpp | 37 ---------------------------- tools/cli/README.md | 3 --- tools/completion/README.md | 3 --- tools/llama-bench/llama-bench.cpp | 40 ------------------------------- tools/server/README.md | 3 --- 5 files changed, 86 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 74241f931285..43052d58d1d1 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -872,17 +872,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context arg.c_str(), e.what(), opt.to_string().c_str())); } } - - // TODO: remove this check after deprecating --mmap|mlock|dio - auto has_arg = [&](std::initializer_list names) { - return std::any_of(names.begin(), names.end(), [&](const char * name) { - return seen_args.count(name); - }); - }; - if (has_arg({"-lm", "--load-mode"}) && - has_arg({"--mlock", "--mmap", "--no-mmap", "-dio", "--direct-io", "-ndio", "--no-direct-io"})) { - LOG_WRN("DEPRECATED: `--load-mode` and `--mlock`/`--mmap`/`--direct-io` should not be combined; only the last flag on the command line will take effect\n"); - } }; // parse all CLI args now, so that -hf is available below for remote preset resolution @@ -2694,32 +2683,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_env("LLAMA_ARG_RPC")); } - add_opt(common_arg( - {"--mlock"}, - "DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing", - [](common_params & params) { - LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n"); - params.load_mode = LLAMA_LOAD_MODE_MLOCK; - } - ).set_env("LLAMA_ARG_MLOCK")); - add_opt(common_arg( - {"--mmap"}, - {"--no-mmap"}, - "DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)", - [](common_params & params, bool value) { - LOG_WRN("DEPRECATED: --mmap and --no-mmap are deprecated. use --load-mode mmap instead\n"); - params.load_mode = value ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE; - } - ).set_env("LLAMA_ARG_MMAP")); - add_opt(common_arg( - {"-dio", "--direct-io"}, - {"-ndio", "--no-direct-io"}, - "DEPRECATED in favor of `--load-mode`: use DirectIO if available", - [](common_params & params, bool value) { - LOG_WRN("DEPRECATED: --direct-io and --no-direct-io are deprecated. use --load-mode dio instead\n"); - params.load_mode = value ? LLAMA_LOAD_MODE_DIRECT_IO : LLAMA_LOAD_MODE_NONE; - } - ).set_env("LLAMA_ARG_DIO")); add_opt(common_arg( {"-lm", "--load-mode"}, "MODE", "model loading mode (default: auto)\n" diff --git a/tools/cli/README.md b/tools/cli/README.md index efe653494dae..77a5e6fe3259 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -55,9 +55,6 @@ | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | -| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) | -| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) | -| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) | | `-lm, --load-mode MODE` | model loading mode (default: auto)
- auto: mmap, unless a device does not support it
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | | `-lzm, --lazy-mode MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)
- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)
- auto: on, but only for tensors larger than 4 GiB
- off: always keep them resident
(env: LLAMA_ARG_LAZY_MODE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) | diff --git a/tools/completion/README.md b/tools/completion/README.md index 702a1c4c2929..08485a95f593 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -138,9 +138,6 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | -| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) | -| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) | -| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) | | `-lm, --load-mode MODE` | model loading mode (default: auto)
- auto: mmap, unless a device does not support it
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | | `-lzm, --lazy-mode MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)
- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)
- auto: on, but only for tensors larger than 4 GiB
- off: always keep them resident
(env: LLAMA_ARG_LAZY_MODE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) | diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 1fff21f701e2..17adda38091f 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -476,8 +476,6 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -dev, --device (default: auto)\n"); printf(" -lm, --load-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str()); printf(" -lzm, --lazy-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str()); - printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); - printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str()); printf(" -ts, --tensor-split (default: 0)\n"); printf(" -ot --override-tensor =;...\n"); @@ -883,44 +881,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { break; } params.flash_attn.insert(params.flash_attn.end(), types.begin(), types.end()); - } else if (arg == "-mmp" || arg == "--mmap") { - if (++i >= argc) { - invalid_param = true; - break; - } - LOG_WRN("DEPRECATED: -mmp and --mmap are deprecated in favour of --load-mode. Please use --load-mode mmap instead.\n"); - auto p = string_split(argv[i], split_delim); - - std::vector modes; - for (const auto & m : p) { - llama_load_mode mode; - if (m) { - mode = LLAMA_LOAD_MODE_MMAP; - } else { - mode = LLAMA_LOAD_MODE_NONE; - } - modes.push_back(mode); - } - params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end()); - } else if (arg == "-dio" || arg == "--direct-io") { - if (++i >= argc) { - invalid_param = true; - break; - } - LOG_WRN("DEPRECATED: -dio and --direct-io are deprecated in favour of --load-mode. Please use --load-mode dio instead.\n"); - auto p = string_split(argv[i], split_delim); - - std::vector modes; - for (const auto & m : p) { - llama_load_mode mode; - if (m) { - mode = LLAMA_LOAD_MODE_DIRECT_IO; - } else { - mode = LLAMA_LOAD_MODE_NONE; - } - modes.push_back(mode); - } - params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end()); } else if (arg == "-embd" || arg == "--embeddings") { if (++i >= argc) { invalid_param = true; diff --git a/tools/server/README.md b/tools/server/README.md index 71ebb95434e4..19090763281a 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -72,9 +72,6 @@ For the full list of features, please refer to [server's changelog](https://gith | `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | -| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) | -| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) | -| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) | | `-lm, --load-mode MODE` | model loading mode (default: auto)
- auto: mmap, unless a device does not support it
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | | `-lzm, --lazy-mode MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)
- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)
- auto: on, but only for tensors larger than 4 GiB
- off: always keep them resident
(env: LLAMA_ARG_LAZY_MODE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) | From 5a4d0fecae272c9caf0b32eb384fa6a58dddb560 Mon Sep 17 00:00:00 2001 From: "Piotr Wilkin (ilintar)" Date: Wed, 9 Sep 2026 12:50:08 +0200 Subject: [PATCH 04/65] CUDA: replace GGML_FA_ALL_QUANTS with GGML_FA_QUANTS, more control over what is compiled (#28079) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * CUDA: add configurable FA quant combinations Assisted-by: Codex * remove all flags but , add runtime fallback with warning for uncompiled combination * Update docs/build.md Co-authored-by: Johannes Gäßler * apply code review comments --------- Co-authored-by: Johannes Gäßler --- docs/build.md | 3 +- ggml/CMakeLists.txt | 2 + ggml/cmake/common.cmake | 71 +++++++++++ ggml/src/ggml-cuda/CMakeLists.txt | 13 +- ggml/src/ggml-cuda/fattn.cu | 200 +++++++++++++++--------------- ggml/src/ggml-cuda/ggml-cuda.cu | 4 +- ggml/src/ggml-hip/CMakeLists.txt | 13 +- ggml/src/ggml-musa/CMakeLists.txt | 13 +- 8 files changed, 185 insertions(+), 134 deletions(-) diff --git a/docs/build.md b/docs/build.md index f794d490b097..28dcbc2e53ea 100644 --- a/docs/build.md +++ b/docs/build.md @@ -300,7 +300,8 @@ The following compilation options are also available to tweak performance: |-------------------------------|------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | GGML_CUDA_FORCE_MMQ | Boolean | false | Force the use of custom matrix multiplication kernels for quantized models instead of FP16 cuBLAS even if there is no int8 tensor core implementation available (affects V100, CDNA and RDNA3+). MMQ kernels are enabled by default on GPUs with int8 tensor core support. With MMQ force enabled, speed for large batch sizes will be worse but VRAM consumption will be lower. | | GGML_CUDA_FORCE_CUBLAS | Boolean | false | Force the use of FP16 cuBLAS instead of custom matrix multiplication kernels for quantized models. There may be issues with numerical overflows (except for V100, CDNA and RDNA4 which use FP32 compute type by default) and memory use will be higher. Prompt processing may become faster on recent datacenter GPUs (the custom kernels were tuned primarily for RTX 3000/4000). | -| GGML_CUDA_FA_ALL_QUANTS | Boolean | false | Compile support for all KV cache quantization type (combinations) for the FlashAttention CUDA kernels. More fine-grained control over KV cache size but compilation takes much longer. | +| GGML_CUDA_FA_QUANTS | `all` or `type_K-type_V` list | q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16 | Select which K/V type combinations to compile the FlashAttention CUDA kernels for. `all` compiles every combination, but compilation takes much longer. Otherwise a `;`-separated list of `type_K-type_V` pairs; f16-f16 is always compiled. Combinations that were not compiled fall back to f16-f16 kernel with a warning. Legal types: f16, bf16, q4_0, q4_1, q5_0, q5_1, q8_0. | +| GGML_CUDA_FA_ALL_QUANTS | Boolean | false | Deprecated alias for `GGML_CUDA_FA_QUANTS=all`. | ## MUSA diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index d76ed8ab0497..ba9bc83b9b08 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -204,6 +204,8 @@ option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copie option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF) option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON) option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF) +set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16" CACHE STRING + "ggml: FlashAttention K-V type combinations to compile, \"all\" or a list such as \"q8_0-q8_0;q8_0-q4_0\"") option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT}) option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON) set (GGML_CUDA_COMPRESSION_MODE "size" CACHE STRING diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake index cb6638833204..25eff7a5ef02 100644 --- a/ggml/cmake/common.cmake +++ b/ggml/cmake/common.cmake @@ -48,3 +48,74 @@ function(ggml_get_system_arch) set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE) endif() endfunction() + +# Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS. +function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS) + set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16) + + string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS) + string(STRIP "${FA_QUANTS}" FA_QUANTS) + if (GGML_CUDA_FA_ALL_QUANTS) + message(WARNING "GGML_CUDA_FA_ALL_QUANTS is deprecated, use GGML_CUDA_FA_QUANTS=all instead") + set(FA_QUANTS all) + endif() + if (NOT FA_QUANTS) + message(FATAL_ERROR "GGML_CUDA_FA_QUANTS must not be empty") + endif() + + if (FA_QUANTS STREQUAL "all") + set(FA_COMBINATIONS "") + foreach (TYPE_V IN LISTS FA_TYPES) + foreach (TYPE_K IN LISTS FA_TYPES) + list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V}) + endforeach() + endforeach() + else() + set(FA_COMBINATIONS f16-f16) + + string(REPLACE "," ";" FA_SELECTED "${FA_QUANTS}") + foreach (COMBINATION IN LISTS FA_SELECTED) + string(STRIP "${COMBINATION}" COMBINATION) + if (NOT COMBINATION MATCHES "^([a-z0-9_]+)-([a-z0-9_]+)$") + message(FATAL_ERROR "GGML_CUDA_FA_QUANTS: \"${COMBINATION}\" is not \"all\" or a - combination") + endif() + set(TYPE_K ${CMAKE_MATCH_1}) + set(TYPE_V ${CMAKE_MATCH_2}) + foreach (TYPE ${TYPE_K} ${TYPE_V}) + if (NOT TYPE IN_LIST FA_TYPES) + message(FATAL_ERROR + "GGML_CUDA_FA_QUANTS: unknown type \"${TYPE}\" in \"${COMBINATION}\", must be one of: ${FA_TYPES}") + endif() + endforeach() + list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V}) + endforeach() + endif() + list(REMOVE_DUPLICATES FA_COMBINATIONS) + + string(REPLACE ";" "," FA_QUANTS_DEFINE "${FA_QUANTS}") + add_compile_definitions(GGML_CUDA_FA_QUANTS="${FA_QUANTS_DEFINE}") + foreach (TYPE_V IN LISTS FA_TYPES) + foreach (TYPE_K IN LISTS FA_TYPES) + if ("${TYPE_K}-${TYPE_V}" IN_LIST FA_COMBINATIONS) + set(COMPILED 1) + else() + set(COMPILED 0) + endif() + string(TOUPPER "GGML_CUDA_FA_${TYPE_K}_${TYPE_V}" COMBINATION_DEF) + add_compile_definitions(${COMBINATION_DEF}=${COMPILED}) + endforeach() + endforeach() + + message(STATUS "FlashAttention K-V type combinations: ${FA_COMBINATIONS}") + + set(SRCS "") + foreach (COMBINATION IN LISTS FA_COMBINATIONS) + set(SRC "${DIR}/template-instances/fattn-vec-instance-${COMBINATION}.cu") + if (NOT EXISTS "${SRC}") + message(FATAL_ERROR "FlashAttention template instance \"${SRC}\" does not exist") + endif() + list(APPEND SRCS "${SRC}") + endforeach() + + set(${OUT_SRCS} ${SRCS} PARENT_SCOPE) +endfunction() diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt index 10828ad8174b..2254090cbab0 100644 --- a/ggml/src/ggml-cuda/CMakeLists.txt +++ b/ggml/src/ggml-cuda/CMakeLists.txt @@ -112,17 +112,8 @@ if (CUDAToolkit_FOUND) file(GLOB SRCS "template-instances/mmf*.cu") list(APPEND GGML_SOURCES_CUDA ${SRCS}) - if (GGML_CUDA_FA_ALL_QUANTS) - file(GLOB SRCS "template-instances/fattn-vec*.cu") - list(APPEND GGML_SOURCES_CUDA ${SRCS}) - add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS) - else() - list(APPEND GGML_SOURCES_CUDA - template-instances/fattn-vec-instance-f16-f16.cu - template-instances/fattn-vec-instance-q4_0-q4_0.cu - template-instances/fattn-vec-instance-q8_0-q8_0.cu - template-instances/fattn-vec-instance-bf16-bf16.cu) - endif() + ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR} SRCS) + list(APPEND GGML_SOURCES_CUDA ${SRCS}) ggml_add_backend_library(ggml-cuda ${GGML_HEADERS_CUDA} diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index ae217fbd9df1..d11a964d59a3 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -374,90 +374,101 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg } } -#define FATTN_VEC_CASE(D, type_K, type_V) \ - { \ - const bool type_K_okay = K->type == (type_K) || (K->type == GGML_TYPE_F32 && (type_K) == GGML_TYPE_F16); \ - const bool type_V_okay = V->type == (type_V) || (V->type == GGML_TYPE_F32 && (type_V) == GGML_TYPE_F16); \ - if (Q->ne[0] == (D) && type_K_okay && type_V_okay) { \ - ggml_cuda_flash_attn_ext_vec_case(ctx, dst); \ - return; \ - } \ - } \ - -#define FATTN_VEC_CASES_ALL_D(type_K, type_V) \ - FATTN_VEC_CASE( 64, type_K, type_V) \ - FATTN_VEC_CASE(128, type_K, type_V) \ - FATTN_VEC_CASE(256, type_K, type_V) \ +#define FATTN_VEC_CASE(D, type_K_case, type_V_case) \ + if constexpr (GGML_CUDA_FA_##type_K_case##_##type_V_case) { \ + const bool type_K_okay = type_K == GGML_TYPE_##type_K_case || (type_K == GGML_TYPE_F32 && GGML_TYPE_##type_K_case == GGML_TYPE_F16); \ + const bool type_V_okay = type_V == GGML_TYPE_##type_V_case || (type_V == GGML_TYPE_F32 && GGML_TYPE_##type_V_case == GGML_TYPE_F16); \ + if (head_size == (D) && type_K_okay && type_V_okay) { \ + return ggml_cuda_flash_attn_ext_vec_case; \ + } \ + } \ + +#define FATTN_VEC_CASES_ALL_D(type_K_case, type_V_case) \ + FATTN_VEC_CASE( 64, type_K_case, type_V_case) \ + FATTN_VEC_CASE(128, type_K_case, type_V_case) \ + FATTN_VEC_CASE(256, type_K_case, type_V_case) \ + +typedef void (* fattn_vec_case_t)(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// Vector kernel for the given head size and K/V types, nullptr if its template instance was not compiled: +static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K, const ggml_type type_V) { + FATTN_VEC_CASES_ALL_D(F16, F16) + FATTN_VEC_CASES_ALL_D(Q4_0, F16) + FATTN_VEC_CASES_ALL_D(Q4_1, F16) + FATTN_VEC_CASES_ALL_D(Q5_0, F16) + FATTN_VEC_CASES_ALL_D(Q5_1, F16) + FATTN_VEC_CASES_ALL_D(Q8_0, F16) + FATTN_VEC_CASES_ALL_D(BF16, F16) + + FATTN_VEC_CASES_ALL_D(F16, Q4_0) + FATTN_VEC_CASES_ALL_D(Q4_0, Q4_0) + FATTN_VEC_CASES_ALL_D(Q4_1, Q4_0) + FATTN_VEC_CASES_ALL_D(Q5_0, Q4_0) + FATTN_VEC_CASES_ALL_D(Q5_1, Q4_0) + FATTN_VEC_CASES_ALL_D(Q8_0, Q4_0) + FATTN_VEC_CASES_ALL_D(BF16, Q4_0) + + FATTN_VEC_CASES_ALL_D(F16, Q4_1) + FATTN_VEC_CASES_ALL_D(Q4_0, Q4_1) + FATTN_VEC_CASES_ALL_D(Q4_1, Q4_1) + FATTN_VEC_CASES_ALL_D(Q5_0, Q4_1) + FATTN_VEC_CASES_ALL_D(Q5_1, Q4_1) + FATTN_VEC_CASES_ALL_D(Q8_0, Q4_1) + FATTN_VEC_CASES_ALL_D(BF16, Q4_1) + + FATTN_VEC_CASES_ALL_D(F16, Q5_0) + FATTN_VEC_CASES_ALL_D(Q4_0, Q5_0) + FATTN_VEC_CASES_ALL_D(Q4_1, Q5_0) + FATTN_VEC_CASES_ALL_D(Q5_0, Q5_0) + FATTN_VEC_CASES_ALL_D(Q5_1, Q5_0) + FATTN_VEC_CASES_ALL_D(Q8_0, Q5_0) + FATTN_VEC_CASES_ALL_D(BF16, Q5_0) + + FATTN_VEC_CASES_ALL_D(F16, Q5_1) + FATTN_VEC_CASES_ALL_D(Q4_0, Q5_1) + FATTN_VEC_CASES_ALL_D(Q4_1, Q5_1) + FATTN_VEC_CASES_ALL_D(Q5_0, Q5_1) + FATTN_VEC_CASES_ALL_D(Q5_1, Q5_1) + FATTN_VEC_CASES_ALL_D(Q8_0, Q5_1) + FATTN_VEC_CASES_ALL_D(BF16, Q5_1) + + FATTN_VEC_CASES_ALL_D(F16, Q8_0) + FATTN_VEC_CASES_ALL_D(Q4_0, Q8_0) + FATTN_VEC_CASES_ALL_D(Q4_1, Q8_0) + FATTN_VEC_CASES_ALL_D(Q5_0, Q8_0) + FATTN_VEC_CASES_ALL_D(Q5_1, Q8_0) + FATTN_VEC_CASES_ALL_D(Q8_0, Q8_0) + FATTN_VEC_CASES_ALL_D(BF16, Q8_0) + + FATTN_VEC_CASES_ALL_D(F16, BF16) + FATTN_VEC_CASES_ALL_D(Q4_0, BF16) + FATTN_VEC_CASES_ALL_D(Q4_1, BF16) + FATTN_VEC_CASES_ALL_D(Q5_0, BF16) + FATTN_VEC_CASES_ALL_D(Q5_1, BF16) + FATTN_VEC_CASES_ALL_D(Q8_0, BF16) + FATTN_VEC_CASES_ALL_D(BF16, BF16) + + return nullptr; +} static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - ggml_tensor * Q = dst->src[0]; - ggml_tensor * K = dst->src[1]; - ggml_tensor * V = dst->src[2]; - -#ifdef GGML_CUDA_FA_ALL_QUANTS - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_F16) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q4_0) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q4_1) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q5_0) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q5_1) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q8_0) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_BF16) -#else - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_BF16) -#endif // GGML_CUDA_FA_ALL_QUANTS + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; - GGML_ABORT("fatal error"); + fattn_vec_case_t vec_case = ggml_cuda_get_fattn_vec_case(Q->ne[0], K->type, V->type); + if (vec_case == nullptr) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("%s: no FlashAttention vector kernel compiled for K/V types %s-%s, converting K and V to f16 instead (slow). " + "Add \"%s-%s\" to GGML_CUDA_FA_QUANTS to compile it.\n", + __func__, ggml_type_name(K->type), ggml_type_name(V->type), ggml_type_name(K->type), ggml_type_name(V->type)); + warned = true; + } + vec_case = ggml_cuda_get_fattn_vec_case(Q->ne[0], GGML_TYPE_F16, GGML_TYPE_F16); + } + GGML_ASSERT(vec_case != nullptr); + vec_case(ctx, dst); } // Best FlashAttention kernel for a specific GPU: @@ -468,20 +479,17 @@ enum best_fattn_kernel { BEST_FATTN_KERNEL_MMA_F16 = 400, }; -static bool ggml_cuda_fattn_kv_type_supported(ggml_type type) { +// K/V types for which there is a vector kernel template instance, other kernels convert these to f16: +static bool ggml_cuda_fattn_kv_type_supported(const ggml_type type) { switch (type) { case GGML_TYPE_F32: case GGML_TYPE_F16: - return true; + case GGML_TYPE_BF16: + case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: -#ifndef GGML_CUDA_FA_ALL_QUANTS - return false; -#endif // GGML_CUDA_FA_ALL_QUANTS - case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: - case GGML_TYPE_BF16: return true; default: return false; @@ -572,12 +580,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_NONE; } -#ifndef GGML_CUDA_FA_ALL_QUANTS - if (K->type != V->type) { - return BEST_FATTN_KERNEL_NONE; - } -#endif // GGML_CUDA_FA_ALL_QUANTS - if (!ggml_cuda_fattn_kv_type_supported(K->type) || !ggml_cuda_fattn_kv_type_supported(V->type)) { return BEST_FATTN_KERNEL_NONE; } @@ -669,6 +671,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * dst) { GGML_ASSERT(dst->op == GGML_OP_FLASH_ATTN_EXT); + const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; @@ -686,10 +689,11 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d need_f16_K = true; need_f16_V = true; break; - case BEST_FATTN_KERNEL_VEC: - need_f16_K = K->type == GGML_TYPE_F32; - need_f16_V = V->type == GGML_TYPE_F32; - break; + case BEST_FATTN_KERNEL_VEC: { + const bool f16_fallback = ggml_cuda_get_fattn_vec_case(Q->ne[0], K->type, V->type) == nullptr; + need_f16_K = K->type == GGML_TYPE_F32 || f16_fallback; + need_f16_V = V->type == GGML_TYPE_F32 || f16_fallback; + } break; case BEST_FATTN_KERNEL_NONE: break; } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 38bd4c9a07e6..5ae3b8d22a36 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -5640,8 +5640,8 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t features.push_back({ "USE_GRAPHS", "1" }); #endif - #ifdef GGML_CUDA_FA_ALL_QUANTS - features.push_back({ "FA_ALL_QUANTS", "1" }); + #ifdef GGML_CUDA_FA_QUANTS + features.push_back({ "FA_QUANTS", GGML_CUDA_FA_QUANTS }); #endif { diff --git a/ggml/src/ggml-hip/CMakeLists.txt b/ggml/src/ggml-hip/CMakeLists.txt index 47f16f56c470..a6a6b7271c54 100644 --- a/ggml/src/ggml-hip/CMakeLists.txt +++ b/ggml/src/ggml-hip/CMakeLists.txt @@ -70,17 +70,8 @@ list(APPEND GGML_SOURCES_ROCM ${SRCS}) file(GLOB SRCS "../ggml-cuda/template-instances/mmf*.cu") list(APPEND GGML_SOURCES_ROCM ${SRCS}) -if (GGML_CUDA_FA_ALL_QUANTS) - file(GLOB SRCS "../ggml-cuda/template-instances/fattn-vec*.cu") - list(APPEND GGML_SOURCES_ROCM ${SRCS}) - add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS) -else() - list(APPEND GGML_SOURCES_ROCM - ../ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu) -endif() +ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR}/../ggml-cuda SRCS) +list(APPEND GGML_SOURCES_ROCM ${SRCS}) ggml_add_backend_library(ggml-hip ${GGML_HEADERS_ROCM} diff --git a/ggml/src/ggml-musa/CMakeLists.txt b/ggml/src/ggml-musa/CMakeLists.txt index faf9790338bb..82b754f41ee5 100644 --- a/ggml/src/ggml-musa/CMakeLists.txt +++ b/ggml/src/ggml-musa/CMakeLists.txt @@ -43,17 +43,8 @@ if (MUSAToolkit_FOUND) add_compile_definitions(GGML_MUSA_MUDNN_COPY) endif() - if (GGML_CUDA_FA_ALL_QUANTS) - file(GLOB SRCS "../ggml-cuda/template-instances/fattn-vec*.cu") - list(APPEND GGML_SOURCES_MUSA ${SRCS}) - add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS) - else() - list(APPEND GGML_SOURCES_MUSA - ../ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu) - endif() + ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR}/../ggml-cuda SRCS) + list(APPEND GGML_SOURCES_MUSA ${SRCS}) set_source_files_properties(${GGML_SOURCES_MUSA} PROPERTIES LANGUAGE CXX) foreach(SOURCE ${GGML_SOURCES_MUSA}) From d4abd573f6a360201799072384ceec6170fdb60c Mon Sep 17 00:00:00 2001 From: "Piotr Wilkin (ilintar)" Date: Wed, 9 Sep 2026 13:25:54 +0200 Subject: [PATCH 05/65] CUDA: size routed MoE MMQ N-tiles from typical expert width on RDNA3 (#28552) Recreated from #24546 --------- Co-authored-by: Carl Philipp Klemm * CUDA: pick MMQ tile size against ncols_opt set on the host side Assisted-by: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011SYPfRhKoUpU3gMsGxq6go --------- Co-authored-by: ravel7524 <58877666+ravel7524@users.noreply.github.com> Co-authored-by: Carl Philipp Klemm --- ggml/src/ggml-cuda/mmq.cu | 11 +++++++++-- ggml/src/ggml-cuda/mmq.cuh | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 9beff0d9b73a..9b6038adff9e 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -171,7 +171,7 @@ void ggml_cuda_mul_mat_q( ne00, ne01, ne1, s01, ne11, s1, ne02, ne12, s02, s12, s2, ne03, ne13, s03, s13, s3, - ne1}; + ne1, ne1}; ggml_cuda_mul_mat_q_switch_type(ctx, args, stream); return; } @@ -244,6 +244,13 @@ void ggml_cuda_mul_mat_q( ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); const int64_t s13 = ne12*s12; + // Each expert only sees ne12*n_expert_used/ne02 tokens on average. + // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12. + int64_t ncols_opt = ne12; + if (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) { + ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02; + } + // Note that ne02 is used instead of ne12 because the number of y channels determines the z dimension of the CUDA grid. const mmq_args args = { src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), dst_d, @@ -251,7 +258,7 @@ void ggml_cuda_mul_mat_q( ne00, ne01, ne_get_rows, s01, ne_get_rows, s1, ne02, ne02, s02, s12, s2, ne03, ne13, s03, s13, s3, - ne12}; + ne12, ncols_opt}; ggml_cuda_mul_mat_q_switch_type(ctx, args, stream); } diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index b4a747720f77..24afedd1432b 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -1376,6 +1376,7 @@ struct mmq_args { int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst; int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst; int64_t ncols_max; + int64_t ncols_opt; // value to optimize the tile size against, launch grid still uses ncols_max }; static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) { @@ -1486,7 +1487,7 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, continue; } - const int ntiles_x = (args.ncols_max + config.J - 1) / config.J; + const int ntiles_x = (args.ncols_opt + config.J - 1) / config.J; if (ntiles_x < ntiles_J_best) { J_best = J; From 4850c7727fa73bbe3098e10ee369fbc3467c445f Mon Sep 17 00:00:00 2001 From: linsen458-spec Date: Wed, 9 Sep 2026 20:27:25 +0800 Subject: [PATCH 06/65] llama : use int32_t for llama_sampler_chain_n return type (#28631) Contributes to #4574 Co-authored-by: linsen --- include/llama.h | 2 +- src/llama-sampler.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/llama.h b/include/llama.h index ef7a012c43a1..3ab935939c6d 100644 --- a/include/llama.h +++ b/include/llama.h @@ -1357,7 +1357,7 @@ extern "C" { LLAMA_API struct llama_sampler * llama_sampler_chain_get( struct llama_sampler * chain, int32_t i); // the total number of samplers in the chain - LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); + LLAMA_API int32_t llama_sampler_chain_n (const struct llama_sampler * chain); // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i); diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cpp index 34a7988262ea..61d28ad8a82e 100644 --- a/src/llama-sampler.cpp +++ b/src/llama-sampler.cpp @@ -1006,7 +1006,7 @@ struct llama_sampler * llama_sampler_chain_remove(struct llama_sampler * chain, return result; } -int llama_sampler_chain_n(const struct llama_sampler * chain) { +int32_t llama_sampler_chain_n(const struct llama_sampler * chain) { const auto * p = (const llama_sampler_chain *) chain->ctx; return p->samplers.size(); From 9cf3bf256b5a50a971a636c36dfe974387140687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 9 Sep 2026 14:59:37 +0200 Subject: [PATCH 07/65] py : bump numpy to 2.4.6 (#28649) --- gguf-py/pyproject.toml | 6 +++--- pyproject.toml | 4 ++-- requirements/requirements-convert_legacy_llama.txt | 2 +- requirements/requirements-gguf_editor_gui.txt | 2 +- requirements/requirements-server-bench.txt | 2 +- requirements/requirements-tool_bench.txt | 2 +- tools/server/tests/requirements.txt | 2 +- ty.toml | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml index d11c34a2186d..b4b0eecffab8 100644 --- a/gguf-py/pyproject.toml +++ b/gguf-py/pyproject.toml @@ -6,8 +6,8 @@ keywords = ["ggml", "gguf", "llama.cpp"] dynamic = ["classifiers"] readme = "README.md" authors = [{name = "GGML", email = "ggml@ggml.ai"}] -requires-python = '>=3.10' -dependencies = ['numpy (>=1.17)', 'tqdm (>=4.27)', 'pyyaml (>=5.1)', 'requests (>=2.25)'] +requires-python = '>=3.11' +dependencies = ['numpy (>=2.4.6)', 'tqdm (>=4.27)', 'pyyaml (>=5.1)', 'requests (>=2.25)'] classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", @@ -35,7 +35,7 @@ packages = [ ] [tool.poetry.dependencies] -python = ">=3.10" +python = ">=3.11" [tool.poetry.group.dev.dependencies] pytest = "^5.2" diff --git a/pyproject.toml b/pyproject.toml index 46cf68ca1a39..a19130d4d8c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,9 +6,9 @@ version = "0.0.0" dynamic = ["classifiers"] readme = "README.md" authors = [{name = "GGML", email = "ggml@ggml.ai"}] -requires-python = '>=3.10,<3.15' +requires-python = '>=3.11,<3.15' dependencies = [ - 'numpy (>=1.26.4,<3.0.0)', + 'numpy (>=2.4.6,<3.0.0)', 'sentencepiece (>=0.1.98,<0.3.0)', 'transformers (==4.57.6)', 'protobuf (>=4.21.0,<5.0.0)', diff --git a/requirements/requirements-convert_legacy_llama.txt b/requirements/requirements-convert_legacy_llama.txt index 28221fad0ce9..63525f25001d 100644 --- a/requirements/requirements-convert_legacy_llama.txt +++ b/requirements/requirements-convert_legacy_llama.txt @@ -1,4 +1,4 @@ -numpy~=1.26.4 +numpy~=2.4.6 sentencepiece>=0.1.98,<0.3.0 transformers==4.57.6 diff --git a/requirements/requirements-gguf_editor_gui.txt b/requirements/requirements-gguf_editor_gui.txt index fd253364e152..f132d6f8b5a1 100644 --- a/requirements/requirements-gguf_editor_gui.txt +++ b/requirements/requirements-gguf_editor_gui.txt @@ -1,3 +1,3 @@ -numpy~=1.26.4 +numpy~=2.4.6 PySide6~=6.9.0 gguf>=0.17.0 diff --git a/requirements/requirements-server-bench.txt b/requirements/requirements-server-bench.txt index fb3b0d2664b0..984e2bdf1d67 100644 --- a/requirements/requirements-server-bench.txt +++ b/requirements/requirements-server-bench.txt @@ -1,5 +1,5 @@ datasets~=4.8.0 matplotlib~=3.10.0 -numpy~=1.26.4 +numpy~=2.4.6 requests~=2.32.3 tqdm~=4.67.1 diff --git a/requirements/requirements-tool_bench.txt b/requirements/requirements-tool_bench.txt index 3e6f824165c4..07a658b6bf81 100644 --- a/requirements/requirements-tool_bench.txt +++ b/requirements/requirements-tool_bench.txt @@ -1,7 +1,7 @@ aiohttp~=3.9.3 pytest~=8.3.3 matplotlib~=3.10.0 -numpy~=1.26.4 +numpy~=2.4.6 openai~=2.14.0 pandas~=2.2.3 prometheus-client~=0.20.0 diff --git a/tools/server/tests/requirements.txt b/tools/server/tests/requirements.txt index 6c256f67d838..409fe674fe04 100644 --- a/tools/server/tests/requirements.txt +++ b/tools/server/tests/requirements.txt @@ -2,7 +2,7 @@ aiohttp~=3.9.3 pytest~=8.3.3 pytest-xdist~=3.6 filelock~=3.16 -numpy~=1.26.4 +numpy~=2.4.6 openai~=2.14.0 prometheus-client~=0.20.0 requests~=2.32.3 diff --git a/ty.toml b/ty.toml index 340b0649d334..fbc403a99ad5 100644 --- a/ty.toml +++ b/ty.toml @@ -1,6 +1,6 @@ [environment] extra-paths = ["./gguf-py", "./examples/model-conversion/scripts", "./tools/server/tests", "./scripts/snapdragon/qdc/tests"] -python-version = "3.10" +python-version = "3.11" [rules] deprecated = "warn" From 4b98ab805a2638121f1671bf572832e07ef13e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 9 Sep 2026 15:56:27 +0200 Subject: [PATCH 08/65] py : lower numpy to 2.2.6 (#28654) * Revert "py : bump numpy to 2.4.6 (#28649)" This reverts commit 9cf3bf256b5a50a971a636c36dfe974387140687. * bump numpy to 2.2.6 --- gguf-py/pyproject.toml | 6 +++--- pyproject.toml | 4 ++-- requirements/requirements-convert_legacy_llama.txt | 2 +- requirements/requirements-gguf_editor_gui.txt | 2 +- requirements/requirements-server-bench.txt | 2 +- requirements/requirements-tool_bench.txt | 2 +- tools/server/tests/requirements.txt | 2 +- ty.toml | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml index b4b0eecffab8..07e6f7fee82d 100644 --- a/gguf-py/pyproject.toml +++ b/gguf-py/pyproject.toml @@ -6,8 +6,8 @@ keywords = ["ggml", "gguf", "llama.cpp"] dynamic = ["classifiers"] readme = "README.md" authors = [{name = "GGML", email = "ggml@ggml.ai"}] -requires-python = '>=3.11' -dependencies = ['numpy (>=2.4.6)', 'tqdm (>=4.27)', 'pyyaml (>=5.1)', 'requests (>=2.25)'] +requires-python = '>=3.10' +dependencies = ['numpy (>=2.2.6)', 'tqdm (>=4.27)', 'pyyaml (>=5.1)', 'requests (>=2.25)'] classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", @@ -35,7 +35,7 @@ packages = [ ] [tool.poetry.dependencies] -python = ">=3.11" +python = ">=3.10" [tool.poetry.group.dev.dependencies] pytest = "^5.2" diff --git a/pyproject.toml b/pyproject.toml index a19130d4d8c0..0383fbc5e6d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,9 +6,9 @@ version = "0.0.0" dynamic = ["classifiers"] readme = "README.md" authors = [{name = "GGML", email = "ggml@ggml.ai"}] -requires-python = '>=3.11,<3.15' +requires-python = '>=3.10,<3.15' dependencies = [ - 'numpy (>=2.4.6,<3.0.0)', + 'numpy (>=2.2.6,<3.0.0)', 'sentencepiece (>=0.1.98,<0.3.0)', 'transformers (==4.57.6)', 'protobuf (>=4.21.0,<5.0.0)', diff --git a/requirements/requirements-convert_legacy_llama.txt b/requirements/requirements-convert_legacy_llama.txt index 63525f25001d..edc945c1cb67 100644 --- a/requirements/requirements-convert_legacy_llama.txt +++ b/requirements/requirements-convert_legacy_llama.txt @@ -1,4 +1,4 @@ -numpy~=2.4.6 +numpy~=2.2.6 sentencepiece>=0.1.98,<0.3.0 transformers==4.57.6 diff --git a/requirements/requirements-gguf_editor_gui.txt b/requirements/requirements-gguf_editor_gui.txt index f132d6f8b5a1..93fe087223eb 100644 --- a/requirements/requirements-gguf_editor_gui.txt +++ b/requirements/requirements-gguf_editor_gui.txt @@ -1,3 +1,3 @@ -numpy~=2.4.6 +numpy~=2.2.6 PySide6~=6.9.0 gguf>=0.17.0 diff --git a/requirements/requirements-server-bench.txt b/requirements/requirements-server-bench.txt index 984e2bdf1d67..0b065b1ea4c1 100644 --- a/requirements/requirements-server-bench.txt +++ b/requirements/requirements-server-bench.txt @@ -1,5 +1,5 @@ datasets~=4.8.0 matplotlib~=3.10.0 -numpy~=2.4.6 +numpy~=2.2.6 requests~=2.32.3 tqdm~=4.67.1 diff --git a/requirements/requirements-tool_bench.txt b/requirements/requirements-tool_bench.txt index 07a658b6bf81..ba865115026d 100644 --- a/requirements/requirements-tool_bench.txt +++ b/requirements/requirements-tool_bench.txt @@ -1,7 +1,7 @@ aiohttp~=3.9.3 pytest~=8.3.3 matplotlib~=3.10.0 -numpy~=2.4.6 +numpy~=2.2.6 openai~=2.14.0 pandas~=2.2.3 prometheus-client~=0.20.0 diff --git a/tools/server/tests/requirements.txt b/tools/server/tests/requirements.txt index 409fe674fe04..5e6dff298f4e 100644 --- a/tools/server/tests/requirements.txt +++ b/tools/server/tests/requirements.txt @@ -2,7 +2,7 @@ aiohttp~=3.9.3 pytest~=8.3.3 pytest-xdist~=3.6 filelock~=3.16 -numpy~=2.4.6 +numpy~=2.2.6 openai~=2.14.0 prometheus-client~=0.20.0 requests~=2.32.3 diff --git a/ty.toml b/ty.toml index fbc403a99ad5..340b0649d334 100644 --- a/ty.toml +++ b/ty.toml @@ -1,6 +1,6 @@ [environment] extra-paths = ["./gguf-py", "./examples/model-conversion/scripts", "./tools/server/tests", "./scripts/snapdragon/qdc/tests"] -python-version = "3.11" +python-version = "3.10" [rules] deprecated = "warn" From 22397c31a00e78f55ae556c41fc78b717c5911bd Mon Sep 17 00:00:00 2001 From: Masato Nakasaka Date: Wed, 9 Sep 2026 07:54:15 -0700 Subject: [PATCH 09/65] vulkan: Convert FILL to distribute workgroups in 2D to avoid exceeding maxComputeWorkGroupCount (#28592) * divide workload to 2D This is to workaround FILL exceeding maxComputeWorkGroupCount for Intel GPUs on Qwen 3.8 flash next * minor change * Fixed comment --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 13 ++++++++++--- ggml/src/ggml-vulkan/vulkan-shaders/fill.comp | 4 +++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index b3a7cb6ab6ac..9eae8dab9c39 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -13748,9 +13748,11 @@ static void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, gg static void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { VK_LOG_DEBUG("ggml_vk_fill(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")"); + const uint64_t n = ggml_nelements(dst); + GGML_ASSERT(n > 0); vk_op_push_constants pc = { - (uint32_t)ggml_nelements(dst), + (uint32_t)n, 1, ggml_get_op_params_f32(dst, 0), 0.0f, @@ -13760,11 +13762,16 @@ static void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_FILL); GGML_ASSERT(pipeline != nullptr); + // Split the task distribution to 2D to avoid exceeding maxComputeWorkGroupCount + const uint32_t total_wg = CEIL_DIV(n, pipeline->wg_denoms[0]); + const uint32_t wg_x = std::min(total_wg, ctx->device->properties.limits.maxComputeWorkGroupCount[0]); + const uint32_t wg_y = CEIL_DIV(total_wg, wg_x); + GGML_ASSERT(wg_y <= ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, false); - std::array elements = { (uint32_t)ggml_nelements(dst), 1, 1 }; - + std::array elements = { wg_x * pipeline->wg_denoms[0], wg_y * pipeline->wg_denoms[1], 1 }; ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp b/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp index a56be76c61c5..b5cc33322078 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp @@ -8,7 +8,9 @@ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) writeonly buffer D {D_TYPE data_d[];}; void main() { - const uint i = gl_GlobalInvocationID.x; + // 2D grid flattening: each x workgroup covers gl_WorkGroupSize.x elements, + // each y workgroup covers gl_NumWorkGroups.x * gl_WorkGroupSize.x elements. + const uint i = (gl_GlobalInvocationID.y * gl_NumWorkGroups.x * gl_WorkGroupSize.x) + gl_GlobalInvocationID.x; if (i >= p.KX) { return; From 6d9c82ea2bb34e277c0664b8dd3434bfb4dcfb27 Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Wed, 9 Sep 2026 08:40:24 -0700 Subject: [PATCH 10/65] hexagon: rope updates (#28628) * hexagon: vectorize RoPE theta cache on v75 * hexagon: vectorize MROPE/IMROPE theta pick * hexagon: tighten NEOX RoPE rotate and aligned tail copy * hex-rope: use inplace rope for all scenarios * hex-rope: remove ctx->spad usage and legacy timers * hex-rope: add kernel params and enforce vtcm reqs at the host * hex-rope: cleanup unused params and tighten the mode checks * hex-rope: add missing ops header --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 131 +++-- ggml/src/ggml-hexagon/htp/hvx-sin-cos.h | 110 ++--- ggml/src/ggml-hexagon/htp/rope-ops.c | 627 +++++++++++++----------- ggml/src/ggml-hexagon/htp/rope-ops.h | 56 +++ tests/test-backend-ops.cpp | 12 + 5 files changed, 555 insertions(+), 381 deletions(-) create mode 100644 ggml/src/ggml-hexagon/htp/rope-ops.h diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index a39df2a878c5..112e9bae6020 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -56,6 +56,7 @@ #include "htp/unary-ops.h" #include "htp/get-rows-ops.h" #include "htp/set-rows-ops.h" +#include "htp/rope-ops.h" #include "htp_iface.h" #include "htp-drv.h" @@ -299,6 +300,12 @@ static void ggml_hexagon_precompute_set_rows_params( struct htp_set_rows_kernel_params * kparams ); +static void ggml_hexagon_precompute_rope_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_rope_kernel_params * kparams +); + static void ggml_hexagon_precompute_fused_mmnx_params( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, @@ -4148,6 +4155,36 @@ static void ggml_hexagon_precompute_set_rows_params( kparams->vtcm_size = vtcm_layout.total_bytes; } +static void ggml_hexagon_precompute_rope_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_rope_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); + + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads); + + kparams->n_threads = n_threads; + kparams->src0_nrows = src0_nrows; + kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread; + kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned; + kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned; + + if (src0_nrows > 0) { + kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); + kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); + } +} + static void ggml_hexagon_precompute_fused_mmnx_params( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, // W0 @@ -4706,56 +4743,82 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s } static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const int32_t * op_params = &op->op_params[0]; + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * src2 = op->src[2]; + const struct ggml_tensor * dst = op; - // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems) - if (op_params[15] % 32 != 0) { + if (!ggml_are_same_shape(src0, dst)) { return false; } - int mode = op_params[2]; + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) { + return false; + } - // n_dims == ne0/2, so the rotation spans the full row - if (mode == GGML_ROPE_TYPE_VISION) { - const int n_dims = op_params[1]; - if (n_dims != (int) (op->src[0]->ne[0] / 2)) { - return false; - } + if (src0->ne[0] <= 0) { + return false; } - if (mode & 1) { + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + if (src0_nrows == 0) { return false; } - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; + const int32_t * op_params = &op->op_params[0]; + const int n_dims = op_params[1]; + const int mode = op_params[2]; + const int n_offs = op_params[15]; - if (src0->type != GGML_TYPE_F32) { - return false; // FIXME: add support for GGML_TYPE_F16 for src0 + if (n_dims <= 0 || n_dims % 2 != 0) { + return false; } - if (dst->type != GGML_TYPE_F32) { + + // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems) + if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) { return false; } - if (src1->type != GGML_TYPE_I32) { + + float freq_base; + memcpy(&freq_base, op_params + 5, sizeof(float)); + if (freq_base <= 0.0f) { return false; } - if (src2) { - if (src2->type != GGML_TYPE_F32) { + + if (mode != GGML_ROPE_TYPE_NORMAL && + mode != GGML_ROPE_TYPE_NEOX && + mode != GGML_ROPE_TYPE_MROPE && + mode != GGML_ROPE_TYPE_VISION && + mode != GGML_ROPE_TYPE_IMROPE) { + return false; + } + + const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0; + + // n_dims == ne0/2, so the rotation spans the full row + if (mode == GGML_ROPE_TYPE_VISION) { + if (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0) { return false; } - int n_dims = op_params[1]; - if (src2->ne[0] < (n_dims / 2)) { + } + + if (is_mrope) { + const int32_t * sections = op_params + 11; + if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) { return false; } } + const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2]; + if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) { + return false; + } + if (src2) { - if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(src2)) { + if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2)) { return false; } - } else { - if (!ggml_is_contiguous(src1)) { + if (src2->ne[0] < (n_dims / 2)) { return false; } } @@ -4768,9 +4831,16 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) { return false; } - return true; - GGML_UNUSED(sess); + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); + + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads); + if (layout.total_bytes > sess->vtcm_size) { + return false; + } + + return true; } static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -5206,6 +5276,11 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg node.node->src[0], node.node->src[1], node.dst(), (struct htp_set_rows_kernel_params *)node.kernel_params ); + } else if (node.opcode == HTP_OP_ROPE) { + ggml_hexagon_precompute_rope_params(sess, + node.node, + (struct htp_rope_kernel_params *)node.kernel_params + ); } computed_nodes.push_back(std::move(node)); } diff --git a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h index c5b9a5d47c17..8648af0e5b95 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h @@ -4,87 +4,75 @@ #include "hvx-base.h" #include "hvx-floor.h" -static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) { - HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); - HVX_Vector const_half = hvx_vec_splat_f32(0.5f); - HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); - HVX_Vector const_one = hvx_vec_splat_f32(1.0f); +// Range-reduce x to y in [-pi/2, pi/2] and the quadrant sign (-1)^n. +// Floor/truncate need IEEE bits, so convert qf32 back to sf before them. +static inline void hvx_vec_sincos_reduce_f32(HVX_Vector x, HVX_Vector * y, HVX_Vector * sign) { + HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); + HVX_Vector const_half = hvx_vec_splat_f32(0.5f); + HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); + HVX_Vector const_one = hvx_vec_splat_f32(1.0f); HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f); + HVX_Vector const_one_i = Q6_V_vsplat_R(1); + + HVX_Vector x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, const_inv_pi)); + x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(x_over_pi, const_half)); - // n = floor(x * (1/pi) + 0.5) - HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half)); + HVX_Vector n_float = hvx_vec_floor_f32(x_over_pi); + HVX_Vector n_int = hvx_vec_truncate_f32(n_float); - // y = x - n * pi - HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi)); + HVX_Vector n_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(n_float, const_pi)); + *y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(x, n_pi)); - // Sign determination: if n is odd, sign is -1.0f, else 1.0f - // half_n = n * 0.5f - HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half); - // floor_half_n = floor(half_n) - HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n); - // is_odd = half_n > floor_half_n - HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n); - // sign = vmux(is_odd, -1.0f, 1.0f) - HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); + HVX_VectorPred is_odd = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(n_int, const_one_i), const_one_i); + *sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); +} - // z = y^2 - HVX_Vector z = hvx_vec_mul_f32_f32(y, y); +static inline void hvx_vec_sincos_f32(HVX_Vector x, HVX_Vector * vcos, HVX_Vector * vsin) { + HVX_Vector y; + HVX_Vector sign; + hvx_vec_sincos_reduce_f32(x, &y, &sign); + + HVX_Vector z = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, y)); - // Chebyshev approximation for cos(y) HVX_Vector c4 = hvx_vec_splat_f32(2.3557242013849433e-05f); HVX_Vector c3 = hvx_vec_splat_f32(-0.0013871428263450528f); HVX_Vector c2 = hvx_vec_splat_f32(0.041665895266688284f); HVX_Vector c1 = hvx_vec_splat_f32(-0.4999999360426369f); HVX_Vector c0 = hvx_vec_splat_f32(0.9999999999071725f); - HVX_Vector cos_y = hvx_vec_add_f32_f32(c3, hvx_vec_mul_f32_f32(z, c4)); - cos_y = hvx_vec_add_f32_f32(c2, hvx_vec_mul_f32_f32(z, cos_y)); - cos_y = hvx_vec_add_f32_f32(c1, hvx_vec_mul_f32_f32(z, cos_y)); - cos_y = hvx_vec_add_f32_f32(c0, hvx_vec_mul_f32_f32(z, cos_y)); - - return hvx_vec_mul_f32_f32(cos_y, sign); -} - -static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) { - HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); - HVX_Vector const_half = hvx_vec_splat_f32(0.5f); - HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); - HVX_Vector const_one = hvx_vec_splat_f32(1.0f); - HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f); - - // n = floor(x * (1/pi) + 0.5) - HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half)); + HVX_Vector cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, c4)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); - // y = x - n * pi - HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi)); - - // Sign determination: if n is odd, sign is -1.0f, else 1.0f - // half_n = n * 0.5f - HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half); - // floor_half_n = floor(half_n) - HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n); - // is_odd = half_n > floor_half_n - HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n); - // sign = vmux(is_odd, -1.0f, 1.0f) - HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); - - // z = y^2 - HVX_Vector z = hvx_vec_mul_f32_f32(y, y); - - // Chebyshev approximation for sin(y) HVX_Vector s4 = hvx_vec_splat_f32(2.642186986152672e-06f); HVX_Vector s3 = hvx_vec_splat_f32(-0.00019825318964070864f); HVX_Vector s2 = hvx_vec_splat_f32(0.00833326283319605f); HVX_Vector s1 = hvx_vec_splat_f32(-0.16666666082087775f); HVX_Vector s0 = hvx_vec_splat_f32(0.999999999915155f); - HVX_Vector sin_y = hvx_vec_add_f32_f32(s3, hvx_vec_mul_f32_f32(z, s4)); - sin_y = hvx_vec_add_f32_f32(s2, hvx_vec_mul_f32_f32(z, sin_y)); - sin_y = hvx_vec_add_f32_f32(s1, hvx_vec_mul_f32_f32(z, sin_y)); - sin_y = hvx_vec_add_f32_f32(s0, hvx_vec_mul_f32_f32(z, sin_y)); - sin_y = hvx_vec_mul_f32_f32(y, sin_y); + HVX_Vector sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, s4)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, sin_y)); + + *vcos = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(cos_y, sign)); + *vsin = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(sin_y, sign)); +} - return hvx_vec_mul_f32_f32(sin_y, sign); +static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) { + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(x, &vcos, &vsin); + return vcos; +} + +static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) { + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(x, &vcos, &vsin); + return vsin; } #endif /* HVX_SIN_COS_H */ diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 6c689824934f..0a4b31ccb1d8 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -17,8 +17,8 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" #include "htp-tensor.h" +#include "rope-ops.h" // Redefined the rope type constants as we can't include ggml.h #define HTP_ROPE_TYPE_NORMAL 0 @@ -27,9 +27,6 @@ #define HTP_ROPE_TYPE_VISION 24 #define HTP_ROPE_TYPE_IMROPE 40 -#define HTP_ROPE_SPAD_NROWS 16 -#define HTP_ROPE_SPAD_BLOCK (HTP_ROPE_SPAD_NROWS/2) - #define htp_rope_preamble \ const uint32_t ne00 = src0->ne[0]; \ const uint32_t ne01 = src0->ne[1]; \ @@ -65,26 +62,27 @@ struct htp_rope_context { float beta_fast; float beta_slow; float theta_scale; + float theta_scale_32; + float theta_powers[32]; float corr_dims[2]; uint32_t src0_nrows_per_thread; - size_t spad_stride; struct htp_ops_context * octx; + uint8_t * vtcm_base; + size_t spad_per_thread; + size_t theta_cache_offset; + size_t src0_row_size; size_t src0_row_stride; size_t dst_row_size; size_t dst_row_stride; size_t src0_row_size_aligned; - size_t dst_row_size_aligned; - size_t theta_cache_offset; uint32_t src0_nrows; struct fastdiv_values div_ne2_ne1; struct fastdiv_values div_ne1; - - uint64_t t_start; }; static float rope_yarn_ramp(const float low, const float high, const int i0) { @@ -112,94 +110,80 @@ static inline void rope_yarn_one(float theta, float freq_scale, float * corr_dim mscale_final *= 1.0f + 0.1f * logf(1.0f / freq_scale); } - cache[i0 + 0] = cosf(theta_final) * mscale_final; - cache[i0 + 1] = sinf(theta_final) * mscale_final; + const uint32_t b = i0 / 64; + const uint32_t k = (i0 % 64) / 2; + cache[b * 64 + k] = cosf(theta_final) * mscale_final; + cache[b * 64 + 32 + k] = sinf(theta_final) * mscale_final; +} + +// 32 thetas -> 32 deinterleaved pairs [cos[32] | sin[32]] at cache[i0]. +static inline void rope_cache_hvx_32(float * cache, uint32_t i0, + HVX_Vector v_theta, + const float * freq_factors, + HVX_Vector v_freq_scale, + HVX_Vector v_mscale) { + if (freq_factors) { + HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2); + v_theta = hvx_vec_mul_f32_f32(v_theta, hvx_vec_inverse_f32(v_ff)); + } + + HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale); + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(v_theta_final, &vcos, &vsin); + vcos = hvx_vec_mul_f32_f32(vcos, v_mscale); + vsin = hvx_vec_mul_f32_f32(vsin, v_mscale); + + if (((uintptr_t) (cache + i0)) % 128 == 0) { + hvx_vmem(cache + i0 + 0) = vcos; + hvx_vmem(cache + i0 + 32) = vsin; + } else { + hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), vcos); + hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), vsin); + } } static __attribute__((noinline)) void rope_cache_init(const float theta_base, const float freq_scale, const float * freq_factors, float * corr_dims, - const uint32_t ne0, + const uint32_t n_cache, const float ext_factor, const float mscale, float * cache, - const float theta_scale) { + const float theta_scale, + const float * theta_powers, + const float theta_scale_32) { // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py -#if __HVX_ARCH__ >= 79 - const bool is_v79_or_newer = true; -#else - const bool is_v79_or_newer = false; -#endif - - if (is_v79_or_newer && ext_factor == 0.0f) { + if (ext_factor == 0.0f) { // Fast path: fully vectorized // We process 32 pairs (64 elements) per iteration. - const uint32_t n_blocks = ne0 / 64; - - // Initialize theta scale powers: [1.0f, theta_scale, theta_scale^2, ..., theta_scale^31] - float __attribute__((aligned(128))) theta_powers[32]; - theta_powers[0] = 1.0f; - for (int j = 1; j < 32; j++) { - theta_powers[j] = theta_powers[j - 1] * theta_scale; - } - HVX_Vector v_theta_powers = hvx_vmem(theta_powers); + const uint32_t n_blocks = n_cache / 64; + HVX_Vector v_theta_powers = hvx_vmemu(theta_powers); HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); - // Base theta starts at theta_base float theta_block = theta_base; - // The scale factor for the next block is theta_scale^32 - float theta_scale_32 = 1.0f; - for (int j = 0; j < 32; j++) { - theta_scale_32 *= theta_scale; - } for (uint32_t b = 0; b < n_blocks; b++) { uint32_t i0 = b * 64; HVX_Vector v_theta_base = hvx_vec_splat_f32(theta_block); HVX_Vector v_theta = hvx_vec_mul_f32_f32(v_theta_base, v_theta_powers); - - if (freq_factors) { - // Load 32 elements of freq_factors - HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2); - HVX_Vector v_inv_ff = hvx_vec_inverse_f32(v_ff); - v_theta = hvx_vec_mul_f32_f32(v_theta, v_inv_ff); - } - - HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale); - - HVX_Vector vcos = hvx_vec_cos_f32(v_theta_final); - HVX_Vector vsin = hvx_vec_sin_f32(v_theta_final); - - vcos = hvx_vec_mul_f32_f32(vcos, v_mscale); - vsin = hvx_vec_mul_f32_f32(vsin, v_mscale); - - HVX_VectorPair vstore = Q6_W_vshuff_VVR(vsin, vcos, -4); - - if (((uintptr_t)cache) % 128 == 0) { - hvx_vmem(cache + i0 + 0) = Q6_V_lo_W(vstore); - hvx_vmem(cache + i0 + 32) = Q6_V_hi_W(vstore); - } else { - hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), Q6_V_lo_W(vstore)); - hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), Q6_V_hi_W(vstore)); - } - + rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale); theta_block *= theta_scale_32; } // Leftovers float theta = theta_block; - for (uint32_t i0 = n_blocks * 64; i0 < ne0; i0 += 2) { + for (uint32_t i0 = n_blocks * 64; i0 < n_cache; i0 += 2) { const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); theta *= theta_scale; } } else { - // Fallback to original scalar loop float theta = theta_base; - for (uint32_t i0 = 0; i0 < ne0; i0 += 2) { + for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) { const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); theta *= theta_scale; @@ -207,6 +191,72 @@ static __attribute__((noinline)) void rope_cache_init(const float theta_base, } } +static inline float mrope_pick_theta(float theta_t, float theta_h, float theta_w, float theta_e, + int sector, const int32_t sections[4], int sec_w, int sec_e, + bool is_imrope) { + if (is_imrope) { + if (sector % 3 == 0 && sector < 3 * sections[0]) { return theta_t; } + else if (sector % 3 == 1 && sector < 3 * sections[1]) { return theta_h; } + else if (sector % 3 == 2 && sector < 3 * sections[2]) { return theta_w; } + else { return theta_e; } + } + if (sector < sections[0]) { return theta_t; } + else if (sector < sec_w) { return theta_h; } + else if (sector < sec_e) { return theta_w; } + else { return theta_e; } +} + +// lane j is 1 when (j % 3) == rem +static const float __attribute__((aligned(128))) mrope_mod3_eq0[32] = { + 1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0 +}; +static const float __attribute__((aligned(128))) mrope_mod3_eq1[32] = { + 0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1 +}; +static const float __attribute__((aligned(128))) mrope_mod3_eq2[32] = { + 0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0 +}; + +static const float __attribute__((aligned(128))) mrope_k_ramp[32] = { + 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, + 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +}; + +static inline HVX_VectorPred mrope_mask_eq1(const float * m) { + return Q6_Q_vcmp_gt_VsfVsf(hvx_vmemu(m), Q6_V_vzero()); +} + +// IMROPE without wrap: theta[k] = pos[k % 3] * scale^k +static inline HVX_Vector mrope_thetas_imrope_mod3(float pos_t, float pos_h, float pos_w, + uint32_t k0, HVX_Vector v_powers, float scale_block) { + const int r = (int) (k0 % 3); + const float * mt = (r == 0) ? mrope_mod3_eq0 : (r == 1) ? mrope_mod3_eq2 : mrope_mod3_eq1; + const float * mh = (r == 0) ? mrope_mod3_eq1 : (r == 1) ? mrope_mod3_eq0 : mrope_mod3_eq2; + + HVX_Vector v = hvx_vec_splat_f32(pos_w); + v = Q6_V_vmux_QVV(mrope_mask_eq1(mh), hvx_vec_splat_f32(pos_h), v); + v = Q6_V_vmux_QVV(mrope_mask_eq1(mt), hvx_vec_splat_f32(pos_t), v); + v = hvx_vec_mul_f32_f32(v, v_powers); + return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block)); +} + +// Contiguous MROPE without wrap: theta[k] = pos[section(k)] * scale^k +static inline HVX_Vector mrope_thetas_contig(float pos_t, float pos_h, float pos_w, float pos_e, + uint32_t k0, int s0, int sec_w, int sec_e, + HVX_Vector v_powers, float scale_block) { + HVX_Vector v_k = hvx_vec_add_f32_f32(hvx_vec_splat_f32((float) k0), hvx_vmemu(mrope_k_ramp)); + HVX_VectorPred lt_s0 = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) s0), v_k); + HVX_VectorPred lt_sw = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_w), v_k); + HVX_VectorPred lt_se = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_e), v_k); + + HVX_Vector v = hvx_vec_splat_f32(pos_e); + v = Q6_V_vmux_QVV(lt_se, hvx_vec_splat_f32(pos_w), v); + v = Q6_V_vmux_QVV(lt_sw, hvx_vec_splat_f32(pos_h), v); + v = Q6_V_vmux_QVV(lt_s0, hvx_vec_splat_f32(pos_t), v); + v = hvx_vec_mul_f32_f32(v, v_powers); + return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block)); +} + // pos_t/h/w/e: the four position ids for this sequence step (t=time, h=height, w=width, e=extra). // sections[4]: number of head dims assigned to each position component. static __attribute__((noinline)) void mrope_cache_init(const float pos_t, @@ -219,23 +269,71 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, const float freq_scale, const float * freq_factors, float * corr_dims, - const uint32_t ne0, + const uint32_t n_cache, const float ext_factor, const float mscale, float * cache, - const float theta_scale) { + const float theta_scale, + const float * theta_powers, + const float theta_scale_32) { const int sect_dims = sections[0] + sections[1] + sections[2] + sections[3]; const int sec_w = sections[0] + sections[1]; const int sec_e = sec_w + sections[2]; + const uint32_t n_pairs = n_cache / 2; + + const bool no_wrap = (sect_dims > 0) && (n_pairs <= (uint32_t) sect_dims); + const bool imrope_mod3 = is_imrope && !indep_sects && no_wrap + && sections[0] > 0 && sections[1] > 0 && sections[2] > 0 + && n_pairs <= (uint32_t) (3 * sections[0]) + && n_pairs <= (uint32_t) (3 * sections[1]) + && n_pairs <= (uint32_t) (3 * sections[2]); + const bool contig = !is_imrope && !indep_sects && no_wrap; + + if (ext_factor == 0.0f && (imrope_mod3 || contig)) { + HVX_Vector v_powers = hvx_vmemu(theta_powers); + HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); + HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); + float scale_block = 1.0f; + const uint32_t n_blocks = n_cache / 64; + + for (uint32_t b = 0; b < n_blocks; b++) { + const uint32_t i0 = b * 64; + const uint32_t k0 = b * 32; + HVX_Vector v_theta = imrope_mod3 + ? mrope_thetas_imrope_mod3(pos_t, pos_h, pos_w, k0, v_powers, scale_block) + : mrope_thetas_contig(pos_t, pos_h, pos_w, pos_e, k0, sections[0], sec_w, sec_e, + v_powers, scale_block); + rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale); + scale_block *= theta_scale_32; + } + + float theta_k = scale_block; + for (uint32_t k = n_blocks * 32; k < n_pairs; k++) { + const uint32_t i0 = 2 * k; + const float pos = mrope_pick_theta(pos_t, pos_h, pos_w, pos_e, + (int) k, sections, sec_w, sec_e, is_imrope); + const float ff = freq_factors ? freq_factors[k] : 1.0f; + rope_yarn_one(pos * theta_k / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); + theta_k *= theta_scale; + } + return; + } float theta_t = pos_t; float theta_h = pos_h; float theta_w = pos_w; float theta_e = pos_e; - for (uint32_t i0 = 0; i0 < ne0; i0 += 2) { - const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; - const int sector = (i0 / 2) % sect_dims; + const bool use_hvx = (ext_factor == 0.0f); + float __attribute__((aligned(128))) thetas[32]; + uint32_t n_thetas = 0; + uint32_t block_i0 = 0; + + HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); + HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); + + for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) { + const int sector = (i0 / 2) % sect_dims; if (indep_sects) { // Reset theta when crossing into a new section. @@ -245,28 +343,34 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, else if (sector == sec_e) { theta_e = pos_e; } } - float theta; - if (is_imrope) { - // Interleaved: sector mod 3 selects component - if (sector % 3 == 0 && sector < 3 * sections[0]) { theta = theta_t; } - else if (sector % 3 == 1 && sector < 3 * sections[1]) { theta = theta_h; } - else if (sector % 3 == 2 && sector < 3 * sections[2]) { theta = theta_w; } - else { theta = theta_e; } + const float theta = mrope_pick_theta(theta_t, theta_h, theta_w, theta_e, + sector, sections, sec_w, sec_e, is_imrope); + + if (use_hvx) { + if (n_thetas == 0) { + block_i0 = i0; + } + thetas[n_thetas++] = theta; + if (n_thetas == 32) { + rope_cache_hvx_32(cache, block_i0, hvx_vmemu(thetas), freq_factors, v_freq_scale, v_mscale); + n_thetas = 0; + } } else { - // Contiguous sections - if (sector < sections[0]) { theta = theta_t; } - else if (sector < sec_w) { theta = theta_h; } - else if (sector < sec_e) { theta = theta_w; } - else { theta = theta_e; } + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); } - rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); - theta_t *= theta_scale; theta_h *= theta_scale; theta_w *= theta_scale; theta_e *= theta_scale; } + + for (uint32_t k = 0; k < n_thetas; k++) { + const uint32_t i0 = block_i0 + 2 * k; + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(thetas[k] / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); + } } #define M_PI 3.1415926535897932384626433 @@ -283,52 +387,54 @@ static void rope_corr_dims(int n_dims, dims[1] = MIN(n_dims - 1, end); } +static inline void hvx_rope_neox_mul(HVX_Vector v0, HVX_Vector v1, HVX_Vector vcos, HVX_Vector vsin, + HVX_Vector * o0, HVX_Vector * o1) { + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, vsin); + *o0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s)); + *o1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c)); +} + +// theta_cache full 32-pair blocks are deinterleaved [cos | sin]. static inline void hvx_rope_neox_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) { const uint32_t he = ne / 2; const uint32_t nvec = he / 32; const uint32_t nloe = he % 32; - for (uint32_t i = 0; i < nvec; i++) { - HVX_Vector v0 = ((const HVX_Vector *) src0)[i]; - HVX_Vector v1 = hvx_vmemu(src0 + he + i * 32); - - HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; - HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; - - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); - - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin)); - - HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); - HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); - - ((HVX_Vector *) dst)[i] = Q6_Vsf_equals_Vqf32(v4); - hvx_vmemu(dst + he + i * 32) = Q6_Vsf_equals_Vqf32(v5); + if (nloe == 0) { + const HVX_Vector * vs = (const HVX_Vector *) src0; + const HVX_Vector * vt = (const HVX_Vector *) theta_cache; + HVX_Vector * vd = (HVX_Vector *) dst; + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector o0, o1; + hvx_rope_neox_mul(vs[i], vs[nvec + i], vt[i * 2 + 0], vt[i * 2 + 1], &o0, &o1); + vd[i] = o0; + vd[nvec + i] = o1; + } + return; } - if (nloe > 0) { - HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32); - HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32); - - HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 0]; - HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 1]; - - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); - - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin)); - - HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); - HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); - - hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v4)); - hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v5)); + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector o0, o1; + hvx_rope_neox_mul(((const HVX_Vector *) src0)[i], + hvx_vmemu(src0 + he + i * 32), + ((const HVX_Vector *) theta_cache)[i * 2 + 0], + ((const HVX_Vector *) theta_cache)[i * 2 + 1], + &o0, &o1); + ((HVX_Vector *) dst)[i] = o0; + hvx_vmemu(dst + he + i * 32) = o1; } + + HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32); + HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); + HVX_Vector o0, o1; + hvx_rope_neox_mul(v0, v1, vcos, vsin, &o0, &o1); + hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), o0); + hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), o1); } static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) { @@ -339,16 +445,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict HVX_Vector v0 = ((const HVX_Vector *) src0)[i * 2 + 0]; HVX_Vector v1 = ((const HVX_Vector *) src0)[i * 2 + 1]; - HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; - HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; + HVX_Vector vcos = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; + HVX_Vector vsin = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); @@ -362,15 +467,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict if (nloe > 0) { if (nloe <= 32) { HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64); - HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(Q6_V_vzero(), v2, -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); @@ -382,16 +487,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64); HVX_Vector v1 = hvx_vmemu(src0 + nvec * 64 + 32); - HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64); - HVX_Vector v3 = hvx_vmemu(theta_cache + nvec * 64 + 32); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); @@ -404,54 +508,23 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict } } -static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { - const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) - #pragma unroll(4) - for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); - float * s = (float *) (src + i * rctx->src0_row_size_aligned); - - hvx_rope_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache); - - // fill the remain channels with data from src tensor - if (n_offs > 0) { - hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs); - } - if (n_offs + rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims); - } - } -} - -static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { - const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) +static void inline rope_basic_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, + uint32_t nr, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - - hvx_rope_neox_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache); - - // fill the remain channels with data from src tensor - if (n_offs > 0) { - hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs); - } - if (n_offs + rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims); - } + hvx_rope_f32_aa(s + n_offs, s + n_offs, rctx->n_dims, theta_cache); } } -static void inline rope_vision_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { +static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, + uint32_t nr, uint32_t ne, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - - hvx_rope_neox_f32_aa(d, s, ne0, theta_cache); + hvx_rope_neox_f32_aa(s + n_offs, s + n_offs, ne, theta_cache); } } @@ -477,20 +550,18 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { return; } - uint64_t tt = HAP_perf_get_qtimer_count(); - const int32_t mode = rctx->mode; // MROPE, IMROPE and VISION use NEOX-style pairing for the rotation const bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE); const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); // VTCM setup - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); + uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread); float * theta_cache = (float *) (src0_spad_base); src0_spad_base = src0_spad_base + rctx->theta_cache_offset; - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); dma_queue * dma_queue = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const int32_t * pos = (const int32_t *) src1->data; const float * freq_factors = src2 ? (const float *) src2->data : NULL; @@ -501,6 +572,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { uint32_t ir = src0_start_row; uint32_t prev_i2 = (uint32_t) -1; + uint32_t cur_slot = 0; for (uint32_t i3 = i3_start; i3 < ne3; i3++) { // batch const uint32_t i2_init = (i3 == i3_start) ? i2_start : 0; @@ -513,35 +585,30 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { const uint32_t nrows = MIN(src0_end_row - ir, ne1 - i1); // Depth before prefetch - uint32_t dma_depth = dma_queue_depth(dma_queue); - - // FARF(HIGH, "rope-block %u: ir %u n-rows %u dma-depth %u : usec %u", ith, ir, nrows, dma_depth, - // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); + const uint32_t dma_depth = dma_queue_depth(dma_queue); - // Prefetch loop - for (uint32_t pnr = 0, pr = 0; pr < nrows && pr < HTP_ROPE_SPAD_NROWS; pr += pnr) { - pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK); + // Prefetch up to 2 blocks + const uint32_t p_nrows = MIN(nrows, 2 * HTP_ROPE_SPAD_BLOCK); + for (uint32_t pr = 0; pr < p_nrows; pr += HTP_ROPE_SPAD_BLOCK) { + const uint32_t pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK); + const uint32_t slot = (cur_slot + pr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + uint8_t * spad_slot = rope_spad_slot(src0_spad_base, slot, rctx->src0_row_size_aligned); + const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + (i1 + pr) * nb01; - uint32_t pi1 = i1 + pr; - uint32_t pir = ir + pr; + // Dummy DMA transaction for sequencing (interleaving wr, rd, wr, rd, ...) + dma_queue_push(dma_queue, dma_make_ptr((void *) dst->data, spad_slot), 0, 0, 0, 0); - // Dummy DMA transaction for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr((void *) dst->data, dst_spad_base + pr * rctx->dst_row_size_aligned), 0, 0, 0); - - const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned; - - // Copy only the row payload while striding the DDR source - dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), + dma_queue_push(dma_queue, dma_make_ptr(spad_slot, src_addr), rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); - - // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); } // Update theta cache if (i2 != prev_i2) { prev_i2 = i2; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); + // VISION rotates the full row; other modes only rotate n_dims. + const uint32_t n_cache = is_vision ? ne0 : (uint32_t) rctx->n_dims; const bool is_mrope = (rctx->mode & HTP_ROPE_TYPE_MROPE) != 0; if (is_mrope) { // src1 holds four position arrays stacked along ne0: @@ -554,66 +621,71 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { (float) pos[i2 + ne2 * 3], rctx->sections, is_imrope, is_vision, rctx->freq_scale, freq_factors, rctx->corr_dims, - ne0, rctx->ext_factor, rctx->attn_factor, - theta_cache, rctx->theta_scale); + n_cache, rctx->ext_factor, rctx->attn_factor, + theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); } else { rope_cache_init(pos[i2], rctx->freq_scale, freq_factors, rctx->corr_dims, - ne0, rctx->ext_factor, rctx->attn_factor, - theta_cache, rctx->theta_scale); + n_cache, rctx->ext_factor, rctx->attn_factor, + theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); } // Skip output DMA transactions from prev block (if any) - // No need to wait for those here since we're explicitly waiting for the latest prefecthes below. - for (uint32_t d=0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); } + for (uint32_t d = 0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); } // Compute loop - for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr, ir += cnr, i1 += cnr) { - // Number of rows to compute - cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t ne = is_vision ? ne0 : rctx->n_dims; + const uint32_t base_i1 = i1; + const uint32_t base_ir = ir; - uint8_t * dst_spad = (uint8_t *) dma_queue_pop(dma_queue).src; - uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_queue).dst; + for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr) { + cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t slot = (cur_slot + cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + const uint32_t cur_ir = base_ir + cr; + const uint32_t cur_i1 = base_i1 + cr; - // FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr, - // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); + dma_queue_pop(dma_queue); + uint8_t * cur_spad = (uint8_t *) dma_queue_pop(dma_queue).dst; - if (is_vision) { - rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); - } else if (is_neox) { - rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir); + if (is_neox || is_vision) { + rope_neox_f32_inplace(rctx, cur_spad, cnr, ne, theta_cache); } else { - rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + rope_basic_f32_inplace(rctx, cur_spad, cnr, theta_cache); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir); - uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1; - - // Write only the row payload while striding the DDR dst - dma_queue_push(dma_queue, dma_make_ptr(dst_addr, dst_spad), - rctx->dst_row_stride, rctx->dst_row_size_aligned, rctx->dst_row_size, cnr); + uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + cur_i1 * nb1; + dma_queue_push(dma_queue, dma_make_ptr(dst_addr, cur_spad), + rctx->dst_row_stride, rctx->src0_row_size_aligned, rctx->dst_row_size, cnr); - // Prefetch more rows (if any) - if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) { - uint32_t pnr = MIN(nrows - (cr + HTP_ROPE_SPAD_NROWS), HTP_ROPE_SPAD_BLOCK); - uint32_t pi1 = i1 + HTP_ROPE_SPAD_NROWS; - uint32_t pir = ir + HTP_ROPE_SPAD_NROWS; + // Prefetch 2 blocks ahead into the slot just freed + if ((cr + 2 * HTP_ROPE_SPAD_BLOCK) < nrows) { + const uint32_t p_cr = cr + 2 * HTP_ROPE_SPAD_BLOCK; + const uint32_t pnr = MIN(nrows - p_cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t p_slot = (cur_slot + p_cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + uint8_t * p_spad = rope_spad_slot(src0_spad_base, p_slot, rctx->src0_row_size_aligned); + const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + (base_i1 + p_cr) * nb01; - const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), + dma_queue_push(dma_queue, dma_make_ptr(p_spad, src_addr), rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); - - // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); } } + + const uint32_t n_chunks = (nrows + HTP_ROPE_SPAD_BLOCK - 1) / HTP_ROPE_SPAD_BLOCK; + cur_slot = (cur_slot + n_chunks) % HTP_ROPE_SPAD_NSLOTS; + + ir += nrows; + i1 += nrows; } } } done: dma_queue_flush(dma_queue); - tt = HAP_perf_get_qtimer_count() - tt; - FARF(HIGH, "rope-f32: %d/%d: (%u:%u) usec %u\n", ith, nth, src0_start_row, src0_end_row, (unsigned) HAP_perf_qtimer_count_to_us(tt)); + FARF(HIGH, "rope-f32: %d/%d: (%u:%u)\n", ith, nth, src0_start_row, src0_end_row); } static int execute_op_rope_f32(struct htp_ops_context * octx) { @@ -624,8 +696,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const struct htp_tensor * src2 = octx->src[2]; const struct htp_tensor * dst = octx->dst; - const char * op_type = "rope-f32"; - switch (octx->op) { case HTP_OP_ROPE: break; @@ -635,48 +705,23 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t ne0 = dst->ne[0]; - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); + const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params; + assert(kparams->n_threads > 0); + assert(octx->ctx->vtcm_size >= kparams->vtcm_size); + const uint32_t ne0 = dst->ne[0]; const size_t src0_row_size = src0->ne[0] * sizeof(float); const size_t src0_row_stride = src0->nb[1]; const size_t dst_row_size = dst->ne[0] * sizeof(float); const size_t dst_row_stride = dst->nb[1]; - // Aligned row sizes for VTCM - const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_stride, VLEN); - const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 256); - - // Calculate spad sizes per thread - size_t src0_spad_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; - size_t dst_spad_per_thread = HTP_ROPE_SPAD_NROWS * dst_row_size_aligned; - size_t spad_per_thread = src0_spad_per_thread + dst_spad_per_thread; - - // Check if we fit in VTCM - size_t total_vtcm_needed = spad_per_thread * n_threads; - if (octx->ctx->vtcm_size < total_vtcm_needed) { - FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, total_vtcm_needed); - return HTP_STATUS_VTCM_TOO_SMALL; - } - - octx->src0_spad.size_per_thread = src0_spad_per_thread; - octx->dst_spad.size_per_thread = dst_spad_per_thread; - octx->src0_spad.size = n_threads * src0_spad_per_thread; - octx->dst_spad.size = n_threads * dst_spad_per_thread; - octx->src1_spad.size = 0; - - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = NULL; octx->src1_spad.src = NULL; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL; - struct htp_rope_context rctx; memset(&rctx, 0, sizeof(struct htp_rope_context)); - rctx.t_start = HAP_perf_get_qtimer_count(); - - rctx.octx = octx; + rctx.octx = octx; + rctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + rctx.spad_per_thread = kparams->spad_per_thread; + rctx.theta_cache_offset = kparams->theta_cache_offset; const int32_t * op_params = &octx->op_params[0]; rctx.n_dims = ((const int32_t *) op_params)[1]; @@ -693,31 +738,29 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { memcpy(&rctx.sections, (int32_t *) op_params + 11, sizeof(int) * 4); rctx.theta_scale = powf(rctx.freq_base, -2.0f / rctx.n_dims); + rctx.theta_powers[0] = 1.0f; + for (int j = 1; j < 32; j++) { + rctx.theta_powers[j] = rctx.theta_powers[j - 1] * rctx.theta_scale; + } + rctx.theta_scale_32 = rctx.theta_powers[31] * rctx.theta_scale; rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims); - rctx.src0_row_size = src0_row_size; - rctx.src0_row_stride = src0_row_stride; - rctx.dst_row_size = dst_row_size; - rctx.dst_row_stride = dst_row_stride; - rctx.src0_row_size_aligned = src0_row_size_aligned; - rctx.dst_row_size_aligned = dst_row_size_aligned; - rctx.theta_cache_offset = theta_cache_size_aligned; + rctx.src0_row_size = src0_row_size; + rctx.src0_row_stride = src0_row_stride; + rctx.dst_row_size = dst_row_size; + rctx.dst_row_stride = dst_row_stride; + rctx.src0_row_size_aligned = kparams->src0_row_size_aligned; - rctx.src0_nrows = src0_nrows; - rctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - - if (src0_nrows > 0) { - rctx.div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); - rctx.div_ne1 = init_fastdiv_values(dst->ne[1]); - } + rctx.src0_nrows = kparams->src0_nrows; + rctx.src0_nrows_per_thread = kparams->src0_nrows_per_thread; + rctx.div_ne2_ne1 = kparams->div_ne2_ne1; + rctx.div_ne1 = kparams->div_ne1; FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - worker_pool_run_func(octx->ctx->worker_pool, rope_job_f32, &rctx, n_threads); - } + work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, kparams->n_threads); return err; } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h new file mode 100644 index 000000000000..476653d05d2b --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/rope-ops.h @@ -0,0 +1,56 @@ +#ifndef HTP_ROPE_OPS_H +#define HTP_ROPE_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" + +#define HTP_ROPE_SPAD_BLOCK 8 +#define HTP_ROPE_SPAD_NSLOTS 4 +#define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS) + +struct htp_rope_kernel_params { + uint32_t n_threads; + uint32_t src0_nrows; + uint32_t src0_nrows_per_thread; + uint32_t vtcm_size; + uint32_t spad_per_thread; + uint32_t theta_cache_offset; + uint32_t src0_row_size_aligned; + + struct fastdiv_values div_ne2_ne1; + struct fastdiv_values div_ne1; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_rope_vtcm_layout { + size_t total_bytes; + size_t bytes_per_thread; + size_t theta_cache_size_aligned; + size_t src0_row_size_aligned; +}; + +static inline void htp_rope_vtcm_layout_build( + struct htp_rope_vtcm_layout * layout, + uint32_t ne00, + uint32_t n_threads +) { + const size_t src0_row_size = ne00 * sizeof(float); + const size_t src0_row_size_aligned = hex_round_up((uint32_t) src0_row_size, 128); + const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256); + + layout->src0_row_size_aligned = src0_row_size_aligned; + layout->theta_cache_size_aligned = theta_cache_size_aligned; + layout->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; + layout->total_bytes = layout->bytes_per_thread * n_threads; +} + +static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) { + return base + (slot * HTP_ROPE_SPAD_BLOCK) * row_size_aligned; +} + +#endif // HTP_ROPE_OPS_H diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 8030186fb496..2deb90f6ab1e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10282,6 +10282,12 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true, 32)); } + // Real-model RoPE: F32 forward, packed Q, 512-token prefill. + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global + for (int v : { 0, 1, 2, 3 }) { for (int dim : { 0, 1, 2, 3, }) { test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v)); @@ -11178,6 +11184,12 @@ static std::vector> make_test_cases_perf() { } } + // Real-model RoPE: F32 forward, packed Q, 512-token prefill. + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global + std::vector> reduce_rows_cases = { { 8192, 1, 1, 1 }, { 8192, 8192, 1, 1 }, From 91f6a6cf361385700bbe15981f0f39909df77498 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Wed, 9 Sep 2026 18:53:39 +0200 Subject: [PATCH 11/65] vulkan: use spec constant for matrix matrix multiplication A-type (#25773) * vulkan: use spec constant for mul mat type_a vulkan: use map for mul_mm shapes cleanup fix indentation fix cm2 and shmem init fix cm2 spec constants fix cm2 bindings consolidate shmem tables and reduce size by type spec constant fix compiler warning fix missing Q2_0 type fix unused warning when integer dot glslc support is missing use minimal shmem size 8 instead of 1 to workaround cm2 compiler bug fix missing Q2_0 type in cm2 matmul fix types * remove LUT quants from unified shader * clean up * restore coopmat2 q4_k/q5_k optimization * split out q4_k/q5_k cm2 shader to fix Ampere regression * revert iq shmem table renames * simplify cm2 code with single uint8_t buffer * fix fp4 extension use switch being overwritten by generic shader * clean up * adapt TQ1_0 changes * adapt #27471 f16 Intel tuning changes --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 1791 +++++++---------- .../vulkan-shaders/dequant_funcs_cm2.glsl | 22 +- .../ggml-vulkan/vulkan-shaders/fa_types.glsl | 42 +- .../vulkan-shaders/flash_attn.comp | 6 +- .../vulkan-shaders/flash_attn_base.glsl | 4 +- .../vulkan-shaders/flash_attn_cm2.comp | 58 +- .../vulkan-shaders/flash_attn_dequant.glsl | 32 +- .../vulkan-shaders/flash_attn_mmq_funcs.glsl | 72 +- .../vulkan-shaders/ggml_type_ids.glsl | 34 + .../vulkan-shaders/iq_shmem_init.glsl | 2 + .../vulkan-shaders/lightning_indexer.comp | 10 +- .../ggml-vulkan/vulkan-shaders/mul_mm.comp | 73 +- .../vulkan-shaders/mul_mm_cm2.comp | 129 +- .../vulkan-shaders/mul_mm_funcs.glsl | 1249 ++++++------ .../src/ggml-vulkan/vulkan-shaders/types.glsl | 54 +- .../vulkan-shaders/vulkan-shaders-gen.cpp | 91 +- 16 files changed, 1785 insertions(+), 1884 deletions(-) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 9eae8dab9c39..235b35c6ceb1 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -258,27 +258,34 @@ typedef std::weak_ptr vk_pipeline_ref; static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline); -struct vk_matmul_pipeline_struct { - vk_pipeline l, m, s; - vk_pipeline a_l, a_m, a_s; - // Returns true when all unaligned pipelines are null. - // We only check for unaligned variants since one of the unaligned pipelines must exist - // while aligned pipelines are optional - bool is_empty() const { - return l == nullptr && m == nullptr && s == nullptr; +struct vk_matmul_pipeline_key { + ggml_type type_a; + ggml_type type_b; + bool mul_mat_id; + bool f16acc; + + bool operator<(const vk_matmul_pipeline_key & o) const { + return std::tie(type_a, type_b, mul_mat_id, f16acc) + < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc); } }; -typedef std::shared_ptr vk_matmul_pipeline; -struct vk_matmul_pipeline2 { - vk_matmul_pipeline2() { - f16acc = std::make_shared(); - f32acc = std::make_shared(); - } - vk_matmul_pipeline f32acc; - vk_matmul_pipeline f16acc; +struct vk_matmul_pipeline_pair { + vk_pipeline unaligned; + vk_pipeline aligned; + uint32_t align; +}; + +struct vk_tile_config { + std::vector warptile; + std::array wg_denoms; + uint32_t align; }; +using matmul_tile_selector_t = std::function& configs)>; + struct vk_device_struct; typedef std::shared_ptr vk_device; typedef std::weak_ptr vk_device_ref; @@ -949,24 +956,9 @@ struct vk_device_struct { vk::DescriptorSetLayout dsl; - vk_matmul_pipeline pipeline_matmul_f32 {}; - vk_matmul_pipeline pipeline_matmul_f32_f16 {}; - vk_matmul_pipeline pipeline_matmul_bf16 {}; - vk_matmul_pipeline2 pipeline_matmul_f16; - vk_matmul_pipeline2 pipeline_matmul_f16_f32; - - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat[GGML_TYPE_COUNT]; - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_COUNT]; - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_COUNT]; - - vk_matmul_pipeline pipeline_matmul_id_f32 {}; - vk_matmul_pipeline pipeline_matmul_id_bf16 {}; - vk_matmul_pipeline2 pipeline_matmul_id_f16; - vk_matmul_pipeline2 pipeline_matmul_id_f16_f32; - - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id[GGML_TYPE_COUNT]; - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_COUNT]; // f16 B-type variant (coopmat1 only) - vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_COUNT]; + std::map> pipeline_matmul; + matmul_tile_selector_t matmul_tile_selector; + matmul_tile_selector_t matmul_id_tile_selector; vk_pipeline pipeline_matmul_split_k_reduce; vk_pipeline pipeline_quantize_q8_1_x4; @@ -4411,8 +4403,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // Warptile layout (indices match mul_mm.comp constantIDs): // [0..9] : BLOCK_SIZE, BM, BN, BK, WM, WN, WMITER, TM, TN, TK // [10] : WARP / required_subgroup_size (read via WARP_SIZE_IDX) - // [11] : ALIGNED (appended by ggml_vk_mul_mm_spec) - // [12,13] : SHMEM_STRIDE_PAD, APPLY_SLM_A_RESHAPE static constexpr size_t WARP_SIZE_IDX = 10; std::vector l_warptile, m_warptile, s_warptile, l_warptile_id, m_warptile_id, s_warptile_id, @@ -4465,9 +4455,9 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // spec constants and tile sizes for quant matmul_id const uint32_t mmqid_bk = device->coopmat2_decode_vector ? 64u : 32u; - l_warptile_mmqid = { 256, 128, 128, mmqid_bk, 1, device->subgroup_size }; - m_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0, device->subgroup_size }; - s_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0, device->subgroup_size }; + l_warptile_mmqid = { 256, 128, 128, mmqid_bk, 1 }; + m_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0 }; + s_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0 }; l_mmqid_wg_denoms = { 128, 128, 1 }; m_mmqid_wg_denoms = { 128, 64, 1 }; s_mmqid_wg_denoms = { 128, 64, 1 }; @@ -4618,22 +4608,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } } - if (!device->pipeline_matmul_f32) { - device->pipeline_matmul_f32 = std::make_shared(); - } - if (!device->pipeline_matmul_f32_f16) { - device->pipeline_matmul_f32_f16 = std::make_shared(); - } - if (!device->pipeline_matmul_id_f32) { - device->pipeline_matmul_id_f32 = std::make_shared(); - } - if (!device->pipeline_matmul_bf16) { - device->pipeline_matmul_bf16 = std::make_shared(); - } - if (!device->pipeline_matmul_id_bf16) { - device->pipeline_matmul_id_bf16 = std::make_shared(); - } - auto const &ggml_vk_create_pipeline = [&](vk_device& device, vk_pipeline& base_pipeline, const char *name, size_t spv_size, const void* spv_data, const char *entrypoint, uint32_t parameter_count, uint32_t push_constant_size, std::array wg_denoms, const std::vector& specialization_constants, uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) { @@ -4842,685 +4816,639 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { return spec; }; + auto const &ggml_vk_mul_mm_spec_quant = [&device](std::vector spec, bool aligned, uint32_t type) { + spec.push_back(aligned ? 1u : 0u); // constantID=11: ALIGNED + spec.push_back(type); // constantID=12: MmTypeA + if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && + device->driver_id == vk::DriverId::eIntelProprietaryWindows) { + spec.push_back(0u); // constantID=13: SHMEM_STRIDE_PAD = 0 + spec.push_back(1u); // constantID=14: APPLY_SLM_A_RESHAPE = true + } + return spec; + }; + + static const ggml_type non_lut_quant_types[] = { + GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, + GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_TQ1_0, GGML_TYPE_TQ2_0, + }; + +#define FOR_EACH_LUT_TYPE_NONFP4(X) \ + X(GGML_TYPE_IQ1_S, iq1_s) \ + X(GGML_TYPE_IQ1_M, iq1_m) \ + X(GGML_TYPE_IQ2_XXS, iq2_xxs) \ + X(GGML_TYPE_IQ2_XS, iq2_xs) \ + X(GGML_TYPE_IQ2_S, iq2_s) \ + X(GGML_TYPE_IQ3_XXS, iq3_xxs) \ + X(GGML_TYPE_IQ3_S, iq3_s) \ + X(GGML_TYPE_IQ4_XS, iq4_xs) \ + X(GGML_TYPE_IQ4_NL, iq4_nl) +#define FOR_EACH_LUT_FP4_TYPE(X) \ + X(GGML_TYPE_MXFP4, mxfp4) \ + X(GGML_TYPE_NVFP4, nvfp4) +#define FOR_EACH_LUT_TYPE(X) \ + FOR_EACH_LUT_TYPE_NONFP4(X) \ + FOR_EACH_LUT_FP4_TYPE(X) + const int mul_mat_id_param_count = 5; + using spec_fn_t = std::function(const std::vector&, bool)>; + auto const &create_mm_pipelines = [&]( + const vk_matmul_pipeline_key& key, + const std::vector& tile_configs, + const std::string& shader_name, size_t spv_len, const void* spv_data, + uint32_t push_constant_size, uint32_t param_count, + const spec_fn_t& spec_fn, + bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0, + bool create_aligned = true, bool pin_subgroup_to_warp = false + ) { + auto& vec = device->pipeline_matmul[key]; + const bool first_call = vec.empty(); + for (size_t i = 0; i < tile_configs.size(); i++) { + const auto& tc = tile_configs[i]; + + // Intel coopmat1 pins the required subgroup size to each warptile's WARP element. + const uint32_t rsgs = pin_subgroup_to_warp ? tc.warptile[WARP_SIZE_IDX] : required_subgroup_size; + const bool rfs = require_full_subgroups || pin_subgroup_to_warp; + + if (first_call) { + vk_matmul_pipeline_pair pair{}; + pair.align = tc.align; + std::string suffix = "_" + std::to_string(i); + pair.unaligned = std::make_shared(); + if (create_aligned) { + pair.aligned = std::make_shared(); + } + vec.push_back(pair); + } + + ggml_vk_create_pipeline(device, vec[i].unaligned, + vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(), + spv_len, spv_data, "main", param_count, push_constant_size, + tc.wg_denoms, spec_fn(tc.warptile, false), 1, + disable_robustness, rfs, rsgs); + + if (vec[i].aligned) { + ggml_vk_create_pipeline(device, vec[i].aligned, + vec[i].aligned->name.empty() ? (shader_name + "_aligned_" + std::to_string(i)).c_str() : vec[i].aligned->name.c_str(), + spv_len, spv_data, "main", param_count, push_constant_size, + tc.wg_denoms, spec_fn(tc.warptile, true), tc.align, + disable_robustness, rfs, rsgs); + } + } + }; + + auto filter_tc = [&](const std::vector& configs, ggml_type type, bool is_id, bool is_int = false) -> std::vector { + std::vector result; + bool enabled[3]; + if (is_int) { + enabled[0] = is_id ? device->mul_mat_id_s_int[type] : device->mul_mat_s_int[type]; + enabled[1] = is_id ? device->mul_mat_id_m_int[type] : device->mul_mat_m_int[type]; + enabled[2] = is_id ? device->mul_mat_id_l_int[type] : device->mul_mat_l_int[type]; + } else { + enabled[0] = is_id ? device->mul_mat_id_s[type] : device->mul_mat_s[type]; + enabled[1] = is_id ? device->mul_mat_id_m[type] : device->mul_mat_m[type]; + enabled[2] = is_id ? device->mul_mat_id_l[type] : device->mul_mat_l[type]; + } + for (size_t i = 0; i < configs.size() && i < 3; i++) { + if (enabled[i]) result.push_back(configs[i]); + } + return result; + }; + + std::vector tc_mm = {{s_warptile, s_wg_denoms, s_align}, {m_warptile, m_wg_denoms, m_align}, {l_warptile, l_wg_denoms, l_align}}; + std::vector tc_mmq = {{s_warptile_mmq, s_mmq_wg_denoms, s_align}, {m_warptile_mmq, m_mmq_wg_denoms, m_align}, {l_warptile_mmq, l_mmq_wg_denoms, l_align}}; + #if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) if (device->coopmat2) { - auto const &ggml_vk_mul_mm_cm2_spec = [](std::vector spec, bool aligned, bool mul_mat_id) { - if (mul_mat_id && spec.size() > 5) { - spec.insert(spec.begin() + 5, aligned ? 1u : 0u); - } else { - spec.push_back(aligned ? 1u : 0u); - } - if (mul_mat_id && spec.size() == 6) { - spec.push_back(32); + auto const &ggml_vk_mul_mm_cm2_spec = [&](std::vector spec, bool aligned, uint32_t type = UINT32_MAX) { + spec.push_back(aligned ? 1u : 0u); // ALIGNED + spec.push_back(device->subgroup_size); // subgroup_size + if (type != UINT32_MAX) { + spec.push_back(type); // MmTypeA + spec.push_back((uint32_t)ggml_type_size((ggml_type)type)); // MmABlockBytes } return spec; }; - // Create 6 variants, {s,m,l}x{unaligned,aligned} -#define CREATE_MM(PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(l_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(m_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(s_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(l_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), l_align, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(m_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), m_align, true); \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(s_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), s_align, true); \ - - // Create 2 variants, {f16,f32} accumulator -#define CREATE_MM2(PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - CREATE_MM(PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - CREATE_MM(PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \ - - CREATE_MM2(pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3) + std::vector tc_mmq_k = {{s_warptile_mmq_k, s_mmq_wg_denoms_k, s_align}, {m_warptile_mmq_k, m_mmq_wg_denoms_k, m_align}, {l_warptile_mmq_k, l_mmq_wg_denoms_k, l_align}}; + std::vector tc_mmqid = {{s_warptile_mmqid, s_mmqid_wg_denoms, s_align}, {m_warptile_mmqid, m_mmqid_wg_denoms, m_align}, {l_warptile_mmqid, l_mmqid_wg_denoms, l_align}}; + + spec_fn_t cm2_spec = [&](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a); }; + + // F16 x F16 + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm2_len, matmul_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (device->coopmat_bf16_support) { - CREATE_MM(pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3) + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); } #endif - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q1_0], matmul_q1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_0], matmul_q2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_0], matmul_q4_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_1], matmul_q4_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_0], matmul_q5_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_1], matmul_q5_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q8_0], matmul_q8_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_K], matmul_q2_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ2_0], matmul_tq2_0_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ1_0], matmul_tq1_0_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q3_K], matmul_q3_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_K], matmul_q4_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_K], matmul_q5_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q6_K], matmul_q6_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_S], matmul_iq1_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_M], matmul_iq1_m_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_S], matmul_iq2_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_S], matmul_iq3_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) + for (const auto type : non_lut_quant_types) { + // regression in unified shader on Ampere + if (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) { + continue; + } + auto& tc = ((type >= GGML_TYPE_Q2_K && type <= GGML_TYPE_Q6_K) || type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0) ? tc_mmq_k : tc_mmq; + spec_fn_t qs = [&, type](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; + create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true); + } + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16", matmul_q4_k_f16_cm2_len, matmul_q4_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16", matmul_q5_k_f16_cm2_len, matmul_q5_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); +#define X_CM2(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmq, TYPE, false); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm2_len, matmul_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + } } + FOR_EACH_LUT_TYPE_NONFP4(X_CM2) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) +#define X_CM2_OCP(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmq, TYPE, false); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm2_len, matmul_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \ + } } + FOR_EACH_LUT_FP4_TYPE(X_CM2_OCP) +#undef X_CM2_OCP } else #endif { - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) - CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3) + FOR_EACH_LUT_FP4_TYPE(X_CM2) } +#undef X_CM2 GGML_ASSERT(device->subgroup_ballot); - CREATE_MM2(pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, 5) + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm2_len, matmul_id_subgroup_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm2_len, matmul_id_subgroup_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (device->coopmat_bf16_support) { - CREATE_MM(pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, 5) + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm2_len, matmul_id_subgroup_bf16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); } #endif - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0], matmul_id_subgroup_tq1_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) + for (const auto type : non_lut_quant_types) { + if (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) { + continue; + } + spec_fn_t qs_id = [&, type](const std::vector& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); }; + create_mm_pipelines({type, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm2_len, matmul_id_subgroup_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + create_mm_pipelines({type, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm2_len, matmul_id_subgroup_quant_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true); + } + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_q4_k_f16_f16acc", matmul_id_subgroup_q4_k_f16_f16acc_cm2_len, matmul_id_subgroup_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_q4_k_f16", matmul_id_subgroup_q4_k_f16_cm2_len, matmul_id_subgroup_q4_k_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_q5_k_f16_f16acc", matmul_id_subgroup_q5_k_f16_f16acc_cm2_len, matmul_id_subgroup_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); + create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_q5_k_f16", matmul_id_subgroup_q5_k_f16_cm2_len, matmul_id_subgroup_q5_k_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); +#define X_CM2_ID(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmqid, TYPE, true); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, true}, tc, "matmul_id_subgroup_" #tstr "_f16_f16acc", matmul_id_subgroup_##tstr##_f16_f16acc_cm2_len, matmul_id_subgroup_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, false}, tc, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm2_len, matmul_id_subgroup_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + } } + FOR_EACH_LUT_TYPE_NONFP4(X_CM2_ID) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16_ocp, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16_ocp, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) +#define X_CM2_ID_OCP(TYPE, tstr) \ + { auto tc = filter_tc(tc_mmqid, TYPE, true); \ + if (!tc.empty()) { \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, true}, tc, "matmul_id_subgroup_" #tstr "_f16_ocp_f16acc", matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm2_len, matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + create_mm_pipelines({TYPE, GGML_TYPE_F16, true, false}, tc, "matmul_id_subgroup_" #tstr "_f16_ocp", matmul_id_subgroup_##tstr##_f16_ocp_cm2_len, matmul_id_subgroup_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \ + } } + FOR_EACH_LUT_FP4_TYPE(X_CM2_ID_OCP) +#undef X_CM2_ID_OCP } else #endif { - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) - CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5) + FOR_EACH_LUT_FP4_TYPE(X_CM2_ID) } -#undef CREATE_MM -#undef CREATE_MM2 +#undef X_CM2_ID } else #endif // defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) #if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) if (device->coopmat_support) { - // Create 6 variants, {s,m,l}x{unaligned,aligned} - // Only Intel needs required_subgroup_size pinned to the warptile's WARP element. -#define REQUIRED_SUBGROUP_SIZE(WARPTILE) (device->vendor_id == VK_VENDOR_ID_INTEL ? (WARPTILE)[WARP_SIZE_IDX] : 0) -#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, true, REQUIRED_SUBGROUP_SIZE(l_ ## WARPTILE)); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, true, REQUIRED_SUBGROUP_SIZE(m_ ## WARPTILE)); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, true, REQUIRED_SUBGROUP_SIZE(s_ ## WARPTILE)); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, true, REQUIRED_SUBGROUP_SIZE(l_ ## WARPTILE)); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, true, REQUIRED_SUBGROUP_SIZE(m_ ## WARPTILE)); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, true, REQUIRED_SUBGROUP_SIZE(s_ ## WARPTILE)); \ - - // Create 2 variants, {f16,f32} accumulator -#define CREATE_MM2(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - if (device->coopmat_acc_f16_support) { \ - CREATE_MM(TYPE, PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - } \ - if (device->coopmat_acc_f32_support) { \ - CREATE_MM(TYPE, PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - } \ + spec_fn_t cm1_spec = [&](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); }; - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16_f32, matmul_f16_f32, wg_denoms, warptile, vk_mat_mat_push_constants, 3, ); -#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) - if (device->coopmat_bf16_support) { - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, ) + // Intel coopmat1 pins each pipeline's required subgroup size to its warptile WARP element. + const bool cm1_pin = device->vendor_id == VK_VENDOR_ID_INTEL; + + // Intel coopmat1 uses a dedicated large-tile config for quant matmul_id. + std::vector tc_mmq_id = tc_mmq; + if (cm1_pin) { + tc_mmq_id[2] = { { 512, 128, 128, 32, 32, 32, 2, device->coopmat_m, device->coopmat_n, device->coopmat_k, 32 }, { 128, 128, 1 }, 32 }; } -#endif - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0], matmul_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1], matmul_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0], matmul_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K], matmul_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0], matmul_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ1_0], matmul_tq1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K], matmul_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K], matmul_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K], matmul_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K], matmul_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S], matmul_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M], matmul_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S], matmul_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S], matmul_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); + auto cm1_create = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, cm1_spec, false, true, 0, true, cm1_pin); + }; + auto cm1_create_quant = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) { + spec_fn_t qs = [&, type_a=key.type_a](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec_quant(wt, a, (uint32_t)type_a); }; + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, false, true, 0, true, cm1_pin); + }; -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - } else + cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_cm1_len, matmul_f32_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_cm1_len, matmul_f32_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + if (device->coopmat_acc_f16_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm1_len, matmul_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, true}, tc_mm, "matmul_f16_f32_f16acc", matmul_f16_f32_f16acc_cm1_len, matmul_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + if (device->coopmat_acc_f32_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm1_len, matmul_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_cm1_len, matmul_f16_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } +#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (device->coopmat_bf16_support) { + cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm1_len, matmul_bf16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } #endif - { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - } - - // f16 B-type dense GEMM pipelines for coopmat1 (used when y_non_contig auto-converts f32->f16) - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q1_0], matmul_q1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_0], matmul_q2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ1_0], matmul_tq1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ2_0], matmul_tq2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_0], matmul_q4_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_1], matmul_q4_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_0], matmul_q5_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_1], matmul_q5_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q8_0], matmul_q8_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_K], matmul_q2_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q3_K], matmul_q3_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_K], matmul_q4_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_K], matmul_q5_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q6_K], matmul_q6_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_S], matmul_iq1_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_M], matmul_iq1_m_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_S], matmul_iq2_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_S], matmul_iq3_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); + for (const auto type : non_lut_quant_types) { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", matmul_quant_f32_f16acc_cm1_len, matmul_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create_quant({type, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm1_len, matmul_quant_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_cm1_len, matmul_quant_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create_quant({type, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_quant_f16", matmul_quant_f16_cm1_len, matmul_quant_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + } + } + // The _f16 variants provide the f16 B-type pipeline used when y_non_contig converts f32->f16. +#define X_CM1(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", matmul_##tstr##_f32_f16acc_cm1_len, matmul_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + cm1_create({TYPE, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm1_len, matmul_##tstr##_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_cm1_len, matmul_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + cm1_create({TYPE, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm1_len, matmul_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } + FOR_EACH_LUT_TYPE_NONFP4(X_CM1) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); +#define X_CM1_OCP(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_ocp_f16acc", matmul_##tstr##_f32_ocp_f16acc_cm1_len, matmul_##tstr##_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + cm1_create({TYPE, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm1_len, matmul_##tstr##_f16_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32_ocp", matmul_##tstr##_f32_ocp_cm1_len, matmul_##tstr##_f32_ocp_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + cm1_create({TYPE, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm1_len, matmul_##tstr##_f16_ocp_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ + } + FOR_EACH_LUT_FP4_TYPE(X_CM1_OCP) +#undef X_CM1_OCP } else #endif { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); + FOR_EACH_LUT_FP4_TYPE(X_CM1) } - - // Intel matmul_id warptile tuning - if (device->vendor_id == VK_VENDOR_ID_INTEL) { - l_warptile_mmq = { 512, 128, 128, 32, 32, 32, 2, device->coopmat_m, device->coopmat_n, device->coopmat_k, 32 }; - l_mmq_wg_denoms = { 128, 128, 1 }; - l_align = 32; //set as BK - } - +#undef X_CM1 GGML_ASSERT(device->subgroup_ballot); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_cm1_len, matmul_id_subgroup_f32_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + if (device->coopmat_acc_f16_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm1_len, matmul_id_subgroup_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_mm, "matmul_id_subgroup_f16_f32_f16acc", matmul_id_subgroup_f16_f32_f16acc_cm1_len, matmul_id_subgroup_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + if (device->coopmat_acc_f32_support) { + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm1_len, matmul_id_subgroup_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_cm1_len, matmul_id_subgroup_f16_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (device->coopmat_bf16_support) { - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm1_len, matmul_id_subgroup_bf16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } #endif - - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0], matmul_id_subgroup_tq1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if (device->ocp_fp4) { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - } else -#endif - { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - } - - // f16 B-type MoE GEMM pipelines for coopmat1 (used when y_non_contig auto-converts f32->f16) - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_TQ1_0], matmul_id_subgroup_tq1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + for (const auto type : non_lut_quant_types) { + if (device->coopmat_acc_f16_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_quant_f32_f16acc", matmul_id_subgroup_quant_f32_f16acc_cm1_len, matmul_id_subgroup_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create_quant({type, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm1_len, matmul_id_subgroup_quant_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + if (device->coopmat_acc_f32_support) { + cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_cm1_len, matmul_id_subgroup_quant_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create_quant({type, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm1_len, matmul_id_subgroup_quant_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + } + // The _f16 variants provide the f16 B-type pipeline used when y_non_contig converts f32->f16. +#define X_CM1_ID(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_f16acc", matmul_id_subgroup_##tstr##_f32_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + cm1_create({TYPE, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_f16acc", matmul_id_subgroup_##tstr##_f16_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_cm1_len, matmul_id_subgroup_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + cm1_create({TYPE, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm1_len, matmul_id_subgroup_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } + FOR_EACH_LUT_TYPE_NONFP4(X_CM1_ID) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); +#define X_CM1_ID_OCP(TYPE, tstr) \ + if (device->coopmat_acc_f16_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_ocp_f16acc", matmul_id_subgroup_##tstr##_f32_ocp_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + cm1_create({TYPE, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_ocp_f16acc", matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } \ + if (device->coopmat_acc_f32_support) { \ + cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_ocp", matmul_id_subgroup_##tstr##_f32_ocp_cm1_len, matmul_id_subgroup_##tstr##_f32_ocp_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + cm1_create({TYPE, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_ocp", matmul_id_subgroup_##tstr##_f16_ocp_cm1_len, matmul_id_subgroup_##tstr##_f16_ocp_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + } + FOR_EACH_LUT_FP4_TYPE(X_CM1_ID_OCP) +#undef X_CM1_ID_OCP } else #endif { - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + FOR_EACH_LUT_FP4_TYPE(X_CM1_ID) } -#undef CREATE_MM2 -#undef CREATE_MM -#undef REQUIRED_SUBGROUP_SIZE +#undef X_CM1_ID } else #endif // defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) - if (device->fp16) { - // Create 6 variants, {s,m,l}x{unaligned,aligned} - // Selects dot2 SPIR-V variant at runtime when device->dot2_f16 is true -#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - - // bf16 scalar path promotes to f32, no dot2 variant -#define CREATE_MM_NODOT2(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - -#define CREATE_MMQ(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->l, #NAMELC "_l", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - if (device->mul_mat ## ID ## _m_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->m, #NAMELC "_m", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, m_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - if (device->mul_mat ## ID ## _s_int[TYPE]) { \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->s, #NAMELC "_s", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - } \ - - // Create 2 variants, {f16,f32} accumulator -#define CREATE_MM2(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - CREATE_MM(TYPE, PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - CREATE_MM(TYPE, PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16_f32, matmul_f16_f32, wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0], matmul_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1], matmul_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0], matmul_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K], matmul_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0], matmul_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ1_0], matmul_tq1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K], matmul_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K], matmul_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K], matmul_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K], matmul_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S], matmul_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M], matmul_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S], matmul_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S], matmul_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - + { + // Helper for subgroup path with dot2 selection and filtering + auto sg_create = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + uint32_t rsgs = 0) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, + [&](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); }, + false, rsgs > 0, rsgs); + }; + auto sg_create_quant = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + uint32_t rsgs = 0) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); + if (!tc.empty()) { + spec_fn_t qs = [&, type_a=key.type_a](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec_quant(wt, a, (uint32_t)type_a); }; + create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, false, rsgs > 0, rsgs); + } + }; #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_0], matmul_q2_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_0], matmul_q4_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_1], matmul_q4_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_0], matmul_q5_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_1], matmul_q5_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q8_0], matmul_q8_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_MXFP4], matmul_mxfp4_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_K], matmul_q2_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q3_K], matmul_q3_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_K], matmul_q4_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_K], matmul_q5_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q6_K], matmul_q6_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0); - } + auto sg_create_mmq = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + uint32_t rsgs = 0) { + auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id, true); + if (!tc.empty()) { + spec_fn_t identity = [](const std::vector& wt, bool) { return wt; }; + create_mm_pipelines(key, tc, name, len, data, pc_size, pc, identity, false, rsgs > 0, rsgs, false); + } + }; #endif - if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0], matmul_id_subgroup_tq1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); + std::vector tc_id = {{s_warptile_id, s_wg_denoms, s_align}, {m_warptile_id, m_wg_denoms, m_align}, {l_warptile_id, l_wg_denoms, l_align}}; + std::vector tc_mmqid = {{s_warptile_mmqid, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid, l_mmq_wg_denoms, l_align}}; + + if (device->fp16) { + // FP16 subgroup path - with dot2 runtime selection + #define SPV_DOT2(NAME) (device->dot2_f16 ? NAME ## _dot2_len : NAME ## _len), (device->dot2_f16 ? NAME ## _dot2_data : NAME ## _data) + #define SPV_DOT2_F16ACC(NAME) (device->dot2_f16 ? NAME ## _dot2_f16acc_len : NAME ## _f16acc_len), (device->dot2_f16 ? NAME ## _dot2_f16acc_data : NAME ## _f16acc_data) + + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", SPV_DOT2(matmul_f32_f32), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", SPV_DOT2(matmul_f32_f16), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", SPV_DOT2_F16ACC(matmul_f16), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", SPV_DOT2(matmul_f16), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, true}, tc_mm, "matmul_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_f16_f32), sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", SPV_DOT2(matmul_f16_f32), sizeof(vk_mat_mat_push_constants), 3); + // BF16 - no dot2 + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_len, matmul_bf16_data, sizeof(vk_mat_mat_push_constants), 3); + + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3); + sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", SPV_DOT2(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3); + } + #define X_SG(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3); \ + sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", SPV_DOT2(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3); + FOR_EACH_LUT_TYPE(X_SG) +#undef X_SG #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); + std::vector tc_mmq_int = {{s_warptile_mmq_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int, l_mmq_wg_denoms, l_align}}; + std::vector tc_mmq_int_k = {{s_warptile_mmq_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q2_0_q8_1", matmul_q2_0_q8_1_len, matmul_q2_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_0_q8_1", matmul_q4_0_q8_1_len, matmul_q4_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_1_q8_1", matmul_q4_1_q8_1_len, matmul_q4_1_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_0_q8_1", matmul_q5_0_q8_1_len, matmul_q5_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_1_q8_1", matmul_q5_1_q8_1_len, matmul_q5_1_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q8_0_q8_1", matmul_q8_0_q8_1_len, matmul_q8_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_mxfp4_q8_1", matmul_mxfp4_q8_1_len, matmul_mxfp4_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q2_k_q8_1", matmul_q2_k_q8_1_len, matmul_q2_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q3_k_q8_1", matmul_q3_k_q8_1_len, matmul_q3_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_len, matmul_q4_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_len, matmul_q5_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_len, matmul_q6_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3); } #endif + + if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", SPV_DOT2(matmul_id_subgroup_f32_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_id, "matmul_id_subgroup_f16_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", SPV_DOT2(matmul_id_subgroup_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_id, "matmul_id_subgroup_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", SPV_DOT2(matmul_id_subgroup_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + // BF16 id - no dot2 + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_len, matmul_id_subgroup_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", SPV_DOT2(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + } + #define X_SG_ID_SUB(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", SPV_DOT2(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + FOR_EACH_LUT_TYPE(X_SG_ID_SUB) +#undef X_SG_ID_SUB +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + std::vector tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}}; + std::vector tc_mmqid_int_k = {{s_warptile_mmqid_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q2_0_q8_1", matmul_id_subgroup_q2_0_q8_1_len, matmul_id_subgroup_q2_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q4_0_q8_1", matmul_id_subgroup_q4_0_q8_1_len, matmul_id_subgroup_q4_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q4_1_q8_1", matmul_id_subgroup_q4_1_q8_1_len, matmul_id_subgroup_q4_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q5_0_q8_1", matmul_id_subgroup_q5_0_q8_1_len, matmul_id_subgroup_q5_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q5_1_q8_1", matmul_id_subgroup_q5_1_q8_1_len, matmul_id_subgroup_q5_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q8_0_q8_1", matmul_id_subgroup_q8_0_q8_1_len, matmul_id_subgroup_q8_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_mxfp4_q8_1", matmul_id_subgroup_mxfp4_q8_1_len, matmul_id_subgroup_mxfp4_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q2_k_q8_1", matmul_id_subgroup_q2_k_q8_1_len, matmul_id_subgroup_q2_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q3_k_q8_1", matmul_id_subgroup_q3_k_q8_1_len, matmul_id_subgroup_q3_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q4_k_q8_1", matmul_id_subgroup_q4_k_q8_1_len, matmul_id_subgroup_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q5_k_q8_1", matmul_id_subgroup_q5_k_q8_1_len, matmul_id_subgroup_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q6_k_q8_1", matmul_id_subgroup_q6_k_q8_1_len, matmul_id_subgroup_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + } +#endif + } else { + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", SPV_DOT2(matmul_id_f32_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_f16_f16acc", SPV_DOT2_F16ACC(matmul_id_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_f16", SPV_DOT2(matmul_id_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_mm, "matmul_id_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", SPV_DOT2(matmul_id_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + // BF16 id - no dot2 + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_len, matmul_id_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", SPV_DOT2(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + #define X_SG_ID(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", SPV_DOT2(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + FOR_EACH_LUT_TYPE(X_SG_ID) +#undef X_SG_ID +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (device->integer_dot_product) { + std::vector tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}}; + std::vector tc_mmqid_int_k = {{s_warptile_mmqid_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q2_0_q8_1", matmul_id_q2_0_q8_1_len, matmul_id_q2_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q4_0_q8_1", matmul_id_q4_0_q8_1_len, matmul_id_q4_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q4_1_q8_1", matmul_id_q4_1_q8_1_len, matmul_id_q4_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q5_0_q8_1", matmul_id_q5_0_q8_1_len, matmul_id_q5_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q5_1_q8_1", matmul_id_q5_1_q8_1_len, matmul_id_q5_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q8_0_q8_1", matmul_id_q8_0_q8_1_len, matmul_id_q8_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_mxfp4_q8_1", matmul_id_mxfp4_q8_1_len, matmul_id_mxfp4_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q2_k_q8_1", matmul_id_q2_k_q8_1_len, matmul_id_q2_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q3_k_q8_1", matmul_id_q3_k_q8_1_len, matmul_id_q3_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q4_k_q8_1", matmul_id_q4_k_q8_1_len, matmul_id_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q5_k_q8_1", matmul_id_q5_k_q8_1_len, matmul_id_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q6_k_q8_1", matmul_id_q6_k_q8_1_len, matmul_id_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } +#endif + } + #undef SPV_DOT2 + #undef SPV_DOT2_F16ACC } else { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_tq2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0], matmul_id_tq1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + // FP32-only fallback path + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_fp32_len, matmul_f32_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_fp32_len, matmul_f32_f16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_fp32_len, matmul_f16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_fp32_len, matmul_f16_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_fp32_len, matmul_bf16_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_fp32_len, matmul_quant_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + } + #define X_SG_FP32(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_fp32_len, matmul_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + FOR_EACH_LUT_TYPE(X_SG_FP32) +#undef X_SG_FP32 #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_0], matmul_id_q2_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + std::vector tc_mmq_int = {{s_warptile_mmq_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int, l_mmq_wg_denoms, l_align}}; + std::vector tc_mmq_int_k = {{s_warptile_mmq_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int_k, l_mmq_wg_denoms, l_align}}; + sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q2_0_q8_1", matmul_q2_0_q8_1_fp32_len, matmul_q2_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_0_q8_1", matmul_q4_0_q8_1_fp32_len, matmul_q4_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_1_q8_1", matmul_q4_1_q8_1_fp32_len, matmul_q4_1_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_0_q8_1", matmul_q5_0_q8_1_fp32_len, matmul_q5_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_1_q8_1", matmul_q5_1_q8_1_fp32_len, matmul_q5_1_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q8_0_q8_1", matmul_q8_0_q8_1_fp32_len, matmul_q8_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q2_k_q8_1", matmul_q2_k_q8_1_fp32_len, matmul_q2_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q3_k_q8_1", matmul_q3_k_q8_1_fp32_len, matmul_q3_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_fp32_len, matmul_q4_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_fp32_len, matmul_q5_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); + sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_fp32_len, matmul_q6_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3); } #endif - } -#undef CREATE_MM2 -#undef CREATE_MMQ -#undef CREATE_MM -#undef CREATE_MM_NODOT2 - } else { - // Create 6 variants, {s,m,l}x{unaligned,aligned} -#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _l[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _m[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - if (device->mul_mat ## ID ## _s[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \ - -#define CREATE_MMQ(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \ - if (device->mul_mat ## ID ## _l_int[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC "_l", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1); \ - if (device->mul_mat ## ID ## _m_int[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC "_m", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, m_ ## WARPTILE, 1); \ - if (device->mul_mat ## ID ## _s_int[TYPE]) \ - ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC "_s", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1); \ - - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_f16.f32acc, matmul_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_f16_f32.f32acc, matmul_f16_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0].f32acc, matmul_q1_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0].f32acc, matmul_q2_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0].f32acc, matmul_q4_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1].f32acc, matmul_q4_1_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0].f32acc, matmul_q5_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1].f32acc, matmul_q5_1_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0].f32acc, matmul_q8_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K].f32acc, matmul_q2_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0].f32acc, matmul_tq2_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ1_0].f32acc, matmul_tq1_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K].f32acc, matmul_q3_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K].f32acc, matmul_q4_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K].f32acc, matmul_q5_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K].f32acc, matmul_q6_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S].f32acc, matmul_iq1_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M].f32acc, matmul_iq1_m_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS].f32acc, matmul_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS].f32acc, matmul_iq2_xs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S].f32acc, matmul_iq2_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS].f32acc, matmul_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S].f32acc, matmul_iq3_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS].f32acc, matmul_iq4_xs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL].f32acc, matmul_iq4_nl_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4].f32acc, matmul_mxfp4_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4].f32acc, matmul_nvfp4_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - if (device->integer_dot_product) { - CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_0].f32acc, matmul_q2_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_0].f32acc, matmul_q4_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_1].f32acc, matmul_q4_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_0].f32acc, matmul_q5_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_1].f32acc, matmul_q5_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q8_0].f32acc, matmul_q8_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, ); - - CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_K].f32acc, matmul_q2_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q3_K].f32acc, matmul_q3_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_K].f32acc, matmul_q4_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_K].f32acc, matmul_q5_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); - CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q6_K].f32acc, matmul_q6_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, ); + if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_fp32_len, matmul_id_subgroup_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_fp32_len, matmul_id_subgroup_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_fp32_len, matmul_id_subgroup_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_fp32_len, matmul_id_subgroup_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16); + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_fp32_len, matmul_id_subgroup_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + } + #define X_SG_ID_SUB_FP32(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_fp32_len, matmul_id_subgroup_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); + FOR_EACH_LUT_TYPE(X_SG_ID_SUB_FP32) +#undef X_SG_ID_SUB_FP32 + } else { + sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", matmul_id_f32_f32_fp32_len, matmul_id_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_f16", matmul_id_f16_fp32_len, matmul_id_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", matmul_id_f16_f32_fp32_len, matmul_id_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_fp32_len, matmul_id_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + for (const auto type : non_lut_quant_types) { + sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", matmul_id_quant_f32_fp32_len, matmul_id_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + } + #define X_SG_ID_FP32(TYPE, tstr) \ + sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", matmul_id_##tstr##_f32_fp32_len, matmul_id_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + FOR_EACH_LUT_TYPE(X_SG_ID_FP32) +#undef X_SG_ID_FP32 + } } -#endif - - if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_subgroup_f16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_subgroup_f16_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_subgroup_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_subgroup_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_subgroup_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_subgroup_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_subgroup_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_subgroup_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_subgroup_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_subgroup_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0].f32acc, matmul_id_subgroup_tq2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0].f32acc, matmul_id_subgroup_tq1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_subgroup_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_subgroup_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_subgroup_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_subgroup_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_subgroup_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_subgroup_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_subgroup_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_subgroup_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_subgroup_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_subgroup_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_subgroup_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_subgroup_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_subgroup_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_subgroup_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_subgroup_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); - } else { - CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_f16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_f16_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - - CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0].f32acc, matmul_id_tq2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0].f32acc, matmul_id_tq1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); - } - } - // reusing CREATE_MM from the fp32 path + } +#undef FOR_EACH_LUT_TYPE +#undef FOR_EACH_LUT_TYPE_NONFP4 +#undef FOR_EACH_LUT_FP4_TYPE + // BF16 fallback for coopmat devices without bf16 coopmat support if ((device->coopmat2 || device->coopmat_support) #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && !device->coopmat_bf16_support #endif ) { - const uint32_t s_warptile_wm = device->subgroup_size == 8 ? 8 : 32; - - // use scalar tile sizes - l_warptile = { 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }; - m_warptile = { 128, 64, 64, 16, mm_warp_8, 32, 2, 4, 2, 1, mm_warp_8 }; - s_warptile = { subgroup_size_32, 32, 32, 16, s_warptile_wm, 32, 2, 2, 2, 1, subgroup_size_8 }; - - l_wg_denoms = {128, 128, 1 }; - m_wg_denoms = { 64, 64, 1 }; - s_wg_denoms = { 32, 32, 1 }; + const uint32_t s_warptile_wm_bf16 = device->subgroup_size == 8 ? 8 : 32; + std::vector tc_bf16_fb = { + {{ subgroup_size_32, 32, 32, 16, s_warptile_wm_bf16, 32, 2, 2, 2, 1, subgroup_size_8 }, {32, 32, 1}, s_align}, + {{ 128, 64, 64, 16, mm_warp_8, 32, 2, 4, 2, 1, mm_warp_8 }, {64, 64, 1}, m_align}, + {{ 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }, {128, 128, 1}, l_align}, + }; + auto tc_bf16_filtered = filter_tc(tc_bf16_fb, GGML_TYPE_BF16, false); + auto tc_bf16_id_filtered = filter_tc(tc_bf16_fb, GGML_TYPE_BF16, true); + spec_fn_t bf16_spec = [&](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); }; + if (!tc_bf16_filtered.empty()) { + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_bf16_filtered, "matmul_bf16", matmul_bf16_fp32_len, matmul_bf16_fp32_data, sizeof(vk_mat_mat_push_constants), 3, bf16_spec); + } + if (!tc_bf16_id_filtered.empty()) { + create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_bf16_id_filtered, "matmul_id_bf16", matmul_id_bf16_fp32_len, matmul_id_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, bf16_spec); + } + } - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); - CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + // Set up tile selector functions + if (device->coopmat2) { + device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t shader_core_count, + const std::vector& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + uint32_t last = (uint32_t)configs.size() - 1; + if (configs.size() == 2) { + uint32_t crossover = configs[0].unaligned->wg_denoms[1]; + return (n > crossover) ? 1 : 0; + } + // 3+ configs: s=0, m=1, l=2 + const uint32_t tiles_l = CEIL_DIV(m, configs[last].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[last].unaligned->wg_denoms[1]); + const uint32_t tiles_m = CEIL_DIV(m, configs[1].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[1].unaligned->wg_denoms[1]); + uint32_t crossover_large = configs[1].unaligned->wg_denoms[1]; + bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count || + (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2); + if (n > crossover_large && prefer_large) return last; + uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1]; + if (n > crossover_medium) return 1; + return 0; + }; + device->matmul_id_tile_selector = [](uint32_t /*m*/, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, + const std::vector& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + uint32_t last = (uint32_t)configs.size() - 1; + if (configs.size() == 2) { + uint32_t crossover = configs[0].unaligned->wg_denoms[1]; + return (n > crossover) ? 1 : 0; + } + uint32_t crossover_large = configs[1].unaligned->wg_denoms[1]; + if (n > crossover_large) return last; + uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1]; + if (n > crossover_medium) return 1; + return 0; + }; + } else { + device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, + const std::vector& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + if (m <= 32 || n <= 32) return 0; + if (configs.size() == 2) return 1; + if (m <= 64 || n <= 64) return 1; + return (uint32_t)configs.size() - 1; + }; + device->matmul_id_tile_selector = device->matmul_tile_selector; } -#undef CREATE_MM // mul mat vec @@ -5591,12 +5519,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f32_f32", arr_dmmv_q5_1_f32_f32_len[reduc], arr_dmmv_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f32_f32", arr_dmmv_q8_0_f32_f32_len[reduc], arr_dmmv_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f32_f32", arr_dmmv_q2_k_f32_f32_len[reduc16], arr_dmmv_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f32_f32", arr_dmmv_tq2_0_f32_f32_len[reduc16], arr_dmmv_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f32_f32", arr_dmmv_tq1_0_f32_f32_len[reduc16], arr_dmmv_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f32_f32", arr_dmmv_q3_k_f32_f32_len[reduc16], arr_dmmv_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f32_f32", arr_dmmv_q4_k_f32_f32_len[reduc16], arr_dmmv_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f32_f32", arr_dmmv_q5_k_f32_f32_len[reduc16], arr_dmmv_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f32_f32", arr_dmmv_q6_k_f32_f32_len[reduc16], arr_dmmv_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f32_f32", arr_dmmv_tq1_0_f32_f32_len[reduc16], arr_dmmv_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f32_f32", arr_dmmv_tq2_0_f32_f32_len[reduc16], arr_dmmv_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32", arr_dmmv_iq1_s_f32_f32_len[reduc16], arr_dmmv_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32", arr_dmmv_iq1_m_f32_f32_len[reduc16], arr_dmmv_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32", arr_dmmv_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); @@ -5620,12 +5548,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f16_f32", arr_dmmv_q5_1_f16_f32_len[reduc], arr_dmmv_q5_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f16_f32", arr_dmmv_q8_0_f16_f32_len[reduc], arr_dmmv_q8_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f16_f32", arr_dmmv_q2_k_f16_f32_len[reduc16], arr_dmmv_q2_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f16_f32", arr_dmmv_tq2_0_f16_f32_len[reduc16], arr_dmmv_tq2_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f16_f32", arr_dmmv_tq1_0_f16_f32_len[reduc16], arr_dmmv_tq1_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f16_f32", arr_dmmv_q3_k_f16_f32_len[reduc16], arr_dmmv_q3_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f16_f32", arr_dmmv_q4_k_f16_f32_len[reduc16], arr_dmmv_q4_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f16_f32", arr_dmmv_q5_k_f16_f32_len[reduc16], arr_dmmv_q5_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f16_f32", arr_dmmv_q6_k_f16_f32_len[reduc16], arr_dmmv_q6_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f16_f32", arr_dmmv_tq1_0_f16_f32_len[reduc16], arr_dmmv_tq1_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f16_f32", arr_dmmv_tq2_0_f16_f32_len[reduc16], arr_dmmv_tq2_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32", arr_dmmv_iq1_s_f16_f32_len[reduc16], arr_dmmv_iq1_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32", arr_dmmv_iq1_m_f16_f32_len[reduc16], arr_dmmv_iq1_m_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32", arr_dmmv_iq2_xxs_f16_f32_len[reduc16], arr_dmmv_iq2_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); @@ -5676,12 +5604,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_1], "mul_mat_vec_id_q5_1_f32", arr_dmmv_id_q5_1_f32_f32_len[reduc], arr_dmmv_id_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q8_0], "mul_mat_vec_id_q8_0_f32", arr_dmmv_id_q8_0_f32_f32_len[reduc], arr_dmmv_id_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_f32", arr_dmmv_id_q2_k_f32_f32_len[reduc16], arr_dmmv_id_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ2_0], "mul_mat_vec_id_tq2_0_f32", arr_dmmv_id_tq2_0_f32_f32_len[reduc16], arr_dmmv_id_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ1_0], "mul_mat_vec_id_tq1_0_f32", arr_dmmv_id_tq1_0_f32_f32_len[reduc16], arr_dmmv_id_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_f32", arr_dmmv_id_q3_k_f32_f32_len[reduc16], arr_dmmv_id_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_f32", arr_dmmv_id_q4_k_f32_f32_len[reduc16], arr_dmmv_id_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_f32", arr_dmmv_id_q5_k_f32_f32_len[reduc16], arr_dmmv_id_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_f32", arr_dmmv_id_q6_k_f32_f32_len[reduc16], arr_dmmv_id_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ1_0], "mul_mat_vec_id_tq1_0_f32", arr_dmmv_id_tq1_0_f32_f32_len[reduc16], arr_dmmv_id_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ2_0], "mul_mat_vec_id_tq2_0_f32", arr_dmmv_id_tq2_0_f32_f32_len[reduc16], arr_dmmv_id_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", arr_dmmv_id_iq1_s_f32_f32_len[reduc16], arr_dmmv_id_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", arr_dmmv_id_iq1_m_f32_f32_len[reduc16], arr_dmmv_id_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", arr_dmmv_id_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); @@ -5743,12 +5671,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q8_0], "dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant_transpose[GGML_TYPE_Q8_0], "dequant_q8_0_transpose", dequant_q8_0_transpose_len, dequant_q8_0_transpose_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_K], "dequant_q2_k", dequant_q2_k_len, dequant_q2_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ2_0], "dequant_tq2_0", dequant_tq2_0_len, dequant_tq2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ1_0], "dequant_tq1_0", dequant_tq1_0_len, dequant_tq1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q3_K], "dequant_q3_k", dequant_q3_k_len, dequant_q3_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_K], "dequant_q4_k", dequant_q4_k_len, dequant_q4_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_K], "dequant_q5_k", dequant_q5_k_len, dequant_q5_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q6_K], "dequant_q6_k", dequant_q6_k_len, dequant_q6_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ1_0], "dequant_tq1_0", dequant_tq1_0_len, dequant_tq1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ2_0], "dequant_tq2_0", dequant_tq2_0_len, dequant_tq2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ1_S], "dequant_iq1_s", dequant_iq1_s_len, dequant_iq1_s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ1_M], "dequant_iq1_m", dequant_iq1_m_len, dequant_iq1_m_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ2_XXS], "dequant_iq2_xxs", dequant_iq2_xxs_len, dequant_iq2_xxs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); @@ -5773,12 +5701,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_1], "get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q8_0], "get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_K], "get_rows_q2_k", get_rows_q2_k_len, get_rows_q2_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ2_0], "get_rows_tq2_0", get_rows_tq2_0_len, get_rows_tq2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ1_0], "get_rows_tq1_0", get_rows_tq1_0_len, get_rows_tq1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q3_K], "get_rows_q3_k", get_rows_q3_k_len, get_rows_q3_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_K], "get_rows_q4_k", get_rows_q4_k_len, get_rows_q4_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_K], "get_rows_q5_k", get_rows_q5_k_len, get_rows_q5_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q6_K], "get_rows_q6_k", get_rows_q6_k_len, get_rows_q6_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ1_0], "get_rows_tq1_0", get_rows_tq1_0_len, get_rows_tq1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ2_0], "get_rows_tq2_0", get_rows_tq2_0_len, get_rows_tq2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ1_S], "get_rows_iq1_s", get_rows_iq1_s_len, get_rows_iq1_s_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ1_M], "get_rows_iq1_m", get_rows_iq1_m_len, get_rows_iq1_m_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ2_XXS], "get_rows_iq2_xxs", get_rows_iq2_xxs_len, get_rows_iq2_xxs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); @@ -5803,12 +5731,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_1], "get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q8_0], "get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_K], "get_rows_q2_k_f32", get_rows_q2_k_f32_len, get_rows_q2_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ2_0], "get_rows_tq2_0_f32", get_rows_tq2_0_f32_len, get_rows_tq2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ1_0], "get_rows_tq1_0_f32", get_rows_tq1_0_f32_len, get_rows_tq1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q3_K], "get_rows_q3_k_f32", get_rows_q3_k_f32_len, get_rows_q3_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_K], "get_rows_q4_k_f32", get_rows_q4_k_f32_len, get_rows_q4_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_K], "get_rows_q5_k_f32", get_rows_q5_k_f32_len, get_rows_q5_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q6_K], "get_rows_q6_k_f32", get_rows_q6_k_f32_len, get_rows_q6_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ1_0], "get_rows_tq1_0_f32", get_rows_tq1_0_f32_len, get_rows_tq1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ2_0], "get_rows_tq2_0_f32", get_rows_tq2_0_f32_len, get_rows_tq2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ1_S], "get_rows_iq1_s_f32", get_rows_iq1_s_f32_len, get_rows_iq1_s_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ1_M], "get_rows_iq1_m_f32", get_rows_iq1_m_f32_len, get_rows_iq1_m_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ2_XXS], "get_rows_iq2_xxs_f32", get_rows_iq2_xxs_f32_len, get_rows_iq2_xxs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); @@ -8079,115 +8007,14 @@ static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: case GGML_TYPE_TQ1_0: - break; - default: - return nullptr; - } - - return ctx->device->pipeline_dequant[type]; -} - -static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { - VK_LOG_DEBUG("ggml_vk_get_mul_mat_mat_pipeline(" << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ", " << prec << ")"); - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_f32; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_f32_f16; - } - if (src0_type == GGML_TYPE_BF16 && src1_type == GGML_TYPE_BF16) { - return ctx->device->pipeline_matmul_bf16; - } - if (prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support)) { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_f16_f32.f16acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_f16.f16acc; - } - } else { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_f16_f32.f32acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_f16.f32acc; - } - } - - // MMQ - if (src1_type == GGML_TYPE_Q8_1) { - vk_matmul_pipeline pipelines = ctx->device->pipeline_dequant_mul_mat_mat_q8_1[src0_type].f32acc; - - if (pipelines->is_empty()) { - return nullptr; - } - - return pipelines; - } - - // f16 B on coopmat1 - if (src1_type == GGML_TYPE_F16 && ctx->device->coopmat_support && !ctx->device->coopmat2) { - vk_matmul_pipeline2& mmp = ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type]; - bool prefer_fp16acc = ctx->device->fp16 && prec == GGML_PREC_DEFAULT; - bool support_fp16acc = !mmp.f16acc->is_empty(); - bool support_fp32acc = !mmp.f32acc->is_empty(); - - if (support_fp16acc && (prefer_fp16acc || !support_fp32acc)) { - return mmp.f16acc; - } else if (support_fp32acc) { - return mmp.f32acc; - } - return nullptr; - } - - if (src1_type != GGML_TYPE_F32 && - !(src1_type == GGML_TYPE_F16 && ctx->device->coopmat_support) && - !ctx->device->coopmat2) { - return nullptr; - } - - switch (src0_type) { - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: case GGML_TYPE_TQ2_0: - case GGML_TYPE_TQ1_0: break; default: return nullptr; } - if (ctx->device->coopmat2) { - assert(src1_type == GGML_TYPE_F16); - return prec == GGML_PREC_DEFAULT ? ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type].f32acc; - } - - if (ctx->device->coopmat_support) { - return (ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT) ? ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f32acc; - } - return (ctx->device->fp16 && prec == GGML_PREC_DEFAULT) ? ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f32acc; + return ctx->device->pipeline_dequant[type]; } static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t num_cols, uint32_t m, uint32_t k) { @@ -8244,8 +8071,8 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: break; default: return nullptr; @@ -8278,103 +8105,6 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * return b_type == GGML_TYPE_F32 ? ctx->device->pipeline_dequant_mul_mat_vec_f32_f32[dmmv_wg][a_type][num_cols-1] : ctx->device->pipeline_dequant_mul_mat_vec_f16_f32[dmmv_wg][a_type][num_cols-1]; } -static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_id_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { - VK_LOG_DEBUG("ggml_vk_get_mul_mat_mat_id_pipeline()"); - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_id_f32; - } - if (src0_type == GGML_TYPE_BF16 && src1_type == GGML_TYPE_BF16) { - return ctx->device->pipeline_matmul_id_bf16; - } - if (prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support)) { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_id_f16_f32.f16acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_id_f16.f16acc; - } - } else { - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return ctx->device->pipeline_matmul_id_f16_f32.f32acc; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return ctx->device->pipeline_matmul_id_f16.f32acc; - } - } - - // MMQ - if (src1_type == GGML_TYPE_Q8_1) { - vk_matmul_pipeline pipelines = ctx->device->pipeline_dequant_mul_mat_mat_id_q8_1[src0_type].f32acc; - - if (pipelines->is_empty()) { - return nullptr; - } - - return pipelines; - } - - // f16 B on coopmat1 - if (src1_type == GGML_TYPE_F16 && ctx->device->coopmat_support && !ctx->device->coopmat2) { - vk_matmul_pipeline2& mmp = ctx->device->pipeline_dequant_mul_mat_mat_id_f16b[src0_type]; - bool prefer_fp16acc = ctx->device->fp16; - bool support_fp16acc = !mmp.f16acc->is_empty(); - bool support_fp32acc = !mmp.f32acc->is_empty(); - - if (support_fp16acc && (prefer_fp16acc || !support_fp32acc)) { - return mmp.f16acc; - } else if (support_fp32acc) { - return mmp.f32acc; - } - return nullptr; - } - - GGML_ASSERT(src1_type == GGML_TYPE_F32 || (ctx->device->coopmat2 && src1_type == GGML_TYPE_F16)); - - switch (src0_type) { - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: - case GGML_TYPE_IQ2_XXS: - case GGML_TYPE_IQ2_XS: - case GGML_TYPE_IQ2_S: - case GGML_TYPE_IQ3_XXS: - case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ4_XS: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: - case GGML_TYPE_TQ1_0: - break; - default: - return nullptr; - } - - vk_matmul_pipeline2& mmp = ctx->device->pipeline_dequant_mul_mat_mat_id[src0_type]; - // XXX TODO 'prec' is not actually allowed in mul_mat_id. - bool prefer_fp16acc = ctx->device->fp16 /*&& prec == GGML_PREC_DEFAULT*/; - bool support_fp16acc = !mmp.f16acc->is_empty(); - bool support_fp32acc = !mmp.f32acc->is_empty(); - - if (support_fp16acc && (prefer_fp16acc || !support_fp32acc)) { - return mmp.f16acc; - } else { - GGML_ASSERT(support_fp32acc); - return mmp.f32acc; - } -} - static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t m, uint32_t k) { VK_LOG_DEBUG("ggml_vk_get_dequantize_mul_mat_vec_id()"); GGML_ASSERT(b_type == GGML_TYPE_F32 || b_type == GGML_TYPE_Q8_1); @@ -8428,8 +8158,8 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: break; default: return nullptr; @@ -9211,58 +8941,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, return split_k; } -static vk_pipeline ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, uint32_t m, uint32_t n, bool aligned, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - - // The q8_1 (integer dot) mmq path uses a different shader with its own - // shared-memory layout, so use the int-specific availability flags. - const bool is_q8_1 = (src1_type == GGML_TYPE_Q8_1); - const bool mm_l = is_q8_1 ? ctx->device->mul_mat_l_int[src0_type] : ctx->device->mul_mat_l[src0_type]; - const bool mm_m = is_q8_1 ? ctx->device->mul_mat_m_int[src0_type] : ctx->device->mul_mat_m[src0_type]; - const bool mm_s = is_q8_1 ? ctx->device->mul_mat_s_int[src0_type] : ctx->device->mul_mat_s[src0_type]; - if (ctx->device->coopmat2) { - const uint32_t shader_core_count = ctx->device->shader_core_count; - const uint32_t tiles_l = CEIL_DIV(m, mmp->a_l->wg_denoms[0]) * CEIL_DIV(n, mmp->a_l->wg_denoms[1]); - const uint32_t tiles_m = CEIL_DIV(m, mmp->a_m->wg_denoms[0]) * CEIL_DIV(n, mmp->a_m->wg_denoms[1]); - - // Use large shader when the N dimension is greater than the medium shader's tile size - uint32_t crossover_large = mmp->m->wg_denoms[1]; - - // Prefer large over medium if either: - // - medium or large tiles would overfill the GPU - // - large tiles with a split_k==3 fits in the GPU and medium tiles with split_k==2 does not - // (medium with split_k==2 is probably better if it fits - more workgroups running and less split_k overhead) - bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count || - // split_k==3 with large tiles likely better than medium tiles with no split_k. - (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2); - - if ((mm_l && (n > crossover_large && prefer_large)) || (!mm_m && !mm_s)) { - return aligned ? mmp->a_l : mmp->l; - } - // Use medium shader when the N dimension is greater than the small shader's tile size - uint32_t crossover_medium = mmp->s->wg_denoms[1]; - if ((mm_m && (n > crossover_medium)) || !mm_s) { - return aligned ? mmp->a_m : mmp->m; - } - return aligned ? mmp->a_s : mmp->s; - } - - if ((mm_s && (m <= 32 || n <= 32)) || (!mm_m && !mm_l)) { - return aligned ? mmp->a_s : mmp->s; - } - if ((mm_m && (m <= 64 || n <= 64)) || !mm_l) { - return aligned ? mmp->a_m : mmp->m; - } - return aligned ? mmp->a_l : mmp->l; -} - -static uint32_t ggml_vk_guess_matmul_pipeline_align(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline(ctx, mmp, m, n, true, src0_type, src1_type); - GGML_ASSERT(pipeline != nullptr && "missing matmul pipeline - check pipeline registration in ggml_vk_load_shaders for this type combo"); - return pipeline->align; -} static void ggml_vk_matmul( ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, @@ -9313,42 +8992,56 @@ static void ggml_vk_matmul( ctx->prealloc_split_k_need_sync = true; } -static vk_pipeline ggml_vk_guess_matmul_id_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, uint32_t m, uint32_t n, bool aligned, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_id_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - - // The q8_1 (integer dot) mmq path uses a different shader with its own - // shared-memory layout, so use the int-specific availability flags. - const bool is_q8_1 = (src1_type == GGML_TYPE_Q8_1); - const bool mm_l = is_q8_1 ? ctx->device->mul_mat_id_l_int[src0_type] : ctx->device->mul_mat_id_l[src0_type]; - const bool mm_m = is_q8_1 ? ctx->device->mul_mat_id_m_int[src0_type] : ctx->device->mul_mat_id_m[src0_type]; - const bool mm_s = is_q8_1 ? ctx->device->mul_mat_id_s_int[src0_type] : ctx->device->mul_mat_id_s[src0_type]; +static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { + if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false; + if (src1_type == GGML_TYPE_Q8_1) return false; + if (src0_type == GGML_TYPE_F16) { + return prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support); + } + // quant types if (ctx->device->coopmat2) { - // Use large shader when the N dimension is greater than the medium shader's tile size - uint32_t crossover_large = mmp->m->wg_denoms[1]; - if ((mm_l && (n > crossover_large)) || (!mm_m && !mm_s)) { - return aligned ? mmp->a_l : mmp->l; - } - // Use medium shader when the N dimension is greater than the small shader's tile size - uint32_t crossover_medium = mmp->s->wg_denoms[1]; - if ((mm_m && (n > crossover_medium)) || !mm_s) { - return aligned ? mmp->a_m : mmp->m; - } - return aligned ? mmp->a_s : mmp->s; + return prec == GGML_PREC_DEFAULT; } - - if ((mm_s && (m <= 32 || n <= 32)) || (!mm_m && !mm_l)) { - return aligned ? mmp->a_s : mmp->s; + if (ctx->device->coopmat_support) { + return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT; } - if ((mm_m && (m <= 64 || n <= 64)) || !mm_l) { - return aligned ? mmp->a_m : mmp->m; + return ctx->device->fp16 && prec == GGML_PREC_DEFAULT; +} + +static const std::vector* ggml_vk_get_mul_mat_mat_pipeline_map( + ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec, bool mul_mat_id = false) { + bool f16acc = ggml_vk_get_mul_mat_mat_f16acc(ctx, src0_type, src1_type, prec); + vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc}; + auto it = ctx->device->pipeline_matmul.find(key); + if (it == ctx->device->pipeline_matmul.end() || it->second.empty()) { + // Try without f16acc + if (f16acc) { + key.f16acc = false; + it = ctx->device->pipeline_matmul.find(key); + if (it != ctx->device->pipeline_matmul.end() && !it->second.empty()) return &it->second; + } + return nullptr; } - return aligned ? mmp->a_l : mmp->l; + return &it->second; } -static uint32_t ggml_vk_guess_matmul_id_pipeline_align(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, ggml_type src0_type, ggml_type src1_type) { - VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")"); - return ggml_vk_guess_matmul_id_pipeline(ctx, mmp, m, n, true, src0_type, src1_type)->align; +static vk_pipeline ggml_vk_guess_matmul_pipeline_map(ggml_backend_vk_context * ctx, + const std::vector& configs, + uint32_t m, uint32_t n, bool aligned, bool mul_mat_id) { + auto& selector = mul_mat_id ? ctx->device->matmul_id_tile_selector : ctx->device->matmul_tile_selector; + uint32_t idx = selector(m, n, 0, ctx->device->shader_core_count, configs); + if (idx >= configs.size()) idx = (uint32_t)configs.size() - 1; + return (aligned && configs[idx].aligned) ? configs[idx].aligned : configs[idx].unaligned; +} + +static uint32_t ggml_vk_guess_matmul_pipeline_align_map(ggml_backend_vk_context * ctx, + const std::vector& configs, + uint32_t m, uint32_t n, bool mul_mat_id) { + auto& selector = mul_mat_id ? ctx->device->matmul_id_tile_selector : ctx->device->matmul_tile_selector; + uint32_t idx = selector(m, n, 0, ctx->device->shader_core_count, configs); + if (idx >= configs.size()) idx = (uint32_t)configs.size() - 1; + return configs[idx].align; } static void ggml_vk_matmul_id( @@ -9657,10 +9350,12 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub src1_uma = d_Qy != nullptr; } + // TODO: Clean up this logic to pick src1 type by capability // Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf const bool x_non_contig = (ctx->device->coopmat2 && src0->type == GGML_TYPE_F32) || !ggml_vk_dim01_contiguous(src0); const bool y_non_contig = (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) || + // coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is used. (ctx->device->coopmat_support && !ctx->device->coopmat2 && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) || (src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) || @@ -9674,31 +9369,31 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; // Check for mmq first - vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; + const std::vector* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; - if (mmp == nullptr) { + if (mmp_map == nullptr) { // Fall back to f16 dequant mul mat - mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); quantize_y = false; } - const bool qx_needs_dequant = mmp == nullptr || x_non_contig; + const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; const bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); if (qx_needs_dequant) { // Fall back to dequant + f16 mulmat - mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); } // Not implemented GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT - const ggml_type effective_src1_type = quantize_y ? GGML_TYPE_Q8_1 : (y_f32_kernel ? GGML_TYPE_F32 : src1->type); + GGML_ASSERT(mmp_map != nullptr); - const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ctx, mmp, ne01, ne11, qx_needs_dequant ? f16_type : src0->type, effective_src1_type)); + const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, ne11, false)); const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && ne11 > 8; - vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline(ctx, mmp, ne01, ne11, aligned, qx_needs_dequant ? f16_type : src0->type, effective_src1_type); + vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, ne11, aligned, false); if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); @@ -10698,7 +10393,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& #endif const bool y_non_contig = y_decode_vector_staging || (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) || - // Intel coopmat1: force f32->f16 conversion so the f16-B-type pipeline is used. + // Intel coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is used. (ctx->device->coopmat_support && !ctx->device->coopmat2 && ctx->device->vendor_id == VK_VENDOR_ID_INTEL && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) || @@ -10710,25 +10405,22 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0; // Check for mmq first - vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_id_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr; + const std::vector* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0], true) : nullptr; - if (mmp == nullptr) { + if (mmp_map == nullptr) { // Fall back to f16 dequant mul mat - mmp = ggml_vk_get_mul_mat_mat_id_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0], true); quantize_y = false; } - const bool qx_needs_dequant = mmp == nullptr || x_non_contig; + const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig; bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig); if (qx_needs_dequant) { // Fall back to dequant + f16 mulmat - mmp = ggml_vk_get_mul_mat_mat_id_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]); + mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0], true); } - const ggml_type effective_src1_type = quantize_y ? GGML_TYPE_Q8_1 : (y_f32_kernel ? GGML_TYPE_F32 : src1->type); - - const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_id_pipeline_align(ctx, mmp, ne01, nei1, qx_needs_dequant ? f16_type : src0->type, effective_src1_type)); // Coopmat2 MUL_MAT_ID BK specialization constants in ggml_vk_load_shaders are at most 64. const uint32_t y_staged_row_stride = ctx->device->coopmat2 && !quantize_y ? ggml_vk_align_size(ne10, 64) : ne10; const bool y_needs_k_padding = ne10 != y_staged_row_stride; @@ -10738,9 +10430,12 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& // Not implemented GGML_ASSERT(y_needs_reformat || !qy_needs_dequant); // NOLINT + GGML_ASSERT(mmp_map != nullptr); + + const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, nei1, true)); const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && nei1 > 8; - vk_pipeline pipeline = ggml_vk_guess_matmul_id_pipeline(ctx, mmp, ne01, nei1, aligned, qx_needs_dequant ? f16_type : src0->type, effective_src1_type); + vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, nei1, aligned, true); if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); @@ -15330,106 +15025,22 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t const size_t y_ne = k * n * batch; const size_t d_ne = m * n * batch; - vk_pipeline p; - std::string shname; - if (shader_size == 0) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->a_s; - shname = "F32_ALIGNED_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->a_s; - shname = "F32_F16_ALIGNED_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_s; - shname = "F16_F32_ALIGNED_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->a_s; - shname = "F16_ALIGNED_S"; - } else { - GGML_ABORT("fatal error"); - } - } else if (shader_size == 1) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->a_m; - shname = "F32_ALIGNED_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->a_m; - shname = "F32_F16_ALIGNED_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_m; - shname = "F16_F32_ALIGNED_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->a_m; - shname = "F16_ALIGNED_M"; - } else { - GGML_ABORT("fatal error"); - } - } else if (shader_size == 2) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->a_l; - shname = "F32_ALIGNED_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->a_l; - shname = "F32_F16_ALIGNED_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_l; - shname = "F16_F32_ALIGNED_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->a_l; - shname = "F16_ALIGNED_L"; - } else { - GGML_ABORT("fatal error"); - } - } else { - GGML_ASSERT(0); - } + ggml_type x_type = std::is_same() ? GGML_TYPE_F32 : GGML_TYPE_F16; + ggml_type y_type = std::is_same() ? GGML_TYPE_F32 : GGML_TYPE_F16; + vk_matmul_pipeline_key mm_test_key{x_type, y_type, false, false}; + auto mm_test_it = ctx->device->pipeline_matmul.find(mm_test_key); + GGML_ASSERT(mm_test_it != ctx->device->pipeline_matmul.end() && !mm_test_it->second.empty()); + auto& mm_test_configs = mm_test_it->second; + GGML_ASSERT(shader_size >= 0 && shader_size < (int)mm_test_configs.size()); + + std::string shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_ALIGNED_" + std::to_string(shader_size); + vk_pipeline p = mm_test_configs[shader_size].aligned ? mm_test_configs[shader_size].aligned : mm_test_configs[shader_size].unaligned; - const size_t kpad = ggml_vk_align_size(k, p->align); + const size_t kpad = ggml_vk_align_size(k, mm_test_configs[shader_size].align); if (k != kpad) { - if (shader_size == 0) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->s; - shname = "F32_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->s; - shname = "F32_F16_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->s; - shname = "F16_F32_S"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->s; - shname = "F16_S"; - } - } else if (shader_size == 1) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->m; - shname = "F32_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->m; - shname = "F32_F16_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->m; - shname = "F16_F32_M"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->m; - shname = "F16_M"; - } - } else if (shader_size == 2) { - if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32->l; - shname = "F32_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f32_f16->l; - shname = "F32_F16_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16_f32.f32acc->l; - shname = "F16_F32_L"; - } else if (std::is_same() && std::is_same()) { - p = ctx->device->pipeline_matmul_f16.f32acc->l; - shname = "F16_L"; - } - } + p = mm_test_configs[shader_size].unaligned; + shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_" + std::to_string(shader_size); } if (split_k > 1) { @@ -15864,46 +15475,34 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, const size_t y_ne = k * n * batch; const size_t d_ne = m * n * batch; - vk_matmul_pipeline2 * pipelines; - - if (mmq) { - pipelines = ctx->device->pipeline_dequant_mul_mat_mat_q8_1; - } else { - pipelines = ctx->device->pipeline_dequant_mul_mat_mat; - } - - const bool fp16acc = ctx->device->fp16; - - vk_pipeline p; - std::string shname; - if (shader_size == 0) { - p = fp16acc ? pipelines[quant].f16acc->a_s : pipelines[quant].f32acc->a_s; - shname = std::string(ggml_type_name(quant)) + "_ALIGNED_S"; - } else if (shader_size == 1) { - p = fp16acc ? pipelines[quant].f16acc->a_m : pipelines[quant].f32acc->a_m; - shname = std::string(ggml_type_name(quant)) + "_ALIGNED_M"; - } else if (shader_size == 2) { - p = fp16acc ? pipelines[quant].f16acc->a_l : pipelines[quant].f32acc->a_l; - shname = std::string(ggml_type_name(quant)) + "_ALIGNED_L"; - } else { - GGML_ASSERT(0); + ggml_type b_type = mmq ? GGML_TYPE_Q8_1 : GGML_TYPE_F32; + bool f16acc = ctx->device->fp16 && !mmq; + vk_matmul_pipeline_key dq_key{quant, b_type, false, f16acc}; + auto dq_it = ctx->device->pipeline_matmul.find(dq_key); + if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) { + if (f16acc) { + dq_key.f16acc = false; + dq_it = ctx->device->pipeline_matmul.find(dq_key); + } + } + if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) { + std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl; + return; + } + auto& dq_configs = dq_it->second; + if (shader_size >= (int)dq_configs.size()) { + std::cerr << "error: shader_size " << shader_size << " >= configs.size() " << dq_configs.size() << " for " << ggml_type_name(quant) << std::endl; + return; } - const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, p->align); + std::string shname = std::string(ggml_type_name(quant)) + "_ALIGNED_" + std::to_string(shader_size); + vk_pipeline p = dq_configs[shader_size].aligned ? dq_configs[shader_size].aligned : dq_configs[shader_size].unaligned; + + const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, dq_configs[shader_size].align); if (mmq || k != kpad) { - if (shader_size == 0) { - p = fp16acc ? pipelines[quant].f16acc->s : pipelines[quant].f32acc->s; - shname = std::string(ggml_type_name(quant)) + "_S"; - } else if (shader_size == 1) { - p = fp16acc ? pipelines[quant].f16acc->m : pipelines[quant].f32acc->m; - shname = std::string(ggml_type_name(quant)) + "_M"; - } else if (shader_size == 2) { - p = fp16acc ? pipelines[quant].f16acc->l : pipelines[quant].f32acc->l; - shname = std::string(ggml_type_name(quant)) + "_L"; - } else { - GGML_ASSERT(0); - } + p = dq_configs[shader_size].unaligned; + shname = std::string(ggml_type_name(quant)) + "_" + std::to_string(shader_size); } if (p == nullptr) { @@ -19464,8 +19063,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: break; default: return false; @@ -19571,8 +19170,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: - case GGML_TYPE_TQ2_0: case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: case GGML_TYPE_I32: return true; default: diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index ef53264a7700..cc6e242a90d5 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -260,6 +260,20 @@ float16_t dequantFuncTQ1_0(const in decodeBufTQ1_0 bl, const in uint blockCoords return bl.block.d * (float16_t(int(xi)) - float16_t(1.0)); } +f16vec4 dequantFuncTQ1_0_v(const in decodeBufTQ1_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) +{ + const uint e = coordInBlock[1]; + f16vec4 v; + [[unroll]] for (uint k = 0u; k < 4u; ++k) { + const uint ee = e + k; + const uint bidx = tq1_0_byte_of(ee); + const uint qbyte = uint(bidx < 48u ? bl.block.qs[bidx] : bl.block.qh[bidx - 48u]); + const uint xi = tq1_0_trit(qbyte, tq1_0_digit_of(ee)); + v[k] = bl.block.d * (float16_t(int(xi)) - float16_t(1.0)); + } + return v; +} + layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ2_0 { block_tq2_0 block; }; @@ -1054,7 +1068,7 @@ float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoords const uint scale = (bl.block.scales[ib32] >> ((idx & 0x10) >> 2)) & 0xf; const uint qs = bl.block.qs[ib8]; const uint qh = bl.block.qh[ib32]; - const uint sign = bl.block.qs[QUANT_K / 8 + ib8] >> (idx & 0x6); + const uint sign = bl.block.qs[QUANT_K_IQ2_S / 8 + ib8] >> (idx & 0x6); const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); @@ -1076,7 +1090,7 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords const uint scale = (bl.block.scales[ib32] >> ((idx & 0x10) >> 2)) & 0xf; const uint qs = bl.block.qs[ib8]; const uint qh = bl.block.qh[ib32]; - const uint sb = uint(bl.block.qs[QUANT_K / 8 + ib8]) >> (idx & 0x6u); + const uint sb = uint(bl.block.qs[QUANT_K_IQ2_S / 8 + ib8]) >> (idx & 0x6u); const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); @@ -1107,7 +1121,7 @@ float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCo uint idx = coordInBlock[1]; const uint iqs = (idx & 0xFC) >> 2; // 0..63 - const uint is = QUANT_K / 4 + ((idx & 0xE0) >> 3);// 8 values + const uint is = QUANT_K_IQ3_XXS / 4 + ((idx & 0xE0) >> 3);// 8 values const float d = float(bl.block.d); const uint qs = bl.block.qs[iqs]; @@ -1130,7 +1144,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo const uint idx = coordInBlock[1]; const uint iqs = idx >> 2; - const uint is = QUANT_K / 4 + ((idx & 0xE0) >> 3); + const uint is = QUANT_K_IQ3_XXS / 4 + ((idx & 0xE0) >> 3); const float d = float(bl.block.d); const uint qs = bl.block.qs[iqs]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl index 6f414ded1230..1e732a9a29cc 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl @@ -1,32 +1,22 @@ #if !defined(GGML_FA_TYPES_COMP) #define GGML_FA_TYPES_COMP -// FaTypeK / FaTypeV spec constant values. These mirror enum ggml_type so the -// host can pass the type directly. Keep in sync with ggml.h. -#define FA_TYPE_F32 0u -#define FA_TYPE_F16 1u -#define FA_TYPE_Q4_0 2u -#define FA_TYPE_Q4_1 3u -#define FA_TYPE_Q5_0 6u -#define FA_TYPE_Q5_1 7u -#define FA_TYPE_Q8_0 8u -#define FA_TYPE_IQ4_NL 20u -#define FA_TYPE_BF16 30u +#include "ggml_type_ids.glsl" // Number of matrix elements per buffer block, derived from the K/V type spec // constant. F32 is treated as a vec4 "block" of 4 floats. F16 uses block size 1 // and bypasses the dequant path entirely. Quants follow their ggml block sizes. uint fa_block_elems(uint ty) { switch (ty) { - case FA_TYPE_F32: return 4u; - case FA_TYPE_F16: return 1u; - case FA_TYPE_Q4_0: return uint(QUANT_K_Q4_0); - case FA_TYPE_Q4_1: return uint(QUANT_K_Q4_1); - case FA_TYPE_Q5_0: return uint(QUANT_K_Q5_0); - case FA_TYPE_Q5_1: return uint(QUANT_K_Q5_1); - case FA_TYPE_Q8_0: return uint(QUANT_K_Q8_0); - case FA_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL); - case FA_TYPE_BF16: return 1u; + case GGML_TYPE_F32: return 4u; + case GGML_TYPE_F16: return 1u; + case GGML_TYPE_Q4_0: return uint(QUANT_K_Q4_0); + case GGML_TYPE_Q4_1: return uint(QUANT_K_Q4_1); + case GGML_TYPE_Q5_0: return uint(QUANT_K_Q5_0); + case GGML_TYPE_Q5_1: return uint(QUANT_K_Q5_1); + case GGML_TYPE_Q8_0: return uint(QUANT_K_Q8_0); + case GGML_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL); + case GGML_TYPE_BF16: return 1u; default: return 1u; } } @@ -36,18 +26,18 @@ uint fa_block_elems(uint ty) { // of int32s per 32-element block on the MMQ K path: ints_per_block == 8 / R. uint fa_quant_r_mmq(uint ty) { switch (ty) { - case FA_TYPE_Q4_0: return uint(QUANT_R_Q4_0); - case FA_TYPE_Q4_1: return uint(QUANT_R_Q4_1); - case FA_TYPE_Q5_0: return uint(QUANT_R_Q5_0); - case FA_TYPE_Q5_1: return uint(QUANT_R_Q5_1); - case FA_TYPE_Q8_0: return uint(QUANT_R_Q8_0); + case GGML_TYPE_Q4_0: return uint(QUANT_R_Q4_0); + case GGML_TYPE_Q4_1: return uint(QUANT_R_Q4_1); + case GGML_TYPE_Q5_0: return uint(QUANT_R_Q5_0); + case GGML_TYPE_Q5_1: return uint(QUANT_R_Q5_1); + case GGML_TYPE_Q8_0: return uint(QUANT_R_Q8_0); default: return 1u; } } bool fa_type_needs_shmem(uint ty) { switch (ty) { - case FA_TYPE_IQ4_NL: return true; + case GGML_TYPE_IQ4_NL: return true; default: return false; } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp index 0c1b6d0673e9..9a12cdfb8817 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp @@ -134,7 +134,7 @@ void main() { // Q8_0 K only needs (qd, _); the asymmetric Q4_*/Q5_* family also stores // the row-sum scaled by qd, used in k_dot_correction. - if (FaTypeK == FA_TYPE_Q8_0) { + if (FaTypeK == GGML_TYPE_Q8_0) { if (buf_iqs == 0) { Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f); } @@ -367,7 +367,7 @@ void main() { // Q4_*/Q5_* take the block-8 fast path when one step covers a full // block; Q8_0 always goes through the per-int get_k_qs* helpers // (its qs is byte-packed, not nibble-packed). - const bool block8_fast = (d_per_step == 8) && (FaTypeK != FA_TYPE_Q8_0); + const bool block8_fast = (d_per_step == 8) && (FaTypeK != GGML_TYPE_Q8_0); if (SHMEM_STAGING != 0) { const uint k_block_idx = (d_tid * (HSK_per_thread / 4) + d_block) / 8; @@ -375,7 +375,7 @@ void main() { k_dm = ACC_TYPEV2(kblocksh[buf_ib].dm); if (block8_fast) { - const bool has_qh = (FaTypeK == FA_TYPE_Q5_0) || (FaTypeK == FA_TYPE_Q5_1); + const bool has_qh = (FaTypeK == GGML_TYPE_Q5_0) || (FaTypeK == GGML_TYPE_Q5_1); [[unroll]] for (uint32_t d = 0; d < 4; d++) { uint vui = kblocksh[buf_ib].qs[d]; k_quants[d ] = int32_t( vui & 0x0F0F0F0F); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl index 0ce4503a8847..a4be1ebf98e3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl @@ -105,8 +105,8 @@ layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];}; #define BLOCK_SIZE_V fa_block_elems(FaTypeV) // F16 reads f16 elements directly from the binding; everything else routes // through dequantize4 / the MMQ helpers to unpack from the packed block layout. -#define USE_DECODE_K (FaTypeK != FA_TYPE_F16) -#define USE_DECODE_V (FaTypeV != FA_TYPE_F16) +#define USE_DECODE_K (FaTypeK != GGML_TYPE_F16) +#define USE_DECODE_V (FaTypeV != GGML_TYPE_F16) #define CEIL_DIV(a, b) (((a) + (b) - 1) / (b)) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp index 317411153087..5a9abe2265fa 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp @@ -40,26 +40,28 @@ layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufFA_ #if !defined(BFLOAT16) float16_t faDecodeK(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeK) { - case FA_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); default: return float16_t(0); } } float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeV) { - case FA_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); default: return float16_t(0); } } @@ -67,26 +69,26 @@ float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const // V=4 vector decode for K/V; dispatches to per-format _v decoders. f16vec4 faDecodeKVector(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeK) { - case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); - case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return f16vec4(0); } } f16vec4 faDecodeVVector(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeV) { - case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); - case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return f16vec4(0); } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl index 8ba4725f3342..4fcf7c1f4f64 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl @@ -121,25 +121,25 @@ layout (binding = 1) readonly buffer K_PACKED_Q5_1_P32 { block_q5_1_packed32 dat FLOAT_TYPEV4 dequantize4(uint ib, uint iqs, uint a_offset, uint binding_idx) { if (binding_idx == BINDING_IDX_K) { switch (FaTypeK) { - case FA_TYPE_F32: FA_DEQUANT4_F32 (k_packed_f32) - case FA_TYPE_Q4_0: FA_DEQUANT4_Q4_0(k_packed_q4_0) - case FA_TYPE_Q4_1: FA_DEQUANT4_Q4_1(k_packed_q4_1) - case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(k_packed_q5_0) - case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(k_packed_q5_1) - case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(k_packed_q8_0) - case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(k_packed_iq4_nl) - case FA_TYPE_BF16: FA_DEQUANT4_BF16(k_packed_bf16) + case GGML_TYPE_F32: FA_DEQUANT4_F32 (k_packed_f32) + case GGML_TYPE_Q4_0: FA_DEQUANT4_Q4_0(k_packed_q4_0) + case GGML_TYPE_Q4_1: FA_DEQUANT4_Q4_1(k_packed_q4_1) + case GGML_TYPE_Q5_0: FA_DEQUANT4_Q5_0(k_packed_q5_0) + case GGML_TYPE_Q5_1: FA_DEQUANT4_Q5_1(k_packed_q5_1) + case GGML_TYPE_Q8_0: FA_DEQUANT4_Q8_0(k_packed_q8_0) + case GGML_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(k_packed_iq4_nl) + case GGML_TYPE_BF16: FA_DEQUANT4_BF16(k_packed_bf16) } } else { switch (FaTypeV) { - case FA_TYPE_F32: FA_DEQUANT4_F32 (v_packed_f32) - case FA_TYPE_Q4_0: FA_DEQUANT4_Q4_0(v_packed_q4_0) - case FA_TYPE_Q4_1: FA_DEQUANT4_Q4_1(v_packed_q4_1) - case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(v_packed_q5_0) - case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(v_packed_q5_1) - case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(v_packed_q8_0) - case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(v_packed_iq4_nl) - case FA_TYPE_BF16: FA_DEQUANT4_BF16(v_packed_bf16) + case GGML_TYPE_F32: FA_DEQUANT4_F32 (v_packed_f32) + case GGML_TYPE_Q4_0: FA_DEQUANT4_Q4_0(v_packed_q4_0) + case GGML_TYPE_Q4_1: FA_DEQUANT4_Q4_1(v_packed_q4_1) + case GGML_TYPE_Q5_0: FA_DEQUANT4_Q5_0(v_packed_q5_0) + case GGML_TYPE_Q5_1: FA_DEQUANT4_Q5_1(v_packed_q5_1) + case GGML_TYPE_Q8_0: FA_DEQUANT4_Q8_0(v_packed_q8_0) + case GGML_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(v_packed_iq4_nl) + case GGML_TYPE_BF16: FA_DEQUANT4_BF16(v_packed_bf16) } } return FLOAT_TYPEV4(0); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl index 6bf10a7cffd2..49900aa5aeb3 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl @@ -4,20 +4,20 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { switch (FaTypeK) { - case FA_TYPE_Q4_0: { + case GGML_TYPE_Q4_0: { uint vui = pack32(u16vec2(k_packed_q4_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 0], k_packed_q4_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 1])); uint shift = (iqs & 0x10) >> 2; vui >>= shift; return int32_t(vui & 0x0F0F0F0F); } - case FA_TYPE_Q4_1: { // uses packed32 alias + case GGML_TYPE_Q4_1: { // uses packed32 alias uint vui = k_packed_q4_1_p32.data[a_offset + ib].qs[(iqs & 0xF) / 4]; uint shift = (iqs & 0x10) >> 2; vui >>= shift; return int32_t(vui & 0x0F0F0F0F); } - case FA_TYPE_Q5_0: { + case GGML_TYPE_Q5_0: { uint vui = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 0], k_packed_q5_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 1])); uint qh = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qh[0], @@ -27,7 +27,7 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { uint qh_bits = (qh >> iqs) & 0xF; return int32_t(vui & 0x0F0F0F0F) | int32_t((qh_bits * 0x02040810u) & 0x10101010u); } - case FA_TYPE_Q5_1: { // qs via packed32, qh via packed16 + case GGML_TYPE_Q5_1: { // qs via packed32, qh via packed16 uint vui = k_packed_q5_1_p32.data[a_offset + ib].qs[(iqs & 0xF) / 4]; uint qh = k_packed_q5_1.data[a_offset + ib].qh; uint shift = (iqs & 0x10) >> 2; @@ -35,7 +35,7 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { uint qh_bits = (qh >> iqs) & 0xF; return int32_t(vui & 0x0F0F0F0F) | int32_t((qh_bits * 0x02040810u) & 0x10101010u); } - case FA_TYPE_Q8_0: { + case GGML_TYPE_Q8_0: { return pack32(i16vec2(k_packed_q8_0.data[a_offset + ib].qs[iqs / 2], k_packed_q8_0.data[a_offset + ib].qs[iqs / 2 + 1])); } @@ -47,11 +47,11 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) { // return (d, 0) so call sites always see the same shape. FLOAT_TYPEV2 get_k_scale(uint ib, uint a_offset) { switch (FaTypeK) { - case FA_TYPE_Q4_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + ib].d), 0.0); - case FA_TYPE_Q4_1: return FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + ib].dm); - case FA_TYPE_Q5_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + ib].d), 0.0); - case FA_TYPE_Q5_1: return FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + ib].dm); - case FA_TYPE_Q8_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + ib].d), 0.0); + case GGML_TYPE_Q4_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + ib].d), 0.0); + case GGML_TYPE_Q4_1: return FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + ib].dm); + case GGML_TYPE_Q5_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + ib].d), 0.0); + case GGML_TYPE_Q5_1: return FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + ib].dm); + case GGML_TYPE_Q8_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + ib].d), 0.0); default: return FLOAT_TYPEV2(0); } } @@ -61,16 +61,16 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c // explicit casts. The bit pattern is what we care about here -- the actual // signed/unsigned interpretation happens downstream in the dot product. switch (FaTypeK) { - case FA_TYPE_Q4_0: { + case GGML_TYPE_Q4_0: { kblocksh[buf_ib].qs[iqs] = int32_t(pack32(u16vec2(k_packed_q4_0.data[a_offset + global_ib].qs[iqs * 2], k_packed_q4_0.data[a_offset + global_ib].qs[iqs * 2 + 1]))); break; } - case FA_TYPE_Q4_1: { + case GGML_TYPE_Q4_1: { kblocksh[buf_ib].qs[iqs] = int32_t(k_packed_q4_1_p32.data[a_offset + global_ib].qs[iqs]); break; } - case FA_TYPE_Q5_0: { + case GGML_TYPE_Q5_0: { kblocksh[buf_ib].qs[iqs] = int32_t(pack32(u16vec2(k_packed_q5_0.data[a_offset + global_ib].qs[iqs * 2], k_packed_q5_0.data[a_offset + global_ib].qs[iqs * 2 + 1]))); if (iqs == 0) { @@ -79,14 +79,14 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c } break; } - case FA_TYPE_Q5_1: { + case GGML_TYPE_Q5_1: { kblocksh[buf_ib].qs[iqs] = int32_t(k_packed_q5_1_p32.data[a_offset + global_ib].qs[iqs]); if (iqs == 0) { kblocksh[buf_ib].qh = k_packed_q5_1.data[a_offset + global_ib].qh; } break; } - case FA_TYPE_Q8_0: { + case GGML_TYPE_Q8_0: { kblocksh[buf_ib].qs[iqs] = pack32(i16vec2(k_packed_q8_0.data[a_offset + global_ib].qs[iqs * 2], k_packed_q8_0.data[a_offset + global_ib].qs[iqs * 2 + 1])); break; @@ -96,11 +96,11 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c if (iqs == 0) { // Q4_0/Q5_0/Q8_0 store dm.x = d; Q4_1/Q5_1 store dm = (d, m) pair. switch (FaTypeK) { - case FA_TYPE_Q4_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + global_ib].d), 0.0); break; - case FA_TYPE_Q4_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + global_ib].dm); break; - case FA_TYPE_Q5_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + global_ib].d), 0.0); break; - case FA_TYPE_Q5_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + global_ib].dm); break; - case FA_TYPE_Q8_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + global_ib].d), 0.0); break; + case GGML_TYPE_Q4_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + global_ib].d), 0.0); break; + case GGML_TYPE_Q4_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + global_ib].dm); break; + case GGML_TYPE_Q5_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + global_ib].d), 0.0); break; + case GGML_TYPE_Q5_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + global_ib].dm); break; + case GGML_TYPE_Q8_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + global_ib].d), 0.0); break; } } } @@ -121,31 +121,31 @@ struct fa_k_qs_block8 { fa_k_qs_block8 get_k_qs_block8(uint ib, uint a_offset) { fa_k_qs_block8 r; uint qh = 0; - if (FaTypeK == FA_TYPE_Q5_0) { + if (FaTypeK == GGML_TYPE_Q5_0) { qh = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qh[0], k_packed_q5_0.data[a_offset + ib].qh[1])); - } else if (FaTypeK == FA_TYPE_Q5_1) { + } else if (FaTypeK == GGML_TYPE_Q5_1) { qh = k_packed_q5_1.data[a_offset + ib].qh; } - const bool has_qh = (FaTypeK == FA_TYPE_Q5_0) || (FaTypeK == FA_TYPE_Q5_1); + const bool has_qh = (FaTypeK == GGML_TYPE_Q5_0) || (FaTypeK == GGML_TYPE_Q5_1); [[unroll]] for (uint32_t d = 0; d < 4; d++) { uint vui = 0; switch (FaTypeK) { - case FA_TYPE_Q4_0: { // packed16 + case GGML_TYPE_Q4_0: { // packed16 vui = pack32(u16vec2(k_packed_q4_0.data[a_offset + ib].qs[d * 2 + 0], k_packed_q4_0.data[a_offset + ib].qs[d * 2 + 1])); break; } - case FA_TYPE_Q4_1: { // packed32 alias + case GGML_TYPE_Q4_1: { // packed32 alias vui = k_packed_q4_1_p32.data[a_offset + ib].qs[d]; break; } - case FA_TYPE_Q5_0: { // packed16 + case GGML_TYPE_Q5_0: { // packed16 vui = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qs[d * 2 + 0], k_packed_q5_0.data[a_offset + ib].qs[d * 2 + 1])); break; } - case FA_TYPE_Q5_1: { // packed32 alias + case GGML_TYPE_Q5_1: { // packed32 alias vui = k_packed_q5_1_p32.data[a_offset + ib].qs[d]; break; } @@ -164,21 +164,21 @@ fa_k_qs_block8 get_k_qs_block8(uint ib, uint a_offset) { int32_t get_k_qs_shmem(const uint buf_ib, const uint pos) { switch (FaTypeK) { - case FA_TYPE_Q4_0: - case FA_TYPE_Q4_1: { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: { uint sub = pos % 4; uint shift = ((pos % 8) >= 4) ? 4u : 0u; return int32_t((uint(kblocksh[buf_ib].qs[sub]) >> shift) & 0x0F0F0F0Fu); } - case FA_TYPE_Q5_0: - case FA_TYPE_Q5_1: { + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: { uint sub = pos % 4; uint shift = ((pos % 8) >= 4) ? 4u : 0u; int32_t result = int32_t((uint(kblocksh[buf_ib].qs[sub]) >> shift) & 0x0F0F0F0Fu); uint qh_bits = (kblocksh[buf_ib].qh >> (pos * 4u)) & 0xFu; return result | int32_t((qh_bits * 0x02040810u) & 0x10101010u); } - case FA_TYPE_Q8_0: { + case GGML_TYPE_Q8_0: { return kblocksh[buf_ib].qs[pos]; } default: return 0; @@ -187,10 +187,10 @@ int32_t get_k_qs_shmem(const uint buf_ib, const uint pos) { ACC_TYPE k_dot_correction(const uint qib, const ACC_TYPEV2 k_dm) { switch (FaTypeK) { - case FA_TYPE_Q4_0: return -ACC_TYPE(8.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; - case FA_TYPE_Q5_0: return -ACC_TYPE(16.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; - case FA_TYPE_Q4_1: - case FA_TYPE_Q5_1: return ACC_TYPE(Qf[qib].ds.y) * k_dm.y; + case GGML_TYPE_Q4_0: return -ACC_TYPE(8.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; + case GGML_TYPE_Q5_0: return -ACC_TYPE(16.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x; + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_1: return ACC_TYPE(Qf[qib].ds.y) * k_dm.y; default: return ACC_TYPE(0.0); } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl new file mode 100644 index 000000000000..0f10c733dd88 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl @@ -0,0 +1,34 @@ +#if !defined(GGML_TYPE_IDS_COMP) +#define GGML_TYPE_IDS_COMP + +// ggml_type enum values — must match ggml.h +#define GGML_TYPE_F32 0u +#define GGML_TYPE_F16 1u +#define GGML_TYPE_Q4_0 2u +#define GGML_TYPE_Q4_1 3u +#define GGML_TYPE_Q5_0 6u +#define GGML_TYPE_Q5_1 7u +#define GGML_TYPE_Q8_0 8u +#define GGML_TYPE_Q2_K 10u +#define GGML_TYPE_Q3_K 11u +#define GGML_TYPE_Q4_K 12u +#define GGML_TYPE_Q5_K 13u +#define GGML_TYPE_Q6_K 14u +#define GGML_TYPE_IQ2_XXS 16u +#define GGML_TYPE_IQ2_XS 17u +#define GGML_TYPE_IQ3_XXS 18u +#define GGML_TYPE_IQ1_S 19u +#define GGML_TYPE_IQ4_NL 20u +#define GGML_TYPE_IQ3_S 21u +#define GGML_TYPE_IQ2_S 22u +#define GGML_TYPE_IQ4_XS 23u +#define GGML_TYPE_IQ1_M 29u +#define GGML_TYPE_BF16 30u +#define GGML_TYPE_TQ1_0 34u +#define GGML_TYPE_TQ2_0 35u +#define GGML_TYPE_MXFP4 39u +#define GGML_TYPE_NVFP4 40u +#define GGML_TYPE_Q1_0 41u +#define GGML_TYPE_Q2_0 42u + +#endif // !defined(GGML_TYPE_IDS_COMP) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl new file mode 100644 index 000000000000..12e50ee9eb9d --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl @@ -0,0 +1,2 @@ +void init_iq_shmem(uvec3 wgsize) { +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp b/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp index ba76ec72ca6c..9b34d8366220 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp @@ -12,9 +12,9 @@ #include "types.glsl" #include "fa_types.glsl" -#define FaTypeV FA_TYPE_F32 +#define FaTypeV GGML_TYPE_F32 -layout(constant_id = 0) const uint FaTypeK = FA_TYPE_F32; +layout(constant_id = 0) const uint FaTypeK = GGML_TYPE_F32; layout(constant_id = 1) const uint FaBlockBytesK = 4; layout(constant_id = 2) const uint SUBGROUP_SIZE = 32; @@ -84,11 +84,11 @@ void main() { const uint k_block_elems = fa_block_elems(FaTypeK); const uint k_elem_bytes = FaBlockBytesK / k_block_elems; - if (FaTypeK == FA_TYPE_F16) { + if (FaTypeK == GGML_TYPE_F16) { k_row[tid] = float(k_f16[k_offset / k_elem_bytes + tid]); - } else if (FaTypeK == FA_TYPE_F32) { + } else if (FaTypeK == GGML_TYPE_F32) { k_row[tid] = k_f32[k_offset / k_elem_bytes + tid]; - } else if (FaTypeK == FA_TYPE_BF16) { + } else if (FaTypeK == GGML_TYPE_BF16) { k_row[tid] = bf16_to_fp32(uint(k_bf16[k_offset / k_elem_bytes + tid])); } else if (4 * tid < HEAD_SIZE) { const uint coord = 4 * tid; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 63c4aaebcb1a..90e4e11cdec7 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -9,6 +9,9 @@ #if defined(DATA_A_IQ1_M) #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #endif +#if !defined(DATA_A_F32) && !defined(DATA_A_F16) && !defined(DATA_A_BF16) +#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require +#endif #if defined(DATA_A_BF16) && defined(COOPMAT) #extension GL_EXT_bfloat16 : enable @@ -28,24 +31,54 @@ #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require #endif +#ifdef MULMAT_QUANT +#include "ggml_type_ids.glsl" +layout (constant_id = 12) const uint MmTypeA = 0; +#endif + #include "types.glsl" #include "dot_product_funcs.glsl" +#ifndef MULMAT_QUANT #ifndef LOAD_VEC_A #define LOAD_VEC_A 1 #endif +#endif #ifndef LOAD_VEC_B #define LOAD_VEC_B 1 #endif layout (constant_id = 11) const uint ALIGNED = 0; +#ifdef MULMAT_QUANT + +uint mm_load_vec_a() { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_1: + return 8u; + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return 4u; + default: + return 2u; + } +} +#endif + #if !defined(TO_FLOAT_TYPE) #define TO_FLOAT_TYPE FLOAT_TYPE #endif layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +#ifndef MULMAT_QUANT layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; #if defined(DATA_A_F32) layout (binding = 0) readonly buffer A_SCALAR {float data_a_scalar[];}; @@ -60,6 +93,30 @@ layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16 #if defined(A_TYPE_PACKED32) layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];}; #endif +#else +// Unpacked struct aliases +layout (binding = 0) readonly buffer BUF_Q1_0 { block_q1_0 data[]; } a_q1_0; +layout (binding = 0) readonly buffer BUF_Q2_0 { block_q2_0 data[]; } a_q2_0; +layout (binding = 0) readonly buffer BUF_Q2_K { block_q2_K data[]; } a_q2_k; +layout (binding = 0) readonly buffer BUF_Q3_K { block_q3_K data[]; } a_q3_k; +layout (binding = 0) readonly buffer BUF_Q4_K { block_q4_K data[]; } a_q4_k; +layout (binding = 0) readonly buffer BUF_Q5_K { block_q5_K data[]; } a_q5_k; +layout (binding = 0) readonly buffer BUF_Q6_K { block_q6_K data[]; } a_q6_k; +layout (binding = 0) readonly buffer BUF_TQ1_0 { block_tq1_0 data[]; } a_tq1_0; +layout (binding = 0) readonly buffer BUF_TQ2_0 { block_tq2_0 data[]; } a_tq2_0; +// Packed16 aliases +layout (binding = 0) readonly buffer BUF_Q4_0_P16 { block_q4_0_packed16 data[]; } a_q4_0_p16; +layout (binding = 0) readonly buffer BUF_Q5_0_P16 { block_q5_0_packed16 data[]; } a_q5_0_p16; +layout (binding = 0) readonly buffer BUF_Q8_0_P16 { block_q8_0_packed16 data[]; } a_q8_0_p16; +layout (binding = 0) readonly buffer BUF_Q3_K_P16 { block_q3_K_packed16 data[]; } a_q3_k_p16; +layout (binding = 0) readonly buffer BUF_Q6_K_P16 { block_q6_K_packed16 data[]; } a_q6_k_p16; +// Packed32 aliases +layout (binding = 0) readonly buffer BUF_Q4_1_P32 { block_q4_1_packed32 data[]; } a_q4_1_p32; +layout (binding = 0) readonly buffer BUF_Q5_1_P32 { block_q5_1_packed32 data[]; } a_q5_1_p32; +layout (binding = 0) readonly buffer BUF_Q2_K_P32 { block_q2_K_packed32 data[]; } a_q2_k_p32; +layout (binding = 0) readonly buffer BUF_Q4_K_P32 { block_q4_K_packed32 data[]; } a_q4_k_p32; +layout (binding = 0) readonly buffer BUF_Q5_K_P32 { block_q5_K_packed32 data[]; } a_q5_k_p32; +#endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 1) readonly buffer B_SCALAR {B_TYPE_SCALAR data_b_scalar[];}; @@ -121,8 +178,13 @@ layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working wit #endif #ifdef COOPMAT +#ifdef MULMAT_QUANT +layout(constant_id = 13) const uint SHMEM_STRIDE_PAD = 4; +layout(constant_id = 14) const bool APPLY_SLM_A_RESHAPE = false; +#else layout(constant_id = 12) const uint SHMEM_STRIDE_PAD = 4; layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false; +#endif #else const uint SHMEM_STRIDE_PAD = 1; const bool APPLY_SLM_A_RESHAPE = false; @@ -141,6 +203,10 @@ shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS]; #include "mul_mm_id_funcs.glsl" #include "mul_mm_funcs.glsl" +#ifdef MULMAT_QUANT +#include "iq_shmem_init.glsl" +#endif + void main() { const uint ic = gl_WorkGroupID.y; @@ -150,7 +216,7 @@ void main() { return; } #endif -#ifdef NEEDS_INIT_IQ_SHMEM +#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT) init_iq_shmem(gl_WorkGroupSize); #endif @@ -200,9 +266,12 @@ void main() { #if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16) const uint LOAD_VEC_A_EFF = (ALIGNED != 0) ? LOAD_VEC_A : 1; const uint LOAD_VEC_BATCH_A = (ALIGNED != 0) ? 1 : 2; -#else +#elif !defined(MULMAT_QUANT) const uint LOAD_VEC_A_EFF = LOAD_VEC_A; const uint LOAD_VEC_BATCH_A = 1; +#else + const uint LOAD_VEC_A_EFF = mm_load_vec_a(); + const uint LOAD_VEC_BATCH_A = 1; #endif const uint LOAD_VEC_B_EFF = (ALIGNED != 0) ? LOAD_VEC_B : 1; const uint LOAD_VEC_BATCH_B = (ALIGNED != 0) ? 1 : 2; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 27f3178e7f26..189788a86257 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -21,6 +21,13 @@ #extension GL_EXT_bfloat16 : enable #endif +#include "ggml_type_ids.glsl" + +#ifdef MULMAT_QUANT +layout (constant_id = 7) const uint MmTypeA = 0; +layout (constant_id = 8) const uint MmABlockBytes = 2; +#endif + #include "types.glsl" #include "utils.glsl" @@ -37,6 +44,24 @@ layout (constant_id = 4) const bool enable_smaller_matrices = false; const uint BNover2 = enable_smaller_matrices ? (BN / 2) : BN; const uint BNover4 = enable_smaller_matrices ? (BN / 4) : BN; layout (constant_id = 5) const uint ALIGNED = 0; +layout (constant_id = 6) const uint subgroup_size = 32; + +#ifdef MULMAT_QUANT + +uint mm_quant_k() { + switch (MmTypeA) { + case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return 32u; + case GGML_TYPE_Q1_0: + return 128u; + case GGML_TYPE_Q2_0: + return 64u; + default: + return 256u; + } +} +#endif layout (push_constant) uniform parameter { @@ -72,21 +97,73 @@ layout (push_constant) uniform parameter } p; +#ifndef MULMAT_QUANT layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; +#else +layout (binding = 0) readonly buffer A {uint8_t data_a[];}; +#endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];}; #endif -#if QUANT_K > 1 +#if defined(MULMAT_QUANT) || QUANT_K > 1 #include "dequant_funcs_cm2.glsl" +#ifndef MULMAT_QUANT +// Per-type path: use the alias set by dequant_funcs_cm2.glsl #if defined(dequantFuncA_v) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) #define DECODEFUNCA , dequantFuncA, dequantFuncA_v #else #define DECODEFUNCA , dequantFuncA #endif #else +layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufA { + uint8_t raw[MmABlockBytes]; +}; + +float16_t mmDecodeA(const in decodeBufA bl_in, const in uint blockCoords[2], const in uint coordInBlock[2]) { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: return dequantFuncQ1_0 (decodeBufQ1_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_0: return dequantFuncQ2_0 (decodeBufQ2_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0 (decodeBufQ4_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1 (decodeBufQ4_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0 (decodeBufQ5_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1 (decodeBufQ5_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0 (decodeBufQ8_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_K: return dequantFuncQ2_K (decodeBufQ2_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q3_K: return dequantFuncQ3_K (decodeBufQ3_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q6_K: return dequantFuncQ6_K (decodeBufQ6_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ1_0: return dequantFuncTQ1_0(decodeBufTQ1_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ2_0: return dequantFuncTQ2_0(decodeBufTQ2_0(bl_in), blockCoords, coordInBlock); + default: return float16_t(0); + } +} + +#ifdef GGML_VULKAN_COOPMAT2_DECODE_VECTOR +f16vec4 mmDecodeA_v(const in decodeBufA bl_in, const in uint blockCoords[2], const in uint coordInBlock[2]) { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: return dequantFuncQ1_0_v (decodeBufQ1_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_0: return dequantFuncQ2_0_v (decodeBufQ2_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v (decodeBufQ4_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v (decodeBufQ4_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v (decodeBufQ5_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v (decodeBufQ5_1 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v (decodeBufQ8_0 (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q2_K: return dequantFuncQ2_K_v (decodeBufQ2_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q3_K: return dequantFuncQ3_K_v (decodeBufQ3_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_Q6_K: return dequantFuncQ6_K_v (decodeBufQ6_K (bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ1_0: return dequantFuncTQ1_0_v(decodeBufTQ1_0(bl_in), blockCoords, coordInBlock); + case GGML_TYPE_TQ2_0: return dequantFuncTQ2_0_v(decodeBufTQ2_0(bl_in), blockCoords, coordInBlock); + default: return f16vec4(0); + } +} +#define DECODEFUNCA , mmDecodeA, mmDecodeA_v +#else +#define DECODEFUNCA , mmDecodeA +#endif +#endif +#else #define DECODEFUNCA #endif @@ -114,7 +191,6 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB { }; uint _ne1; -layout (constant_id = 6) const uint subgroup_size = 32; shared uvec4 ballots_sh[BLOCK_SIZE / subgroup_size]; B_TYPE decodeFuncB(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2]) @@ -246,6 +322,10 @@ void load_row_ids_hoisted(uint expert_idx, uint ic) { } #endif +#ifdef MULMAT_QUANT +#include "iq_shmem_init.glsl" +#endif + void main() { const uint tid = gl_LocalInvocationIndex; const uint ic = gl_WorkGroupID.y; @@ -264,7 +344,7 @@ void main() { #endif #endif -#ifdef NEEDS_INIT_IQ_SHMEM +#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT) init_iq_shmem(gl_WorkGroupSize); #endif @@ -305,22 +385,33 @@ void main() { const uint end_k = min(p.K, (ik + 1) * p.k_split); #endif +#ifdef MULMAT_QUANT + const uint qk = mm_quant_k(); +#else + const uint qk = QUANT_K; +#endif + #ifdef MUL_MAT_ID - uint pos_a = expert_idx * (p.batch_stride_a / QUANT_K); + uint pos_a = expert_idx * (p.batch_stride_a / qk); uint pos_b = 0; #else - uint pos_a = batch_idx_a * (p.batch_stride_a / QUANT_K); + uint pos_a = batch_idx_a * (p.batch_stride_a / qk); uint pos_b = batch_idx * p.batch_stride_b; uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches; #endif - uint stride_a = p.stride_a / QUANT_K; +#ifdef MULMAT_QUANT + // pos_a is a byte offset into the raw buffer; strides stay in block units + pos_a *= MmABlockBytes; +#endif + + uint stride_a = p.stride_a / qk; uint stride_b = p.stride_b; // Hint to the compiler that values are aligned (want 16B alignment). // Quants are always block-aligned, no alignment needed. if (ALIGNED != 0) { -#if QUANT_K == 1 +#if !defined(MULMAT_QUANT) && QUANT_K == 1 stride_a &= ~7; #endif stride_b &= ~7; @@ -335,10 +426,8 @@ void main() { #endif tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutD = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV); -#if QUANT_K > 1 - tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K); - tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, QUANT_K); -#endif + tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, qk); + tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, qk); #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR) tensorLayoutB = setTensorLayoutBlockSizeNV(tensorLayoutB, 1, BK); #endif @@ -368,19 +457,19 @@ void main() { const uint START_ALIGN_K = 256; // For Qi_K (block size 256), unroll whole 256 element tiles. // For legacy quants (block size 32), unroll 8x. - const uint UNROLL_K = (QUANT_K == 256) ? 256 : (BK * 8); + const uint UNROLL_K = (qk == 256) ? 256 : (BK * 8); const uint unroll_count = UNROLL_K / BK; // Detect a fast path where all loads are entirely in bounds and no clamping is required if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 && -#if QUANT_K == 1 +#if !defined(MULMAT_QUANT) && QUANT_K == 1 (stride_a % 8) == 0 && #endif (stride_b % 8) == 0) { // Hint to the compiler that values are aligned (want 16B alignment) start_k &= ~(START_ALIGN_K-1); stride_b &= ~7; -#if QUANT_K == 1 +#if !defined(MULMAT_QUANT) && QUANT_K == 1 stride_a &= ~7; #endif @@ -551,10 +640,10 @@ void main() { [[dont_unroll]] for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - if ((block_k % QUANT_K) == 0) { + if ((block_k % qk) == 0) { store_scales(tid); } - if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) { + if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) { fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false); } @@ -595,10 +684,10 @@ void main() { [[dont_unroll]] for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - if ((block_k % QUANT_K) == 0) { + if ((block_k % qk) == 0) { store_scales(tid); } - if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) { + if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) { fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false); } @@ -639,10 +728,10 @@ void main() { [[dont_unroll]] for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) { - if ((block_k % QUANT_K) == 0) { + if ((block_k % qk) == 0) { store_scales(tid); } - if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) { + if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) { fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index bdc70af140a3..dd05fb1bde69 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -20,669 +20,698 @@ void store_a(uint m, uint k_pair, FLOAT_TYPEV2 value) { void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uint idx_m, const uint block, const uint end_k) { #if defined(DATA_A_F32) || defined(DATA_A_F16) #if LOAD_VEC_A == 8 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]); - store_a(col, k_pair, aa[0].xy); - store_a(col, k_pair + 1, aa[0].zw); - store_a(col, k_pair + 2, aa[1].xy); - store_a(col, k_pair + 3, aa[1].zw); - return; - } + if (ALIGNED != 0) { + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]); + store_a(col, k_pair, aa[0].xy); + store_a(col, k_pair + 1, aa[0].zw); + store_a(col, k_pair + 2, aa[1].xy); + store_a(col, k_pair + 3, aa[1].zw); + return; + } #elif LOAD_VEC_A == 4 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]); - store_a(col, k_pair, aa.xy); - store_a(col, k_pair + 1, aa.zw); - return; - } + if (ALIGNED != 0) { + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]); + store_a(col, k_pair, aa.xy); + store_a(col, k_pair + 1, aa.zw); + return; + } #endif - const uint idx = pos_a + col * p.stride_a + row * 2; - if (idx_m < p.M && block + row * 2 + 1 < end_k) { - store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], - data_a_scalar[idx + 1])); - } else if (idx_m < p.M && block + row * 2 < end_k) { - store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f)); - } else { - store_a(col, row, FLOAT_TYPEV2(0.0f)); - } + const uint idx = pos_a + col * p.stride_a + row * 2; + if (idx_m < p.M && block + row * 2 + 1 < end_k) { + store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], + data_a_scalar[idx + 1])); + } else if (idx_m < p.M && block + row * 2 < end_k) { + store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f)); + } else { + store_a(col, row, FLOAT_TYPEV2(0.0f)); + } #elif defined(DATA_A_BF16) #if LOAD_VEC_A == 4 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx])); - store_a(col, k_pair, aa.xy); - store_a(col, k_pair + 1, aa.zw); - return; - } + if (ALIGNED != 0) { + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx])); + store_a(col, k_pair, aa.xy); + store_a(col, k_pair + 1, aa.zw); + return; + } #endif - const uint idx = pos_a + col * p.stride_a + row * 2; - if (idx_m < p.M && block + row * 2 + 1 < end_k) { - store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), - TO_FLOAT_TYPE(data_a_scalar[idx + 1]))); - } else if (idx_m < p.M && block + row * 2 < end_k) { - store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f)); - } else { - store_a(col, row, FLOAT_TYPEV2(0.0f)); - } -#elif defined(DATA_A_Q4_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const float d = float(data_a_packed16[ib].d); - const uint vui = uint(data_a_packed16[ib].qs[2*iqs]) | (uint(data_a_packed16[ib].qs[2*iqs + 1]) << 16); - const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d; - const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d; - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); -#elif defined(DATA_A_Q4_1) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const vec2 dm = vec2(data_a_packed32[ib].dm); - const uint vui = data_a_packed32[ib].qs[iqs]; - const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y; - const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y; - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); -#elif defined(DATA_A_Q5_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const float d = float(data_a_packed16[ib].d); - const uint uint_qh = uint(data_a_packed16[ib].qh[1]) << 16 | uint(data_a_packed16[ib].qh[0]); - const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); - const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); - - const uint vui = uint(data_a_packed16[ib].qs[iqs]); - const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d; - store_a(col, row, FLOAT_TYPEV2(v.xz)); - store_a(col, row + 8, FLOAT_TYPEV2(v.yw)); -#elif defined(DATA_A_Q5_1) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const vec2 dm = vec2(data_a_packed32[ib].dm); - const uint uint_qh = data_a_packed32[ib].qh; - const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10); - const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10); - const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10); - const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10); - - const uint vui = data_a_packed32[ib].qs[iqs]; - const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y; - const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y; - - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, FLOAT_TYPEV2(v0.xz)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw)); -#elif defined(DATA_A_Q8_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const float d = float(data_a_packed16[ib].d); - const i8vec2 v0 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs])).xy; // vec4 used due to #12147 - const i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy; - const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); -#elif defined(DATA_A_Q1_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 16; - const uint iqs = idx & 0xfu; - - const float d = float(data_a[ib].d); - const uint bits = uint(data_a[ib].qs[iqs]); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d)); - store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d)); - store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d)); -#elif defined(DATA_A_Q2_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 16; - const uint iqs = idx & 0xfu; - - const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib].d); - const uint bits = uint(data_a[ib].qs[iqs]); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f))); - store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f))); -#elif defined(DATA_A_Q2_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15 - const uint scalesi = iqs / 8; // 0..15 - const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 - - const vec4 qs = vec4(unpack8((data_a_packed32[ib].qs[qsi / 2] >> qsshift) & 0x03030303)); - const uint scales = data_a[ib].scales[scalesi]; - const vec2 dm = vec2(data_a[ib].dm); - - const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); -#elif defined(DATA_A_TQ1_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = (idx % 128) * 2; // element 0,2,4..254 - - const float d = float(data_a[ib].d); - vec2 v; - for (uint kk = 0u; kk < 2u; ++kk) { - const uint e = iqs + kk; - const uint bidx = tq1_0_byte_of(e); - const uint qbyte = uint(bidx < 48u ? data_a[ib].qs[bidx] - : data_a[ib].qh[bidx - 48u]); - v[kk] = d * (float(tq1_0_trit(qbyte, tq1_0_digit_of(e))) - 1.0); - } - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); -#elif defined(DATA_A_TQ2_0) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 - - const uint qsi = (iqs / 128) * 32 + (iqs % 32); // byte pair start - const uint shift = 2 * ((iqs % 128) / 32); // 0,2,4,6 - - const uvec2 qs = uvec2(data_a[ib].qs[qsi], data_a[ib].qs[qsi + 1]); - const float d = float(data_a[ib].d); - - const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); -#elif defined(DATA_A_Q3_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = idx % 128; // 0..127 - - const uint n = iqs / 64; // 0,1 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62 - const uint hmi = (iqs % 16) * 2; // 0,2,4..30 - const uint j = (iqs % 64) / 4; // 0..3 - const uint is = iqs / 8; // 0..15 - const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3 - const uint qsshift = halfsplit * 2; // 0,2,4,6 - - const int8_t us = int8_t(((data_a[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF) - | (((data_a[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4)); - const float dl = float(data_a[ib].d) * float(us - 32); - - const vec2 qs = vec2(unpack8((uint(data_a_packed16[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy); - const vec2 hm = vec2(unpack8(((uint(data_a_packed16[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy); - - store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(dl * (qs.x - hm.x), - dl * (qs.y - hm.y))); -#elif defined(DATA_A_Q4_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint n = iqs / 32; // 0,1,2,3 - const uint b = (iqs % 32) / 16; // 0,1 - const uint is = 2 * n + b; // 0..7 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 - - const vec2 loadd = vec2(data_a[ib].dm); - - const uvec3 scales = uvec3(data_a_packed32[ib].scales[0], - data_a_packed32[ib].scales[1], - data_a_packed32[ib].scales[2]); - const uint scalesoffs = (is & 3) * 8; - - const uint scidx0 = (is < 4) ? 0 : 2; - const uint scidxshift0 = scalesoffs; - const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - const uint mbidx0 = (is < 4) ? 1 : 2; - const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; - const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - - const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); - const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); - - const float d = loadd.x * sc; - const float m = -loadd.y * mbyte; - - const vec4 q = vec4(unpack8((data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); - store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); -#elif defined(DATA_A_Q5_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint n = iqs / 32; // 0,1,2,3 - const uint b = (iqs % 32) / 16; // 0,1 - const uint is = 2 * n + b; // 0..7 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 - const uint qhi = (iqs % 16) * 2; // 0,2,4..30 - - const vec2 loadd = vec2(data_a[ib].dm); - - const uvec3 scales = uvec3(data_a_packed32[ib].scales[0], - data_a_packed32[ib].scales[1], - data_a_packed32[ib].scales[2]); - const uint scalesoffs = (is & 3) * 8; - - const uint scidx0 = (is < 4) ? 0 : 2; - const uint scidxshift0 = scalesoffs; - const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - const uint mbidx0 = (is < 4) ? 1 : 2; - const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; - const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const uint idx = pos_a + col * p.stride_a + row * 2; + if (idx_m < p.M && block + row * 2 + 1 < end_k) { + store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), + TO_FLOAT_TYPE(data_a_scalar[idx + 1]))); + } else if (idx_m < p.M && block + row * 2 < end_k) { + store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f)); + } else { + store_a(col, row, FLOAT_TYPEV2(0.0f)); + } +#elif defined(DATA_A_IQ1_S) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; - const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); - const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); - - const float d = loadd.x * sc; - const float m = -loadd.y * mbyte; - - const uint qs = (data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F; - const uint qh = ((data_a_packed32[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4; - const vec4 q = vec4(unpack8(qs | qh)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); - store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); -#elif defined(DATA_A_Q6_K) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = idx % 128; // 0..127 - - const uint n = iqs / 64; // 0,1 - const uint b = ((iqs % 64) / 32) * 4; // 0,4 - const uint is_b = (iqs % 16) / 8; // 0,1 - const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 - const uint is = 8 * n + qhshift + is_b; // 0..15 - const uint qsi = n * 32 + (iqs % 32); // 0..63 - const uint qhi = n * 16 + (iqs % 16); // 0..31 - - const float dscale = float(data_a[ib].d) * float(data_a[ib].scales[is]); + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 32; - const uint ql = (uint(data_a_packed16[ib].ql[qsi]) >> b) & 0x0F0F; - const uint qh = (uint(data_a_packed16[ib].qh[qhi]) >> qhshift) & 0x0303; - const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale; + const float d = float(data_a[ib].d); + const uint qh = data_a[ib].qh[ib32]; + const uint qs = data_a[ib].qs[ib8]; + const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); + const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); - store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(q.x, q.y)); -#elif defined(DATA_A_IQ1_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 32; - - const float d = float(data_a[ib].d); - const uint qh = data_a[ib].qh[ib32]; - const uint qs = data_a[ib].qs[ib8]; - const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); - const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); - - const uint k_pair = row * LOAD_VEC_A / 2; - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - } + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); + + } #elif defined(DATA_A_IQ1_M) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib8 = idx % 32; - const uint ib16 = ib8 / 2; - - const uint16_t[4] scales = data_a[ib].scales; - const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; - const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); - const uint sc = scales[ib8 / 8]; - const uint qs = data_a[ib].qs[ib8]; - const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1)); - const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); - const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); - - const uint k_pair = row * LOAD_VEC_A / 2; - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - } + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib8 = idx % 32; + const uint ib16 = ib8 / 2; + + const uint16_t[4] scales = data_a[ib].scales; + const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; + const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); + const uint sc = scales[ib8 / 8]; + const uint qs = data_a[ib].qs[ib8]; + const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1)); + const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); + const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); + + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); + + } #elif defined(DATA_A_IQ2_XXS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 4; - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[8 * ib32 + ib8]; - const uint signs = pack32(u8vec4( - data_a[ib].qs[8*ib32 + 4], - data_a[ib].qs[8*ib32 + 5], - data_a[ib].qs[8*ib32 + 6], - data_a[ib].qs[8*ib32 + 7] - )); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); - const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2xxs_grid[qs]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 4; + + const float d = float(data_a[ib].d); + const uint qs = data_a[ib].qs[8 * ib32 + ib8]; + const uint signs = pack32(u8vec4( + data_a[ib].qs[8*ib32 + 4], + data_a[ib].qs[8*ib32 + 5], + data_a[ib].qs[8*ib32 + 6], + data_a[ib].qs[8*ib32 + 7] + )); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); + const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2xxs_grid[qs]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + #elif defined(DATA_A_IQ2_XS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib32 = (idx % 32) / 4; // 0..7 - const uint ib8 = idx % 4; // 0..3 - - const float d = float(data_a[ib].d); - const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uint qs = data_a[ib].qs[4 * ib32 + ib8]; - const uint sign7 = qs >> 9; - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2xs_grid[qs & 511]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 4; + + const float d = float(data_a[ib].d); + const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uint qs = data_a[ib].qs[4 * ib32 + ib8]; + const uint sign7 = qs >> 9; + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2xs_grid[qs & 511]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + #elif defined(DATA_A_IQ2_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 32; // 8 values per idx - const uint ib8 = idx % 32; // 0..31 - const uint ib32 = ib8 / 4; // 0..7 - - const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const uint qs = data_a[ib].qs[ib8]; - const uint qh = data_a[ib].qh[ib32]; - const uint qhshift = 2 * (ib8 % 4); - const uint sign = data_a[ib].qs[QUANT_K / 8 + ib8]; - - const float d = float(data_a[ib].d); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib8 = idx % 32; + const uint ib32 = ib8 / 4; + + const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const uint qs = data_a[ib].qs[ib8]; + const uint qh = data_a[ib].qh[ib32]; + const uint qhshift = 2 * (ib8 % 4); + const uint sign = data_a[ib].qs[QUANT_K_IQ2_S / 8 + ib8]; + + const float d = float(data_a[ib].d); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + + store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + + store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + + store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); + #elif defined(DATA_A_IQ3_XXS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = idx % 64; // 0..63 - const uint is = QUANT_K / 4 + 4 * (iqs / 8); // 8 values - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[iqs]; - const uint signs = pack32(u16vec2( - data_a_packed16[ib].qs[is/2], - data_a_packed16[ib].qs[is/2+1] - )); - const float db = d * 0.5 * (0.5 + (signs >> 28)); - const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); - const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); - const uint grid = iq3xxs_grid[qs]; - const vec4 v = db * vec4(unpack8(grid)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 64; + const uint iqs = idx % 64; + const uint is = QUANT_K_IQ3_XXS / 4 + 4 * (iqs / 8); + + const float d = float(data_a[ib].d); + const uint qs = data_a[ib].qs[iqs]; + const uint signs = pack32(u16vec2( + data_a_packed16[ib].qs[is/2], + data_a_packed16[ib].qs[is/2+1] + )); + const float db = d * 0.5 * (0.5 + (signs >> 28)); + const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); + const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); + const uint grid = iq3xxs_grid[qs]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + + store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); + #elif defined(DATA_A_IQ3_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = idx % 64; // 0..63 - const uint iqh = iqs / 8; - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[iqs]; - const uint qh = data_a[ib].qh[iqh]; - const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2))); - const uint scale = data_a[ib].scales[iqs / 16]; - const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); - const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; - const vec4 v = db * vec4(unpack8(grid)); - - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 64; + const uint iqs = idx % 64; + const uint iqh = iqs / 8; + + const float d = float(data_a[ib].d); + const uint qs = data_a[ib].qs[iqs]; + const uint qh = data_a[ib].qh[iqh]; + const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2))); + const uint scale = data_a[ib].scales[iqs / 16]; + const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); + const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + + store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); + #elif defined(DATA_A_IQ4_XS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; - const uint ib = idx / 64; // 4 values per idx - const uint ib32 = (idx % 64) / 8; // 0..7 - const uint iq = 4 * ib32 + (idx % 4); + const uint ib = idx / 64; + const uint ib32 = (idx % 64) / 8; + const uint iq = 4 * ib32 + (idx % 4); - const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; - const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3; - const uint qshift = idx & 4; - u8vec4 qs = unpack8((uint(data_a_packed32[ib].qs[iq]) >> qshift) & 0x0F0F0F0F); + const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; + const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3; + const uint qshift = idx & 4; + u8vec4 qs = unpack8((uint(data_a_packed32[ib].qs[iq]) >> qshift) & 0x0F0F0F0F); - const float d = float(data_a[ib].d); - const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]); + const float d = float(data_a[ib].d); + const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]); - const uint k_pair = row * LOAD_VEC_A / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); #elif defined(DATA_A_IQ4_NL) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; - const uint ib = idx / 8; - const uint iqs = idx & 0x07; + const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d); + const uint vui = uint(data_a_packed16[ib].qs[iqs]); - const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d); - const uint vui = uint(data_a_packed16[ib].qs[iqs]); + store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], + kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); + + store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], + kvalues_iq4nl[vui >> 12])); - const uint k_pair = row * LOAD_VEC_A / 4; - store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], - kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); - store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], - kvalues_iq4nl[vui >> 12])); #elif defined(DATA_A_MXFP4) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; - const uint ib = idx / 8; - const uint iqs = (idx & 0x07) * 2; + const uint ib = idx / 8; + const uint iqs = (idx & 0x07) * 2; - const uint vui = uint(data_a[ib].qs[iqs]); - const uint vui2 = uint(data_a[ib].qs[iqs+1]); + const uint vui = uint(data_a[ib].qs[iqs]); + const uint vui2 = uint(data_a[ib].qs[iqs+1]); #ifdef USE_OCP_FP4 - const float d = e8m0_to_fp32(data_a[ib].e); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); - store_a(col, row + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); + const float d = e8m0_to_fp32(data_a[ib].e); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col, k_pair, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); #else - const float d = e8m0_to_fp32(data_a[ib].e) * 0.5; - store_a(col, row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - store_a(col, row + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); + const float d = e8m0_to_fp32(data_a[ib].e) * 0.5; + store_a(col, k_pair, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + + store_a(col, k_pair + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); + #endif #elif defined(DATA_A_NVFP4) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint ib = idx / 16u; - const uint sub = (idx & 0xC) >> 2; - const uint iqs = (idx & 0xF) * 2; - const uint vui = uint(data_a[ib].qs[iqs]); - const uint vui2 = uint(data_a[ib].qs[iqs+1]); - - // lo and hi nibbles are 8 elements apart, which doesn't quite line up with - // how the thread mapping and buf_idx calculation works for other types. - const uint eff_row = (row & 3) + (row & ~3) * 2; + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + const uint eff_row = (row & 3) + (row & ~3) * 2; + + const uint ib = idx / 16u; + const uint sub = (idx & 0xC) >> 2; + const uint iqs = (idx & 0xF) * 2; + const uint vui = uint(data_a[ib].qs[iqs]); + const uint vui2 = uint(data_a[ib].qs[iqs+1]); + #ifdef USE_OCP_FP4 - const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub])); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); - store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); + const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub])); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); + store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); #else - const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5; - store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); + const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5; + store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); #endif +#else + if (MmTypeA == GGML_TYPE_Q4_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const float d = float(a_q4_0_p16.data[ib].d); + const uint vui = uint(a_q4_0_p16.data[ib].qs[2*iqs]) | (uint(a_q4_0_p16.data[ib].qs[2*iqs + 1]) << 16); + const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d; + const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d; + + store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); + } else if (MmTypeA == GGML_TYPE_Q4_1) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const vec2 dm = vec2(a_q4_1_p32.data[ib].dm); + const uint vui = a_q4_1_p32.data[ib].qs[iqs]; + const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y; + const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y; + + store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); + } else if (MmTypeA == GGML_TYPE_Q5_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const float d = float(a_q5_0_p16.data[ib].d); + const uint uint_qh = uint(a_q5_0_p16.data[ib].qh[1]) << 16 | uint(a_q5_0_p16.data[ib].qh[0]); + const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); + const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); + + const uint vui = uint(a_q5_0_p16.data[ib].qs[iqs]); + const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d; + + store_a(col, k_pair, FLOAT_TYPEV2(v.xz)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v.yw)); + } else if (MmTypeA == GGML_TYPE_Q5_1) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const vec2 dm = vec2(a_q5_1_p32.data[ib].dm); + const uint uint_qh = a_q5_1_p32.data[ib].qh; + const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10); + const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10); + const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10); + const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10); + + const uint vui = a_q5_1_p32.data[ib].qs[iqs]; + const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y; + const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y; + + store_a(col, k_pair, FLOAT_TYPEV2(v0.xz)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz)); + store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw)); + store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw)); + } else if (MmTypeA == GGML_TYPE_Q8_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const float d = float(a_q8_0_p16.data[ib].d); + const i8vec2 v0 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs])).xy; // vec4 used due to #12147 + const i8vec2 v1 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs + 1])).xy; + const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; + + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); + } else if (MmTypeA == GGML_TYPE_Q1_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 16; + const uint iqs = idx & 0xfu; + + const float d = float(a_q1_0.data[ib].d); + const uint bits = uint(a_q1_0.data[ib].qs[iqs]); + + store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d)); + store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d)); + store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d)); + store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d)); + } else if (MmTypeA == GGML_TYPE_Q2_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 16; + const uint iqs = idx & 0xfu; + + const FLOAT_TYPE d = FLOAT_TYPE(a_q2_0.data[ib].d); + const uint bits = uint(a_q2_0.data[ib].qs[iqs]); + + store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f))); + store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f))); + } else if (MmTypeA == GGML_TYPE_Q2_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = (idx % 64) * 2; // 0,2,4..126 + + const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15 + const uint scalesi = iqs / 8; // 0..15 + const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 + + const vec4 qs = vec4(unpack8((a_q2_k_p32.data[ib].qs[qsi / 2] >> qsshift) & 0x03030303)); + const uint scales = a_q2_k.data[ib].scales[scalesi]; + const vec2 dm = vec2(a_q2_k.data[ib].dm); + + const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4); + + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); + } else if (MmTypeA == GGML_TYPE_Q3_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = idx % 128; // 0..127 + + const uint n = iqs / 64; // 0,1 + const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62 + const uint hmi = (iqs % 16) * 2; // 0,2,4..30 + const uint j = (iqs % 64) / 4; // 0..3 + const uint is = iqs / 8; // 0..15 + const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3 + const uint qsshift = halfsplit * 2; // 0,2,4,6 + + const int8_t us = int8_t(((a_q3_k.data[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF) + | (((a_q3_k.data[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4)); + const float dl = float(a_q3_k.data[ib].d) * float(us - 32); + + const vec2 qs = vec2(unpack8((uint(a_q3_k_p16.data[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy); + const vec2 hm = vec2(unpack8(((uint(a_q3_k_p16.data[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy); + + store_a(col, k_pair, FLOAT_TYPEV2(dl * (qs.x - hm.x), + dl * (qs.y - hm.y))); + + } else if (MmTypeA == GGML_TYPE_Q4_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = (idx % 64) * 2; // 0,2,4..126 + + const uint n = iqs / 32; // 0,1,2,3 + const uint b = (iqs % 32) / 16; // 0,1 + const uint is = 2 * n + b; // 0..7 + const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 + + const vec2 loadd = vec2(a_q4_k.data[ib].dm); + + const uvec3 scales = uvec3(a_q4_k_p32.data[ib].scales[0], + a_q4_k_p32.data[ib].scales[1], + a_q4_k_p32.data[ib].scales[2]); + const uint scalesoffs = (is & 3) * 8; + + const uint scidx0 = (is < 4) ? 0 : 2; + const uint scidxshift0 = scalesoffs; + const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const uint mbidx0 = (is < 4) ? 1 : 2; + const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; + const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + + const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); + const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); + + const float d = loadd.x * sc; + const float m = -loadd.y * mbyte; + + const vec4 q = vec4(unpack8((a_q4_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F)); + + store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); + store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); + } else if (MmTypeA == GGML_TYPE_Q5_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; // 4 values per idx + const uint iqs = (idx % 64) * 2; // 0,2,4..126 + + const uint n = iqs / 32; // 0,1,2,3 + const uint b = (iqs % 32) / 16; // 0,1 + const uint is = 2 * n + b; // 0..7 + const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 + const uint qhi = (iqs % 16) * 2; // 0,2,4..30 + + const vec2 loadd = vec2(a_q5_k.data[ib].dm); + + const uvec3 scales = uvec3(a_q5_k_p32.data[ib].scales[0], + a_q5_k_p32.data[ib].scales[1], + a_q5_k_p32.data[ib].scales[2]); + const uint scalesoffs = (is & 3) * 8; + + const uint scidx0 = (is < 4) ? 0 : 2; + const uint scidxshift0 = scalesoffs; + const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const uint mbidx0 = (is < 4) ? 1 : 2; + const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; + const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + + const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); + const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); + + const float d = loadd.x * sc; + const float m = -loadd.y * mbyte; + + const uint qs = (a_q5_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F; + const uint qh = ((a_q5_k_p32.data[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4; + const vec4 q = vec4(unpack8(qs | qh)); + + store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); + store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); + } else if (MmTypeA == GGML_TYPE_Q6_K) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = idx % 128; // 0..127 + + const uint n = iqs / 64; // 0,1 + const uint b = ((iqs % 64) / 32) * 4; // 0,4 + const uint is_b = (iqs % 16) / 8; // 0,1 + const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 + const uint is = 8 * n + qhshift + is_b; // 0..15 + const uint qsi = n * 32 + (iqs % 32); // 0..63 + const uint qhi = n * 16 + (iqs % 16); // 0..31 + + const float dscale = float(a_q6_k.data[ib].d) * float(a_q6_k.data[ib].scales[is]); + + const uint ql = (uint(a_q6_k_p16.data[ib].ql[qsi]) >> b) & 0x0F0F; + const uint qh = (uint(a_q6_k_p16.data[ib].qh[qhi]) >> qhshift) & 0x0303; + const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale; + + store_a(col, k_pair, FLOAT_TYPEV2(q.x, q.y)); + } else if (MmTypeA == GGML_TYPE_TQ1_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 + + const float d = float(a_tq1_0.data[ib].d); + vec2 v; + for (uint kk = 0u; kk < 2u; ++kk) { + const uint e = iqs + kk; + const uint bidx = tq1_0_byte_of(e); + const uint qbyte = uint(bidx < 48u ? a_tq1_0.data[ib].qs[bidx] + : a_tq1_0.data[ib].qh[bidx - 48u]); + v[kk] = d * (float(tq1_0_trit(qbyte, tq1_0_digit_of(e))) - 1.0); + } + + const uint k_pair = row * mm_load_vec_a() / 2; + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + } else if (MmTypeA == GGML_TYPE_TQ2_0) { + const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; + + const uint ib = idx / 128; // 2 values per idx + const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 + + const uint qsi = (iqs / 128) * 32 + (iqs % 32); // byte pair start + const uint shift = 2 * ((iqs % 128) / 32); // 0,2,4,6 + + const uvec2 qs = uvec2(a_tq2_0.data[ib].qs[qsi], a_tq2_0.data[ib].qs[qsi + 1]); + const float d = float(a_tq2_0.data[ib].d); + + const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0); + + const uint k_pair = row * mm_load_vec_a() / 2; + store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); + } #endif } #if !defined(MUL_MAT_ID) void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint idx_n, const uint block, const uint end_k) { #if LOAD_VEC_B == 8 - if (ALIGNED != 0) { - // Not supported for b_type bf16 because bf16mat2x4 does not exist - const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; - FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); - buf_b[buf_idx + 0] = bb[0].xy; - buf_b[buf_idx + 1] = bb[0].zw; - buf_b[buf_idx + 2] = bb[1].xy; - buf_b[buf_idx + 3] = bb[1].zw; - return; - } + if (ALIGNED != 0) { + // Not supported for b_type bf16 because bf16mat2x4 does not exist + const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); + buf_b[buf_idx + 0] = bb[0].xy; + buf_b[buf_idx + 1] = bb[0].zw; + buf_b[buf_idx + 2] = bb[1].xy; + buf_b[buf_idx + 3] = bb[1].zw; + return; + } #elif LOAD_VEC_B == 4 - if (ALIGNED != 0) { - const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + if (ALIGNED != 0) { + const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; #if defined(DATA_B_BF16) - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); #else - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); #endif - buf_b[buf_idx + 0] = bb.xy; - buf_b[buf_idx + 1] = bb.zw; - return; - } + buf_b[buf_idx + 0] = bb.xy; + buf_b[buf_idx + 1] = bb.zw; + return; + } #endif - const uint idx = pos_b + col * p.stride_b + row * 2; - const uint buf_idx = col * SHMEM_STRIDE + row; - if (idx_n < p.N && block + row * 2 + 1 < end_k) { - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), - TO_FLOAT_TYPE(data_b_scalar[idx + 1])); - } else if (idx_n < p.N && block + row * 2 < end_k) { - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); - } else { - buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); - } + const uint idx = pos_b + col * p.stride_b + row * 2; + const uint buf_idx = col * SHMEM_STRIDE + row; + if (idx_n < p.N && block + row * 2 + 1 < end_k) { + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), + TO_FLOAT_TYPE(data_b_scalar[idx + 1])); + } else if (idx_n < p.N && block + row * 2 < end_k) { + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); + } else { + buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); + } } #else void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint ic, const uint _ne1, const uint block, const uint end_k) { #if LOAD_VEC_B == 8 - if (ALIGNED != 0) { - // Not supported for b_type bf16 because bf16mat2x4 does not exist - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; - FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); - buf_b[buf_idx + 0] = bb[0].xy; - buf_b[buf_idx + 1] = bb[0].zw; - buf_b[buf_idx + 2] = bb[1].xy; - buf_b[buf_idx + 3] = bb[1].zw; - return; - } + if (ALIGNED != 0) { + // Not supported for b_type bf16 because bf16mat2x4 does not exist + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); + buf_b[buf_idx + 0] = bb[0].xy; + buf_b[buf_idx + 1] = bb[0].zw; + buf_b[buf_idx + 2] = bb[1].xy; + buf_b[buf_idx + 3] = bb[1].zw; + return; + } #elif LOAD_VEC_B == 4 - if (ALIGNED != 0) { - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + if (ALIGNED != 0) { + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; + const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; #if defined(DATA_B_BF16) - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); #else - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); #endif - buf_b[buf_idx + 0] = bb.xy; - buf_b[buf_idx + 1] = bb.zw; - return; - } + buf_b[buf_idx + 0] = bb.xy; + buf_b[buf_idx + 1] = bb.zw; + return; + } #endif - const uint row_i = ic * BN + col; - const uint buf_idx = col * SHMEM_STRIDE + row; - if (row_i < _ne1 && block + row * 2 + 1 < end_k) { - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), - TO_FLOAT_TYPE(data_b_scalar[idx + 1])); - } else if (row_i < _ne1 && block + row * 2 < end_k) { - const u16vec2 row_idx = row_ids[col]; - const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); - } else { - buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); - } + const uint row_i = ic * BN + col; + const uint buf_idx = col * SHMEM_STRIDE + row; + if (row_i < _ne1 && block + row * 2 + 1 < end_k) { + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), + TO_FLOAT_TYPE(data_b_scalar[idx + 1])); + } else if (row_i < _ne1 && block + row * 2 < end_k) { + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2; + buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); + } else { + buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); + } } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index a19c7f2f4e9f..21d601d0eb61 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -955,6 +955,7 @@ shared uint16_t iq1s_grid[2048]; shared uint32_t iq1s_grid_gpu[2048]; #endif +#if defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -978,6 +979,17 @@ void init_iq_shmem(uvec3 wgsize) barrier(); } #endif +#endif + +#if defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) +#if defined(DATA_A_IQ2_S) +shared uvec2 iq2s_grid[1024]; +#elif defined(DATA_A_IQ2_XS) +shared uvec2 iq2xs_grid[512]; +#else +shared uvec2 iq2xxs_grid[256]; +#endif +#endif #define QUANT_K_IQ2_XXS 256 #define QUANT_R_IQ2_XXS 1 @@ -1063,8 +1075,7 @@ const uvec2[256] iq2xxs_grid_const = { uvec2(0x08080808, 0x2b2b082b), uvec2(0x08192b08, 0x2b2b1908), uvec2(0x19190808, 0x2b2b2b08), uvec2(0x08081908, 0x2b2b2b19) }; -shared uvec2 iq2xxs_grid[256]; - +#if defined(DATA_A_IQ2_XXS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1076,12 +1087,15 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ2_XXS) #define QUANT_K QUANT_K_IQ2_XXS #define QUANT_R QUANT_R_IQ2_XXS #define A_TYPE block_iq2_xxs #define A_TYPE_PACKED16 block_iq2_xxs_packed16 #endif +#endif #define QUANT_K_IQ2_XS 256 #define QUANT_R_IQ2_XS 1 @@ -1233,8 +1247,7 @@ const uvec2 iq2xs_grid_const[512] = { uvec2(0x082b2b08, 0x2b2b2b2b), uvec2(0x082b2b2b, 0x2b2b2b2b), uvec2(0x2b190819, 0x2b2b2b2b), uvec2(0x2b2b2b2b, 0x2b2b2b2b), }; -shared uvec2 iq2xs_grid[512]; - +#if defined(DATA_A_IQ2_XS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1246,12 +1259,15 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ2_XS) #define QUANT_K QUANT_K_IQ2_XS #define QUANT_R QUANT_R_IQ2_XS #define A_TYPE block_iq2_xs #define A_TYPE_PACKED16 block_iq2_xs_packed16 #endif +#endif #define QUANT_K_IQ2_S 256 #define QUANT_R_IQ2_S 1 @@ -1533,8 +1549,7 @@ const uvec2 iq2s_grid_const[1024] = { uvec2(0x082b082b, 0x2b2b2b2b), uvec2(0x082b2b08, 0x2b2b2b2b), uvec2(0x2b082b08, 0x2b2b2b2b), uvec2(0x2b2b2b2b, 0x2b2b2b2b) }; -shared uvec2 iq2s_grid[1024]; - +#if defined(DATA_A_IQ2_S) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1546,12 +1561,23 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ2_S) #define QUANT_K QUANT_K_IQ2_S #define QUANT_R QUANT_R_IQ2_S #define A_TYPE block_iq2_s #define A_TYPE_PACKED16 block_iq2_s_packed16 #endif +#endif + +#if defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) +#if defined(DATA_A_IQ3_S) +shared uint32_t iq3s_grid[512]; +#else +shared uint32_t iq3xxs_grid[256]; +#endif +#endif #define QUANT_K_IQ3_XXS 256 #define QUANT_R_IQ3_XXS 1 @@ -1605,8 +1631,7 @@ const uint32_t iq3xxs_grid_const[256] = { 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, }; -shared uint32_t iq3xxs_grid[256]; - +#if defined(DATA_A_IQ3_XXS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1618,12 +1643,15 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ3_XXS) #define QUANT_K QUANT_K_IQ3_XXS #define QUANT_R QUANT_R_IQ3_XXS #define A_TYPE block_iq3_xxs #define A_TYPE_PACKED16 block_iq3_xxs_packed16 #endif +#endif #define QUANT_K_IQ3_S 256 #define QUANT_R_IQ3_S 1 @@ -1715,8 +1743,7 @@ const uint32_t iq3s_grid_const[512] = { 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101, }; -shared uint32_t iq3s_grid[512]; - +#if defined(DATA_A_IQ3_S) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1728,12 +1755,15 @@ void init_iq_shmem(uvec3 wgsize) } barrier(); } +#endif +#if defined(DATA_A_IQ3_S) #define QUANT_K QUANT_K_IQ3_S #define QUANT_R QUANT_R_IQ3_S #define A_TYPE block_iq3_s #define A_TYPE_PACKED16 block_iq3_s_packed16 #endif +#endif #define QUANT_K_IQ4_XS 256 #define QUANT_R_IQ4_XS 1 @@ -1847,6 +1877,7 @@ const int8_t kvalues_iq4nl_const[16] = { shared FLOAT_TYPE kvalues_iq4nl[16]; +#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { @@ -1857,6 +1888,7 @@ void init_iq_shmem(uvec3 wgsize) barrier(); } #endif +#endif #if defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4) #if !defined(USE_OCP_FP4) @@ -1886,7 +1918,7 @@ float ue4m3_to_fp32_build(uint u) { } #endif -#if !defined(USE_OCP_FP4) +#if (defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4)) && !defined(USE_OCP_FP4) #define NEEDS_INIT_IQ_SHMEM void init_iq_shmem(uvec3 wgsize) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index ea4851b7ab2d..30fe0884e547 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -246,6 +246,17 @@ bool is_iq_quant(const std::string& type_name) { return string_starts_with(type_name, "iq"); } +bool is_lut_quant(const std::string& type_name) { + return is_iq_quant(type_name) || type_name == "mxfp4" || type_name == "nvfp4"; +} + +std::string lut_load_vec_a(const std::string& type_name) { + if (type_name == "iq1_s" || type_name == "iq1_m" || type_name == "iq2_xxs" || type_name == "iq2_xs" || type_name == "iq2_s") { + return "8"; + } + return "4"; +} + static const char path_separator = '/'; std::string join_paths(const std::string& path1, const std::string& path2) { @@ -583,20 +594,28 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c } for (const auto& tname : type_names) { - std::string load_vec_quant = "2"; - if ((tname == "q1_0") || (tname == "q4_0") || (tname == "q4_1") || (tname == "q5_1") || (tname == "iq1_s") || (tname == "iq1_m") || (tname == "iq2_xxs") || (tname == "iq2_xs") || (tname == "iq2_s")) - load_vec_quant = "8"; - else if ((tname == "q2_0") || (tname == "q5_0") || (tname == "q8_0") || (tname == "q2_k") || (tname == "q4_k") || (tname == "q5_k") || (tname == "iq3_xxs") || (tname == "iq3_s") || (tname == "iq4_xs") || (tname == "iq4_nl") || (tname == "mxfp4") || (tname == "nvfp4")) - load_vec_quant = "4"; - if (tname == "bf16") { continue; } - std::string data_a_key = "DATA_A_" + to_uppercase(tname); - // For aligned matmul loads - std::string load_vec_a = (coopmat2 || tname == "f32" || tname == "f16" || tname == "bf16") ? load_vec : load_vec_quant; + // Float types keep per-type compilation (different accumulation loop structure) + if (tname == "f32" || tname == "f16") { + std::string data_a_key = "DATA_A_" + to_uppercase(tname); + + const std::map float_type_dict = { + {"FLOAT_TYPE", FLOAT_TYPE(1, tname)}, + {"FLOAT_TYPEV2", FLOAT_TYPE(2, tname)}, + {"FLOAT_TYPEV4", FLOAT_TYPE(4, tname)}, + {"FLOAT_TYPEV8", FLOAT_TYPE(8, tname)}, + }; + + if (!coopmat2) { + string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } + continue; + } + std::string data_a_key = "DATA_A_" + to_uppercase(tname); const std::map float_type_dict = { {"FLOAT_TYPE", FLOAT_TYPE(1, tname)}, {"FLOAT_TYPEV2", FLOAT_TYPE(2, tname)}, @@ -604,30 +623,52 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c {"FLOAT_TYPEV8", FLOAT_TYPE(8, tname)}, }; - // don't generate f32 variants for coopmat2 - if (!coopmat2) { - string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); +#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) + if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4")) { + string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc); } +#endif - if (tname != "f16" && tname != "f32") { - string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); - } + if (is_lut_quant(tname)) { + std::string lva = lut_load_vec_a(tname); + + string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); -#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) - if ((coopmat || coopmat2) && (tname == "mxfp4" || tname == "nvfp4")) { if (!coopmat2) { - string_to_spv(shader_name + "_" + tname + "_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } + +#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) + if ((tname == "mxfp4" || tname == "nvfp4") && (coopmat || coopmat2)) { + string_to_spv(shader_name + "_" + tname + "_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (!coopmat2) { + string_to_spv(shader_name + "_" + tname + "_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } } - string_to_spv(shader_name + "_" + tname + "_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); - } #endif + continue; + } -#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) - // Integer dot mmq performs better with f32 accumulators (different shader, skip for dot2) - if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4")) { - string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc); + // dedicated shader needed due to regression on Ampere + if (coopmat2 && (tname == "q4_k" || tname == "q5_k")) { + string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } + } + + // Quant shader: one SPIR-V for all quant types, selected via MmTypeA spec constant + { + const std::map quant_float_type_dict = { + {"FLOAT_TYPE", FLOAT_TYPE(1, "q4_0")}, + {"FLOAT_TYPEV2", FLOAT_TYPE(2, "q4_0")}, + {"FLOAT_TYPEV4", FLOAT_TYPE(4, "q4_0")}, + {"FLOAT_TYPEV8", FLOAT_TYPE(8, "q4_0")}, + }; + + string_to_spv(shader_name + "_quant_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + + if (!coopmat2) { + string_to_spv(shader_name + "_quant_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); } -#endif } } From 434ddbbc0e30522e897670681e503b797c12b7c1 Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:46:28 +0000 Subject: [PATCH 12/65] ci: fix sanitizer tests (#28583) --- .github/workflows/build-riscv.yml | 80 ---------------------------- .github/workflows/build-sanitize.yml | 2 - 2 files changed, 82 deletions(-) diff --git a/.github/workflows/build-riscv.yml b/.github/workflows/build-riscv.yml index 13f2576b9f08..23a64454e9db 100644 --- a/.github/workflows/build-riscv.yml +++ b/.github/workflows/build-riscv.yml @@ -106,83 +106,3 @@ jobs: wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256 - - ubuntu-riscv64-native-sanitizer: - runs-on: ubuntu-24.04-riscv - - continue-on-error: true - - strategy: - matrix: - sanitizer: [ADDRESS, THREAD, UNDEFINED] - build_type: [Debug] - - steps: - - name: Install dependencies - run: | - # Set gcc-14 and g++-14 as the default compilers - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100 - - git lfs install - - - name: GCC version check - run: | - gcc --version - g++ --version - - - name: Clone - id: checkout - uses: actions/checkout@v6 - - # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation - #- name: ccache - # uses: ggml-org/ccache-action@v1.2.24 - # with: - # key: riscv-ubuntu-native-sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }} - # evict-old-files: 1d - # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - - - name: Build - id: cmake_build - if: ${{ matrix.sanitizer != 'THREAD' }} - run: | - cmake -B build \ - -DLLAMA_OPENSSL=OFF \ - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DGGML_OPENMP=ON \ - -DLLAMA_BUILD_EXAMPLES=ON \ - -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_TESTS=OFF \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ - -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \ - -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 - - cmake --build build --config ${{ matrix.build_type }} -j $(nproc) - - - name: Build (no OpenMP) - id: cmake_build_no_openmp - if: ${{ matrix.sanitizer == 'THREAD' }} - run: | - cmake -B build \ - -DLLAMA_OPENSSL=OFF \ - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DGGML_OPENMP=OFF \ - -DLLAMA_BUILD_EXAMPLES=ON \ - -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_TESTS=OFF \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \ - -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \ - -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 - - cmake --build build --config ${{ matrix.build_type }} -j $(nproc) - - - name: Test - id: cmake_test - run: | - cd build - ctest -L main --verbose --timeout 900 diff --git a/.github/workflows/build-sanitize.yml b/.github/workflows/build-sanitize.yml index 189b5c0fe7b7..89fcff71d445 100644 --- a/.github/workflows/build-sanitize.yml +++ b/.github/workflows/build-sanitize.yml @@ -101,8 +101,6 @@ jobs: - name: Test id: cmake_test - # skip run in Debug - very slow - if: ${{ matrix.sanitizer != 'UNDEFINED' }} run: | cd build ctest -L main -E tokenizer --verbose --timeout 900 From d7e86430a7d5fa4a0a7ee8bfb24d413d87bb240e Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Thu, 10 Sep 2026 14:49:56 +0800 Subject: [PATCH 13/65] model: fix all granite family parameter counts (#28643) * model: fix all granite family parameter counts Signed-off-by: Aaron Teo * model: fix additional include, add missing `A` prefix for active experts Signed-off-by: Aaron Teo * model: fix code alignment, rm unused 40 block case Signed-off-by: Aaron Teo --------- Signed-off-by: Aaron Teo --- src/llama-model.cpp | 2 ++ src/llama-model.h | 2 ++ src/models/granite-hybrid.cpp | 2 +- src/models/granite-moe.cpp | 3 +-- src/models/granite.cpp | 11 ++++++++++- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 54009b3696a5..d10b60afd9fd 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -936,6 +936,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_17B_128E: return "17Bx128E (Maverick)"; case LLM_TYPE_A13B: return "A13B"; case LLM_TYPE_1B_A400M: return "1B.A400M"; + case LLM_TYPE_3B_A800M: return "3B.A800M"; case LLM_TYPE_7B_A1B: return "7B.A1B"; case LLM_TYPE_8B_A1B: return "8B.A1B"; case LLM_TYPE_7_9B_A1_3B: return "7.9B.A1.3B"; @@ -946,6 +947,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_26B_A4B: return "26B.A4B"; case LLM_TYPE_30B_A3B: return "30B.A3B"; case LLM_TYPE_31B_A3_5B: return "31B.A3.5B"; + case LLM_TYPE_32B_A9B: return "32B.A9B"; case LLM_TYPE_35B_A3B: return "35B.A3B"; case LLM_TYPE_48B_A3B: return "48B.A3B"; case LLM_TYPE_75B_A9B: return "75B.A9B"; diff --git a/src/llama-model.h b/src/llama-model.h index c0cc4065567a..a02b30ca7ada 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -117,6 +117,7 @@ enum llm_type { LLM_TYPE_17B_128E, // llama4 Maverick LLM_TYPE_A13B, LLM_TYPE_1B_A400M, // Granite3 MoE + LLM_TYPE_3B_A800M, // Granite3 MoE LLM_TYPE_7B_A1B, LLM_TYPE_8B_A1B, // lfm2moe LLM_TYPE_7_9B_A1_3B, // Ling-3.0-tiny @@ -127,6 +128,7 @@ enum llm_type { LLM_TYPE_26B_A4B, // Gemma4 LLM_TYPE_30B_A3B, LLM_TYPE_31B_A3_5B, + LLM_TYPE_32B_A9B, // Granite4 Hybrid LLM_TYPE_35B_A3B, // Qwen3.5 LLM_TYPE_48B_A3B, // Kimi Linear LLM_TYPE_75B_A9B, // Nemotron 3 Puzzle diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp index 8a8f7e19ff08..c177ae78756e 100644 --- a/src/models/granite-hybrid.cpp +++ b/src/models/granite-hybrid.cpp @@ -30,7 +30,7 @@ void llama_model_granite_hybrid::load_arch_hparams(llama_model_loader & ml) { case 768: type = LLM_TYPE_350M; break; case 1536: type = (hparams.n_ff() == 512 ? LLM_TYPE_7B_A1B : LLM_TYPE_1B); break; case 2048: case 2560: type = LLM_TYPE_3B; break; - case 4096: type = LLM_TYPE_32B; break; + case 4096: type = LLM_TYPE_32B_A9B; break; default: type = LLM_TYPE_UNKNOWN; } diff --git a/src/models/granite-moe.cpp b/src/models/granite-moe.cpp index 156553edfd04..febe1bfa7aa4 100644 --- a/src/models/granite-moe.cpp +++ b/src/models/granite-moe.cpp @@ -9,8 +9,7 @@ void llama_model_granite_moe::load_arch_hparams(llama_model_loader & ml) { switch (hparams.n_layer()) { case 24: type = LLM_TYPE_1B_A400M; break; - case 32: type = LLM_TYPE_3B; break; - case 40: type = LLM_TYPE_3B; break; + case 32: type = LLM_TYPE_3B_A800M; break; // Add additional layer/vocab/etc checks here for other model sizes default: type = LLM_TYPE_UNKNOWN; } diff --git a/src/models/granite.cpp b/src/models/granite.cpp index 9e9f97e94dca..60d463aedab0 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -38,7 +38,16 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) { switch (hparams.n_layer()) { case 32: type = LLM_TYPE_3B; break; - case 40: type = LLM_TYPE_3B; break; + case 40: { + switch (hparams.n_embd) { + case 2048: type = LLM_TYPE_2B; break; + case 2560: type = LLM_TYPE_3B; break; + case 4096: type = LLM_TYPE_8B; break; + default: type = LLM_TYPE_UNKNOWN; + } + break; + } + case 64: type = LLM_TYPE_30B; break; // Add additional layer/vocab/etc checks here for other model sizes default: type = LLM_TYPE_UNKNOWN; } From f1b6fbf35cfa010b0a8d6301fdfccbb7f41bd903 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Thu, 10 Sep 2026 14:50:28 +0800 Subject: [PATCH 14/65] ggml-cpu(s390x): add Q1_0 vector intrinsic support (#28606) * ggml-cpu: add `ggml_vec_dot_q1_0_q8_0` support Signed-off-by: Aaron Teo * ggml-cpu: clean up variable naming for understanding Signed-off-by: Aaron Teo * docs: update support for Q1_0 Signed-off-by: Aaron Teo --------- Signed-off-by: Aaron Teo --- docs/build-s390x.md | 3 +- ggml/src/ggml-cpu/arch-fallback.h | 1 - ggml/src/ggml-cpu/arch/s390/quants.c | 68 ++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/docs/build-s390x.md b/docs/build-s390x.md index 4568d5010f6c..005dd2983459 100644 --- a/docs/build-s390x.md +++ b/docs/build-s390x.md @@ -243,6 +243,7 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl | FP32 | ✅ | ✅ | ❓ | | FP16 | ✅ | ✅ | ❓ | | BF16 | ✅ | ✅ | ❓ | +| Q1_0 | ✅ | ❓ | ❓ | | Q4_0 | ✅ | ❓ | ❓ | | Q4_1 | ✅ | ❓ | ❓ | | MXFP4 | ✅ | ❓ | ❓ | @@ -272,4 +273,4 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl - 🚫 - acceleration unavailable, will still run using scalar implementation - ❓ - acceleration unknown, please contribute if you can test it yourself -Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Feb 15, 2026. +Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Sep 8, 2026. diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 152e0bac99b0..98ef5e1405f9 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -247,7 +247,6 @@ // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 -#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K diff --git a/ggml/src/ggml-cpu/arch/s390/quants.c b/ggml/src/ggml-cpu/arch/s390/quants.c index d3436c24b5f3..70f2882d830d 100644 --- a/ggml/src/ggml-cpu/arch/s390/quants.c +++ b/ggml/src/ggml-cpu/arch/s390/quants.c @@ -146,6 +146,74 @@ void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, i //===================================== Dot products ================================= +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK1_0; // 128 + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + +#if defined(__VXE__) || defined(__VXE2__) + float32x4_t v_sumf = vec_splats(0.0f); + + const uint8x16_t v_zero = vec_splats((uint8_t)0x00); // zero + const uint8x16_t v_bias = vec_splats((uint8_t)0x80); // bias from signed to unsigned + // v ^ 0x80 == v + 128 + + const uint8x16_t v_idx = (const uint8x16_t){ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 }; + const uint8x16_t v_bit = (const uint8x16_t){ 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128 }; + + for (int i = 0; i < nb; ++i) { + const uint8x16_t v_x = vec_xl(0, (const uint8_t *)x[i].qs); + const float32x4_t v_xd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d)); + + for (int k = 0; k < 4; ++k) { + // sub-block k holds elements 32k .. 32k+31 + const block_q8_0 * GGML_RESTRICT yb = &y[i*4 + k]; + const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(yb->d)); + + const uint8x16_t v_xrl = vec_perm(v_x, v_x, vec_add(v_idx, vec_splats((uint8_t)(k*4 + 0)))); + const uint8x16_t v_xrh = vec_perm(v_x, v_x, vec_add(v_idx, vec_splats((uint8_t)(k*4 + 2)))); + + // isolate each lane's bit, then set all ones where that bit is clear, the -d case + const int8x16_t v_ml = (int8x16_t)vec_cmpeq(vec_and(v_xrl, v_bit), v_zero); + const int8x16_t v_mh = (int8x16_t)vec_cmpeq(vec_and(v_xrh, v_bit), v_zero); + + const int8x16_t v_yl = vec_xl(0, (const int8_t *)yb->qs); + const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)yb->qs); + + // weights are only +1 or -1, so negate y + const int8x16_t v_ysl = vec_sub(vec_xor(v_yl, v_ml), v_ml); + const int8x16_t v_ysh = vec_sub(vec_xor(v_yh, v_mh), v_mh); + + // bias to unsigned, then vec_sum4 adds each group of 4 bytes into one word + const uint32x4_t v_p = vec_add(vec_sum4(vec_xor((uint8x16_t)v_ysl, v_bias), v_zero), + vec_sum4(vec_xor((uint8x16_t)v_ysh, v_bias), v_zero)); + + // each word summed 8 biased bytes, so take back 8 * 128 + const int32x4_t v_xy = vec_sub((int32x4_t)v_p, vec_splats((int32_t)1024)); + + // apply both block scales and add into the running total + v_sumf = vec_madd(vec_float(v_xy), vec_mul(v_xd, v_yd), v_sumf); + } + } + + *s = vec_hsum_f32x4(v_sumf); +#else + UNUSED(nb); + UNUSED(x); + UNUSED(y); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; From 4ea6d1bb6dac161f70be728983e2cd58e4d9246f Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Thu, 10 Sep 2026 14:51:17 +0800 Subject: [PATCH 15/65] ggml-cpu(s390x): add repack support for q4_0 (#28667) ggml-cpu: clean comments Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/CMakeLists.txt | 4 +- ggml/src/ggml-cpu/arch-fallback.h | 3 - ggml/src/ggml-cpu/arch/s390/repack.cpp | 223 +++++++++++++++++++++++++ ggml/src/ggml-cpu/repack.cpp | 5 + 4 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 ggml/src/ggml-cpu/arch/s390/repack.cpp diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 17540faa66d3..1c7338eea49c 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -520,7 +520,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name) elseif (GGML_SYSTEM_ARCH STREQUAL "s390x") message(STATUS "s390x detected") list(APPEND GGML_CPU_SOURCES - ggml-cpu/arch/s390/quants.c) + ggml-cpu/arch/s390/quants.c + ggml-cpu/arch/s390/repack.cpp + ) # for native compilation if (GGML_NATIVE) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 98ef5e1405f9..2b9a426577c7 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -259,11 +259,9 @@ #define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K // repack.cpp -#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 -#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K @@ -279,7 +277,6 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 -#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K diff --git a/ggml/src/ggml-cpu/arch/s390/repack.cpp b/ggml/src/ggml-cpu/arch/s390/repack.cpp new file mode 100644 index 000000000000..3990a6b0487e --- /dev/null +++ b/ggml/src/ggml-cpu/arch/s390/repack.cpp @@ -0,0 +1,223 @@ +#define GGML_COMMON_IMPL_CPP +#define GGML_COMMON_DECL_CPP +#include "ggml-common.h" +#include "ggml-backend-impl.h" + +#include "ggml-impl.h" +#include "ggml-cpu.h" +#include "ggml-cpu-impl.h" +#include "simd-mappings.h" +#include "traits.h" + +#include +#include +#include + +#define GGML_CPU_CLANG_WORKAROUND +#include "../../repack.h" + +#define UNUSED GGML_UNUSED + +void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(QK8_0 == 32); + assert(k % QK8_0 == 0); + const int nb = k / QK8_0; + + block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; + +#if defined(__VXE__) || defined(__VXE2__) + float32x4_t v_src[4][8]; + float id[4]; + + for (int i = 0; i < nb; i++) { + float32x4_t v_asrc[8]; + float32x4_t v_amax[8]; + + for (int row_iter = 0; row_iter < 4; row_iter++) { + for (int j = 0; j < 8; j++) v_src[row_iter][j] = vec_xl(0, x + row_iter * k + i * 32 + 4 * j); + for (int j = 0; j < 8; j++) v_asrc[j] = vec_abs(v_src[row_iter][j]); + + for (int j = 0; j < 4; j++) v_amax[2 * j] = vec_max(v_asrc[2 * j], v_asrc[2 * j + 1]); + for (int j = 0; j < 2; j++) v_amax[4 * j] = vec_max(v_amax[4 * j], v_amax[4 * j + 2]); + for (int j = 0; j < 1; j++) v_amax[8 * j] = vec_max(v_amax[8 * j], v_amax[8 * j + 4]); + + const float amax = MAX(MAX(vec_extract(v_amax[0], 0), vec_extract(v_amax[0], 1)), + MAX(vec_extract(v_amax[0], 2), vec_extract(v_amax[0], 3))); + + const float d = amax / ((1 << 7) - 1); + id[row_iter] = d ? 1.0f / d : 0.0f; + + y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d); + } + + for (int j = 0; j < 8; j++) { + /* Uses non-default rounding for vec_signed or vec_round */ + const int32x4_t v_qs0 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[0][j], id[0]), 4, 1)); + const int32x4_t v_qs1 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[1][j], id[1]), 4, 1)); + const int32x4_t v_qs2 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[2][j], id[2]), 4, 1)); + const int32x4_t v_qs3 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[3][j], id[3]), 4, 1)); + + const int16x8_t v_qs01 = vec_packs(v_qs0, v_qs1); + const int16x8_t v_qs23 = vec_packs(v_qs2, v_qs3); + + vec_xst(vec_packs(v_qs01, v_qs23), 0, y[i].qs + 16 * j); + } + } +#else + UNUSED(nb); + UNUSED(y); + ggml_quantize_mat_q8_0_4x4_generic(x, vy, k); +#endif +} + +static inline int16x8_t vxe_dot_acc(const int8x16_t v_x, const int8x16_t v_y, const int16x8_t v_acc) { + return vec_meadd(v_x, v_y, vec_moadd(v_x, v_y, v_acc)); +} + +static inline int8x16_t vxe_splat_granule(const int8_t * qs) { + uint32_t g; + memcpy(&g, qs, sizeof(g)); + return (int8x16_t)vec_splats(g); +} + +static inline int32x4_t vxe_fold(const int16x8_t v_sumi) { + const int16x8_t v_ones = vec_splats((int16_t)1); + return vec_add(vec_mule(v_sumi, v_ones), vec_mulo(v_sumi, v_ones)); +} + +void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK8_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + +#if defined(__VXE__) || defined(__VXE2__) + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + float * res_ptr = s; + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb); + + float32x4_t v_sumf = vec_splats(0.0f); + + for (int l = 0; l < nb; l++) { + const int8_t * x_qs = b_ptr[l].qs; + + const int8x16_t v_x0 = vec_xl( 0, x_qs); + const int8x16_t v_x1 = vec_xl(16, x_qs); + const int8x16_t v_x2 = vec_xl(32, x_qs); + const int8x16_t v_x3 = vec_xl(48, x_qs); + + const int8x16_t v_x0l = vec_sra(vec_sl(v_x0, 4), 4); + const int8x16_t v_x1l = vec_sra(vec_sl(v_x1, 4), 4); + const int8x16_t v_x2l = vec_sra(vec_sl(v_x2, 4), 4); + const int8x16_t v_x3l = vec_sra(vec_sl(v_x3, 4), 4); + + const int8x16_t v_x0h = vec_sra(v_x0, 4); + const int8x16_t v_x1h = vec_sra(v_x1, 4); + const int8x16_t v_x2h = vec_sra(v_x2, 4); + const int8x16_t v_x3h = vec_sra(v_x3, 4); + + const int8_t * y_lo = a_ptr[l].qs; + const int8_t * y_hi = y_lo + qk / 2; + + int16x8_t v_sumi = vec_splats((int16_t)0); + + v_sumi = vxe_dot_acc(v_x0l, vxe_splat_granule(y_lo + 0), v_sumi); + v_sumi = vxe_dot_acc(v_x1l, vxe_splat_granule(y_lo + 4), v_sumi); + v_sumi = vxe_dot_acc(v_x2l, vxe_splat_granule(y_lo + 8), v_sumi); + v_sumi = vxe_dot_acc(v_x3l, vxe_splat_granule(y_lo + 12), v_sumi); + + v_sumi = vxe_dot_acc(v_x0h, vxe_splat_granule(y_hi + 0), v_sumi); + v_sumi = vxe_dot_acc(v_x1h, vxe_splat_granule(y_hi + 4), v_sumi); + v_sumi = vxe_dot_acc(v_x2h, vxe_splat_granule(y_hi + 8), v_sumi); + v_sumi = vxe_dot_acc(v_x3h, vxe_splat_granule(y_hi + 12), v_sumi); + + const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(a_ptr[l].d)); + const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d); + const float32x4_t v_d = vec_mul(v_yd, v_xd); + + v_sumf = vec_madd(vec_float(vxe_fold(v_sumi)), v_d, v_sumf); + } + + vec_xst(v_sumf, 0, res_ptr + x * ncols_interleaved); + } +#else + UNUSED(nb); + UNUSED(ncols_interleaved); + ggml_gemv_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +#endif +} + +void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK8_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(nr % 4 == 0); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + +#if defined(__VXE__) || defined(__VXE2__) + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb); + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb); + + float32x4_t v_sumf[4]; + for (int m = 0; m < 4; m++) { + v_sumf[m] = vec_splats(0.0f); + } + + for (int l = 0; l < nb; l++) { + int16x8_t v_sumi0 = vec_splats((int16_t)0); + int16x8_t v_sumi1 = vec_splats((int16_t)0); + int16x8_t v_sumi2 = vec_splats((int16_t)0); + int16x8_t v_sumi3 = vec_splats((int16_t)0); + + for (int k = 0; k < 4; k++) { + const int8x16_t v_x = vec_xl(0, b_ptr[l].qs + 16 * k); + const int8x16_t v_xl = vec_sra(vec_sl(v_x, 4), 4); + const int8x16_t v_xh = vec_sra(v_x, 4); + + const int8_t * y_lo = a_ptr[l].qs + 16 * k; + const int8_t * y_hi = y_lo + qk / 2 * 4; + + v_sumi0 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 0), v_sumi0); + v_sumi1 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 4), v_sumi1); + v_sumi2 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 8), v_sumi2); + v_sumi3 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 12), v_sumi3); + + v_sumi0 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 0), v_sumi0); + v_sumi1 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 4), v_sumi1); + v_sumi2 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 8), v_sumi2); + v_sumi3 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 12), v_sumi3); + } + + const float32x4_t v_yd = __lzs_f16cx4_load(a_ptr[l].d); + const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d); + + v_sumf[0] = vec_madd(vec_float(vxe_fold(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]); + v_sumf[1] = vec_madd(vec_float(vxe_fold(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]); + v_sumf[2] = vec_madd(vec_float(vxe_fold(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]); + v_sumf[3] = vec_madd(vec_float(vxe_fold(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]); + } + + for (int m = 0; m < 4; m++) { + vec_xst(v_sumf[m], 0, s + (y * 4 + m) * bs + x * ncols_interleaved); + } + } + } +#else + UNUSED(nb); + UNUSED(ncols_interleaved); + ggml_gemm_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +#endif +} diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index 9689ca3ced8f..f5e419c1ecd2 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -4586,6 +4586,11 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons return &q4_0_4x4_q8_0; } } + if (ggml_cpu_has_vxe()) { + if (cur->ne[1] % 4 == 0) { + return &q4_0_4x4_q8_0; + } + } if (ggml_cpu_has_riscv_v()) { #if defined __riscv_zvfh switch (__riscv_vlenb() * 8) { From 72797e89198ab564fd0e6baa54ab196e8dd1d884 Mon Sep 17 00:00:00 2001 From: Cordell Blanchard <55163549+CordellBlanchard@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:06:07 -0400 Subject: [PATCH 16/65] vulkan : add command-buffer debug labels for GPU profilers (#28101) * vulkan : add command-buffer debug labels for GPU profilers Co-authored-by: gabby-zy Assisted-by: Claude Code * vulkan : close the queue debug label with the label struct --------- Co-authored-by: gabby-zy --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 135 +++++++++++++++++++++++++-- 1 file changed, 128 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 235b35c6ceb1..cefe186feeaf 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -844,6 +844,8 @@ struct vk_device_struct { std::mutex compile_mutex; std::condition_variable compile_cv; + uint32_t debug_cmdbuf_idx {}; + vk::PhysicalDevice physical_device; vk::PhysicalDeviceProperties properties; std::string name; @@ -2209,6 +2211,8 @@ struct vk_context_struct { std::vector out_memcpys; std::vector memsets; + std::vector debug_labels; + vk_command_pool * p {}; }; typedef std::shared_ptr vk_context; @@ -8332,6 +8336,95 @@ template const T *push_constant_data(const std::array{1.0f, 1.0f, 1.0f, 1.0f}; + vk_instance.pfn_vkCmdBeginDebugUtilsLabelEXT(buf, reinterpret_cast(&label)); +} + +// no-op unless GGML_VK_DEBUG_MARKERS is set +struct ggml_vk_debug_label { + // at most one of these is set, depending on the scope the label was opened in + vk_context_struct * subctx {}; + vk_queue_handle * qhandle {}; + + // one region per dispatch, e.g. "matmul_q4_k_f32_f16acc_aligned_m (192,8,1)". + // RGP cannot recover the pipeline name on its own, it only has the hash + ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2) { + if (!vk_instance.debug_utils_support || ctx->s == nullptr) { + return; + } + begin(ctx, pipeline_name + " (" + std::to_string(wg0) + "," + std::to_string(wg1) + "," + std::to_string(wg2) + ")"); + } + + // one region per graph node + // fused nodes are joined with '+', e.g. "RMS_NORM+MUL+ROPE Qcur-19" + ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused) { + if (!vk_instance.debug_utils_support || ctx->s == nullptr) { + return; + } + std::string name = ggml_op_name(cgraph->nodes[node_idx]->op); + for (int i = 1; i <= n_fused; i++) { + name += "+"; + name += ggml_op_name(cgraph->nodes[node_idx + i]->op); + } + name += " "; + name += cgraph->nodes[node_idx]->name; + begin(ctx, name); + } + + // one region per graph evaluation, opened on the queue instead of a command buffer + // so it spans every submit the evaluation makes + ggml_vk_debug_label(vk_queue_handle * handle, const char * name) { + if (!vk_instance.debug_utils_support || handle == nullptr) { + return; + } + vk::DebugUtilsLabelEXT label = {}; + label.pLabelName = name; + label.color = std::array{1.0f, 1.0f, 1.0f, 1.0f}; + + qhandle = handle; + std::lock_guard guard(*qhandle); + vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(qhandle->queue, reinterpret_cast(&label)); + } + + // call before the command buffer can end, the destructor covers the rest + void close() { + if (subctx != nullptr) { + // close on the current command buffer, which may differ from the one begin used + if (subctx->s != nullptr) { + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(subctx->s->buffer->buf); + } + subctx->debug_labels.pop_back(); + subctx = nullptr; + } + if (qhandle != nullptr) { + std::lock_guard guard(*qhandle); + vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT(qhandle->queue); + qhandle = nullptr; + } + } + + ~ggml_vk_debug_label() { + close(); + } + + ggml_vk_debug_label(const ggml_vk_debug_label &) = delete; + ggml_vk_debug_label & operator=(const ggml_vk_debug_label &) = delete; + +private: + // the constructors check this too, so the name is not built when markers are off + void begin(vk_context & ctx, const std::string & name) { + if (!vk_instance.debug_utils_support || ctx->s == nullptr) { + return; + } + subctx = ctx.get(); + subctx->debug_labels.push_back(name); + ggml_vk_cmd_label_begin(subctx->s->buffer->buf, subctx->debug_labels.back().c_str()); + } +}; + template static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list const& descriptor_buffer_infos, const T &push_constants, std::array elements) { const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]); @@ -8361,7 +8454,10 @@ static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& 0, { descriptor_set }, {}); - subctx->s->buffer->buf.dispatch(wg0, wg1, wg2); + { + ggml_vk_debug_label dbg(subctx, pipeline->name, wg0, wg1, wg2); + subctx->s->buffer->buf.dispatch(wg0, wg1, wg2); + } } static void ggml_vk_ctx_end(vk_context& ctx) { @@ -8370,6 +8466,15 @@ static void ggml_vk_ctx_end(vk_context& ctx) { return; } + // close open labels so this buffer is balanced; reopened in ggml_vk_ctx_begin + if (vk_instance.debug_utils_support) { + for (size_t i = 0; i < ctx->debug_labels.size(); i++) { + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(ctx->s->buffer->buf); + } + // the enclosing per-command-buffer region + vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(ctx->s->buffer->buf); + } + ctx->s->buffer->buf.end(); ctx->s = nullptr; } @@ -8382,6 +8487,17 @@ static void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) { subctx->seqs.push_back({ ggml_vk_begin_submission(device, *subctx->p) }); subctx->s = subctx->seqs[subctx->seqs.size() - 1].data(); + + if (vk_instance.debug_utils_support) { + // outermost region, one per command buffer, so the gaps between submits stand out + const std::string name = "submit " + std::to_string(device->debug_cmdbuf_idx++); + ggml_vk_cmd_label_begin(subctx->s->buffer->buf, name.c_str()); + + // reopen labels left open when the previous command buffer was submitted + for (const std::string & label : subctx->debug_labels) { + ggml_vk_cmd_label_begin(subctx->s->buffer->buf, label.c_str()); + } + } } static vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) { @@ -15971,6 +16087,9 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr } } + // closed explicitly below, and by the destructor on the early returns + ggml_vk_debug_label dbg(compute_ctx, cgraph, node_idx, ctx->num_additional_fused_ops); + switch (node->op) { case GGML_OP_REPEAT: ggml_vk_repeat(ctx, compute_ctx, src0, node); @@ -16375,6 +16494,9 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr return false; } + // the submit path below can end the command buffer, so close the region first + dbg.close(); + ctx->tensor_ctxs[node_idx] = compute_ctx; #if defined(GGML_VULKAN_CHECK_RESULTS) @@ -17841,14 +17963,13 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->device->diag_prev_end = -1; if (vk_instance.debug_utils_support) { - vk::DebugUtilsLabelEXT dul = {}; - dul.pLabelName = "ggml_backend_vk_graph_compute"; - dul.color = std::array{1.0f, 1.0f, 1.0f, 1.0f}; - - std::lock_guard guard(*ctx->device->compute_queue->handle); - vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(ctx->device->compute_queue->handle->queue, reinterpret_cast(&dul)); + ctx->device->debug_cmdbuf_idx = 0; } + // queue scope, so it encloses every submit this evaluation makes. + // closed when the function returns + ggml_vk_debug_label queue_dbg(ctx->device->compute_queue->handle.get(), "ggml_backend_vk_graph_compute"); + ctx->prealloc_size_add_rms_partials_offset = 0; ctx->do_add_rms_partials = false; ctx->do_add_rms_partials_offset_calculation = false; From 311d4211bf1611ff7ca6b67035a4a07c79766efc Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:55:46 +0200 Subject: [PATCH 17/65] memory : avoid allocating V cache for indexer (it's not used) (#28330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stanisław Szymczyk --- src/llama-memory-hybrid-idx.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp index 93b468784a33..3972ce9ce293 100644 --- a/src/llama-memory-hybrid-idx.cpp +++ b/src/llama-memory-hybrid-idx.cpp @@ -55,6 +55,10 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx( // K-shift must not rotate them while the stream copies in the same update still apply hparams_idx.rope_type = LLAMA_ROPE_TYPE_NONE; + // fool llama_kv_cache into thinking this is a MLA cache, so it won't cache V tensors + hparams_idx.n_embd_head_k_mla_impl = model.hparams.indexer_head_size; + hparams_idx.n_embd_head_v_mla_impl = model.hparams.indexer_head_size; + LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); return new llama_kv_cache( From 8c322d5bc4107f05fda262cb06e239b66958ade0 Mon Sep 17 00:00:00 2001 From: Kartik Gulia Date: Thu, 10 Sep 2026 16:11:57 +0530 Subject: [PATCH 18/65] convert : expand Nemotron H conversion fix (#28689) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * override function for n_h_l * narrow change for extracting nested attribute * simpler change; combines has_moe_params * Apply suggestion from @CISC Co-authored-by: Sigbjørn Skjæret --------- Co-authored-by: Sigbjørn Skjæret --- conversion/nemotron.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conversion/nemotron.py b/conversion/nemotron.py index c7adb2e27aca..65728d554485 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -216,14 +216,14 @@ def __init__(self, *args, **kwargs): hparams = kwargs.pop("hparams", None) if hparams is None: hparams = ModelBase.load_hparams(args[0], self.is_mistral_format) - has_moe_params = ( - "num_experts_per_tok" in hparams - or (isinstance(hparams.get("llm_config"), dict) and "num_experts_per_tok" in hparams["llm_config"]) - ) + llm_config = {**hparams, **(hparams.get("llm_config") or {})} + + has_moe_params = "num_experts_per_tok" in llm_config + layers_block_type = llm_config.get("layers_block_type") + if has_moe_params: self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE self.is_moe = True - layers_block_type = hparams.get("layers_block_type") if layers_block_type is not None: hparams["num_hidden_layers"] = len(layers_block_type) From 3ff67eb43d362b6720a4f1fc745931fd3e8a78d6 Mon Sep 17 00:00:00 2001 From: Julian Pscheid Date: Thu, 10 Sep 2026 03:42:23 -0700 Subject: [PATCH 19/65] vulkan: fall back to shared-memory reduction for dmmv on PowerVR (#28341) The Imagination proprietary Vulkan compiler returns VK_ERROR_UNKNOWN from vkCreateComputePipelines for every dequant mul_mat_vec shader built with the subgroup-only reduction that requires a subgroup size >= 16. That covers the k-quants, the i-quants, TQ2_0, MXFP4 and NVFP4. ggml rethrows, so the first generated token of any such model kills the process. Reproduced on a Pixel 11 Pro (PowerVR C-Series CXTP-48-1536 MC1, driver 1.662.3024, subgroup size 128, min 32, max 128). The failure is independent of subgroup size: 32, 64 and 128 all fail, as does dropping the full-subgroups flag and the required-subgroup-size pNext. The legacy quants, which use the plain subgroup reduction, compile and run fine. The shared-memory reduction variant compiles and matches the CPU reference for q2_K, q3_K, q4_K, q5_K and q6_K. The hybrid variant also compiles but costs 27% of token throughput (3.78 vs 5.20 t/s on Qwen3.5-2B-Q4_K_M). --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index cefe186feeaf..870fa115589b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -5480,8 +5480,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { uint32_t rm_iq = 2 * rm_kq; const bool use_subgroups = device->subgroup_arithmetic; + // The Imagination proprietary compiler rejects the subgroup-only dequant mul_mat_vec + // shaders that require a subgroup size >= 16; fall back to shared-memory reduction. + const bool is_imagination_proprietary = + device->driver_id == vk::DriverId::eImaginationProprietary; // Ensure a subgroup size >= 16 is available - const bool use_subgroups16 = use_subgroups && subgroup_min_size_16; + const bool use_subgroups16 = use_subgroups && subgroup_min_size_16 && !is_imagination_proprietary; const uint32_t subgroup_size = (device->vendor_id == VK_VENDOR_ID_INTEL && device->subgroup_size_control && device->subgroup_min_size <= 16 && device->subgroup_max_size >= 16) ? 16 : device->subgroup_size; const uint32_t subgroup_size16 = std::max(subgroup_size, 16u); From e5a8d439cef31f27fad6938233da10dae1ba5631 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 10 Sep 2026 13:43:43 +0300 Subject: [PATCH 20/65] tests : drop SYCL special-casing in test-backend-ops.cpp (#28688) --- tests/test-backend-ops.cpp | 139 ++++--------------------------------- 1 file changed, 15 insertions(+), 124 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 2deb90f6ab1e..503998883223 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -462,17 +462,9 @@ static std::string var_to_str(ggml_scale_mode mode) { #define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) #define VARS_TO_STR17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) VAR_TO_STR(a) + "," + VARS_TO_STR16(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) -#ifdef GGML_USE_SYCL -static bool inline _isinf(float f) { - return (*(uint32_t *)&f & 0x7fffffff) == 0x7f800000; -} -#else -static bool inline _isinf(float f) { return std::isinf(f); } -#endif - // accept FLT_MAX as infinity static bool isinf_or_max(float f) { - return _isinf(f) || f == FLT_MAX || f == -FLT_MAX; + return std::isinf(f) || f == FLT_MAX || f == -FLT_MAX; } static bool ggml_is_view_op(enum ggml_op op) { @@ -4831,51 +4823,6 @@ struct test_mul_mat : public test_case { } }; -#define P 1.0f -#define N -1.0f - -// constant Hadamard matrix via Paley I construction -static constexpr float H12[12][12] = { - { P, P, P, P, P, P, P, P, P, P, P, P }, - { P, N, P, N, P, P, P, N, N, N, P, N }, - { P, N, N, P, N, P, P, P, N, N, N, P }, - { P, P, N, N, P, N, P, P, P, N, N, N }, - { P, N, P, N, N, P, N, P, P, P, N, N }, - { P, N, N, P, N, N, P, N, P, P, P, N }, - { P, N, N, N, P, N, N, P, N, P, P, P }, - { P, P, N, N, N, P, N, N, P, N, P, P }, - { P, P, P, N, N, N, P, N, N, P, N, P }, - { P, P, P, P, N, N, N, P, N, N, P, N }, - { P, N, P, P, P, N, N, N, P, N, N, P }, - { P, P, N, P, P, P, N, N, N, P, N, N } -}; - -static constexpr float H20[20][20] = { - { P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P }, - { P, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N }, - { P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P }, - { P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P }, - { P, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N }, - { P, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N }, - { P, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N }, - { P, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N }, - { P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P }, - { P, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N }, - { P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P }, - { P, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N }, - { P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P }, - { P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P }, - { P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P }, - { P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P }, - { P, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N }, - { P, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N }, - { P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P }, - { P, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N } -}; - -#undef P -#undef N - // GGML_HINT_SRC0_IS_HADAMARD struct test_mul_mat_hadamard : public test_mul_mat { test_mul_mat_hadamard(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, @@ -4900,58 +4847,20 @@ struct test_mul_mat_hadamard : public test_mul_mat { void initialize_tensors(ggml_context * ctx) override { for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (strcmp(t->name, "a") == 0) { - const int64_t n_cols = t->ne[0]; - const int64_t n_rows = ggml_nrows(t); + const int64_t n_cols = t->ne[0]; + const int64_t n_rows = ggml_nrows(t); std::vector data(n_cols * n_rows); - float scale = 1.0f / sqrtf((float) n_cols); - - auto is_pow2 = [](const int64_t a) { - return (a > 0) && ((a & (a - 1)) == 0); - }; -#ifdef GGML_USE_SYCL - const bool is_kronecker = - ((n_cols % 12 == 0) && is_pow2(n_cols / 12)) || ((n_cols % 20 == 0) && is_pow2(n_cols / 20)); -#else - const bool is_kronecker = false; -#endif - if (is_kronecker) { - const int64_t B = (n_cols % 12 == 0 && is_pow2(n_cols / 12)) ? 12 : 20; - for (int64_t r = 0; r < n_rows; r++) { - float * row_data = data.data() + r * n_cols; - const int64_t r_mod = r % n_cols; - const int64_t r_b = r_mod / B; - const int64_t r_m = r_mod % B; - - for (int64_t i = 0; i < n_cols; i++) { - const int64_t c_b = i / B; - const int64_t c_m = i % B; - - int pop = 0; - int64_t val = r_b & c_b; - while (val) { - pop += (val & 1); - val >>= 1; - } - const float sign_m = (pop % 2 == 0) ? 1.0f : -1.0f; - const float sign_b = (B == 12) ? H12[c_m][r_m] : H20[c_m][r_m]; - - row_data[i] = scale * sign_b * sign_m; - } - } - } - - else if (is_pow2(n_cols)) { - for (int64_t r = 0; r < n_rows; r++) { - float * row_data = data.data() + r * n_cols; - for (int64_t i = 0; i < n_cols; i++) { - int pop_cnt = 0; - int64_t val = r & i; - while (val) { - pop_cnt += (val & 1); - val >>= 1; - } - row_data[i] = (pop_cnt % 2 == 0) ? scale : -scale; + float scale = 1.0f / sqrtf((float)n_cols); + for (int64_t r = 0; r < n_rows; r++) { + float * row_data = data.data() + r * n_cols; + for (int64_t i = 0; i < n_cols; i++) { + int pop = 0; + int64_t val = r & i; + while (val) { + pop += (val & 1); + val >>= 1; } + row_data[i] = (pop % 2 == 0) ? scale : -scale; } } ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float)); @@ -9716,16 +9625,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 512, 256)); // many rows test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 32, 1, 32)); // too small (N<64) test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 1024, 1, 1024)); // too big (N>512) -#ifdef GGML_USE_SYCL - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 384, 1, 384)); // m=12 (N=384) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 384, 32, 384)); // m=12 (batch) - test_cases.emplace_back( - new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 384, 4, 384, { 2, 3 })); // m=12 (multi-dim) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 768, 1, 768)); // m=12 (N=768) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 640, 1, 640)); // m=20 (N=640) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 640, 32, 640)); // m=20 (batch) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 1280, 1, 1280)); // m=20 (N=1280) -#endif + #if 0 // > 4GB A matrix. Too slow to be enabled by default. test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 900000, 3, 2592, {1, 1}, {1, 1})); @@ -11011,16 +10911,7 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 2048, 128)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 2048, 256)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 512, 2048, 512)); -#ifdef GGML_USE_SYCL - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 384, 1, 384)); // m=12 (N=384) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 384, 32, 384)); // m=12 (batch) - test_cases.emplace_back( - new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 384, 4, 384, { 2, 3 })); // m=12 (multi-dim) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 768, 1, 768)); // m=12 (N=768) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 640, 1, 640)); // m=20 (N=640) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 640, 32, 640)); // m=20 (batch) - test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 1280, 1, 1280)); // m=20 (N=1280) -#endif + test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 4, 4 }, { 32, 64, 4, 4 })); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 128, 128, 4, 2 }, { 32, 128, 4, 2 })); // qwen3next with CHUNK_SIZE 64 From c32d1dabe819002ca8aa3a885aca4057f5968e3d Mon Sep 17 00:00:00 2001 From: Gaurav Garg Date: Thu, 10 Sep 2026 17:42:40 +0530 Subject: [PATCH 21/65] tests : increase tolerance for Add fusion tests (#28691) --- tests/test-backend-ops.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 503998883223..0a6516e61454 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -3328,6 +3328,14 @@ struct test_bin_bcast : public test_case { return op == ggml_div; } + double max_nmse_err() override { + if (op == ggml_add && type == GGML_TYPE_F16 && nf > 1) { + // Fused ADDs can keep FP32 intermediates while the CPU rounds each ADD to FP16. + return 1e-6; + } + return test_case::max_nmse_err(); + } + double max_maa_err() override { return op == ggml_add ? 1e-4 : 1e-3; } From d344123fe2de081a72e02d6869360dfcbc0b528b Mon Sep 17 00:00:00 2001 From: Iggy Jackson Date: Thu, 10 Sep 2026 06:09:35 -0700 Subject: [PATCH 22/65] models: clean up some dead switch branches in old models (#28669) Some of these if statements were copypastaed in a former refactor and never cleaned up to remove the cases that could never happen anymore. The only thing that's shared between these relatives anymore is llama_model_bert::graph::graph, so the rest of the code doesn't need the conditionals. --- src/models/bert.cpp | 30 ++++++++----------------- src/models/jina-bert-v3.cpp | 28 ++++-------------------- src/models/nomic-bert-moe.cpp | 24 ++++---------------- src/models/nomic-bert.cpp | 41 +++++++++-------------------------- 4 files changed, 27 insertions(+), 96 deletions(-) diff --git a/src/models/bert.cpp b/src/models/bert.cpp index ca0281d306b1..9cc03dc56ad0 100644 --- a/src/models/bert.cpp +++ b/src/models/bert.cpp @@ -29,15 +29,13 @@ void llama_model_bert::load_arch_tensors(llama_model_loader &) { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_BERT) { - pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); + pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); - cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); - cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); + cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); + cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); - cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - } + cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); + cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0); tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0); @@ -53,20 +51,10 @@ void llama_model_bert::load_arch_tensors(llama_model_loader &) { layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0); layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0); - if (hparams.moe_every_n_layers > 0 && i % hparams.moe_every_n_layers == 1) { - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - } else { - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); - layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); - layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); - - if (arch == LLM_ARCH_NOMIC_BERT) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - } - } + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0); layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}, 0); diff --git a/src/models/jina-bert-v3.cpp b/src/models/jina-bert-v3.cpp index 1c974a6f16cc..78cf9d835976 100644 --- a/src/models/jina-bert-v3.cpp +++ b/src/models/jina-bert-v3.cpp @@ -19,16 +19,6 @@ void llama_model_jina_bert_v3::load_arch_tensors(llama_model_loader &) { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_BERT) { - pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); - - cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); - cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); - - cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - } - tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0); tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0); @@ -43,20 +33,10 @@ void llama_model_jina_bert_v3::load_arch_tensors(llama_model_loader &) { layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0); layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0); - if (hparams.moe_every_n_layers > 0 && i % hparams.moe_every_n_layers == 1) { - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - } else { - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); - layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); - layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); - - if (arch == LLM_ARCH_NOMIC_BERT) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - } - } + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0); layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}, 0); diff --git a/src/models/nomic-bert-moe.cpp b/src/models/nomic-bert-moe.cpp index da4b62919bb9..924af5e0bc40 100644 --- a/src/models/nomic-bert-moe.cpp +++ b/src/models/nomic-bert-moe.cpp @@ -4,12 +4,10 @@ void llama_model_nomic_bert_moe::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); ml.get_key(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers, 0); - if (hparams.n_layer() == 12 && hparams.n_embd == 768) { - if (arch == LLM_ARCH_NOMIC_BERT) { - type = LLM_TYPE_137M; - } else if (arch == LLM_ARCH_NOMIC_BERT_MOE && hparams.moe_every_n_layers == 2) { - type = LLM_TYPE_475M; - } + switch (hparams.n_layer()) { + case 12: + type = LLM_TYPE_475M; break; + default: type = LLM_TYPE_UNKNOWN; } } @@ -22,16 +20,6 @@ void llama_model_nomic_bert_moe::load_arch_tensors(llama_model_loader &) { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_BERT) { - pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); - - cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); - cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); - - cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - } - tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0); tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0); @@ -55,10 +43,6 @@ void llama_model_nomic_bert_moe::load_arch_tensors(llama_model_loader &) { layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); - - if (arch == LLM_ARCH_NOMIC_BERT) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - } } layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0); diff --git a/src/models/nomic-bert.cpp b/src/models/nomic-bert.cpp index e7fc72286a6d..509787286efa 100644 --- a/src/models/nomic-bert.cpp +++ b/src/models/nomic-bert.cpp @@ -1,15 +1,12 @@ #include "models.h" void llama_model_nomic_bert::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); - ml.get_key(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers, 0); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); - if (hparams.n_layer() == 12 && hparams.n_embd == 768) { - if (arch == LLM_ARCH_NOMIC_BERT) { - type = LLM_TYPE_137M; - } else if (arch == LLM_ARCH_NOMIC_BERT_MOE && hparams.moe_every_n_layers == 2) { - type = LLM_TYPE_475M; - } + switch (hparams.n_layer()) { + case 12: + type = LLM_TYPE_137M; break; + default: type = LLM_TYPE_UNKNOWN; } } @@ -22,16 +19,6 @@ void llama_model_nomic_bert::load_arch_tensors(llama_model_loader &) { tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_token_types}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_BERT) { - pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, n_ctx_train}, 0); - - cls = create_tensor(tn(LLM_TENSOR_CLS, "weight"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED); - cls_b = create_tensor(tn(LLM_TENSOR_CLS, "bias"), {n_embd}, TENSOR_NOT_REQUIRED); - - cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, "bias"), {hparams.n_cls_out}, TENSOR_NOT_REQUIRED); - } - tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0); tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd}, 0); @@ -46,20 +33,12 @@ void llama_model_nomic_bert::load_arch_tensors(llama_model_loader &) { layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0); layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0); - if (hparams.moe_every_n_layers > 0 && i % hparams.moe_every_n_layers == 1) { - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0); - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - } else { - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); - layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); - layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED); - if (arch == LLM_ARCH_NOMIC_BERT) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - } - } + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0); layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}, 0); From 41fc7584f0c1d72d9cc1ac46ccae8defc1587f0f Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Thu, 10 Sep 2026 15:44:40 +0200 Subject: [PATCH 23/65] scripts : use sed instead of grep for version parsing [no ci] (#28700) This commit updates the version parsing in make-release-checks.sh to use sed instead of grep. The motivation for this is that currently when running this script on macos it errors: ```console $ ./scripts/make-release-checks.sh --dry-run grep: invalid option -- P usage: grep [-abcdDEFGHhIiJLlMmnOopqRSsUVvwXxZz] [-A num] [-B num] [-C[num]] [-e pattern] [-f file] [--binary-files=value] [--color=when] [--context[=num]] [--directories=action] [--label] [--line-buffered] [--null] [pattern] [file ...] ``` With the changes in this commit it is possible to run this without failure. --- scripts/make-release-checks.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/make-release-checks.sh b/scripts/make-release-checks.sh index 32c193745b5c..d78fa1457c3a 100755 --- a/scripts/make-release-checks.sh +++ b/scripts/make-release-checks.sh @@ -22,9 +22,9 @@ for arg in "$@"; do esac done -MAJOR=$(grep "set(LLAMA_VERSION_MAJOR" "$REPO_ROOT/CMakeLists.txt" | grep -oP '\d+') -MINOR=$(grep "set(LLAMA_VERSION_MINOR" "$REPO_ROOT/CMakeLists.txt" | grep -oP '\d+') -PATCH=$(grep "set(LLAMA_VERSION_PATCH" "$REPO_ROOT/CMakeLists.txt" | grep -oP '\d+') +MAJOR=$(grep "set(LLAMA_VERSION_MAJOR" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*MAJOR \([0-9]*\).*/\1/') +MINOR=$(grep "set(LLAMA_VERSION_MINOR" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*MINOR \([0-9]*\).*/\1/') +PATCH=$(grep "set(LLAMA_VERSION_PATCH" "$REPO_ROOT/CMakeLists.txt" | sed 's/.*PATCH \([0-9]*\).*/\1/') VERSION="v${MAJOR}.${MINOR}.${PATCH}" echo "Determined version: ${VERSION}" if [[ -n "${GITHUB_OUTPUT:-}" ]]; then @@ -91,9 +91,9 @@ else fi fi -MAJOR=$(grep "set(GGML_VERSION_MAJOR" "$REPO_ROOT/ggml/CMakeLists.txt" | grep -oP '\d+') -MINOR=$(grep "set(GGML_VERSION_MINOR" "$REPO_ROOT/ggml/CMakeLists.txt" | grep -oP '\d+') -PATCH=$(grep "set(GGML_VERSION_PATCH" "$REPO_ROOT/ggml/CMakeLists.txt" | grep -oP '\d+') +MAJOR=$(grep "set(GGML_VERSION_MAJOR" "$REPO_ROOT/ggml/CMakeLists.txt" | sed 's/.*MAJOR \([0-9]*\).*/\1/') +MINOR=$(grep "set(GGML_VERSION_MINOR" "$REPO_ROOT/ggml/CMakeLists.txt" | sed 's/.*MINOR \([0-9]*\).*/\1/') +PATCH=$(grep "set(GGML_VERSION_PATCH" "$REPO_ROOT/ggml/CMakeLists.txt" | sed 's/.*PATCH \([0-9]*\).*/\1/') GGML_VERSION="v${MAJOR}.${MINOR}.${PATCH}" echo "Local ggml version: ${GGML_VERSION}" From fa6769818708afd9807b22183ccda112fd563427 Mon Sep 17 00:00:00 2001 From: Jesus Gulfo Date: Thu, 10 Sep 2026 10:10:55 -0500 Subject: [PATCH 24/65] spec: fix failed to decode mtmd chunk with DFlash (#28587) * speculative: fix failed to decode mtmd chunk with DFlash When using DFlash w/ vision models, the drafter memory fails to allocate new tokens because images report a fixed offset. Stop copying them to allow the drafter to continue. * address PR feedback limit M-RoPE skip to images only, allow audio to pass through. Clean up comments to align to the updated implementation --- common/speculative.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index 2db381d58086..b7811b853e1d 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1094,8 +1094,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { // Target prefill may contain token IDs or multimodal embeddings. Both // produce the target-layer features used to seed the draft KV cache, so - // skipping the embedding batches leaves a hole in the draft's cache and - // the next injection fails to initialize. + // embeddings are injected too, except the pinned ones skipped below. // TODO: revisit after https://github.com/ggml-org/llama.cpp/pull/24669 is merged const bool has_tokens = batch_in.token != nullptr; const bool has_embeddings = batch_in.embd != nullptr; @@ -1131,6 +1130,13 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { } const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1; + // an M-RoPE image pins all its rows to one position, so a windowed draft + // cache cannot free cells for it - skip it, the draft can jump over the gap + const bool pos_pinned = batch_in.pos[i_batch_beg[seq_id]] == batch_in.pos[i_batch_end[seq_id]]; + if (has_embeddings && n_rows > 1 && pos_pinned) { + continue; + } + for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) { const int32_t n_chunk = std::min(n_ubatch, n_rows - offset); From 18c17b4d66444c014cdc2f1d48335f343c0135f9 Mon Sep 17 00:00:00 2001 From: shivamkumard-ctrl Date: Thu, 10 Sep 2026 21:40:55 +0530 Subject: [PATCH 25/65] ci : Update WoA CUDA 13.4 release to use 13.4.1 GA redistributables (#28687) - Move Windows ARM64 CUDA 13.4 builds from the Developer Preview archives to the 13.4.1 GA redistributables --- .github/actions/windows-setup-cuda/action.yml | 24 +++++++++---------- .github/workflows/release.yml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/actions/windows-setup-cuda/action.yml b/.github/actions/windows-setup-cuda/action.yml index 917513b85eac..2048740a2549 100644 --- a/.github/actions/windows-setup-cuda/action.yml +++ b/.github/actions/windows-setup-cuda/action.yml @@ -137,19 +137,19 @@ runs: run: | mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" choco install unzip -y - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cccl-windows-x86_64-13.3.4.1.2-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_crt-windows-x86_64-13.4.46-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_nvcc-windows-x86_64-13.4.46-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/libnvvm-windows-x86_64-13.4.46-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_cudart-windows-arm64-13.4.46-archive.zip" - curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/libcublas-windows-arm64-13.7.0.10-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive.zip" unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.1.2-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.10-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1101fef157bc..9b77c2d97d80 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1717,7 +1717,7 @@ jobs: - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip) - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip) - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip) - - [Windows arm64 (CUDA 13) (preview)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip) + - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip) - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip) - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip) - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip) From 52d42686560a9e8f441f9b9780c8890c37d2802d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Thu, 10 Sep 2026 18:11:32 +0200 Subject: [PATCH 26/65] ci : add self-hosted-gpu-cuda and server-sanitize to hf-jobs (#28693) --- .github/workflows/build-self-hosted.yml | 34 ++++++++++++++++- .github/workflows/server-sanitize.yml | 50 ++++++++++++++++--------- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index ccfe2a604645..fda4879e2149 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -58,18 +58,48 @@ env: jobs: gpu-cuda: - runs-on: [self-hosted, Linux, NVIDIA] + runs-on: "hf-jobs-t4-small:cuda13" steps: - name: Clone id: checkout uses: actions/checkout@v6 + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y cmake libssl-dev time unzip wget python3 python3-venv python3-pip + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + with: + key: self-hosted-gpu-cuda + folder: llama.cpp + hf_bucket: ggml-org/cache + - name: Test id: ggml-ci run: | nvidia-smi - GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + GG_BUILD_CUDA=1 CUDACXX=/usr/local/cuda/bin/nvcc bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: self-hosted-gpu-cuda + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true gpu-rocm: runs-on: [self-hosted, Linux, AMD] diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml index 77549ee8717a..52e175f834d1 100644 --- a/.github/workflows/server-sanitize.yml +++ b/.github/workflows/server-sanitize.yml @@ -32,6 +32,8 @@ on: ] env: + # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302) + HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} LLAMA_ARG_LOG_COLORS: 1 LLAMA_ARG_LOG_PREFIX: 1 LLAMA_ARG_LOG_TIMESTAMPS: 1 @@ -43,7 +45,7 @@ concurrency: jobs: server: - runs-on: [self-hosted, CPU, Linux, llama-server] + runs-on: hf-jobs-cpu-upgrade strategy: matrix: @@ -52,20 +54,6 @@ jobs: fail-fast: false steps: - #- name: Dependencies - # id: depends - # run: | - # sudo apt-get update - # sudo apt-get -y install \ - # build-essential \ - # xxd \ - # git \ - # cmake \ - # curl \ - # wget \ - # language-pack-en \ - # libssl-dev - - name: Clone id: checkout uses: actions/checkout@v6 @@ -73,6 +61,24 @@ jobs: fetch-depth: 0 ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y build-essential cmake python3-full + + - name: ccache + uses: ggml-org/ccache-action@v1.2.24 + with: + restore: false + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + with: + key: server-sanitize + folder: llama.cpp + hf_bucket: ggml-org/cache + - name: Build id: cmake_build run: | @@ -87,9 +93,17 @@ jobs: -DLLAMA_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }} cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server - - name: Python setup - id: setup_python - uses: actions/setup-python@v7 + - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: server-sanitize + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ggml-org/cache + save: true - name: Install Python dependencies run: | From 6788edb4f325c1cb4210997eb79edcab2e27aeaa Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Thu, 10 Sep 2026 12:20:18 -0500 Subject: [PATCH 27/65] vulkan: small M matrix optimizations for qwen (#28457) * vulkan: optimize m=1 mul_mat by swapping A/B * vulkan: Improve small M perf Allow split_k with small M. Make small vs med tile selection (for coopmat2) depend on M, not just N. --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 27 +++++++++++++++++++-------- tests/test-backend-ops.cpp | 10 +++++++++- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 870fa115589b..34400a1b7ab5 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -5424,8 +5424,9 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count || (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2); if (n > crossover_large && prefer_large) return last; - uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1]; - if (n > crossover_medium) return 1; + uint32_t crossover_medium_m = configs[0].unaligned->wg_denoms[0]; + uint32_t crossover_medium_n = configs[0].unaligned->wg_denoms[1]; + if (m > crossover_medium_m && n > crossover_medium_n) return 1; return 0; }; device->matmul_id_tile_selector = [](uint32_t /*m*/, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, @@ -9027,7 +9028,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, } uint32_t split_k = 1; - if (ctx->device->shader_core_count != 0 && m >= pipeline->wg_denoms[0] && n >= pipeline->wg_denoms[1]) { + if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) { // If k is 'large' and the SMs will fill less than halfway, use split_k. uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]); uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]); @@ -9780,10 +9781,10 @@ static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_ GGML_UNUSED(m); } -static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { +static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, bool swap_inputs = false) { ggml_tensor * dst = cgraph->nodes[node_idx]; - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * src0 = dst->src[swap_inputs ? 1 : 0]; + const ggml_tensor * src1 = dst->src[swap_inputs ? 0 : 1]; VK_LOG_DEBUG("ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -9802,8 +9803,8 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint64_t ne12 = src1->ne[2]; const uint64_t ne13 = src1->ne[3]; - const uint64_t ne20 = dst->ne[0]; - const uint64_t ne21 = dst->ne[1]; + const uint64_t ne20 = dst->ne[swap_inputs ? 1 : 0]; + const uint64_t ne21 = dst->ne[swap_inputs ? 0 : 1]; // const uint64_t ne22 = dst->ne[2]; // const uint64_t ne23 = dst->ne[3]; @@ -10417,6 +10418,16 @@ static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, c src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { ggml_vk_mul_mat_vec_nc_f16_f32(ctx, subctx, cgraph, node_idx); + // With one output row, B^T*A has the same flat output as A^T*B. + } else if (ctx->num_additional_fused_ops == 0 && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && + (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16 || ggml_is_quantized(src1->type)) && + dst->ne[0] == 1 && dst->ne[1] > mul_mat_vec_max_cols && + src0->ne[2] == 1 && src0->ne[3] == 1 && + src1->ne[2] == 1 && src1->ne[3] == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) && + get_misalign_bytes(ctx, src0) == 0 && get_misalign_bytes(ctx, src1) == 0 && get_misalign_bytes(ctx, dst) == 0) { + ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx, true); // mul_mat_vec supports batching ne12*ne13 when ne11==1, or treating ne11 as the batch size (up to four) // when ne12 and ne13 are one. } else if ((dst->ne[1] == 1 || (dst->ne[1] <= mul_mat_vec_max_cols && src1->ne[2] * src1->ne[3] == 1)) && diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 0a6516e61454..3428629fd462 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9683,9 +9683,17 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); // m == 1, with n on both sides of MMVF_MAX_BATCH_SIZE (8): mmvf below, operand swap above - for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) { + for (int64_t n : {1, 7, 8, 9, 16, 127, 128, 511, 512}) { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1})); } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 512, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 512, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 509, 2051, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 509, 2051, {1, 1}, {1, 1})); + + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 31, 509, 2051, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 32, 509, 2112, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 32, 509, 2112, {1, 1}, {1, 1})); #if 0 { From 50182a53fa2c26bd2a7fc31d855231effdc2f4ad Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Thu, 10 Sep 2026 12:21:29 -0500 Subject: [PATCH 28/65] vulkan: use add_alloc_dep to enable topk_moe fusion for prefill (#28422) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 97 +++++++++++++++++----------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 34400a1b7ab5..8e1cf3ff3774 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -18328,39 +18328,31 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg bool need_disable = false; - // topk_moe often overwrites the source, but for a given row all the src values are - // loaded before anything is stored. If there's only one row, this is safe, so treat - // this as a special case. - bool is_topk_moe_single_row = ctx->fused_topk_moe_mode != TOPK_MOE_COUNT && - ggml_nrows(cgraph->nodes[i]->src[0]) == 1; - - if (!is_topk_moe_single_row) { - for (int j = 0; j < 2; ++j) { - ggml_tensor *dst = output_nodes[j]; - if (!dst) { - continue; - } - // Loop over all srcs of all nodes in the fusion. If the src overlaps - // the destination and the src is not an intermediate node that's being - // elided, then disable fusion. - for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) { - for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { - ggml_tensor *src = cgraph->nodes[i + k]->src[s]; - if (!src || src->op == GGML_OP_NONE) { - continue; - } - if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) { - bool found = false; - for (int n = 0; n < k; ++n) { - if (cgraph->nodes[i + n] == src) { - found = true; - break; - } - } - if (!found) { - need_disable = true; + for (int j = 0; j < 2; ++j) { + ggml_tensor *dst = output_nodes[j]; + if (!dst) { + continue; + } + // Loop over all srcs of all nodes in the fusion. If the src overlaps + // the destination and the src is not an intermediate node that's being + // elided, then disable fusion. + for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) { + for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor *src = cgraph->nodes[i + k]->src[s]; + if (!src || src->op == GGML_OP_NONE) { + continue; + } + if (ggml_vk_tensors_overlap(src, dst, op_srcs_fused_elementwise[k])) { + bool found = false; + for (int n = 0; n < k; ++n) { + if (cgraph->nodes[i + n] == src) { + found = true; + break; } } + if (!found) { + need_disable = true; + } } } } @@ -18372,6 +18364,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->fused_topk_moe_scale = false; ctx->fused_topk_qsa = false; ctx->fused_rms_norm_mode = RMS_NORM_COUNT; + fusion_string = nullptr; } } @@ -18474,7 +18467,6 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // Sort the graph for improved parallelism. static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params) { - GGML_UNUSED(params); VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)"); ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; @@ -18560,19 +18552,50 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * return false; }; - if (keep_pattern(topk_moe_early_softmax_norm)) { + auto const &add_pattern_alloc_deps = [&](const std::initializer_list &pattern, int last_node) { + // Keep external inputs alive through the fused output. + std::set seen; + for (size_t j = 0; j < pattern.size(); ++j) { + ggml_tensor * node = graph->nodes[first_unused + j]; + for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor * src = node->src[s]; + if (src && seen.insert(src).second) { + params->add_alloc_dep(params->user_data, src, graph->nodes[last_node]); + } + } + seen.insert(node); + } + }; + + auto const &keep_topk_moe_pattern = [&](const std::initializer_list &pattern) -> bool { + if (!match_pattern(pattern, first_unused)) { + return false; + } + + int last_node = first_unused + (int) pattern.size() - 1; + // Some TOPK_MOE variants fuse a trailing scale. + if (last_node + 1 < graph->n_nodes && graph->nodes[last_node + 1]->op == GGML_OP_SCALE) { + last_node++; + } + + add_pattern_alloc_deps(pattern, last_node); + + return keep_pattern(pattern); + }; + + if (keep_topk_moe_pattern(topk_moe_early_softmax_norm)) { continue; } - if (keep_pattern(topk_moe_sigmoid_norm_bias)) { + if (keep_topk_moe_pattern(topk_moe_sigmoid_norm_bias)) { continue; } - if (keep_pattern(topk_moe_sqrt_softplus_norm_bias)) { + if (keep_topk_moe_pattern(topk_moe_sqrt_softplus_norm_bias)) { continue; } - if (keep_pattern(topk_moe_early_softmax)) { + if (keep_topk_moe_pattern(topk_moe_early_softmax)) { continue; } - if (keep_pattern(topk_moe_late_softmax)) { + if (keep_topk_moe_pattern(topk_moe_late_softmax)) { continue; } if (keep_pattern(snake_pattern)) { From 28ff0958291ce3465fabd7bd679d4b0edd742bd9 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Thu, 10 Sep 2026 12:22:46 -0500 Subject: [PATCH 29/65] vulkan: use CPU writes in ggml_backend_vk_cpy_tensor_async if the context is idle (#28618) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 8e1cf3ff3774..b28fdc9bbf49 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -17135,6 +17135,22 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba return false; } + // If the backend is idle, use a CPU copy to avoid GPU synchronization overhead. + static constexpr size_t max_cpu_copy_size = 128 * 1024; + const bool src_backend_synchronous = backend_src->iface.synchronize == nullptr; + const bool transfer_idle = !ctx->device->async_use_transfer_queue || + ctx->transfer_semaphore_last_submitted == ctx->transfer_semaphore.value; + const bool backend_idle = ctx->compute_ctx.expired() && ctx->transfer_ctx.expired() && + !ctx->submit_pending && !ctx->almost_ready_fence_pending && transfer_idle; + const bool dst_host_coherent = + (dst_buf->memory_property_flags & (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent)) == + (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); + + if ((backend_src == backend_dst || src_backend_synchronous) && backend_idle && dst_host_coherent && ggml_nbytes(src) <= max_cpu_copy_size) { + ggml_vk_buffer_write(dst_buf, vk_tensor_offset(dst) + dst->view_offs, src->data, ggml_nbytes(src)); + return true; + } + vk_context cpy_ctx; if (ctx->device->async_use_transfer_queue) { cpy_ctx = ggml_vk_get_transfer_ctx(ctx); @@ -17147,7 +17163,6 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba src->data, ggml_nbytes(src)); } - GGML_UNUSED(backend_src); return false; } From df03399b885831b2a1603b3abb0d8c156808e363 Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Thu, 10 Sep 2026 11:25:40 -0700 Subject: [PATCH 30/65] opencl: add A8 Q4_0 mm binary kernel support (#28268) --- ggml/src/ggml-opencl/CMakeLists.txt | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 274 +++++++++++++++++- .../gemv_noshuffle_q4_0_f32_32b_trans.cl | 137 +++++++++ 3 files changed, 402 insertions(+), 10 deletions(-) create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 37e565ef4ff9..716577bb77e0 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -170,6 +170,7 @@ set(GGML_OPENCL_KERNELS gemv_noshuffle_q4_0_f32 gemv_noshuffle_q4_0_f32_spec gemm_noshuffle_q4_0_f32 + gemv_noshuffle_q4_0_f32_32b_trans gemv_noshuffle_q4_1_f32 gemm_noshuffle_q4_1_f32 gemv_noshuffle_q5_0_f32 diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 3002835e8aea..231be2cf3ac4 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1160,6 +1160,8 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) + cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; cl_kernel kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; @@ -3787,6 +3789,43 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr; + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_0_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_0_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = + clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + } + } + // gemm_noshuffle_q4_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -6725,11 +6764,10 @@ struct ggml_tensor_extra_cl_q4_0 { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } - // Currently, q_img and d_img are only initialized when SMALL_ALLOC is - // enabled. They point to the images in ggml_backend_opencl_buffer_context. - // So, there is no need to release them here. - // TODO: initialize them for non SMALL_PATH path, or remove them. - d_img = nullptr; + if (d_img != nullptr) { + CL_CHECK(clReleaseMemObject(d_img)); + d_img = nullptr; + } size_q = 0; size_d = 0; } @@ -8311,6 +8349,20 @@ inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *b qh_img_width <= backend_ctx->image_max_buffer_size; } +inline bool use_q4_0_ila_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + // The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it // is SLOWER than the route it replaces, not because it is unsafe. It was first // parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that @@ -9573,10 +9625,34 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(K % 32 == 0); - // Transpose q as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K / 8, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K / 32, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + wimg_fmt = { CL_R, CL_HALF_FLOAT }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 32; + wimg_desc.buffer = extra->d; + CL_CHECK((extra->d_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q and d as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; @@ -11104,7 +11180,11 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); + } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); cl_uchar mask_0F = 0x0F; @@ -18347,6 +18427,166 @@ static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + int M = ne01; + int N = ne1; + int K = ne00; + + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &M)); + + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[3] = { wavesize, 4, 1 }; + size_t global_work_size[3] = { (size_t)CEIL_DIV(M, 64) * 64, 4, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + const int gemm_tile_n = 64; + int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1); + + cl_mem a_img = extra0_q4_0->q_img; + cl_mem s_img = extra0_q4_0->d_img; + GGML_ASSERT(a_img && s_img && "ILA Q4_0 weight images missing; set_tensor should have built them"); + + // Pad B through a zero-filled scratch buffer when N needs + // padding, since the GEMM kernel always reads a full N-tile. + const bool need_pad = N_pad > N; + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + if (need_pad) { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, + (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), + 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, + offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + } else { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + } + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = need_pad ? (size_t)K * N_pad : (size_t)K * N; + img_desc.buffer = need_pad ? b_padded : b_sub_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + int line_stride_matrix_A_in_bytes = M * 4; + int line_stride_matrix_S_in_bytes = M * 2; + int line_stride_matrix_B_in_bytes = K * 4; + int line_stride_matrix_C_in_bytes = M * 4; + + int c_offset_for_kernel = 0; + int b_offset_for_kernel = 0; + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &s_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -18399,6 +18639,20 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + const bool use_ila = use_q4_0_ila_kernels(backend_ctx, src0); + + if (use_ila) { + if (use_q40_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q40_MC3 is bypassed by Q4_0 binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst); + return; + } + if (ne1 == 1 || use_q40_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl new file mode 100644 index 000000000000..565285f4b293 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl @@ -0,0 +1,137 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK4_0 32 +#define N_SIMDGROUP 4 + +#define dequantizeBlockAccum_ila_1row_hi(total_sum, bits4, scale, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sum += ((bits4.s0 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sum += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sum += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sum += (((bits4.s0 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sum += ((bits4.s1 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sum += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sum += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sum += (((bits4.s1 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sum += ((bits4.s2 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sum += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sum += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sum += (((bits4.s2 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sum += ((bits4.s3 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sum += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sum += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sum += (((bits4.s3 & 0xF000) >> 12) - 8) * scale * shared_y; + +#define dequantizeBlockAccum_ila_1row_lo(total_sum, bits4, scale, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sum += ((bits4.s4 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sum += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sum += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sum += (((bits4.s4 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sum += ((bits4.s5 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sum += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sum += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sum += (((bits4.s5 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sum += ((bits4.s6 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sum += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sum += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sum += (((bits4.s6 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sum += ((bits4.s7 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sum += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sum += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sum += (((bits4.s7 & 0xF000) >> 12) - 8) * scale * shared_y; + + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q4_0_f32_32b_trans( + __read_only image1d_buffer_t src0_q, + global half * src0_d, + __read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + __private uint4 regA; + __private half regS; + __private float8 regB; + __private float totalSum = 0.0f; + + for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) { + regS = src0_d[k * M + gid]; + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + regA.s0 = read_imageui(src0_q, ((k * 4 + 0) * M + gid)).x; + regA.s1 = read_imageui(src0_q, ((k * 4 + 1) * M + gid)).x; + regA.s2 = read_imageui(src0_q, ((k * 4 + 2) * M + gid)).x; + regA.s3 = read_imageui(src0_q, ((k * 4 + 3) * M + gid)).x; + + dequantizeBlockAccum_ila_1row_hi(totalSum, as_ushort8(regA), regS, regB); + dequantizeBlockAccum_ila_1row_lo(totalSum, as_ushort8(regA), regS, regB); + } + + __local float reduceLM[SIMDGROUP_WIDTH * 3]; + if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum; + if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum; + if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum; + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + if (gid < M) { + dst[gid] = totalSum; + } + } +} From 481c65f091f74c5e7089dd0a3a1cc6b50cced31e Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Fri, 11 Sep 2026 00:44:13 -0500 Subject: [PATCH 31/65] vulkan: fix data race and OOB access in argsort(large) (#28705) argsort had a data race in the inner loop, which VVL caught. But I don't think this was causing failures in practice. argsort_large has OOB accesses which might explain the failures in CI, but I couldn't reproduce it locally and I don't think it's a convincing explanation of the failures. --- .../ggml-vulkan/vulkan-shaders/argsort.comp | 28 +++++++++++-------- .../vulkan-shaders/argsort_large.comp | 5 +++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp b/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp index 0fc2b9b72535..4ba63f7aee1f 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp @@ -33,7 +33,11 @@ void argsort(bool needs_bounds_check, const uint row) { const uint row_offset = row * p.ncols; // initialize indices - dst_row[col] = ivec2(col, floatBitsToInt(data_a[row_offset + col])); + ivec2 value = ivec2(col, 0); + if (!needs_bounds_check || col < p.ncols) { + value.y = floatBitsToInt(data_a[row_offset + col]); + } + dst_row[col] = value; barrier(); uint num_outer_loop_iters = NCOLS_PADDED_LOG2; @@ -42,18 +46,20 @@ void argsort(bool needs_bounds_check, const uint row) { [[unroll]] for (uint j = k / 2, inner_idx = 0; inner_idx < num_inner_loop_iters; j /= 2, inner_idx++) { const int ixj = int(col ^ j); - int idx_0 = (col & k) == 0 ? col : ixj; - int idx_1 = (col & k) == 0 ? ixj : col; + if (ixj > col) { + int idx_0 = (col & k) == 0 ? col : ixj; + int idx_1 = (col & k) == 0 ? ixj : col; - ivec2 sh_idx_0 = dst_row[idx_0]; - ivec2 sh_idx_1 = dst_row[idx_1]; - bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false; - bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false; + ivec2 sh_idx_0 = dst_row[idx_0]; + ivec2 sh_idx_1 = dst_row[idx_1]; + bool idx_0_oob = needs_bounds_check ? sh_idx_0.x >= p.ncols : false; + bool idx_1_oob = needs_bounds_check ? sh_idx_1.x >= p.ncols : false; - if ((idx_0_oob || - (!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) && (ixj > col)) { - dst_row[idx_0] = sh_idx_1; - dst_row[idx_1] = sh_idx_0; + if (idx_0_oob || + (!idx_1_oob && intBitsToFloat(sh_idx_0.y) > intBitsToFloat(sh_idx_1.y))) { + dst_row[idx_0] = sh_idx_1; + dst_row[idx_1] = sh_idx_0; + } } barrier(); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp b/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp index 920bac6bb899..b2df44137488 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp @@ -42,7 +42,10 @@ void argsort(bool needs_bounds_check, const uint row) { [[unroll]] for (int u = 0; u < WG_UNROLL_FACTOR; ++u) { uint c = u*BLOCK_SIZE + col; if (c < p.ncols_padded) { - ivec2 v = ivec2(c, floatBitsToInt(data_a[row_offset + c])); + ivec2 v = ivec2(c, 0); + if (!needs_bounds_check || c < p.ncols) { + v.y = floatBitsToInt(data_a[row_offset + c]); + } tmp_idx[idx_offset + c] = v; } } From 451b89bae0c4b1dd612eb503ceace906c01ddcc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Fri, 11 Sep 2026 07:56:18 +0200 Subject: [PATCH 32/65] ci : key cache to sanitizer matrix (#28708) --- .github/workflows/server-sanitize.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml index 52e175f834d1..11237cf51b18 100644 --- a/.github/workflows/server-sanitize.yml +++ b/.github/workflows/server-sanitize.yml @@ -75,7 +75,7 @@ jobs: - name: ccache-buckets-restore uses: ./.github/actions/ccache-buckets with: - key: server-sanitize + key: server-sanitize-${{ matrix.sanitizer }} folder: llama.cpp hf_bucket: ggml-org/cache @@ -99,7 +99,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: server-sanitize + key: server-sanitize-${{ matrix.sanitizer }} folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache From 16378d93f94012d4228c8c7683adce3f286aee5d Mon Sep 17 00:00:00 2001 From: "Piotr Wilkin (ilintar)" Date: Fri, 11 Sep 2026 09:58:20 +0200 Subject: [PATCH 33/65] CUDA/HIP: Flash Attention tuning (gfx1201) (#28102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * HIP: enable mma FA for head size 256 on RDNA4, tune configs Assisted-by: Claude Assisted-by: Codex * HIP: prefer whole-tile FA grids over stream-k on AMD WMMA Assisted-by: Claude Assisted-by: Codex * revise stream_k logic * revise kernel selection logic --------- Co-authored-by: Johannes Gäßler --- ggml/src/ggml-cuda/fattn-common.cuh | 19 ++++++++++++++----- ggml/src/ggml-cuda/fattn-mma-f16.cuh | 6 +++--- ggml/src/ggml-cuda/fattn.cu | 23 +++++++++++++++++++++-- tests/test-backend-ops.cpp | 16 ++++++++++++++++ 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 7442bc22af20..48b631e60fb4 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -1133,12 +1133,21 @@ void launch_fattn( dim3 blocks_num; if (stream_k) { - // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup. - const int max_blocks = max_blocks_per_sm*nsm; - const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; - const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); + auto should_use_stream_k = [](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) { + const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; + const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); - const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || amd_wmma_available(cc) || tiles_efficiency_percent < 75; + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE) { + return true; + } + if (amd_wmma_available(cc) && DKQ == 64) { + return true; // TODO better configuration + } + return tiles_efficiency_percent < 75; + }; + + const int max_blocks = max_blocks_per_sm*nsm; + const bool use_stream_k = should_use_stream_k(cc, ntiles_dst, max_blocks, Q->ne[0]); blocks_num.x = ntiles_dst; blocks_num.y = 1; diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index bc5060e813e5..578f6cf79c25 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -158,8 +158,8 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 2, 32, 128, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 2, 32, 128, 128, 128, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 256, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2, 64, 128, 128, 64, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 160, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2, 32, 160, 128, 128, 1, true); @@ -1826,7 +1826,7 @@ static __global__ void flash_attn_ext_f16( #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING #if defined(AMD_WMMA_AVAILABLE) - if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 128) { + if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) { NO_DEVICE_CODE; return; } diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index d11a964d59a3..ceb4727931d4 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -221,6 +221,24 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con } } + // On RDNA it is preferable to minimize wasted compute vs. duplicate I/O for the mask. + if (amd_wmma_available(cc)) { + if (use_gqa_opt && gqa_ratio % 8 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 4 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 2 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + } + if (use_gqa_opt && gqa_ratio > 4) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); return; @@ -646,8 +664,9 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const } } - // AMD WMMA is always faster than the tile kernel if the full tile width of 16 can be utilized. - if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 128) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[1] * gqa_ratio_eff > 8) { + // AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized. + if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 && + Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) { return BEST_FATTN_KERNEL_MMA_F16; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3428629fd462..ef4fc30cecb7 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10574,6 +10574,13 @@ static std::vector> make_test_cases_eval() { GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); } + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 512, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 16, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {2, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {4, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // dense-allocated (non-view) quant K/V at batch >= 64, in cache and native layouts test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}, false)); @@ -11031,6 +11038,15 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 4096, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 65536, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 65536, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 131072, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 131072, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + for (int kv : { 4096, 8192, 16384,32768, 65536, }) { for (int hs : { 64, 128, 256, 576, }) { const int hsv = hs == 576 ? 512 : hs; From b0dcb8192b201e402ec3eff524e55450f8070e3e Mon Sep 17 00:00:00 2001 From: Jesus Gulfo Date: Fri, 11 Sep 2026 03:33:26 -0500 Subject: [PATCH 34/65] server: fix speculation after an image (#28715) * server: fix speculation after an image Pass the actual position to the drafter after an image, instead of the token count. Affects every drafter, not just DFlash. * rename draft n_past to pos0 n_past is used to denote number of tokens and this parameter is meant to be a position --- common/speculative.cpp | 20 +++++++++---------- common/speculative.h | 2 +- .../speculative-simple/speculative-simple.cpp | 2 +- tools/server/server-context.cpp | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index b7811b853e1d..7c8a06365cd7 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -296,7 +296,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true); } int ret = llama_decode(ctx_dft, batch); @@ -355,7 +355,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { continue; } - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); + common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true); } if (batch.n_tokens == 0) { @@ -1197,7 +1197,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { common_sampler_reset(smpls[seq_id].get()); - const int32_t n = (int32_t) dp.n_past; + const int32_t n = (int32_t) dp.pos0; const int32_t n_draft = params.n_max; @@ -1621,7 +1621,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); - common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes); i_last[seq_id] = batch.n_tokens - 1; @@ -1635,16 +1635,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { while (n_drafting > 0) { // each step decodes under a different head, i.e. a different decoder layer, and - // KV is per layer. process() filled this layer's KV only for positions < n_past + // KV is per layer. process() filled this layer's KV only for positions < pos0 // (prompt + accepted prefix) — nothing in the draft region yet. so reset the - // draft region (the seq_rm lower bound is n_past, leaving the prompt KV intact) + // draft region (the seq_rm lower bound is pos0, leaving the prompt KV intact) // and select head i so it rebuilds its own layer's KV there; decoding just the // latest token would leave its attention reading cells only another head wrote. if (chain_heads) { auto * mem_dft = llama_get_memory(ctx_dft); for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { if (drafting[seq_id]) { - llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].n_past, -1); + llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].pos0, -1); } } llama_set_nextn_layer_offset(ctx_dft, i); @@ -1710,17 +1710,17 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const int n_rows = (int) result.size() + 1; // id_last + tokens drafted so far for (int t = 0; t < n_rows; ++t) { const llama_token tok = (t == 0) ? dp.id_last : result[t - 1]; - common_batch_add(batch, tok, dp.n_past + t, { seq_id }, t == n_rows - 1); + common_batch_add(batch, tok, dp.pos0 + t, { seq_id }, t == n_rows - 1); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, chain_h[seq_id].data() + (size_t) t * n_embd, row_bytes); } } else if (is_mem_shared) { // note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens // ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37 - common_batch_add(batch, id, dp.n_past, { seq_id }, true); + common_batch_add(batch, id, dp.pos0, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } else { - common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); + common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } diff --git a/common/speculative.h b/common/speculative.h index 22505891f7ef..c968750e2d80 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -61,7 +61,7 @@ struct common_speculative_draft_params { // can be used to constraint the max draft based on the remaining context size int32_t n_max = -1; - llama_pos n_past; + llama_pos pos0; llama_token id_last; // TODO: remove in the future by keeping track of the prompt from the _begin() call and the consecutive accept calls diff --git a/examples/speculative-simple/speculative-simple.cpp b/examples/speculative-simple/speculative-simple.cpp index 487ae03abfa7..863af5a2c71a 100644 --- a/examples/speculative-simple/speculative-simple.cpp +++ b/examples/speculative-simple/speculative-simple.cpp @@ -188,7 +188,7 @@ int main(int argc, char ** argv) { common_speculative_get_draft_params(spec, seq_id) = { /* .drafting = */ true, /* .n_max = */ n_draft_max, - /* .n_past = */ n_past, + /* .pos0 = */ n_past, /* .id_last = */ id_last, /* .prompt = */ &prompt_tgt, /* .result = */ &draft, // output diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index fe068d3e9104..b6835e43459e 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3028,7 +3028,7 @@ struct server_context_impl { common_speculative_get_draft_params(spec.get(), slot.id) = { /* .drafting = */ true, /* .n_max = */ n_draft_max, - /* .n_past = */ slot.prompt.n_tokens(), + /* .pos0 = */ slot.prompt.tokens.pos_next(), /* .id_last = */ slot.sampled, /* .prompt = */ &slot.spec_prompt, /* .result = */ &slot.spec_draft, From 5cdd3d1dad5cbb7107b3e9f6d23239ba88ac0123 Mon Sep 17 00:00:00 2001 From: Logan Chu Date: Fri, 11 Sep 2026 02:02:31 -0700 Subject: [PATCH 35/65] =?UTF-8?q?model=20:=20fix=20MTP=20context=20kv=20ca?= =?UTF-8?q?che=20allocation=20for=20deepseek2,=20glm4moe,=20=E2=80=A6=20(#?= =?UTF-8?q?28630)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * model : fix MTP context kv cache allocation for deepseek2, glm4moe, cohere2moe architectures (#28626) * model: add inverse architecture gating and comprehensive architecture testing for mtp layer filtering * model : slim NextN filter comment, drop test-llama-archs changes --- src/llama-model.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index d10b60afd9fd..f9e9a8bcb06e 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2644,9 +2644,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } - if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA || - arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_DEEPSEEK32) && - hparams.n_layer_nextn > 0) { + // don't filter when n_layer_nextn is repurposed for a router layer the trunk attends + // or when a model is entirely n_layer_nextn layers and has no trunk + if (hparams.n_layer_nextn > 0 && hparams.n_layer() > 0 && hparams.router_layer < 0) { if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } else { From aac810230f9ef0cf73a47c56e46e87d0988be348 Mon Sep 17 00:00:00 2001 From: Foad Abo Dahood <32059146+masterFoad@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:30:20 +0300 Subject: [PATCH 36/65] metal : fix idle threads in the remaining iq mul_mv kernels for ne00 < 1024 (#28692) * metal : fix idle threads in the remaining iq mul_mv kernels for ne00 < 1024 Generalize the row split from #28086 to the six other kernels that use the same lane-to-block mapping: iq1_s, iq1_m, iq2_xxs, iq2_xs, iq2_s and iq3_s. Each of them assigns one 32-element chunk per thread, so when a row has fewer than 32 chunks the rest of the simdgroup is idle. When nb32 < 32 and nb32 divides 32, 32/nb32 threads now share each chunk and each takes a slice of the rows, reusing the FC_mul_mv_split function constant and the dispatch wrapper introduced for iq3_xxs. The plain path is untouched: wide matrices keep one thread per chunk and N_R0_ = 4. Only the split path uses N_R0__SPLIT = 8. The K-quants have the same idle-thread issue but a different lane mapping, so they are left for a separate change. * metal : offset the src0 row pointer once in the iq mul_mv kernels q2, dh, sc, qh and signs are all derived from xr, so the row slice offset only has to be applied to xr. * metal : fold iq mul_mv row split into offset0 Compute row0 and row1 before initializing the source pointers and apply the row slice directly to offset0. This keeps x and its derived pointers on the existing path while applying the split row offset once. --- ggml/src/ggml-metal/ggml-metal-device.cpp | 72 +++++ ggml/src/ggml-metal/ggml-metal-impl.h | 6 + ggml/src/ggml-metal/kernels/mul_mv.metal | 308 ++++++++++++++++------ 3 files changed, 304 insertions(+), 82 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 1137c5f6da79..bf3d07e781df 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -932,12 +932,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { @@ -957,21 +969,45 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { @@ -1177,12 +1213,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { @@ -1202,21 +1250,45 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 1fe947633ed3..28a9ba101c74 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -62,18 +62,23 @@ #define N_R0_IQ1_S 4 #define N_SG_IQ1_S 2 +#define N_R0_IQ1_S_SPLIT 8 #define N_R0_IQ1_M 4 #define N_SG_IQ1_M 2 +#define N_R0_IQ1_M_SPLIT 8 #define N_R0_IQ2_XXS 4 #define N_SG_IQ2_XXS 2 +#define N_R0_IQ2_XXS_SPLIT 8 #define N_R0_IQ2_XS 4 #define N_SG_IQ2_XS 2 +#define N_R0_IQ2_XS_SPLIT 8 #define N_R0_IQ2_S 4 #define N_SG_IQ2_S 2 +#define N_R0_IQ2_S_SPLIT 8 #define N_R0_IQ3_XXS 4 #define N_SG_IQ3_XXS 2 @@ -81,6 +86,7 @@ #define N_R0_IQ3_S 4 #define N_SG_IQ3_S 2 +#define N_R0_IQ3_S_SPLIT 8 #define N_R0_IQ4_NL 2 #define N_SG_IQ4_NL 2 diff --git a/ggml/src/ggml-metal/kernels/mul_mv.metal b/ggml/src/ggml-metal/kernels/mul_mv.metal index fbe8398ea0f2..8e2df276549f 100644 --- a/ggml/src/ggml-metal/kernels/mul_mv.metal +++ b/ggml/src/ggml-metal/kernels/mul_mv.metal @@ -1889,8 +1889,19 @@ void kernel_mul_mv_iq2_xxs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -1898,8 +1909,6 @@ void kernel_mul_mv_iq2_xxs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); { @@ -1912,11 +1921,9 @@ void kernel_mul_mv_iq2_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -1928,7 +1935,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl( device const uint16_t * q2 = xr->qs + 4 * ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; device const uint8_t * aux8 = (device const uint8_t *)q2; const uint32_t aux32 = q2[2] | (q2[3] << 16); @@ -1948,7 +1955,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl( q2 += args.nb01/2; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -1961,6 +1968,23 @@ void kernel_mul_mv_iq2_xxs_f32_impl( } } +template +void kernel_mul_mv_iq2_xxs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_xxs_f32")]] kernel void kernel_mul_mv_iq2_xxs_f32( constant ggml_metal_kargs_mul_mv & args, @@ -1971,7 +1995,7 @@ kernel void kernel_mul_mv_iq2_xxs_f32( uint3 tgpig[[threadgroup_position_in_grid]], ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_xxs_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template @@ -1997,8 +2021,19 @@ void kernel_mul_mv_iq2_xs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2006,8 +2041,6 @@ void kernel_mul_mv_iq2_xs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512); { @@ -2020,11 +2053,9 @@ void kernel_mul_mv_iq2_xs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2037,7 +2068,7 @@ void kernel_mul_mv_iq2_xs_f32_impl( device const uint8_t * sc = xr->scales + ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const uint8_t ls1 = sc[0] & 0xf; const uint8_t ls2 = sc[0] >> 4; @@ -2066,7 +2097,7 @@ void kernel_mul_mv_iq2_xs_f32_impl( sc += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2079,6 +2110,23 @@ void kernel_mul_mv_iq2_xs_f32_impl( } } +template +void kernel_mul_mv_iq2_xs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_xs_f32")]] kernel void kernel_mul_mv_iq2_xs_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2090,7 +2138,7 @@ kernel void kernel_mul_mv_iq2_xs_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_xs_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } // FC_mul_mv_split: for nb32 < 32 (nb32 divides 32), 32/nb32 threads share each chunk and each takes a slice of the rows @@ -2117,8 +2165,19 @@ void kernel_mul_mv_iq3_xxs_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq3_xxs * x = (device const block_iq3_xxs *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2126,8 +2185,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint32_t * svalues = (threadgroup uint32_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); { @@ -2140,15 +2197,6 @@ void kernel_mul_mv_iq3_xxs_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const short ntx = FC_mul_mv_split ? nb32 : 32; - const short nrep = 32 / ntx; - - const short ix = tiisg % ntx; - const short irep = tiisg / ntx; - - const short row0 = (nr0 * irep ) / nrep; - const short row1 = (nr0 * (irep + 1)) / nrep; - device const float * y4 = y + 32 * ix; for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { @@ -2160,9 +2208,9 @@ void kernel_mul_mv_iq3_xxs_f32_impl( const int ib = ib32 % (QK_K / 32); device const block_iq3_xxs * xr = x + ibl; - device const uint8_t * q3 = xr->qs + 8 * ib + (uint64_t) row0*args.nb01; - device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib + (uint64_t) row0*args.nb01/2; - device const half * dh = &xr->d + (uint64_t) row0*args.nb01/2; + device const uint8_t * q3 = xr->qs + 8 * ib; + device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib; + device const half * dh = &xr->d; for (short row = row0; row < row1; row++) { const float db = dh[0]; @@ -2253,8 +2301,19 @@ void kernel_mul_mv_iq3_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq3_s * x = (device const block_iq3_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2262,8 +2321,6 @@ void kernel_mul_mv_iq3_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - threadgroup uint32_t * svalues = (threadgroup uint32_t *) shmem; { int nval = 8; @@ -2272,11 +2329,9 @@ void kernel_mul_mv_iq3_s_f32_impl( threadgroup_barrier(mem_flags::mem_threadgroup); } - const int ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2291,7 +2346,7 @@ void kernel_mul_mv_iq3_s_f32_impl( device const uint8_t * signs = xr->signs + 4 * ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf)); @@ -2315,7 +2370,7 @@ void kernel_mul_mv_iq3_s_f32_impl( signs += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2328,6 +2383,23 @@ void kernel_mul_mv_iq3_s_f32_impl( } } +template +void kernel_mul_mv_iq3_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq3_s_f32")]] kernel void kernel_mul_mv_iq3_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2339,7 +2411,7 @@ kernel void kernel_mul_mv_iq3_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq3_s_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template @@ -2365,8 +2437,19 @@ void kernel_mul_mv_iq2_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq2_s * x = (device const block_iq2_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2374,8 +2457,6 @@ void kernel_mul_mv_iq2_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - //threadgroup uint64_t * svalues = (threadgroup uint64_t *) shmem; //{ // int nval = 32; @@ -2384,11 +2465,9 @@ void kernel_mul_mv_iq2_s_f32_impl( // threadgroup_barrier(mem_flags::mem_threadgroup); //} - const short ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } @@ -2403,7 +2482,7 @@ void kernel_mul_mv_iq2_s_f32_impl( device const uint8_t * signs = qs + QK_K/8; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { const float db = dh[0]; const float d1 = db * (0.5f + (sc[0] & 0xf)); const float d2 = db * (0.5f + (sc[0] >> 4)); @@ -2428,7 +2507,7 @@ void kernel_mul_mv_iq2_s_f32_impl( signs += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2441,6 +2520,23 @@ void kernel_mul_mv_iq2_s_f32_impl( } } +template +void kernel_mul_mv_iq2_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq2_s_f32")]] kernel void kernel_mul_mv_iq2_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2452,7 +2548,7 @@ kernel void kernel_mul_mv_iq2_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + kernel_mul_mv_iq2_s_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } template @@ -2478,8 +2574,19 @@ void kernel_mul_mv_iq1_s_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2487,13 +2594,9 @@ void kernel_mul_mv_iq1_s_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - device const float * y4 = y + 32 * ix; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { float sumy = 0; for (short i = 0; i < 32; ++i) { yl[i] = y4[i]; @@ -2508,7 +2611,7 @@ void kernel_mul_mv_iq1_s_f32_impl( device const uint16_t * qh = xr->qh + ib; device const half * dh = &xr->d; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700))); constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700))); @@ -2528,7 +2631,7 @@ void kernel_mul_mv_iq1_s_f32_impl( qh += args.nb01/2; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2541,6 +2644,23 @@ void kernel_mul_mv_iq1_s_f32_impl( } } +template +void kernel_mul_mv_iq1_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq1_s_f32")]] kernel void kernel_mul_mv_iq1_s_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2551,7 +2671,7 @@ kernel void kernel_mul_mv_iq1_s_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_s_f32_disp(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } template @@ -2577,8 +2697,19 @@ void kernel_mul_mv_iq1_m_f32_impl( const uint i12 = im%FC_mul_mv_ne12; const uint i13 = im/FC_mul_mv_ne12; - const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0); device const float * y = (device const float *) (src1 + offset1); @@ -2586,15 +2717,11 @@ void kernel_mul_mv_iq1_m_f32_impl( float yl[32]; float sumf[nr0]={0.f}; - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - device const float * y4 = y + 32 * ix; iq1m_scale_t scale; - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { float4 sumy = {0.f}; for (short i = 0; i < 8; ++i) { yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; @@ -2611,7 +2738,7 @@ void kernel_mul_mv_iq1_m_f32_impl( device const uint8_t * qh = xr->qh + 2 * ib; device const uint16_t * sc = (device const uint16_t *)xr->scales; - for (short row = 0; row < nr0; row++) { + for (short row = row0; row < row1; row++) { scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); @@ -2637,7 +2764,7 @@ void kernel_mul_mv_iq1_m_f32_impl( qh += args.nb01; } - y4 += 32 * 32; + y4 += 32 * ntx; } device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; @@ -2650,6 +2777,23 @@ void kernel_mul_mv_iq1_m_f32_impl( } } +template +void kernel_mul_mv_iq1_m_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + [[host_name("kernel_mul_mv_iq1_m_f32")]] kernel void kernel_mul_mv_iq1_m_f32( constant ggml_metal_kargs_mul_mv & args, @@ -2660,7 +2804,7 @@ kernel void kernel_mul_mv_iq1_m_f32( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + kernel_mul_mv_iq1_m_f32_disp(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); } template @@ -3239,13 +3383,13 @@ template [[host_name("kernel_mul_mv_id_q3_K_f32")]] kernel kernel_mul_mv_id_t template [[host_name("kernel_mul_mv_id_q4_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q5_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_q6_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; template [[host_name("kernel_mul_mv_id_tq2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; From a2878d30df0130dde503a7d9ba30d3d21bd71b9f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 11 Sep 2026 12:41:54 +0300 Subject: [PATCH 37/65] metal : single-source fusion table + fusion debug rework (#28164) * metal : rework fusion patterns into a single table All fusable op patterns for the Metal backend are now declared once in a fusion table (ggml-metal-fuse.cpp) and consumed by both the graph optimizer (ggml_metal_fuse_max, packing) and the op encoders (ggml_metal_fuse_next, compute). The two phases share the same pattern table plus ggml_can_fuse_subgraph_ext for the structural checks, and differ only in the mode used for the pattern check (STRUCTURAL at optimize time, since tensors are not allocated yet, and FULL at compute time, including Metal buffer placement). This also protects the snake activation (MUL + SIN + SQR + MUL + ADD) from being reordered during graph optimization, which was previously unprotected. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : fix absolute output indices in fusion patterns ggml_can_fuse_subgraph_ext expects the outputs array to contain absolute graph node indices (it indexes cgraph->nodes[outputs[i]]), but the fusion table query was passing a relative index (n_ops - 1). As a result the last node of every pattern was not recognized as an output and was subjected to the elidable use-count check, which failed for essentially all fusions. This silently disabled the norm/MUL fusion and caused a ~5% token-generation regression. Pass the absolute graph index of the last node instead. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : fuse gated_delta_net with cache cpy Add GGML_METAL_FUSE_GDN_CACHE to the fusion table: when the gated_delta_net kernel is followed by a cpy that scatters its recurrent state snapshots into the KV cache, the kernel writes the snapshots straight into the cache buffer and the trailing cpy is elided. The gdn output has other consumers (the attn scores view), so unlike the elision-chain patterns this is not a simple chain: a 'raw' flag on the fusion pattern skips the generic chain/shape and ggml_can_fuse_subgraph_ext checks, making the pattern-specific check callback the sole validator. Packing (ggml_metal_fuse_max) now matches on the same view-transparent node sequence that the compute phase uses, so the gdn + cache cpy group is packed along with any intermediate views and stays adjacent through the reorder. The fused cpy is a view consumer of the gdn (it writes the cache directly), so its mem-range is skipped in the encoder; the skip is restricted to CPY nodes consuming the previous fused node through a view so other fusions are unaffected. Add test_gated_delta_net_cache_fusion and register 5 cases. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : drop is_view_consumer mem-range skip The is_view_consumer skip was carried over from the upstream gated_delta_net cache-fusion draft, but it is not needed: keeping the elided cpy's mem-range in the concurrency tracker only ever adds a (conservative) memory barrier at the fusion point. It can never remove a barrier, so it cannot introduce a race. The worst case is one spurious barrier per gdn+cache-cpy fusion, which is within run-to-run noise on Qwen3.5-0.8B Q8_0. Dropping the check keeps the mem-range loop uniform for all fused groups. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : rename gated_delta_net fused state output args Rename the fused cache-write kernel argument to match the rest of the kargs: state_out_stride -> nb_out (and widen it to uint64_t), and the local buffer id bid_state_out -> bid_out. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : rename raw fusion flag to unsafe raw did not convey that the flag opts a fusion pattern out of the generic elision-chain safety net (ggml_can_fuse_subgraph_ext + chain/shape checks). rename it to 'unsafe' to make explicit that the pattern's check callback is the sole validator and must re-establish the safety guarantees itself. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : tidy fusion pattern checks and table - const-correct ggml_metal_fuse_outputs buffer - annotate unused check-callback parameters - drop a redundant size_t cast - align the ops/table initializers and add blank-line separation Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * metal : add generic fusion stats via ad-hoc proc-address API Add a device-owned fusion context that lets a test tool count how many times each fusion pattern fires and toggle fusion. It is exposed through the ad-hoc ggml_backend_reg_get_proc_address mechanism with generic names so the testing tool is backend-agnostic: - ggml_backend_fusion_stats_init: start collecting fusion stats; when a context is created afterwards it registers the labels/counters and encodes single-threaded (n_cb == 0) so the counters are race-free - ggml_backend_fusion_stats_reset / _get_stats / _set_enabled The context lives on the metal device (not on the last backend context), so counters accumulate across contexts and reads are always consistent. The enable/disable toggle is initialized from GGML_METAL_FUSION_DISABLE and can be overridden by the test through set_enabled. Labels are synthesized from the fuse table via ggml_metal_fuse_label (e.g. "GATED_DELTA_NET+CPY"). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : add fusion count regression test with per-backend baseline test-fusion runs every dummy model generated by test-llama-archs on a single backend (single-threaded encoding, n_cb == 0) with fusion enabled and disabled, and for each mode (prefill / decode) reports the per-fusion counters and the NMSE between the fused and unfused logits, plus the NMSE against a CPU reference. A fusion pattern that silently stops matching (or fires when it should not) is caught as a regression by comparing the counters against a committed per-backend TSV baseline: - --record writes the golden baseline, --check (default) validates it - the unfused run doubles as a control: its counters must be all-zero - NMSE is skipped when it is NaN or the arch is already broken on the device (e.g. plamo2 on Metal), so the count check is the hard gate - baseline counts depend only on graph structure, not weights (verified stable across weight seeds) - the fusion stats API is resolved through the ad-hoc get_proc_address mechanism with generic names; a backend that does not export it makes the test fail with an error The committed MTL0.tsv baseline covers 110 dummy archs (298 rows). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : rename fusion api helpers to match stats_init signature Align the test with the ad-hoc fusion stats API: fusion_stats_init no longer takes an enable bool (stats are turned on by calling it), so the proc-address wrappers and typedefs are renamed to the api_* convention. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : rename backend to device in fusion test CLI The fusion test operates on a compute device (e.g. MTL0), not a backend, so rename the --backend argument to --device and the backend_name variable to device_name. Keep "backend" where it refers to the ggml backend interface (the ad-hoc proc-address mechanism). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : add --model and --help to fusion test --model FILE runs the fusion regression test over a single model file instead of enumerating a --models DIR. --models and --model are mutually exclusive. Also add a --help/-h option that prints the usage. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : use backend base name for fusion baseline output The fusion test is invoked with a specific device name (e.g. MTL0), but its output - the recorded baseline and the header it writes - should be named after the backend base name (e.g. MTL, via ggml_backend_reg_name), since the counters depend on the backend, not on the specific device index. Rename the committed baseline to MTL.tsv. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : run fusion test from ci instead of ctest The fusion test needs Metal and generates a lot of dummy models, so it does not belong in the generic ctest suite. Move it to ci/run.sh as gg_run_test_fusion, gated on GG_BUILD_METAL like gg_run_test_llama_archs_tensor_split: it generates the dummy models with test-llama-archs -o and then validates the fusion counts against the committed baseline. test-fusion.cpp is still built (llama_build) but no longer registered as a ctest. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : align fusion baseline TSV columns Pad the TSV fields to fixed widths so the columns line up regardless of the variable arch and fusion-label lengths, and trim each field on parse so the padded file is still accepted. Regenerate the committed MTL.tsv baseline in the padded format (data unchanged, verified identical modulo padding). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : widen label column and align fusion TSV header Give the label column more room (28 chars) and fix the column header widths so they match the data rows (moe/mode/label), keeping the header aligned with the values. Regenerate the MTL.tsv baseline in the new format (data unchanged). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : switch fusion baseline from TSV to CSV Use comma-separated values like the rest of the project, keeping the padded, aligned columns. Split on ',' and trim on parse. Rename the committed baseline to MTL.csv (data unchanged, verified identical modulo padding/separator). Update the ci/run.sh check path accordingly. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * cont : rebase + update MTL stats * tests : avoid graph reallocations for some archs * metal : tidy fusion debugging context and op init - simplify the shared fusion debugging context comments - shorten the ggml_metal_fusion struct comment - align the ggml_metal_fuse struct fields and comments - move the fusion parameter of ggml_metal_op_init right after dev Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : dedup fusion baseline into any mode prefill and decode always produce the same per-graph fusion count, so store a single row per label with mode = "any" and the per-graph count instead of two rows. this halves the baseline size and keeps the check stable. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * ci : move fusion model generation to a separate step the dummy models generated by test-llama-archs are reused by other tests, so generate them once in their own step instead of inside test_fusion. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : bump nmse thold * models : fix plamo2 graph * tests : remove "skip" logic from test-fusion * tests : set qwen3tts dummy vocab to codec head size the dummy qwen3tts model used a vocab of 4096 while the codec head is 3072, so the graph padded the output with -inf which made the NMSE in test-fusion produce NaN. use the exact codec head size instead so the padding is not generated at all. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * tests : regen fusion baseline reflect the plamo2 graph fix, which changed its fusion pattern split (RMS_NORM+MUL 11->10, RMS_NORM+MUL+ADD 3->4; same total). Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * ci : skip dummy model generation on OpenVINO test-llama-archs does not build on the OpenVINO platform, so do not try to generate the dummy models there. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-0731 * cont : minor * tests : enable test-llama-archs on windows * cont : disable on windows + workaround * metal : naming nits * test-fusion : add instructions to update baseline * context : fix Kimi-K3 graph reserve * fusion : update MTL * cont : fix naming * metal : rework fusion info storage Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * metal : align fusion info API Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * metal : use opaque fusion handle in ad-hoc API Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * ci : move fusion test to dedicated workflow Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * cont : run only on ggml changes * cont : simplify * fusion : remove multi-output stuff for now * ci : fix typo --- .github/workflows/fusion.yml | 67 +++ ci/run.sh | 30 + ggml/src/ggml-metal/CMakeLists.txt | 1 + ggml/src/ggml-metal/ggml-metal-common.cpp | 47 +- ggml/src/ggml-metal/ggml-metal-context.h | 1 + ggml/src/ggml-metal/ggml-metal-context.m | 55 +- ggml/src/ggml-metal/ggml-metal-device.h | 5 + ggml/src/ggml-metal/ggml-metal-device.m | 17 + ggml/src/ggml-metal/ggml-metal-fusion.cpp | 502 ++++++++++++++++ ggml/src/ggml-metal/ggml-metal-fusion.h | 104 ++++ ggml/src/ggml-metal/ggml-metal-impl.h | 1 + ggml/src/ggml-metal/ggml-metal-ops.cpp | 262 ++++---- ggml/src/ggml-metal/ggml-metal-ops.h | 7 +- ggml/src/ggml-metal/ggml-metal.cpp | 42 ++ .../ggml-metal/kernels/gated_delta_net.metal | 20 +- src/llama-context.cpp | 2 + src/models/minimax-01.cpp | 1 + src/models/plamo2.cpp | 10 +- src/models/qwen3vl.cpp | 1 + tests/.gitignore | 1 + tests/CMakeLists.txt | 4 +- tests/fusion/MTL.csv | 154 +++++ tests/test-backend-ops.cpp | 123 ++++ tests/test-fusion.cpp | 565 ++++++++++++++++++ tests/test-llama-archs.cpp | 3 +- tests/test-save-load-state.cpp | 4 +- 26 files changed, 1794 insertions(+), 235 deletions(-) create mode 100644 .github/workflows/fusion.yml create mode 100644 ggml/src/ggml-metal/ggml-metal-fusion.cpp create mode 100644 ggml/src/ggml-metal/ggml-metal-fusion.h create mode 100644 tests/fusion/MTL.csv create mode 100644 tests/test-fusion.cpp diff --git a/.github/workflows/fusion.yml b/.github/workflows/fusion.yml new file mode 100644 index 000000000000..ad7d5ab60e01 --- /dev/null +++ b/.github/workflows/fusion.yml @@ -0,0 +1,67 @@ +name: Fusion + +on: + workflow_dispatch: # allows manual triggering + push: + branches: + - master + paths: [ + '.github/workflows/fusion.yml', + 'ggml/**', + 'tests/fusion/**', + 'tests/test-fusion.cpp' + ] + + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/fusion.yml', + 'ggml/**', + 'tests/fusion/**', + 'tests/test-fusion.cpp' + ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +env: + GGML_NLOOP: 3 + GGML_N_THREADS: 1 + LLAMA_ARG_LOG_COLORS: 1 + LLAMA_ARG_LOG_PREFIX: 1 + LLAMA_ARG_LOG_TIMESTAMPS: 1 + +jobs: + # TODO: add jobs for other backends as they adopt the fusion debug API + metal: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Build + id: cmake_build + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_OPENSSL=OFF \ + -DGGML_SCHED_NO_REALLOC=ON \ + -DGGML_BLAS=OFF \ + -DGGML_METAL=ON + time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu) + time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu) + + - name: Generate models + id: generate_models + run: | + rm -rf build-ci-models && mkdir -p build-ci-models + ./build/bin/test-llama-archs -o build-ci-models + + - name: Test fusion + id: test_fusion + run: | + ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv diff --git a/ci/run.sh b/ci/run.sh index 5463597274ec..294cbe57bb42 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -334,6 +334,35 @@ function gg_sum_test_llama_archs_tensor_split { gg_printf '```\n' } +# test_llama_archs_models + +function gg_run_test_llama_archs_models { + cd ${SRC} + + set -e + + # TODO: fix and re-enable `test-llama-archs` on OpenVINO + # TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists + if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then + rm -rf build-ci-models && mkdir -p build-ci-models + + # generate the dummy models used by the model-dependent tests + ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1 + fi + + set +e +} + +function gg_sum_test_llama_archs_models { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'Generates the dummy models used by the model-dependent tests\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '```\n' + gg_printf '%s\n' "$(cat $OUT/${ci}.log)" + gg_printf '```\n' +} + # test_scripts function gg_run_test_scripts { @@ -790,6 +819,7 @@ ret=0 test $ret -eq 0 && gg_run ctest_debug test $ret -eq 0 && gg_run ctest_release +test $ret -eq 0 && gg_run test_llama_archs_models test $ret -eq 0 && gg_run test_llama_archs_tensor_split if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txt index a661e710a2f2..e7afdb69572f 100644 --- a/ggml/src/ggml-metal/CMakeLists.txt +++ b/ggml/src/ggml-metal/CMakeLists.txt @@ -10,6 +10,7 @@ ggml_add_backend_library(ggml-metal ggml-metal-device.cpp ggml-metal-common.cpp ggml-metal-context.m + ggml-metal-fusion.cpp ggml-metal-ops.cpp ggml-metal-tuning.cpp ) diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cpp index 6f1638a1147e..05755eb3b261 100644 --- a/ggml/src/ggml-metal/ggml-metal-common.cpp +++ b/ggml/src/ggml-metal/ggml-metal-common.cpp @@ -1,4 +1,5 @@ #include "ggml-metal-common.h" +#include "ggml-metal-fusion.h" #include "ggml.h" #include "ggml-impl.h" @@ -390,59 +391,31 @@ static std::vector ggml_metal_graph_optimize_reorder(const std::vectorn_nodes; - enum ggml_op ops[MAX_FUSE]; - std::vector nodes; nodes.reserve(gf->n_nodes); // fuse nodes: // we don't want to make reorders that break fusing, so we first pack all fusable tensors // and perform the reorder over the fused nodes. after the reorder is done, we unfuse + // + // the fusable sequences are declared in the fusion table (ggml-metal-fuse.cpp), so the + // packing here is driven by the same patterns that the op encoders will later use for (int i = 0; i < n; i++) { node_info node = { /*.node =*/ gf->nodes[i], /*.fused =*/ {}, }; - // fuse only ops that start with these operations - // can be expanded when needed - if (node.op() == GGML_OP_ADD || - node.op() == GGML_OP_NORM || - node.op() == GGML_OP_RMS_NORM) { - ops[0] = node.op(); - - int f = i + 1; - while (f < n && f < i + MAX_FUSE) { - // conservatively allow fusing only these ops - // can be expanded when needed - if (gf->nodes[f]->op != GGML_OP_ADD && - gf->nodes[f]->op != GGML_OP_MUL && - gf->nodes[f]->op != GGML_OP_NORM && - gf->nodes[f]->op != GGML_OP_RMS_NORM) { - break; - } - ops[f - i] = gf->nodes[f]->op; - f++; - } - - f -= i; - for (; f > 1; f--) { - if (ggml_can_fuse(gf, i, ops, f)) { - break; - } - } + const int f = ggml_metal_fusion_max(gf, i); - // add the fused tensors into the node info so we can unfuse them later - for (int k = 1; k < f; k++) { - ++i; + // add the fused tensors into the node info so we can unfuse them later + for (int k = 1; k < f; k++) { + ++i; - // the .dst() becomes the last fused tensor - node.add_fused(gf->nodes[i]); - } + // the .dst() becomes the last fused tensor + node.add_fused(gf->nodes[i]); } nodes.push_back(std::move(node)); diff --git a/ggml/src/ggml-metal/ggml-metal-context.h b/ggml/src/ggml-metal/ggml-metal-context.h index abf4b06ed2ab..b538b1ad20a1 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.h +++ b/ggml/src/ggml-metal/ggml-metal-context.h @@ -33,6 +33,7 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx); void ggml_metal_set_n_cb (ggml_metal_t ctx, int n_cb); void ggml_metal_set_abort_callback (ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data); + bool ggml_metal_supports_family (ggml_metal_t ctx, int family); void ggml_metal_capture_next_compute(ggml_metal_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index 6cdc4006bc51..bf4fe2dcd519 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -6,6 +6,7 @@ #import "ggml-metal-impl.h" #import "ggml-metal-common.h" #import "ggml-metal-ops.h" +#import "ggml-metal-fusion.h" #import @@ -36,15 +37,12 @@ // additional, inference-time compiled pipelines ggml_metal_pipelines_t pipelines_ext; - bool use_fusion; bool use_concurrency; bool use_graph_optimize; int debug_graph; - int debug_fusion; - // how many times a given op was fused - uint64_t fuse_cnt[GGML_OP_COUNT]; + struct ggml_metal_fusion_info * finfo; // capture state int capture_compute; @@ -139,7 +137,6 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); - res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil; res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil; { @@ -147,20 +144,19 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { res->debug_graph = val ? atoi(val) : 0; } - { - const char * val = getenv("GGML_METAL_FUSION_DEBUG"); - res->debug_fusion = val ? atoi(val) : 0; - } - res->use_graph_optimize = true; if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) { res->use_graph_optimize = false; } - memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt)); + res->finfo = ggml_metal_device_get_fusion_info(dev); + if (ggml_metal_fusion_info_stats(res->finfo)) { + ggml_metal_fusion_info_labels_init(res->finfo); + res->n_cb = 0; + } - GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false"); + GGML_LOG_INFO("%s: use fusion = %s\n", __func__, ggml_metal_fusion_info_enabled(res->finfo) ? "true" : "false"); GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false"); GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false"); @@ -222,15 +218,18 @@ void ggml_metal_free(ggml_metal_t ctx) { ctx->pipelines_ext = nil; } - if (ctx->debug_fusion > 0) { + if (ggml_metal_fusion_info_debug(ctx->finfo) > 0) { GGML_LOG_DEBUG("%s: fusion stats:\n", __func__); - for (int i = 0; i < GGML_OP_COUNT; i++) { - if (ctx->fuse_cnt[i] == 0) { + + const int n_fusions = ggml_metal_fusion_info_n_fusions(ctx->finfo); + for (int i = 0; i < n_fusions; i++) { + const uint64_t count = ggml_metal_fusion_info_count(ctx->finfo, i); + if (count == 0) { continue; } // note: cannot use ggml_log here - GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_op_name((enum ggml_op) i), ctx->fuse_cnt[i]); + GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_metal_fusion_info_label(ctx->finfo, i), count); } } @@ -481,10 +480,17 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * @autoreleasepool { ctx->gf = gf; - ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); - ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; + if (ctx->n_cb == 0) { + // single-threaded encoding: the whole graph is encoded by one command buffer + ctx->n_nodes_0 = gf->n_nodes; + ctx->n_nodes_1 = 0; + ctx->n_nodes_per_cb = 0; + } else { + ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); + ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; - ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + } if (ctx->capture_compute >= 0) { ctx->capture_compute--; @@ -682,6 +688,12 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { } void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { + // when fusion stats are collected the graph must be encoded by a single thread so the + // counters are race-free; override whatever the caller requested + if (ggml_metal_fusion_info_stats(ctx->finfo)) { + n_cb = 0; + } + if (ctx->n_cb != n_cb) { ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS); @@ -717,13 +729,12 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { ctx->dev, cmd_buf, ctx->gf, + ctx->finfo, idx_start, idx_end, - ctx->use_fusion, ctx->use_concurrency, ctx->capture_compute, - ctx->debug_graph, - ctx->debug_fusion); + ctx->debug_graph); for (int idx = 0; idx < ggml_metal_op_n_nodes(ctx_op); ++idx) { const int res = ggml_metal_op_encode(ctx_op, idx); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 31fc07d44d47..ced33aadfbd4 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -325,6 +325,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev); +struct ggml_metal_fusion_info; + +// the device-owned fusion debugging context (NULL unless fusion debugging is enabled) +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev); + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index afd6f521011e..5654c500406c 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1,4 +1,5 @@ #import "ggml-metal-device.h" +#import "ggml-metal-fusion.h" #import "ggml-impl.h" #import "ggml-backend-impl.h" @@ -896,6 +897,9 @@ void ggml_metal_encoder_end_encoding(ggml_metal_encoder_t encoder) { struct ggml_metal_device_props props; + // shared fusion debugging context + struct ggml_metal_fusion_info * finfo; + // virtual address for GPU memory allocations atomic_uintptr_t addr_virt; }; @@ -1274,6 +1278,13 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { dev->props.max_working_set_size = dev->mtl_device.maxBufferLength; } + { + const char * val = getenv("GGML_METAL_FUSION_DEBUG"); + dev->finfo = ggml_metal_fusion_info_init( + getenv("GGML_METAL_FUSION_DISABLE") == nil, + val ? atoi(val) : 0); + } + snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device); const char * gpu_name = [[dev->mtl_device name] UTF8String]; if (n_devices > 1) { @@ -1348,6 +1359,8 @@ void ggml_metal_device_free(ggml_metal_device_t dev) { assert(dev != NULL); @autoreleasepool { + ggml_metal_fusion_info_free(dev->finfo); + ggml_metal_rsets_free(dev->rsets); ggml_metal_library_free(dev->library); @@ -1935,6 +1948,10 @@ static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) { dev->props.has_tensor = false; } +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev) { + return dev->finfo; +} + // // device buffers // diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp new file mode 100644 index 000000000000..ac3ac0414825 --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp @@ -0,0 +1,502 @@ +#include "ggml-metal-fusion.h" + +#include "ggml-backend-impl.h" +#include "ggml-metal-device.h" + +#include +#include +#include + +// ---- helpers ------------------------------------------------------------- + +// true if two tensors live in the same Metal buffer +static bool ggml_metal_fusion_same_buffer(const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b) { + return false; + } + + ggml_backend_buffer_t ba = a->view_src ? a->view_src->buffer : a->buffer; + ggml_backend_buffer_t bb = b->view_src ? b->view_src->buffer : b->buffer; + + ggml_metal_buffer_t ca = (ggml_metal_buffer_t) ba->context; + ggml_metal_buffer_t cb = (ggml_metal_buffer_t) bb->context; + + return ggml_metal_buffer_get_id(ca, a).metal == ggml_metal_buffer_get_id(cb, b).metal; +} + +// ---- pattern checks ------------------------------------------------------ + +// NORM/RMS_NORM + MUL + ADD: the weight/bias of each fused step must match the norm input +// width, be contiguous rows, and the fused outputs must stay F32 +static bool ggml_metal_fusion_check_norm( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(mode); + + GGML_ASSERT(fusion->n_ops >= 2); + + for (int j = 1; j < fusion->n_ops; j++) { + // the fused MUL/ADD must read the previous node as src0 + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + // the weight/bias must have the same row width as the norm input + if (nodes[j]->src[1]->ne[0] != nodes[0]->ne[0]) { + return false; + } + + if (!ggml_is_contiguous_rows(nodes[j]->src[1])) { + return false; + } + + if (nodes[j]->type != GGML_TYPE_F32) { + return false; + } + } + + return true; +} + +// ADD x N: each ADD reads the previous ADD as src0, and all addends must share layout +// (and, in FULL mode, live in the same Metal buffer) +static bool ggml_metal_fusion_check_add_chain( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_ASSERT(fusion->n_ops >= 2); + + for (int j = 1; j < fusion->n_ops; j++) { + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + if (!ggml_are_same_layout(nodes[j]->src[1], nodes[j - 1]->src[1])) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!ggml_metal_fusion_same_buffer(nodes[j]->src[1], nodes[0]->src[1])) { + return false; + } + } + } + + return true; +} + +// GATED_DELTA_NET + CPY: the trailing cpy scatters the gdn state snapshots into the recurrent +// cache, so the gdn kernel writes them straight to the cache and the cpy is elided. +// mirrors ggml_metal_op_can_fuse_gdn_cache (PR #25788). the gdn output has other consumers (the +// attn scores view), so unlike the other patterns this is not an elision chain: the structural +// checks live entirely in this callback (unsafe = true). +static bool ggml_metal_fusion_check_gdn_cache( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + + const ggml_tensor * gdn = nodes[0]; + const ggml_tensor * cpy = nodes[1]; + + // the kernel skips the snapshot tail, so the gdn output must not be a graph output + if (gdn->type != GGML_TYPE_F32 || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + if (cpy->op != GGML_OP_CPY || (cpy->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + const int64_t S_v = gdn->src[2]->ne[0]; + const int64_t H = gdn->src[2]->ne[1]; + const int64_t n_tokens = gdn->src[2]->ne[2]; + const int64_t n_seqs = gdn->src[2]->ne[3]; + const int64_t K = ggml_get_op_params_i32(gdn, 0); + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + const int64_t D = S_v * S_v * H; + const int64_t n_written = std::min(n_tokens, K); + + const ggml_tensor * src = cpy->src[0]; // gdn snapshot tail view + const ggml_tensor * dst = cpy->src[1]; // cache view + + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != gdn || + src->view_offs != tail_off || !ggml_is_contiguous(src)) { + return false; + } + + const int64_t expected_ne[GGML_MAX_DIMS] = { D, n_seqs, n_written, 1 }; + if (dst->type != GGML_TYPE_F32 || + !std::equal(expected_ne, expected_ne + GGML_MAX_DIMS, dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || + dst->nb[1] != ggml_row_size(GGML_TYPE_F32, D)) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + // the cache must be allocated so the kernel can write straight to its buffer + if (dst->data == nullptr) { + return false; + } + } + + return true; +} + +// MUL + SIN + SQR + MUL + ADD (snake activation) +static bool ggml_metal_fusion_check_snake( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(mode); + + const ggml_tensor * mul0 = nodes[0]; + const ggml_tensor * sin_node = nodes[1]; + const ggml_tensor * sqr = nodes[2]; + const ggml_tensor * mul1 = nodes[3]; + const ggml_tensor * add = nodes[4]; + + // x carries the full activation shape, a is the broadcast operand + const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + + // mul1 reads sqr and inv_b in either operand order + const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + + // closure check: the trailing add reads the same x as the leading mul + const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; + + // x is in the supported whitelist and every chain intermediate shares x's type. + // a and inv_b bind as device const float * in the kernel, so they stay F32. + const bool types_ok = + (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && + (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && + (mul0->type == x->type) && (sin_node->type == x->type) && + (sqr->type == x->type) && (mul1->type == x->type) && + (add->type == x->type); + + // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D + const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; + const bool dim_ok = + (x->ne[2] == 1) && (x->ne[3] == 1) && + (add->ne[2] == 1) && (add->ne[3] == 1) && + (a->ne[2] == 1) && (a->ne[3] == 1) && + (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); + + // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous + const bool contig_ok = + ggml_is_contiguous(x) && ggml_is_contiguous(add) && + ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); + + return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; +} + +// ---- patterns ------------------------------------------------------------ + +static const ggml_op ops_norm_mul[] = { GGML_OP_NORM, GGML_OP_MUL }; +static const ggml_op ops_norm_mul_add[] = { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD }; +static const ggml_op ops_rms_norm_mul[] = { GGML_OP_RMS_NORM, GGML_OP_MUL }; +static const ggml_op ops_rms_norm_mul_add[] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }; + +static const ggml_op ops_add_2[] = { GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_3[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_4[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_5[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_6[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_add_7[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const ggml_op ops_snake[] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }; + +static const ggml_op ops_gdn_cache[] = { GGML_OP_GATED_DELTA_NET, GGML_OP_CPY }; + +static const ggml_metal_fusion ggml_metal_fusions[] = { + { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, 2, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, 3, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, 2, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, 3, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, 2, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, 3, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, 4, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, 5, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, 6, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, 7, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_SNAKE, ops_snake, 5, false, ggml_metal_fusion_check_snake }, + { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, 2, true, ggml_metal_fusion_check_gdn_cache }, +}; + +const ggml_metal_fusion * ggml_metal_fusion_all(int * n) { + *n = (int) sizeof(ggml_metal_fusions) / sizeof(ggml_metal_fusions[0]); + + return ggml_metal_fusions; +} + +// ---- shared fusion info --------------------------------------------------- + +static std::string ggml_metal_fusion_label(const ggml_metal_fusion * fusion) { + GGML_ASSERT(fusion != nullptr); + + std::string label; + for (int j = 0; j < fusion->n_ops; j++) { + if (j > 0) { + label += '+'; + } + label += ggml_op_name(fusion->ops[j]); + } + return label; +} + +struct ggml_metal_fusion_info { + std::vector labels; + std::vector counts; + bool enabled; + bool stats; + bool labels_set; + int debug; +}; + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug) { + ggml_metal_fusion_info * finfo = new ggml_metal_fusion_info; + finfo->enabled = enabled; + finfo->stats = debug > 0; + finfo->labels_set = false; + finfo->debug = debug; + + if (finfo->stats) { + ggml_metal_fusion_info_labels_init(finfo); + } + + return finfo; +} + +void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo) { + delete finfo; +} + +bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo) { + return finfo->enabled; +} + +bool ggml_metal_fusion_info_stats(const struct ggml_metal_fusion_info * finfo) { + return finfo->stats; +} + +int ggml_metal_fusion_info_debug(const struct ggml_metal_fusion_info * finfo) { + return finfo->debug; +} + +int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo) { + return (int) finfo->labels.size(); +} + +const char * ggml_metal_fusion_info_label(const struct ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->labels.size()); + return finfo->labels[idx].c_str(); +} + +uint64_t ggml_metal_fusion_info_count(const struct ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->counts.size()); + return finfo->counts[idx]; +} + +void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion) { + if (!finfo->stats || fusion == nullptr) { + return; + } + + int n = 0; + const ggml_metal_fusion * all = ggml_metal_fusion_all(&n); + + int idx = -1; + for (int i = 0; i < n; i++) { + if (&all[i] == fusion) { + idx = i; + break; + } + } + + if (idx >= 0 && idx < (int) finfo->counts.size()) { + finfo->counts[idx]++; + } +} + +void ggml_metal_fusion_info_set_enabled(struct ggml_metal_fusion_info * finfo, bool enabled) { + finfo->enabled = enabled; +} + +void ggml_metal_fusion_info_labels_init(struct ggml_metal_fusion_info * finfo) { + if (finfo->labels_set) { + return; + } + + int n = 0; + const ggml_metal_fusion * all = ggml_metal_fusion_all(&n); + + finfo->labels.clear(); + finfo->counts.assign(n, 0); + finfo->labels.reserve(n); + + for (int i = 0; i < n; i++) { + finfo->labels.emplace_back(ggml_metal_fusion_label(&all[i])); + } + + finfo->labels_set = true; +} + +void ggml_metal_fusion_info_stats_init(struct ggml_metal_fusion_info * finfo) { + finfo->stats = true; + ggml_metal_fusion_info_labels_init(finfo); +} + +void ggml_metal_fusion_info_stats_reset(struct ggml_metal_fusion_info * finfo) { + std::fill(finfo->counts.begin(), finfo->counts.end(), 0); +} + +int ggml_metal_fusion_info_stats_get(const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n) { + const int n_fusions = (int) finfo->labels.size(); + + if (labels == nullptr) { + return n_fusions; + } + + const int n_fill = std::min(n, n_fusions); + for (int i = 0; i < n_fill; i++) { + labels[i] = finfo->labels[i].c_str(); + if (counts != nullptr) { + counts[i] = finfo->counts[i]; + } + } + + return n_fill; +} + +// ---- queries ------------------------------------------------------------- + +// find the longest pattern matching the node sequence starting at idx +// (idx is a position in node_idxs, which maps to graph node indices) +const ggml_metal_fusion * ggml_metal_fusion_next( + const ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out) { + int n = 0; + const ggml_metal_fusion * all = ggml_metal_fusion_all(&n); + + const ggml_metal_fusion * res = nullptr; + int best = 1; + + for (int i = 0; i < n; i++) { + const ggml_metal_fusion * fusion = &all[i]; + + // only look for a longer match than the current best + if (fusion->n_ops <= best) { + continue; + } + if (idx + fusion->n_ops > n_idxs) { + continue; + } + + const ggml_tensor * nodes[GGML_METAL_FUSION_MAX]; + + // the op sequence must match exactly + bool ok = true; + for (int j = 0; j < fusion->n_ops; j++) { + nodes[j] = gf->nodes[node_idxs[idx + j]]; + if (nodes[j]->op != fusion->ops[j]) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + if (!fusion->unsafe) { + // common element-wise chain constraints: each node reads the previous one, + // and all nodes have the same shape + for (int j = 1; j < fusion->n_ops && ok; j++) { + if (nodes[j]->src[0] != nodes[j - 1] && nodes[j]->src[1] != nodes[j - 1]) { + ok = false; + break; + } + if (!ggml_are_same_shape(nodes[j], nodes[j - 1])) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + // all current fusions are single-output elision chains, so the last node is the only output + // TODO: multi-output fusions: store pattern-relative offsets in the table and translate them here + int outputs_buf[1]; + outputs_buf[0] = node_idxs[idx + fusion->n_ops - 1]; + + // structural subgraph checks (op sequence, elidable uses, view containment) + if (!ggml_can_fuse_subgraph_ext(gf, node_idxs + idx, fusion->n_ops, fusion->ops, outputs_buf, 1)) { + continue; + } + } + + // pattern-specific checks (the sole validator for unsafe patterns) + if (fusion->check && !fusion->check(fusion, nodes, mode)) { + continue; + } + + best = fusion->n_ops; + res = fusion; + } + + *n_out = best; + + return res; +} + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused, chaining patterns back-to-back. matching runs on the same filtered (view +// transparent) node sequence that the compute phase uses, so the returned count is the raw index +// span from idx to the last matched node (intermediate views are packed along). +int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) { + // an empty/view node cannot start a pattern - pack it alone + if (ggml_op_is_empty(gf->nodes[idx]->op) || ggml_is_empty(gf->nodes[idx])) { + return 1; + } + + // collect the non-empty node indices starting at idx + int idxs[GGML_METAL_FUSION_MAX]; + int n_idxs = 0; + for (int i = idx; i < gf->n_nodes && n_idxs < GGML_METAL_FUSION_MAX; i++) { + if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) { + idxs[n_idxs++] = i; + } + } + if (n_idxs == 0) { + return 1; + } + + int total = 0; + int i_f = 0; + + while (i_f < n_idxs && total < GGML_METAL_FUSION_MAX) { + int len = 1; + const ggml_metal_fusion * fusion = ggml_metal_fusion_next(gf, idxs, n_idxs, i_f, GGML_METAL_FUSION_STRUCTURAL, &len); + if (!fusion || total + len > GGML_METAL_FUSION_MAX) { + break; + } + + total += len; + i_f += len; + } + + if (i_f == 0) { + return 1; + } + + // map the matched non-empty nodes back to the raw index span (views are included) + return std::min(GGML_METAL_FUSION_MAX, idxs[i_f - 1] - idx + 1); +} diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.h new file mode 100644 index 000000000000..e8515bdeca3d --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.h @@ -0,0 +1,104 @@ +// single source of truth for the fusions supported by the Metal backend +// +// every fusable subgraph is declared exactly once as a ggml_metal_fusion entry in +// the table in ggml-metal-fusion.cpp. both the graph optimizer (ggml_metal_fusion_max) +// and the op encoders (ggml_metal_fusion_next) consult this same table, so the two +// phases can never disagree about what can be fused. + +#pragma once + +#include "ggml-impl.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// the maximum number of nodes that can be fused in a single kernel +// (also the maximum length of a packed fusion group during graph optimization) +#define GGML_METAL_FUSION_MAX 16 + +typedef enum ggml_metal_fusion_mode { + // structural checks only; used by the graph optimizer, at which point the graph + // tensors are not allocated yet, so buffer placement cannot be verified + GGML_METAL_FUSION_STRUCTURAL = 0, + // full checks, including buffer placement; used by the op encoders + GGML_METAL_FUSION_FULL, +} ggml_metal_fusion_mode; + +// identifier of each fusion pattern so the op encoders know which kernel to use +typedef enum ggml_metal_fusion_id { + GGML_METAL_FUSION_NONE = 0, + GGML_METAL_FUSION_NORM_MUL, // NORM/RMS_NORM + MUL + GGML_METAL_FUSION_NORM_MUL_ADD, // NORM/RMS_NORM + MUL + ADD + GGML_METAL_FUSION_ADD_CHAIN, // ADD x N (N in [2, 7]) + GGML_METAL_FUSION_SNAKE, // MUL + SIN + SQR + MUL + ADD + GGML_METAL_FUSION_GDN_CACHE, // GATED_DELTA_NET + CPY (write snapshots into the recurrent cache) +} ggml_metal_fusion_id; + +struct ggml_metal_fusion { + ggml_metal_fusion_id id; + + const enum ggml_op * ops; // op sequence (fixed length) + int n_ops; // number of ops + + // if unsafe: the generic chain/shape + ggml_can_fuse_subgraph checks are skipped and the + // check callback below is the sole validator (used for patterns that are not elision chains, + // e.g. the gdn + cache-cpy write-through fusion) + bool unsafe; + + // extra backend constraints on top of ggml_can_fuse_subgraph + // nodes[j] is the j-th node of the pattern + bool (*check)(const struct ggml_metal_fusion * fusion, + const struct ggml_tensor * const * nodes, + ggml_metal_fusion_mode mode); +}; + +typedef struct ggml_metal_fusion ggml_metal_fusion; + +// the single table of all fusions supported by the Metal backend +const ggml_metal_fusion * ggml_metal_fusion_all(int * n); + +// ---- shared fusion info --------------------------------------------------- + +// shared fusion debugging context, owned by the device; newly created backend contexts for that +// device register with it so the fusion counters are race-free and accumulate across contexts. +struct ggml_metal_fusion_info; // defined in ggml-metal-fusion.cpp + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug); +void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo); + +bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo); +bool ggml_metal_fusion_info_stats (const struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_debug (const struct ggml_metal_fusion_info * finfo); + +int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo); +const char * ggml_metal_fusion_info_label (const struct ggml_metal_fusion_info * finfo, int idx); +uint64_t ggml_metal_fusion_info_count (const struct ggml_metal_fusion_info * finfo, int idx); + +void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion); +void ggml_metal_fusion_info_set_enabled (struct ggml_metal_fusion_info * finfo, bool enabled); + +void ggml_metal_fusion_info_stats_init ( struct ggml_metal_fusion_info * finfo); +void ggml_metal_fusion_info_stats_reset( struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_stats_get (const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n); +void ggml_metal_fusion_info_labels_init( struct ggml_metal_fusion_info * finfo); + +// compute phase: longest fusion starting at idx (a position in node_idxs) that matches in `mode`. +// returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes consumed. +const ggml_metal_fusion * ggml_metal_fusion_next( + const struct ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out); + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused, chaining patterns back-to-back. returns at least 1. +int ggml_metal_fusion_max(const struct ggml_cgraph * gf, int idx); + +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 28a9ba101c74..7ad21341e4dc 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -985,6 +985,7 @@ typedef struct { uint64_t nb1; uint64_t nb2; uint64_t nb3; + uint64_t nb_out; // 0 => snapshots are appended after the attn scores (unfused) } ggml_metal_kargs_gated_delta_net; typedef struct { diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 3db8bca43752..b4e87cb2cc2a 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -7,6 +7,7 @@ #include "ggml-metal-impl.h" #include "ggml-metal-common.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" #include "ggml-metal-tuning.h" #include @@ -31,24 +32,22 @@ struct ggml_metal_op { ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { this->dev = dev; this->lib = ggml_metal_device_get_library(dev); this->enc = ggml_metal_encoder_init(cmd_buf, use_concurrency); this->mem_ranges = ggml_mem_ranges_init(debug_graph); + this->finfo = finfo; this->idx_start = idx_start; this->idx_end = idx_end; - this->use_fusion = use_fusion; this->use_concurrency = use_concurrency; this->use_capture = use_capture; this->debug_graph = debug_graph; - this->debug_fusion = debug_fusion; this->gf = gf; idxs.reserve(gf->n_nodes); @@ -78,15 +77,24 @@ struct ggml_metal_op { return ggml_graph_node(gf, idxs[i]); } - bool can_fuse(int i0, const ggml_op * ops, int n_ops) const { - assert(use_fusion); + // consult the fusion table for the longest pattern starting at i0 + // returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes + const ggml_metal_fusion * can_fuse(int i0, enum ggml_metal_fusion_mode mode, int * n_out) const { + assert(use_fusion()); assert(i0 >= 0 && i0 < n_nodes()); - if (i0 + n_ops > n_nodes()) { - return false; - } + return ggml_metal_fusion_next(gf, idxs.data(), (int) idxs.size(), i0, mode, n_out); + } + + // whether to attempt fusion; the toggle lives in the shared fusion debugging context owned + // by the device (initialized from GGML_METAL_FUSION_DISABLE, overridable by the test) + bool use_fusion() const { + return ggml_metal_fusion_info_enabled(finfo); + } - return ggml_can_fuse_ext(gf, idxs.data() + i0, ops, n_ops); + // record that a fusion fired, indexed by the matching table entry + void count_fusions(const ggml_metal_fusion * fusion) const { + ggml_metal_fusion_info_count_fusion(finfo, fusion); } ggml_metal_device_t dev; @@ -94,12 +102,13 @@ struct ggml_metal_op { ggml_metal_encoder_t enc; ggml_mem_ranges_t mem_ranges; - bool use_fusion; + // shared fusion debugging context + ggml_metal_fusion_info * finfo; + bool use_concurrency; bool use_capture; int debug_graph; - int debug_fusion; private: ggml_cgraph * gf; @@ -115,24 +124,22 @@ ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { ggml_metal_op_t res = new ggml_metal_op( dev, cmd_buf, gf, + finfo, idx_start, idx_end, - use_fusion, use_concurrency, use_capture, - debug_graph, - debug_fusion); + debug_graph); return res; } @@ -1868,6 +1875,8 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; + const bool use_fusion = ctx->use_fusion(); + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -1880,6 +1889,31 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op); + // when fused with the trailing cache cpy, the snapshots are written straight into the + // recurrent cache and the cpy is skipped (see GGML_METAL_FUSION_GDN_CACHE) + ggml_metal_buffer_id bid_out = ggml_metal_get_buffer_id(op); + uint64_t nb_out = 0; + int n_fuse = 1; + + if (use_fusion) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + + if (fusion && fusion->id == GGML_METAL_FUSION_GDN_CACHE) { + const ggml_tensor * dst_cache = ctx->node(idx + 1)->src[1]; // cache view + + bid_out = ggml_metal_get_buffer_id(dst_cache); + nb_out = dst_cache->nb[2]/sizeof(float); + n_fuse = 2; + + ctx->count_fusions(fusion); + + if (debug_fusion > 1) { + GGML_LOG_DEBUG("%s: fuse: GATED_DELTA_NET + CPY\n", __func__); + } + } + } + int ida = 0; ggml_metal_kargs_gated_delta_net args = { @@ -1918,23 +1952,25 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { /*.nb1 =*/ nb1, /*.nb2 =*/ nb2, /*.nb3 =*/ nb3, + /*.nb_out =*/ nb_out, }; ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); // args ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); // q ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); // k ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); // v ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); // gate ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // beta ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst (attn) + ggml_metal_encoder_set_buffer (enc, bid_out, ida++); // state_out const int nsg = pipeline.nsg; ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1); - return 1; + return n_fuse; } int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) { @@ -3718,56 +3754,20 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { return 1; } -// Snake activation autofuse: mul -> sin -> sqr -> mul -> add -static bool ggml_metal_op_can_fuse_snake(ggml_metal_op_t ctx, int idx) { - static constexpr ggml_op snake_ops[5] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }; - - if (ctx->node(idx)->op != GGML_OP_MUL || !ctx->can_fuse(idx, snake_ops, 5)) { - return false; - } - - const ggml_tensor * mul0 = ctx->node(idx + 0); - const ggml_tensor * sin_node = ctx->node(idx + 1); - const ggml_tensor * sqr = ctx->node(idx + 2); - const ggml_tensor * mul1 = ctx->node(idx + 3); - const ggml_tensor * add = ctx->node(idx + 4); - - // x carries the full activation shape, a is the broadcast operand - const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; - const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; +int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { + int n_fuse = 1; + const ggml_metal_fusion * fusion = nullptr; - // mul1 reads sqr and inv_b in either operand order - const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; - - // closure check: the trailing add reads the same x as the leading mul - const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; - - // x is in the supported whitelist and every chain intermediate shares x's type. - // a and inv_b bind as device const float * in the kernel, so they stay F32. - const bool types_ok = - (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && - (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && - (mul0->type == x->type) && (sin_node->type == x->type) && - (sqr->type == x->type) && (mul1->type == x->type) && - (add->type == x->type); - // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D - const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; - const bool dim_ok = - (x->ne[2] == 1) && (x->ne[3] == 1) && - (add->ne[2] == 1) && (add->ne[3] == 1) && - (a->ne[2] == 1) && (a->ne[3] == 1) && - (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); - // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous - const bool contig_ok = - ggml_is_contiguous(x) && ggml_is_contiguous(add) && - ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); - - return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; -} + if (ctx->use_fusion()) { + int n = 1; + fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + n_fuse = n; -int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { - if (ctx->use_fusion && ggml_metal_op_can_fuse_snake(ctx, idx)) { - return ggml_metal_op_snake_fused(ctx, idx); + // snake activation autofuse: mul -> sin -> sqr -> mul -> add + if (fusion && fusion->id == GGML_METAL_FUSION_SNAKE) { + ctx->count_fusions(fusion); + return ggml_metal_op_snake_fused(ctx, idx); + } } ggml_tensor * op = ctx->node(idx); @@ -3775,9 +3775,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -3822,57 +3822,19 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { /*.o1 =*/ { bid_src1.offs }, }; - ggml_op fops[8]; - - int n_fuse = 1; - // c[0] = add(a, b[0]) // c[1] = add(c[0], b[1]) // c[2] = add(c[1], b[2]) // ... - if (use_fusion) { - fops[0] = GGML_OP_ADD; - fops[1] = GGML_OP_ADD; - fops[2] = GGML_OP_ADD; - fops[3] = GGML_OP_ADD; - fops[4] = GGML_OP_ADD; - fops[5] = GGML_OP_ADD; - fops[6] = GGML_OP_ADD; - fops[7] = GGML_OP_ADD; - - // note: in metal, we sometimes encode the graph in parallel so we have to avoid fusing ops - // across splits. idx_end indicates the last node in the current split - for (n_fuse = 0; n_fuse <= 6; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; - } - - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); - - if (f0 != f1->src[0]) { - break; - } - - // b[0] === b[1] === ... - if (!ggml_are_same_layout(f0->src[1], f1->src[1])) { - break; - } - - // only fuse ops if src1 is in the same Metal buffer - ggml_metal_buffer_id bid_fuse = ggml_metal_get_buffer_id(f1->src[1]); - if (bid_fuse.metal != bid_src1.metal) { - break; - } - - //ctx->fuse_cnt[ops[n_fuse + 1]->op]++; - - args.o1[n_fuse + 1] = bid_fuse.offs; + if (use_fusion && fusion && fusion->id == GGML_METAL_FUSION_ADD_CHAIN) { + // the offsets of the fused addends are relative to the start of the src1 buffer + for (int i = 1; i < n_fuse; i++) { + args.o1[i] = ggml_metal_get_buffer_id(ctx->node(idx + i)->src[1]).offs; } - ++n_fuse; + ctx->count_fusions(fusion); - if (debug_fusion > 1 && n_fuse > 1) { + if (debug_fusion > 1) { GGML_LOG_DEBUG("%s: fuse: ADD x %d\n", __func__, n_fuse); } } @@ -4080,9 +4042,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -4110,8 +4072,6 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { /*.nbf3 =*/ { nb03 }, }; - ggml_op fops[8]; - int n_fuse = 1; ggml_metal_buffer_id bid_fuse[2] = { bid_src0, bid_src0 }; @@ -4120,55 +4080,35 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { // d[1] = mul(d[0], b) // d[2] = add(d[1], c) if (use_fusion) { - fops[0] = op->op; - fops[1] = GGML_OP_MUL; - fops[2] = GGML_OP_ADD; + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); - for (n_fuse = 0; n_fuse <= 1; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; - } + if (fusion && (fusion->id == GGML_METAL_FUSION_NORM_MUL || fusion->id == GGML_METAL_FUSION_NORM_MUL_ADD)) { + n_fuse = n; - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); + ctx->count_fusions(fusion); - if (f0 != f1->src[0]) { - break; - } + for (int i = 1; i < n_fuse; i++) { + const ggml_tensor * fn = ctx->node(idx + i); - if (f1->src[1]->ne[0] != op->ne[0]) { - break; - } + bid_fuse[i - 1] = ggml_metal_get_buffer_id(fn->src[1]); - if (!ggml_is_contiguous_rows(f1->src[1])) { - break; - } + args.nef1[i] = fn->src[1]->ne[1]; + args.nef2[i] = fn->src[1]->ne[2]; + args.nef3[i] = fn->src[1]->ne[3]; - if (f1->type != GGML_TYPE_F32) { - break; + args.nbf1[i] = fn->src[1]->nb[1]; + args.nbf2[i] = fn->src[1]->nb[2]; + args.nbf3[i] = fn->src[1]->nb[3]; } - //ctx->fuse_cnt[f1->op]++; - - bid_fuse[n_fuse] = ggml_metal_get_buffer_id(f1->src[1]); - - args.nef1[n_fuse + 1] = f1->src[1]->ne[1]; - args.nef2[n_fuse + 1] = f1->src[1]->ne[2]; - args.nef3[n_fuse + 1] = f1->src[1]->ne[3]; - - args.nbf1[n_fuse + 1] = f1->src[1]->nb[1]; - args.nbf2[n_fuse + 1] = f1->src[1]->nb[2]; - args.nbf3[n_fuse + 1] = f1->src[1]->nb[3]; - } - - ++n_fuse; - - if (debug_fusion > 1 && n_fuse > 1) { - if (n_fuse == 2) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); - } - if (n_fuse == 3) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + if (debug_fusion > 1) { + if (n_fuse == 2) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); + } + if (n_fuse == 3) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + } } } } diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index f8fe50b468e4..4dd8ce7af679 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -8,17 +8,18 @@ extern "C" { typedef struct ggml_metal_op * ggml_metal_op_t; +struct ggml_metal_fusion; // forward decl (ggml-metal-device.h) + ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, struct ggml_cgraph * gf, + struct ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion); + int debug_graph); void ggml_metal_op_free(ggml_metal_op_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 3bd6abd06fdc..4cbec8645ab9 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -4,6 +4,7 @@ #include "ggml-backend-impl.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" #include "ggml-metal-context.h" #include "ggml-metal-ops.h" #include "ggml-metal-tuning.h" @@ -906,6 +907,30 @@ static const char * ggml_backend_metal_tuning_device_token(ggml_backend_dev_t de return ggml_metal_device_id_token(ggml_metal_device_get_props(ctx_dev)->device_id); } +// generic fusion debugging API (ad-hoc proc-address mechanism): the test resolves the device +// fusion context once and passes that opaque handle to the rest of the functions +typedef void * ggml_backend_fusion_t; + +static ggml_backend_fusion_t ggml_backend_metal_fusion_get(ggml_backend_dev_t dev) { + return ggml_metal_device_get_fusion_info((ggml_metal_device_t)dev->context); +} + +static void ggml_backend_metal_fusion_stats_init(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_init((struct ggml_metal_fusion_info *) finfo); +} + +static void ggml_backend_metal_fusion_stats_reset(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_reset((struct ggml_metal_fusion_info *) finfo); +} + +static int ggml_backend_metal_fusion_stats_get(ggml_backend_fusion_t finfo, const char ** labels, uint64_t * counts, int n) { + return ggml_metal_fusion_info_stats_get((struct ggml_metal_fusion_info *) finfo, labels, counts, n); +} + +static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, bool enabled) { + ggml_metal_fusion_info_set_enabled((struct ggml_metal_fusion_info *) finfo, enabled); +} + static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *)ggml_backend_metal_get_features; @@ -928,6 +953,23 @@ static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const if (strcmp(name, "ggml_backend_metal_tuning_device_token") == 0) { return (void *)ggml_backend_metal_tuning_device_token; } + // generic fusion debugging API (ad-hoc proc-address mechanism, not part of the official + // ggml backend interface yet; a backend that adopts it exports these exact names) + if (strcmp(name, "ggml_backend_fusion_get") == 0) { + return (void *)ggml_backend_metal_fusion_get; + } + if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) { + return (void *)ggml_backend_metal_fusion_stats_init; + } + if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) { + return (void *)ggml_backend_metal_fusion_stats_reset; + } + if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) { + return (void *)ggml_backend_metal_fusion_stats_get; + } + if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) { + return (void *)ggml_backend_metal_fusion_set_enabled; + } return NULL; diff --git a/ggml/src/ggml-metal/kernels/gated_delta_net.metal b/ggml/src/ggml-metal/kernels/gated_delta_net.metal index 8422d8e29f8b..5e4861ece360 100644 --- a/ggml/src/ggml-metal/kernels/gated_delta_net.metal +++ b/ggml/src/ggml-metal/kernels/gated_delta_net.metal @@ -15,6 +15,7 @@ kernel void kernel_gated_delta_net_impl( device const char * b, device const char * s, device char * dst, + device char * dst_fuse, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -65,6 +66,12 @@ kernel void kernel_gated_delta_net_impl( // per-(seq,head) offset within a slot const uint state_out_base = (i23*args.ne21 + i21)*S_v*S_v + i20*S_v; + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + attn_size; + const uint slot_stride = fused ? (uint)args.nb_out : state_size_per_snap; + for (short t = 0; t < args.ne22; t++) { float s_k = 0.0f; @@ -116,7 +123,7 @@ kernel void kernel_gated_delta_net_impl( if (K > 1) { const int target_slot = (int)args.ne22 - 1 - (int)t; if (target_slot >= 0 && target_slot < (int)K) { - device float * dst_state = (device float *) (dst) + attn_size + (uint)target_slot * state_size_per_snap + state_out_base; + device float * dst_state = (device float *)state_out + (uint)target_slot * slot_stride + state_out_base; FOR_UNROLL (short j = 0; j < NSG; j++) { const short is = tx*NSG + j; dst_state[is] = ls[j]; @@ -126,7 +133,7 @@ kernel void kernel_gated_delta_net_impl( } if (K == 1) { - device float * dst_state = (device float *) (dst) + attn_size + state_out_base; + device float * dst_state = (device float *)state_out + state_out_base; FOR_UNROLL (short j = 0; j < NSG; j++) { const short is = tx*NSG + j; dst_state[is] = ls[j]; @@ -158,6 +165,7 @@ kernel void kernel_gated_delta_net_impl( device const char * b, device const char * s, device char * dst, + device char * dst_fuse, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -230,7 +238,13 @@ kernel void kernel_gated_delta_net_impl( dst_attn += args.ne21*S_v; } - device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20; + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + args.ne23*args.ne22*args.ne21*S_v; + const uint slot_stride = fused ? (uint)args.nb_out : S_v*S_v; + + device float * dst_state = (device float *)state_out + (i23*args.ne21 + i21)*slot_stride + i20; device T * dstt_state = (device T *) (dst_state); FOR_UNROLL (short j = 0; j < NSG; j++) { diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 21501574a911..6334f3ccab30 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -666,7 +666,9 @@ void llama_context::sched_reserve() { // need to implement a more robust mechanism that tries a few different inputs and analyzes the results ggml_cgraph * gf = nullptr; switch (model.arch) { + case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_MINIMAX_01: + // [TAG_RESERVE_DIAG_DECAY] // the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which // makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1` gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc); diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cpp index 361114acc327..9fa2e8fc01cd 100644 --- a/src/models/minimax-01.cpp +++ b/src/models/minimax-01.cpp @@ -229,6 +229,7 @@ llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_ ggml_set_input(inp->inp_k_decay); cb(inp->inp_k_decay, "k_decay_exp", -1); + // [TAG_RESERVE_DIAG_DECAY] inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs); ggml_set_input(inp->inp_diag_decay); cb(inp->inp_diag_decay, "diag_decay_exp", -1); diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cpp index d946b3cff6da..ba1cea1465fd 100644 --- a/src/models/plamo2.cpp +++ b/src/models/plamo2.cpp @@ -142,6 +142,11 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il); } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + residual = ggml_get_rows(ctx0, residual, inp_out_ids); + } + // post_mixer_norm cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_post_norm", il); @@ -167,11 +172,6 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_para cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il); cb(cur, "ffn_post_norm", il); - if (il == n_layer - 1 && inp_out_ids) { - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - residual = ggml_get_rows(ctx0, residual, inp_out_ids); - } - // residual connection cur = ggml_add(ctx0, cur, residual); cb(cur, "ffn_residual", il); diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 5596620f0782..30c08ed35e3f 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -18,6 +18,7 @@ void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) { int64_t n_vocab_out = n_vocab; if (arch == LLM_ARCH_QWEN3TTS) { + // [TAG_LLAMA_N_VOCAB_OUT] n_vocab_out = 3072; } diff --git a/tests/.gitignore b/tests/.gitignore index 52b292b1f878..04095c9ddba6 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,6 +1,7 @@ * !*.* !snapshots/ +!fusion/ *.o ggml-common.h **/*.swp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5531c4ce3ce9..920c58c738e4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -196,7 +196,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) # llama_build_and_test(test-double-float.cpp) # SLOW - llama_build_and_test(test-llama-archs.cpp) + llama_build(test-llama-archs.cpp) set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-models/") file(MAKE_DIRECTORY "${MODEL_DIR}") @@ -255,6 +255,8 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) ARGS --models "${MODEL_DIR}" ) set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED generate-models) + + llama_build(test-fusion.cpp) endif() llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple-tokenize.cpp) diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv new file mode 100644 index 000000000000..067316abfe11 --- /dev/null +++ b/tests/fusion/MTL.csv @@ -0,0 +1,154 @@ +# test-fusion baseline for device MTL +# arch ,moe ,mode ,label , count +arcee ,0 ,any ,RMS_NORM+MUL , 5 +arctic ,0 ,any ,RMS_NORM+MUL , 7 +baichuan ,0 ,any ,RMS_NORM+MUL , 5 +bailingmoe ,1 ,any ,ADD+ADD , 2 +bailingmoe ,1 ,any ,RMS_NORM+MUL , 5 +bailingmoe2 ,1 ,any ,ADD+ADD , 1 +bailingmoe2 ,1 ,any ,RMS_NORM+MUL , 9 +bailingmoe3 ,1 ,any ,ADD+ADD , 1 +bailingmoe3 ,1 ,any ,GATED_DELTA_NET+CPY , 1 +bailingmoe3 ,1 ,any ,RMS_NORM+MUL , 8 +bloom ,0 ,any ,NORM+MUL+ADD , 6 +chatglm ,0 ,any ,RMS_NORM+MUL , 5 +codeshell ,0 ,any ,NORM+MUL+ADD , 5 +cogvlm ,0 ,any ,RMS_NORM+MUL , 5 +command-r ,0 ,any ,NORM+MUL , 3 +dbrx ,0 ,any ,NORM+MUL , 5 +deci ,0 ,any ,RMS_NORM+MUL , 5 +deepseek ,0 ,any ,ADD+ADD , 1 +deepseek ,0 ,any ,RMS_NORM+MUL , 5 +deepseek2 ,0 ,any ,ADD+ADD , 1 +deepseek2 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek32 ,0 ,any ,ADD+ADD , 1 +deepseek32 ,0 ,any ,NORM+MUL+ADD , 2 +deepseek32 ,0 ,any ,RMS_NORM+MUL , 9 +deepseek4 ,0 ,any ,RMS_NORM+MUL , 20 +dots1 ,0 ,any ,ADD+ADD , 1 +dots1 ,0 ,any ,RMS_NORM+MUL , 9 +dream ,0 ,any ,RMS_NORM+MUL , 5 +ernie4_5-moe ,1 ,any ,ADD+ADD , 1 +ernie4_5-moe ,1 ,any ,RMS_NORM+MUL , 5 +ernie4_5 ,0 ,any ,RMS_NORM+MUL , 5 +exaone ,0 ,any ,RMS_NORM+MUL , 5 +exaone4 ,0 ,any ,RMS_NORM+MUL , 5 +exaone4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +falcon ,0 ,any ,ADD+ADD , 2 +falcon ,0 ,any ,NORM+MUL+ADD , 5 +falcon-h1 ,0 ,any ,ADD+ADD , 2 +falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9 +gemma ,0 ,any ,RMS_NORM+MUL , 5 +gemma2 ,0 ,any ,RMS_NORM+MUL , 5 +gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +glm-dsa ,0 ,any ,ADD+ADD , 1 +glm-dsa ,0 ,any ,NORM+MUL+ADD , 2 +glm-dsa ,0 ,any ,RMS_NORM+MUL , 9 +glm4 ,0 ,any ,RMS_NORM+MUL , 5 +glm4 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +glm4moe ,1 ,any ,ADD+ADD , 1 +glm4moe ,1 ,any ,RMS_NORM+MUL , 9 +gpt-oss ,0 ,any ,RMS_NORM+MUL , 5 +gpt2 ,0 ,any ,NORM+MUL+ADD , 5 +gptneox ,0 ,any ,NORM+MUL+ADD , 5 +granite ,0 ,any ,RMS_NORM+MUL , 5 +granite ,0 ,any ,RMS_NORM+MUL , 5 +granitehybrid ,0 ,any ,RMS_NORM+MUL , 6 +granitemoe ,1 ,any ,RMS_NORM+MUL , 5 +granitemoe ,1 ,any ,RMS_NORM+MUL , 5 +grok ,0 ,any ,RMS_NORM+MUL , 5 +grok ,0 ,any ,RMS_NORM+MUL+ADD , 4 +grovemoe ,1 ,any ,ADD+ADD , 2 +grovemoe ,1 ,any ,RMS_NORM+MUL , 9 +hunyuan-dense ,0 ,any ,RMS_NORM+MUL , 9 +hunyuan-moe ,1 ,any ,ADD+ADD , 2 +hunyuan-moe ,1 ,any ,RMS_NORM+MUL , 9 +hunyuan_vl ,0 ,any ,RMS_NORM+MUL , 9 +hy_v3 ,0 ,any ,ADD+ADD , 2 +hy_v3 ,0 ,any ,RMS_NORM+MUL , 9 +hy_v4 ,0 ,any ,NORM+MUL+ADD , 1 +hy_v4 ,0 ,any ,RMS_NORM+MUL , 9 +internlm2 ,0 ,any ,RMS_NORM+MUL , 5 +jais ,0 ,any ,NORM+MUL+ADD , 5 +jais2 ,0 ,any ,NORM+MUL+ADD , 5 +jamba ,0 ,any ,RMS_NORM+MUL , 8 +kimi-k3 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-k3 ,0 ,any ,RMS_NORM+MUL , 17 +kimi-linear ,0 ,any ,ADD+ADD , 1 +kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1 +kimi-linear ,0 ,any ,RMS_NORM+MUL , 7 +lfm2 ,0 ,any ,RMS_NORM+MUL , 7 +lfm2moe ,1 ,any ,RMS_NORM+MUL , 7 +llada ,0 ,any ,RMS_NORM+MUL , 5 +llada-moe ,1 ,any ,RMS_NORM+MUL , 9 +llama ,0 ,any ,RMS_NORM+MUL , 5 +llama ,0 ,any ,RMS_NORM+MUL , 5 +llama4 ,0 ,any ,ADD+ADD , 2 +llama4 ,0 ,any ,RMS_NORM+MUL , 9 +maincoder ,0 ,any ,RMS_NORM+MUL , 9 +mamba ,0 ,any ,RMS_NORM+MUL , 3 +mamba2 ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,RMS_NORM+MUL , 5 +minicpm ,0 ,any ,RMS_NORM+MUL , 5 +minicpm3 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-01 ,0 ,any ,RMS_NORM+MUL , 6 +minimax-m2 ,0 ,any ,RMS_NORM+MUL , 9 +minimax-m3 ,0 ,any ,ADD+ADD , 1 +minimax-m3 ,0 ,any ,RMS_NORM+MUL , 11 +mistral3 ,0 ,any ,RMS_NORM+MUL , 5 +mistral3 ,0 ,any ,RMS_NORM+MUL , 5 +mistral4 ,0 ,any ,ADD+ADD , 1 +mistral4 ,0 ,any ,RMS_NORM+MUL , 9 +mpt ,0 ,any ,NORM+MUL+ADD , 5 +nanbeige ,0 ,any ,RMS_NORM+MUL , 5 +nemotron ,0 ,any ,NORM+MUL+ADD , 5 +nemotron_h ,0 ,any ,RMS_NORM+MUL , 5 +nemotron_h_moe ,1 ,any ,RMS_NORM+MUL , 5 +olmoe ,1 ,any ,RMS_NORM+MUL , 9 +openelm ,0 ,any ,RMS_NORM+MUL , 9 +orion ,0 ,any ,NORM+MUL+ADD , 5 +paddleocr ,0 ,any ,RMS_NORM+MUL , 5 +pangu-embedded ,0 ,any ,RMS_NORM+MUL , 5 +phi2 ,0 ,any ,ADD+ADD , 2 +phi2 ,0 ,any ,NORM+MUL+ADD , 3 +phi3 ,0 ,any ,RMS_NORM+MUL , 5 +phimoe ,1 ,any ,RMS_NORM+MUL+ADD , 5 +plamo ,0 ,any ,ADD+ADD , 2 +plamo ,0 ,any ,RMS_NORM+MUL , 3 +plamo2 ,0 ,any ,RMS_NORM+MUL , 10 +plamo2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +pockettts ,0 ,any ,NORM+MUL+ADD , 5 +qwen ,0 ,any ,RMS_NORM+MUL , 5 +qwen2 ,0 ,any ,RMS_NORM+MUL , 5 +qwen2moe ,1 ,any ,ADD+ADD , 2 +qwen2moe ,1 ,any ,RMS_NORM+MUL , 5 +qwen2vl ,0 ,any ,RMS_NORM+MUL , 5 +qwen3 ,0 ,any ,RMS_NORM+MUL , 9 +qwen35 ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen35 ,0 ,any ,RMS_NORM+MUL , 8 +qwen35moe ,1 ,any ,ADD+ADD , 2 +qwen35moe ,1 ,any ,GATED_DELTA_NET+CPY , 1 +qwen35moe ,1 ,any ,RMS_NORM+MUL , 8 +qwen3moe ,1 ,any ,RMS_NORM+MUL , 9 +qwen3next ,0 ,any ,ADD+ADD , 2 +qwen3next ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen3next ,0 ,any ,RMS_NORM+MUL , 8 +qwen3tts ,0 ,any ,RMS_NORM+MUL , 9 +qwen3vl ,0 ,any ,RMS_NORM+MUL , 9 +qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9 +qwen4exp ,0 ,any ,ADD+ADD+ADD , 5 +qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9 +qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1 +qwen4exp ,0 ,any ,RMS_NORM+MUL , 5 +refact ,0 ,any ,RMS_NORM+MUL , 5 +refact ,0 ,any ,RMS_NORM+MUL , 5 +rnd1 ,0 ,any ,RMS_NORM+MUL , 9 +seed_oss ,0 ,any ,RMS_NORM+MUL , 5 +smallthinker ,0 ,any ,RMS_NORM+MUL , 5 +smollm3 ,0 ,any ,RMS_NORM+MUL , 5 +stablelm ,0 ,any ,NORM+MUL , 4 +stablelm ,0 ,any ,NORM+MUL+ADD , 5 +starcoder ,0 ,any ,NORM+MUL+ADD , 5 +starcoder2 ,0 ,any ,NORM+MUL+ADD , 5 +talkie ,0 ,any ,ADD+ADD , 2 +xverse ,0 ,any ,RMS_NORM+MUL , 5 diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index ef4fc30cecb7..15c42a10815f 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4638,6 +4638,122 @@ struct test_gated_delta_net : public test_case { } }; +// GGML_OP_GATED_DELTA_NET + GGML_OP_CPY (recurrent cache fusion) +struct test_gated_delta_net_cache_fusion : public test_case { + const ggml_type type; + + const int64_t head_count; + const int64_t head_size; + const int64_t n_seq_tokens; + const int64_t n_seqs; + const int64_t K; // snapshot slot count (>1) + + ggml_tensor * cpy_node = nullptr; + + std::string vars() override { + return VARS_TO_STR6(type, head_count, head_size, n_seq_tokens, n_seqs, K); + } + + test_gated_delta_net_cache_fusion(ggml_type type = GGML_TYPE_F32, + int64_t head_count = 4, int64_t head_size = 32, int64_t n_seq_tokens = 2, int64_t n_seqs = 1, + int64_t K = 2) + : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), K(K) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t S_v = head_size; + const int64_t H_v = head_count; + const int64_t H_k = head_count; + const int64_t D = S_v * S_v * H_v; + const int64_t n_written = std::min(n_seq_tokens, K); + + ggml_tensor * q = ggml_new_tensor_4d(ctx, type, head_size, H_k, n_seq_tokens, n_seqs); + ggml_tensor * k = ggml_new_tensor_4d(ctx, type, head_size, H_k, n_seq_tokens, n_seqs); + ggml_tensor * v = ggml_new_tensor_4d(ctx, type, head_size, H_v, n_seq_tokens, n_seqs); + ggml_set_name(q, "q"); + ggml_set_name(k, "k"); + ggml_set_name(v, "v"); + ggml_tensor * g = ggml_new_tensor_4d(ctx, type, 1, H_v, n_seq_tokens, n_seqs); + ggml_tensor * beta = ggml_new_tensor_4d(ctx, type, 1, H_v, n_seq_tokens, n_seqs); + ggml_tensor * state = ggml_new_tensor_4d(ctx, type, head_size, head_size, H_v, n_seqs); + ggml_set_name(g, "g"); + ggml_set_name(beta, "beta"); + ggml_set_name(state, "state"); + + q = ggml_l2_norm(ctx, q, 1e-6f); + k = ggml_l2_norm(ctx, k, 1e-6f); + + ggml_tensor * gdn_out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K); + ggml_set_name(gdn_out, "gdn_out"); + + // attn scores view (first part of the gdn output) + ggml_tensor * attn = ggml_view_4d(ctx, gdn_out, + S_v, H_v, n_seq_tokens, n_seqs, + ggml_row_size(gdn_out->type, S_v), + ggml_row_size(gdn_out->type, S_v * H_v), + ggml_row_size(gdn_out->type, S_v * H_v * n_seq_tokens), 0); + ggml_set_name(attn, "attn"); + + // snapshot tail view [D, n_seqs, n_written] + const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs; + ggml_tensor * src = ggml_view_3d(ctx, gdn_out, + D, n_seqs, n_written, + ggml_row_size(gdn_out->type, D), + ggml_row_size(gdn_out->type, D * n_seqs), + ggml_row_size(gdn_out->type, attn_score_elems)); + + // recurrent cache view [D, n_seqs, n_written] + ggml_tensor * cache = ggml_new_tensor_3d(ctx, type, D, n_seqs, n_written); + ggml_set_name(cache, "cache"); + ggml_tensor * dst = ggml_view_3d(ctx, cache, + D, n_seqs, n_written, + ggml_row_size(cache->type, D), + ggml_row_size(cache->type, D * n_seqs), 0); + + ggml_tensor * cpy = ggml_cpy(ctx, src, dst); + ggml_set_name(cpy, "gdn_cache_cpy"); + cpy_node = cpy; + + // read the cpy output (not the plain dst view, which would not pull the cpy into the graph) + // so that neither the gdn nor the cpy is the graph output + ggml_tensor * out = ggml_sum(ctx, cpy); + return out; + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "GATED_DELTA_NET_CACHE_FUSION"; + } + + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return { cpy_node }; } + + uint64_t op_flops(ggml_tensor * t) override { + GGML_UNUSED(t); + const uint64_t S_v = head_size; + const uint64_t H_v = head_count; + const uint64_t T = n_seq_tokens; + const uint64_t B = n_seqs; + return (4ull*S_v + 2ull*S_v*S_v) * H_v * T * B; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (ggml_is_view_op(t->op)) { continue; } + if (strcmp(t->name, "g") == 0) { + init_tensor_uniform(t, -20.0f, -1e-4f); + } else if (strcmp(t->name, "beta") == 0) { + init_tensor_uniform(t, 0.0f, 1.0f); + } else if (strcmp(t->name, "v") == 0) { + init_tensor_uniform(t, -0.3f, 5.0f); + } else if (strcmp(t->name, "cache") == 0) { + init_tensor_uniform(t, 0.0f, 0.0f); + } else { + init_tensor_uniform(t); + } + } + } +}; + // GGML_OP_GATED_LINEAR_ATTN struct test_gla : public test_case { const ggml_type type; @@ -10741,6 +10857,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 8, 1, 1, false, false, /*K=*/3)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 16, 2, 1, false, false, /*K=*/4)); + // gdn + cache cpy fusion (K > 1) + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 2, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 64, 4, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 4, 1, 4)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 8, 32, 4, 2, 4)); + test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 8, 1, 4)); + #if 0 // these tests are disabled to save execution time, sbut they can be handy for debugging test_cases.emplace_back(new test_llama(2, true)); diff --git a/tests/test-fusion.cpp b/tests/test-fusion.cpp new file mode 100644 index 000000000000..467248f0f101 --- /dev/null +++ b/tests/test-fusion.cpp @@ -0,0 +1,565 @@ +// test-fusion: verify the backend fusion logic against a per-device baseline. +// +// for every dummy model generated by test-llama-archs, the tool runs the model on a single +// device with fusion enabled and disabled, and reports: +// - the per-fusion-type counters for each mode (prefill / decode, merged into "any" when the +// per-graph counts match) +// - the NMSE between the fused and unfused logits +// - the NMSE between the device and a CPU reference +// +// the per-fusion-type counters are compared against a per-device baseline file (CSV) so a +// fusion pattern that silently stops matching (or fires when it should not) is caught as a +// regression. +// +// usage: +// test-fusion --models DIR --device MTL0 --record baseline.csv # generate a baseline +// test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it +// test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model + +#include "common.h" +#include "log.h" +#include "llama-cpp.h" + +#include "ggml.h" +#include "gguf.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// generic fusion debugging API, resolved through the ad-hoc get_proc_address mechanism +// (not part of the official ggml backend interface yet). a backend that adopts fusion debugging +// exports these exact names. +typedef void * ggml_backend_fusion_t; + +typedef ggml_backend_fusion_t ( * fusion_get_t) (ggml_backend_dev_t); +typedef void ( * fusion_stats_init_t) (ggml_backend_fusion_t); +typedef void ( * fusion_stats_reset_t) (ggml_backend_fusion_t); +typedef int ( * fusion_stats_get_t) (ggml_backend_fusion_t, const char **, uint64_t *, int); +typedef void ( * fusion_set_enabled_t) (ggml_backend_fusion_t, bool); + +static bool silent_model_load_progress(float, void *) { + return true; +} + +struct gguf_context_ptr { + gguf_context * ctx; + gguf_context_ptr(gguf_context * c) : ctx(c) {} + ~gguf_context_ptr() { if (ctx) { gguf_free(ctx); } } + gguf_context * get() const { return ctx; } + gguf_context_ptr(const gguf_context_ptr &) = delete; + gguf_context_ptr & operator=(const gguf_context_ptr &) = delete; +}; + +// NMSE between two vectors (same as tests/test-llama-archs.cpp) +static double nmse(const std::vector & a, const std::vector & b) { + GGML_ASSERT(a.size() == b.size()); + double mse_a_b = 0.0; + double mse_a_0 = 0.0; + + for (size_t i = 0; i < a.size(); i++) { + const float a_i = a[i]; + const float b_i = b[i]; + + mse_a_b += (a_i - b_i) * (a_i - b_i); + mse_a_0 += a_i * a_i; + } + + return mse_a_b / mse_a_0; +} + +// deterministic token sequence +static std::vector get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) { + std::mt19937 gen(seed); + std::uniform_int_distribution<> dis(0, n_vocab - 1); + std::vector ret; + ret.reserve(n_tokens); + for (uint32_t i = 0; i < n_tokens; i++) { + ret.push_back(dis(gen)); + } + return ret; +} + +// trim leading/trailing whitespace (used when parsing padded CSV columns) +static std::string trim(const std::string & s) { + const size_t b = s.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) { + return ""; + } + const size_t e = s.find_last_not_of(" \t\r\n"); + return s.substr(b, e - b + 1); +} + +static std::string get_arch(const std::string & path) { + gguf_init_params params = { /*no_alloc=*/true, /*ctx=*/nullptr }; + gguf_context_ptr ctx(gguf_init_from_file(path.c_str(), params)); + if (!ctx.get()) { + throw std::runtime_error("failed to read gguf: " + path); + } + const int idx = gguf_find_key(ctx.get(), "general.architecture"); + if (idx < 0) { + return "unknown"; + } + const char * val = gguf_get_val_str(ctx.get(), idx); + return val ? val : "unknown"; +} + +static llama_model_ptr load_model(const std::string & path, ggml_backend_dev_t dev) { + llama_model_params model_params = llama_model_default_params(); + model_params.progress_callback = silent_model_load_progress; + std::vector devs = { dev, nullptr }; + model_params.devices = devs.data(); + model_params.split_mode = LLAMA_SPLIT_MODE_LAYER; + + llama_model_ptr model(llama_model_load_from_file(path.c_str(), model_params)); + if (!model) { + throw std::runtime_error("failed to load model: " + path); + } + return model; +} + +// a fresh context (fresh state) from an already-loaded model +static llama_context_ptr create_ctx(llama_model * model, int n_ubatch) { + llama_context_params ctx_params = llama_context_default_params(); + ctx_params.n_ctx = 0; + ctx_params.n_threads = 4; + ctx_params.n_threads_batch = 4; + ctx_params.n_ubatch = n_ubatch; + ctx_params.n_batch = n_ubatch; + + llama_context_ptr lctx(llama_init_from_model(model, ctx_params)); + if (!lctx) { + throw std::runtime_error("failed to init context"); + } + return lctx; +} + +// decode all tokens in one batch; returns the logits of every token +static std::vector decode_prefill(llama_model * model, llama_context * lctx, const std::vector & tokens) { + const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + llama_batch batch = llama_batch_init(tokens.size(), 0, 1); + for (size_t i = 0; i < tokens.size(); i++) { + common_batch_add(batch, tokens[i], i, { 0 }, true); + } + batch.n_tokens = tokens.size(); + if (llama_decode(lctx, batch)) { + llama_batch_free(batch); + throw std::runtime_error("prefill decode failed"); + } + + std::vector ret; + ret.reserve(tokens.size() * n_vocab); + for (size_t i = 0; i < tokens.size(); i++) { + const float * logits_ith = llama_get_logits_ith(lctx, i); + for (uint32_t j = 0; j < n_vocab; j++) { + ret.push_back(logits_ith[j]); + } + } + llama_batch_free(batch); + return ret; +} + +// decode one token at a time; returns the logits of the last token of each step +static std::vector decode_gen(llama_model * model, llama_context * lctx, const std::vector & tokens) { + const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + llama_batch batch = llama_batch_init(1, 0, 1); + std::vector ret; + for (size_t i = 0; i < tokens.size(); i++) { + common_batch_clear(batch); + common_batch_add(batch, tokens[i], i, { 0 }, true); + if (llama_decode(lctx, batch)) { + llama_batch_free(batch); + throw std::runtime_error("decode failed"); + } + const float * logits = llama_get_logits_ith(lctx, 0); + for (uint32_t j = 0; j < n_vocab; j++) { + ret.push_back(logits[j]); + } + } + llama_batch_free(batch); + return ret; +} + +static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_fusion_t finfo, + std::vector & labels, std::vector & counts) { + const int n = api_stats_get(finfo, nullptr, nullptr, 0); + labels.assign(n, nullptr); + counts.assign(n, 0); + api_stats_get(finfo, labels.data(), counts.data(), n); +} + +// one row of the per-label report +struct fusion_row { + std::string arch; + bool moe; + std::string mode; + std::string label; + uint64_t count_fused; + uint64_t count_unfused; + uint64_t expected; + double nmse_fus; + double nmse_dev; + bool ok_count; // counts match the baseline + bool ok_nmse; // nmse within epsilon +}; + +static void usage(const char * argv0) { + printf("%s: verify fusion counts on a device against a per-device baseline\n\n", argv0); + printf("usage: %s [options]\n\n", argv0); + printf("options:\n"); + printf(" --models DIR run over all .gguf models in a directory\n"); + printf(" --model FILE run over a single model file (mutually exclusive with --models)\n"); + printf(" --device NAME device to run on (e.g. MTL0, CPU)\n"); + printf(" --record CSV write the golden baseline\n"); + printf(" --check CSV validate the counters against a baseline (default)\n"); + printf(" -h, --help show this message and exit\n"); +} + +int main(int argc, char ** argv) { + std::string models_dir; + std::string model_file; + std::string device_name; + std::string record_path; + std::string check_path; + + for (int i = 1; i < argc; i++) { + const std::string arg = argv[i]; + const auto next = [&](const char * name) -> std::string { + if (i + 1 >= argc) { + LOG_ERR("%s: %s requires an argument\n", __func__, name); + exit(1); + } + return argv[++i]; + }; + if (arg == "-h" || arg == "--help") { + usage(argv[0]); + exit(0); + } + if (arg == "--models") { models_dir = next("--models"); } + else if (arg == "--model") { model_file = next("--model"); } + else if (arg == "--device"){ device_name = next("--device"); } + else if (arg == "--record"){ record_path = next("--record"); } + else if (arg == "--check") { check_path = next("--check"); } + else { + LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str()); + return 1; + } + } + + if (device_name.empty()) { + LOG_ERR("%s: --device NAME is required\n", __func__); + return 1; + } + if (models_dir.empty() && model_file.empty()) { + LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__); + return 1; + } + if (!models_dir.empty() && !model_file.empty()) { + LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__); + return 1; + } + if (!record_path.empty() && !check_path.empty()) { + LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__); + return 1; + } + + std::vector models; + if (!model_file.empty()) { + if (!std::filesystem::is_regular_file(model_file)) { + LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str()); + return 1; + } + models.push_back(model_file); + } else { + if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { + LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str()); + return 1; + } + for (const auto & entry : std::filesystem::directory_iterator(models_dir)) { + if (entry.is_regular_file() && entry.path().extension() == ".gguf") { + models.push_back(entry.path().string()); + } + } + std::sort(models.begin(), models.end()); + + if (models.empty()) { + LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str()); + return 1; + } + } + + common_init(); + ggml_backend_load_all(); + + ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str()); + if (!dev) { + LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n", + __func__, device_name.c_str()); + return 0; + } + + // resolve the generic fusion debugging functions through the ad-hoc get_proc_address + // mechanism; a backend that does not adopt fusion debugging exports none of them + auto * reg = ggml_backend_dev_backend_reg(dev); + + // output naming uses the backend base name (e.g. "MTL") rather than the specific device + // name (e.g. "MTL0") the test was invoked with + const std::string base_name = ggml_backend_reg_name(reg); + + auto api_get = (fusion_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_get"); + auto api_stats_init = (fusion_stats_init_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_init"); + auto api_stats_reset = (fusion_stats_reset_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_reset"); + auto api_stats_get = (fusion_stats_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_get"); + auto api_set_enabled = (fusion_set_enabled_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_set_enabled"); + + if (!api_get || !api_stats_init || !api_set_enabled || !api_stats_reset || !api_stats_get) { + LOG_ERR("%s: device '%s' does not export the generic fusion debugging API " + "(ggml_backend_fusion_*) - cannot run the fusion regression test\n", + __func__, device_name.c_str()); + return 1; + } + + ggml_backend_fusion_t finfo = api_get(dev); + + // enable fusions stats + api_stats_init(finfo); + + const bool has_counts = true; + + // load the baseline (if any): key arch|moe|mode|label -> expected count + std::map baseline; + if (!check_path.empty()) { + std::ifstream in(check_path); + if (!in) { + LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str()); + return 1; + } + std::string line; + while (std::getline(in, line)) { + if (line.empty() || line[0] == '#') { + continue; + } + std::vector cols; + size_t pos = 0; + while ((pos = line.find(',')) != std::string::npos) { + cols.push_back(trim(line.substr(0, pos))); + line.erase(0, pos + 1); + } + cols.push_back(trim(line)); + if (cols.size() != 5) { + continue; + } + baseline[cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3]] = std::stoull(cols[4]); + } + } + + std::vector rows; + + LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str()); + + const size_t seed = 1; + + for (const auto & model_path : models) { + const std::string arch = get_arch(model_path); + const bool moe = arch.find("moe") != std::string::npos; + + llama_model_ptr model; + llama_model_ptr model_cpu; + uint32_t n_vocab = 0; + try { + model = load_model(model_path, dev); + model_cpu = load_model(model_path, ggml_backend_dev_by_name("CPU")); + n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get())); + } catch (const std::exception & e) { + LOG_ERR("%s: %s: %s\n", __func__, model_path.c_str(), e.what()); + continue; + } + + struct mode_cfg { + std::string name; + std::vector (*decode)(llama_model *, llama_context *, const std::vector &); + int n_tokens; + int n_graphs; // graph runs per mode (prefill=1, decode=16) + }; + const mode_cfg modes[] = { + { "prefill", decode_prefill, 32, 1 }, + { "decode", decode_gen, 16, 16 }, + }; + + // per-label, per-mode data for this model; prefill and decode are merged into a single + // "any" row when their per-graph counts match + struct mode_data { + bool present; + uint64_t count_fused; // per graph + uint64_t count_unfused; // per graph + double nmse_fus; + double nmse_dev; + bool ok_nmse; + }; + std::map> mdata; + + for (int mi = 0; mi < 2; mi++) { + const mode_cfg & mode = modes[mi]; + const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed); + + // CPU reference for this mode (fresh context, fresh state) + std::vector logits_cpu; + try { + llama_context_ptr ctx = create_ctx(model_cpu.get(), 32); + logits_cpu = mode.decode(model_cpu.get(), ctx.get(), tokens); + } catch (const std::exception & e) { + LOG_WRN("%s: %s: cpu reference: %s\n", __func__, model_path.c_str(), e.what()); + } + + // fused run on a fresh context (fresh state) + std::vector logits_fused; + std::vector labels; + std::vector counts_fused; + { + llama_context_ptr ctx = create_ctx(model.get(), 32); + if (has_counts) { + api_set_enabled(finfo, true); + api_stats_reset(finfo); + } + logits_fused = mode.decode(model.get(), ctx.get(), tokens); + if (has_counts) { + read_counts(api_stats_get, finfo, labels, counts_fused); + } + } + + // unfused run on another fresh context (fresh state) + std::vector logits_unfused; + std::vector counts_unfused; + { + llama_context_ptr ctx = create_ctx(model.get(), 32); + if (has_counts) { + api_set_enabled(finfo, false); + api_stats_reset(finfo); + } + logits_unfused = mode.decode(model.get(), ctx.get(), tokens); + if (has_counts) { + read_counts(api_stats_get, finfo, labels, counts_unfused); + } + } + + const double nmse_fus = nmse(logits_fused, logits_unfused); + const double nmse_dev = logits_cpu.empty() ? 0.0 : nmse(logits_fused, logits_cpu); + + if (has_counts) { + for (int i = 0; i < (int) labels.size(); i++) { + const uint64_t fused = counts_fused[i] / mode.n_graphs; + const uint64_t unfused = counts_unfused[i] / mode.n_graphs; + if (fused == 0 && unfused == 0) { + continue; + } + auto & d = mdata[labels[i]][mi]; + d.present = true; + d.count_fused = fused; + d.count_unfused = unfused; + d.nmse_fus = nmse_fus; + d.nmse_dev = nmse_dev; + d.ok_nmse = nmse_fus <= 1e-4; + } + } else { + rows.push_back({ arch, moe, mode.name, "?", 0, 0, 0, nmse_fus, nmse_dev, true, nmse_fus <= 1e-4 }); + } + } + + // build the per-label rows, merging prefill and decode into "any" when the per-graph + // counts match (they always do for the deterministic fusion table) + if (has_counts) { + for (auto & kv : mdata) { + const std::string & label = kv.first; + const auto & d = kv.second; + const bool both = d[0].present && d[1].present; + const bool match = both && d[0].count_fused == d[1].count_fused; + + if (match) { + // one "any" row; use the worst NMSE across the two modes + const std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label; + const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key) : 0; + const bool ok_count = check_path.empty() || d[0].count_fused == expected; + const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse; + const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus); + const double nmse_dev = std::max(d[0].nmse_dev, d[1].nmse_dev); + rows.push_back({ arch, moe, "any", label, d[0].count_fused, d[0].count_unfused, + expected, nmse_fus, nmse_dev, ok_count, ok_nmse }); + } else { + // counts differ - keep a separate row per mode + for (int mi = 0; mi < 2; mi++) { + if (!d[mi].present) { + continue; + } + const mode_data & a = d[mi]; + const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" + modes[mi].name + "|" + label; + const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key) : 0; + const bool ok_count = check_path.empty() || a.count_fused == expected; + rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused, + expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse }); + } + } + } + } + + LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str()); + } + + // print the report + { + std::ofstream out(record_path); + std::ostream & os = record_path.empty() ? std::cout : out; + if (!record_path.empty()) { + os << "# test-fusion baseline for device " << base_name << "\n"; + os << "# " << std::left + << std::setw(18) << "arch" << ',' + << std::setw(4) << "moe" << ',' + << std::setw(8) << "mode" << ',' + << std::setw(28) << "label" << ',' + << std::right << std::setw(7) << "count" << '\n'; + } + + LOG_INF("%-20s %-4s %-8s %-22s %7s %7s %7s %10s %10s %s\n", + "arch", "moe", "mode", "label", "fused", "unfused", "expected", "nmse_fus", "nmse_dev", "status"); + int n_ok = 0; + int n_bad = 0; + for (const auto & r : rows) { + const bool ok = r.ok_count && r.ok_nmse; + const char * status = ok ? "ok" : "FAIL"; + if (ok) { n_ok++; } else { n_bad++; } + LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n", + r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label.c_str(), + (unsigned long long) r.count_fused, (unsigned long long) r.count_unfused, + (unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status); + if (!record_path.empty()) { + os << std::left + << std::setw(20) << r.arch << ',' + << std::setw(4) << (r.moe ? "1" : "0") << ',' + << std::setw(8) << r.mode << ',' + << std::setw(28) << r.label << ',' + << std::right << std::setw(7) << r.count_fused << '\n'; + } + } + LOG_INF("summary: %d ok, %d failed\n", n_ok, n_bad); + if (!record_path.empty()) { + LOG_INF("%s: baseline written to '%s'\n", __func__, record_path.c_str()); + } + + if (n_bad && !models_dir.empty() && !check_path.empty()) { + LOG_WRN("%s: if the fusion counts are expected to change, run with --record to update the baseline:\n" + "\n" + "./bin/test-llama-archs -o %s\n" + "%s --device %s --models %s --record %s\n", + __func__, models_dir.c_str(), argv[0], device_name.c_str(), models_dir.c_str(), check_path.c_str()); + } + + return n_bad; + } +} diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index dbed9846f9d9..3496f72e4949 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -128,7 +128,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } else if (arch == LLM_ARCH_CHAMELEON) { n_vocab = 10240; } else if (arch == LLM_ARCH_QWEN3TTS) { - n_vocab = 4096; // must be >= the hard-coded codec head size (3072) + //n_vocab = 4096; // must be >= the hard-coded codec head size (3072) + n_vocab = 3072; // TODO: should be 4096, but user code cannot get `n_vocab_out` yet [TAG_LLAMA_N_VOCAB_OUT] } uint32_t n_head_kv = n_head; diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cpp index 6179e6c10848..74d1ba6c213b 100644 --- a/tests/test-save-load-state.cpp +++ b/tests/test-save-load-state.cpp @@ -109,7 +109,7 @@ static bool test_seq_rm_isolated( for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) { llama_batch_ptr batch(n_tokens, 0, 1); for (size_t i = 0; i < n_tokens; ++i) { - common_batch_add(batch.get(), tokens[i], i, { seq_id }, false); + common_batch_add(batch.get(), tokens[i], i, { seq_id }, i == n_tokens - 1); } if (llama_decode(ctx.get(), batch.get())) { @@ -373,7 +373,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_ auto decode_one = [&](llama_token tok, int pos, llama_seq_id seq) { llama_batch_ptr batch(1, 0, 1); - common_batch_add(batch.get(), tok, pos, { seq }, false); + common_batch_add(batch.get(), tok, pos, { seq }, true); return llama_decode(ctx.get(), batch.get()) == 0; }; From 1dfe94e04875bbab710c1fbcb092ae2901b16d1b Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 11 Sep 2026 12:59:43 +0200 Subject: [PATCH 38/65] common : fix typo in speculative.cpp comment [no ci] (#28750) --- common/speculative.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index 7c8a06365cd7..77dfe9535f3f 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1493,7 +1493,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const int32_t n_tokens = batch_in.n_tokens; - // remember the frist and last batch index for each sequence + // remember the first and last batch index for each sequence std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1); std::fill(i_batch_end.begin(), i_batch_end.end(), -1); From 3bcfeb700fce9ff38a050dcd3f6a856319e948ba Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 11 Sep 2026 13:01:29 +0200 Subject: [PATCH 39/65] cmake : add PCH and unity build to improve build times (#28091) * scripts : add initial profiling script (wip) * src : add precompile headers (PCH) for models.h * common : add common.h as PCH * ggml : add PCH for ggml-impl.h * mtmd : use PCH for models.h * scripts : add script to build with Server/Tools/Tests * server : add PCH for common.h * docs: add profiling progress notes (wip) * ggml : add exclude for GCC + SVE on ARM Refs: https://github.com/ggml-org/llama.cpp/actions/runs/33393906061/job/99493756214?pr=28091 * ggml : attempt to fix use of std::hardware_destructive_inference_size Refs: https://github.com/ggml-org/llama.cpp/actions/runs/33396221677/job/99501265689?pr=28091 * squash! ggml : attempt to fix use of std::hardware_destructive_inference_size Add a version check for GCC 12 to conditionally apply the `-Winterference-size` pragma. * editorconfig : exclude profiling reports dir This directory will not be included in the merge later and this commit can be ignore at that point. Just fixing to keep CI happy. * ggml : skip PCH for gcc on non-x86 architectures * tests : add PCH for peg-parser/tests.h There are 7 peg-parser tests that can share one PCH instead of then each parsing the full tests.h. * common : add PCH for chat.h * docs : update linux build profiling full results Just updating after a number of PCH additions. These are not exact figures and will vary a bit from run to run, but they give a general idea of the performance impact of PCH. * cmake : introduce unity build for models This commit introduces a unity build for the models to improve compilation time. The improvements were roughly the following: ```console +------------------------+-----+------------+------------+------------+ | Build | TUs | Frontend | Backend | Total | +------------------------+-----+------------+------------+------------+ | Full, master | 396 | 811.0 s | 692.2 s | 1,503.2 s | | Full, with PCH | 405 | 380.0 s | 664.7 s | 1,044.7 s | | Full, with PCH + UB | 264 | 357.7 s | 635.7 s | 993.4 s | +------------------------+-----+------------+------------+------------+ TU = Translation Unit. Full = includes Server, Tools, and Tests. PCH = precompiled headers. UB = unity build for models. ``` * docs : update linux profiling table with unitiy build results * docs : update mac profiling results to include unity build [no ci] * docs: remove profiling reports * scripts : merge build profile scripts into one script I was lazy before and just copied the first script to enable Tests, Server, and Tools. This now merges them into a single script. * Revert "editorconfig : exclude profiling reports dir" [no ci] This reverts commit 2922a12118a0730d2f7632bcba265b44a0856c59. * src : rename ggml_view_2d_slice to gemma3n_view_2d_slice This is to be consistent with the rename in gemma4.cpp which was required to avoid a name clash. * cmake : add build profile script for windows [no ci] This commit adds a port of the scripts/build-profile.sh script to windows powershell. This was developed on Windows on ARM but should work on X64 as well but needs to be tested there as well. --- common/CMakeLists.txt | 2 + docs/build-profiling.md | 122 +++++++++++++++++++++++++++ ggml/src/ggml-cpu/CMakeLists.txt | 6 ++ ggml/src/ggml-cpu/ops.h | 8 ++ scripts/build-profile.ps1 | 136 +++++++++++++++++++++++++++++++ scripts/build-profile.sh | 122 +++++++++++++++++++++++++++ src/CMakeLists.txt | 75 +++++++++-------- src/models/gemma3n.cpp | 20 ++--- src/models/gemma4.cpp | 4 +- tests/CMakeLists.txt | 2 + tools/mtmd/CMakeLists.txt | 7 ++ tools/server/CMakeLists.txt | 2 + 12 files changed, 462 insertions(+), 44 deletions(-) create mode 100644 docs/build-profiling.md create mode 100644 scripts/build-profile.ps1 create mode 100755 scripts/build-profile.sh diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 1506bf6479ea..9a43911d3547 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -134,6 +134,8 @@ set_target_properties(${TARGET} PROPERTIES target_include_directories(${TARGET} PUBLIC .) target_link_libraries (${TARGET} PUBLIC vendor::nlohmann vendor::sheredom) target_compile_features (${TARGET} PUBLIC cxx_std_17) +target_precompile_headers (${TARGET} PRIVATE common.h) +target_precompile_headers (${TARGET} PRIVATE chat.h) if (LLAMA_SUBPROCESS) target_compile_definitions(${TARGET} PUBLIC LLAMA_SUBPROCESS) diff --git a/docs/build-profiling.md b/docs/build-profiling.md new file mode 100644 index 000000000000..839e7cca4cd5 --- /dev/null +++ b/docs/build-profiling.md @@ -0,0 +1,122 @@ +## Build profiling +This page is a working document for analyzing the current build and try to +identify ways to improve the build time. + +### Requirements +The profiling script requires clang to be used as the compiler tool chain and +also requires that ClangBuildAnalyzer is installed. + +Mac: +```console +brew install clang-build-analyzer +``` + +Linux: +```console +git clone https://github.com/aras-p/ClangBuildAnalyzer.git +cd ClangBuildAnalyzer +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build -j$(nproc) +sudo cp build/ClangBuildAnalyzer /usr/local/bin/ +``` + +Windows: install LLVM/clang and Ninja (e.g. via the +[LLVM releases page](https://github.com/llvm/llvm-project/releases) and +`winget install Ninja-build.Ninja`), then build ClangBuildAnalyzer the same +way as on Linux: +```console +git clone https://github.com/aras-p/ClangBuildAnalyzer.git +cd ClangBuildAnalyzer +cmake -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release +cmake --build build --config Release +``` +Then add `ClangBuildAnalyzer\build` to `PATH`. + +### Usage +Mac/Linux: +```console +$ ./scripts/build-profile.sh +``` + +Windows: +```console +> .\scripts\build-profile.ps1 +``` + +Both accept `--full`/`-Full` (include Server, Tools, and Tests) and a jobs +override (`-jN` / `-Jobs N`). + +Note: on Windows, `cmake` defaults to the Visual Studio generator, which +ignores `CMAKE_C_COMPILER`/`CMAKE_CXX_COMPILER` and silently falls back to +MSVC. `build-profile.ps1` passes `-G Ninja` so clang is actually used, this +is required on ARM64. + +### Linux (Ubuntu 24.04) + +Environment: +- Clang: 18.1.3 (Ubuntu clang version 18.1.3 (1ubuntu1)) +- libstdc++: GCC 13.3.0 (Ubuntu 13.3.0-6ubuntu2~24.04.1) +- Target: x86_64-pc-linux-gnu + +```console ++------------------------+-----+------------+------------+------------+ +| Build | TUs | Frontend | Backend | Total | ++------------------------+-----+------------+------------+------------+ +| Minimal, master | 249 | 468.2 s | 270.3 s | 738.5 s | +| Minimal, with PCH | 253 | 177.1 s | 265.8 s | 442.9 s | +| Full, master | 396 | 811.0 s | 692.2 s | 1,503.2 s | +| Full, with PCH | 405 | 380.0 s | 664.7 s | 1,044.7 s | +| Full, with PCH + UB | 264 | 357.7 s | 635.7 s | 993.4 s | ++------------------------+-----+------------+------------+------------+ + +PCH = precompiled header. +Full = includes building Server, Tools, and Tests. +UB = unity build for models +``` +Note that the number of translation units (TUs) increases when using precompiled +headers — each PCH target adds one extra TU for the precompilation step itself. + +### Mac (Apple M3) + +Environment: +- Clang: Apple clang version 17.0.0 (clang-1700.3.19.1) +- libc++: ships with Apple clang 17.0.0 (Xcode toolchain) +- Target: arm64-apple-macosx15.6 + +```console ++------------------------+-----+------------+------------+------------+ +| Build | TUs | Frontend | Backend | Total | ++------------------------+-----+------------+------------+------------+ +| Minimal, master | 256 | 154.5 s | 94.8 s | 249.3 s | +| Minimal, with PCH | 261 | 65.9 s | 90.0 s | 155.9 s | +| Full, master | 407 | 265.7 s | 209.7 s | 475.4 s | +| Full, with PCH | 414 | 154.6 s | 197.5 s | 352.1 s | +| Full, with PCH + UB | 274 | 143.0 s | 192.2 s | 335.2 s | ++------------------------+-----+------------+------------+------------+ + +PCH = precompiled header. +Full = includes building Server, Tools, and Tests. +UB = unity build for models +``` + +### Windows (ARM64) + +Environment: +- Clang: clang version 22.1.8 (LLVM, `C:\Program Files\LLVM`) +- STL: MSVC STL (Visual Studio 2022 Build Tools 14.44.35207) +- Target: aarch64-pc-windows-msvc + +```console ++------------------------+-----+------------+------------+------------+ +| Build | TUs | Frontend | Backend | Total | ++------------------------+-----+------------+------------+------------+ +| Minimal, master | 249 | 159.4 s | 82.2 s | 241.6 s | +| Full, master | 373 | 337.2 s | 167.4 s | 504.6 s | +| Minimal, with PCH + UB | 113 | 62.3 s | 82.4 s | 144.7 s | +| Full, with PCH + UB | 240 | 233.0 s | 185.1 s | 418.1 s | ++------------------------+-----+------------+------------+------------+ + +PCH = precompiled header. +Full = includes building Server, Tools, and Tests. +UB = unity build for models +``` diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 1c7338eea49c..83088e147133 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -675,6 +675,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name) target_compile_options(${GGML_CPU_NAME} PRIVATE ${ARCH_FLAGS}) target_compile_definitions(${GGML_CPU_NAME} PRIVATE ${ARCH_DEFINITIONS}) + if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT GGML_SYSTEM_ARCH STREQUAL "x86") + message(STATUS "Skipping PCH for ${GGML_CPU_NAME}: GCC PCH is only enabled for x86 (arch: ${GGML_SYSTEM_ARCH})") + else() + target_precompile_headers(${GGML_CPU_NAME} PRIVATE ggml-impl.h) + endif() + if (EMSCRIPTEN) set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS "-msimd128") endif() diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index 4c1642a67603..ce2b3e870b91 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -18,7 +18,15 @@ #endif #endif +// -Winterference-size was introduced in GCC 12 +#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winterference-size" +#endif static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float); +#if defined(__cplusplus) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12 +#pragma GCC diagnostic pop +#endif // Work buffer size for im2col operations in CONV2D #define GGML_IM2COL_WORK_SIZE (16 * 1024 * 1024) diff --git a/scripts/build-profile.ps1 b/scripts/build-profile.ps1 new file mode 100644 index 000000000000..410ead39d5cc --- /dev/null +++ b/scripts/build-profile.ps1 @@ -0,0 +1,136 @@ +# Compile-time profiling using clang -ftime-trace + ClangBuildAnalyzer. +# +# Usage: +# .\scripts\build-profile.ps1 [-Full] [-Jobs N] +# +# -Full : include Server, Tools, and Tests (default: minimal build) +# -Jobs : number of parallel jobs (default: all cores) +# +# Requires ClangBuildAnalyzer: +# https://github.com/aras-p/ClangBuildAnalyzer + +param( + [switch]$Full, + [int]$Jobs = [Environment]::ProcessorCount +) + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$RootDir = Split-Path -Parent $ScriptDir + +if ($Full) { + $BuildDir = Join-Path $RootDir "build-profile-full" + $Report = Join-Path $BuildDir "profile-report-full.txt" +} else { + $BuildDir = Join-Path $RootDir "build-profile-baseline" + $Report = Join-Path $BuildDir "profile-report.txt" +} + +$OutputBin = Join-Path $BuildDir "clang_analysis.bin" + +if (-not (Get-Command clang++ -ErrorAction SilentlyContinue)) { + Write-Error "clang++ not found" + exit 1 +} + +if (-not (Get-Command ninja -ErrorAction SilentlyContinue)) { + Write-Error "ninja not found (required so cmake does not fall back to the Visual Studio/MSVC generator)" + exit 1 +} + +if (-not (Get-Command ClangBuildAnalyzer -ErrorAction SilentlyContinue)) { + Write-Error "ClangBuildAnalyzer not found`n https://github.com/aras-p/ClangBuildAnalyzer/releases" + exit 1 +} + +$ClangVer = (clang++ --version | Select-Object -First 1) +Write-Host "compiler : $ClangVer" +Write-Host "build dir: $BuildDir" +Write-Host "output : $OutputBin" +Write-Host "jobs : $Jobs" +Write-Host "" + +if (Get-Command ccache -ErrorAction SilentlyContinue) { + Write-Host "clearing ccache..." + ccache -C -z +} + +$env:CCACHE_DISABLE = "1" + +$TestsFlag = if ($Full) { "ON" } else { "OFF" } +$ToolsFlag = if ($Full) { "ON" } else { "OFF" } +$ServerFlag = if ($Full) { "ON" } else { "OFF" } + +cmake --fresh ` + -S $RootDir ` + -B $BuildDir ` + -G "Ninja" ` + -DCMAKE_BUILD_TYPE=Release ` + -DCMAKE_C_COMPILER=clang ` + -DCMAKE_CXX_COMPILER=clang++ ` + -DCMAKE_C_FLAGS="-ftime-trace" ` + -DCMAKE_CXX_FLAGS="-ftime-trace" ` + -DGGML_CCACHE=OFF ` + -DGGML_OPENMP=ON ` + -DGGML_NATIVE=OFF ` + "-DLLAMA_BUILD_TESTS=$TestsFlag" ` + -DLLAMA_BUILD_EXAMPLES=OFF ` + "-DLLAMA_BUILD_TOOLS=$ToolsFlag" ` + "-DLLAMA_BUILD_SERVER=$ServerFlag" ` + -DLLAMA_BUILD_APP=OFF + +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +$StrayTrace = Join-Path $RootDir "-.json" +if (Test-Path $StrayTrace) { + Remove-Item $StrayTrace -Force +} + +Write-Host "" +Write-Host "Initializing ClangBuildAnalyzer..." +ClangBuildAnalyzer --start $BuildDir +Write-Host "" + +Write-Host "building..." +Write-Host "" + +$StartTime = Get-Date + +cmake --build $BuildDir --clean-first -j $Jobs + +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +$Elapsed = (Get-Date) - $StartTime + +Write-Host "" +Write-Host ("build time: {0}s ({1}m {2}s)" -f [int]$Elapsed.TotalSeconds, [int]$Elapsed.TotalMinutes, $Elapsed.Seconds) +Write-Host "" + +Write-Host "Aggregating profile metrics..." +ClangBuildAnalyzer --stop $BuildDir $OutputBin | Out-Null + +Write-Host "" +Write-Host ("=" * 80) + +$TUs = "?" +if (Test-Path $Report) { + $Match = Select-String -Path $Report -Pattern "Compilation \((\d+)" | Select-Object -First 1 + if ($Match) { $TUs = $Match.Matches[0].Groups[1].Value } +} + +ClangBuildAnalyzer --analyze $OutputBin | Tee-Object -FilePath $Report + +Write-Host "" +Write-Host "translation units: $TUs" +Write-Host "" +Write-Host "largest trace files (top 20 by size):" + +Get-ChildItem -Path $BuildDir -Recurse -Filter "*.json" | + Where-Object { $_.Name -ne "compile_commands.json" } | + Sort-Object Length -Descending | + Select-Object -First 20 | + ForEach-Object { "{0,8:F1} KB {1}" -f ($_.Length / 1024), $_.FullName } + +Write-Host "" +Write-Host "ClangBuildAnalyzer report was generated: $Report" diff --git a/scripts/build-profile.sh b/scripts/build-profile.sh new file mode 100755 index 000000000000..94299498909b --- /dev/null +++ b/scripts/build-profile.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Compile-time profiling using clang -ftime-trace + ClangBuildAnalyzer. +# +# Usage: +# ./scripts/build-profile.sh [--full] [-jN] +# +# --full: include Server, Tools, and Tests (default: minimal build) +# -jN : number of parallel jobs (default: all cores) +# +# Requires ClangBuildAnalyzer: +# macOS: brew install clang-build-analyzer +# Linux: https://github.com/aras-p/ClangBuildAnalyzer.git + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +FULL=0 +JOBS="-j$(nproc 2>/dev/null || sysctl -n hw.ncpu)" + +for arg in "$@"; do + case "${arg}" in + --full) FULL=1 ;; + -j*) JOBS="${arg}" ;; + *) echo "error: unknown argument: ${arg}" >&2; exit 1 ;; + esac +done + +if [ "${FULL}" -eq 1 ]; then + BUILD_DIR="${ROOT_DIR}/build-profile-full" + REPORT="${BUILD_DIR}/profile-report-full.txt" +else + BUILD_DIR="${ROOT_DIR}/build-profile-baseline" + REPORT="${BUILD_DIR}/profile-report.txt" +fi + +OUTPUT_BIN="${BUILD_DIR}/clang_analysis.bin" + +if ! command -v clang++ &>/dev/null; then + echo "error: clang++ not found" >&2 + exit 1 +fi + +if ! command -v ClangBuildAnalyzer &>/dev/null; then + echo "error: ClangBuildAnalyzer not found" >&2 + echo " brew install clangbuildanalyzer (macOS)" >&2 + echo " or: https://github.com/aras-p/ClangBuildAnalyzer/releases" >&2 + exit 1 +fi + +CLANG_VER=$(clang++ --version | head -1) +echo "compiler : ${CLANG_VER}" +echo "build dir: ${BUILD_DIR}" +echo "output : ${OUTPUT_BIN}" +echo "jobs : ${JOBS}" +echo + +if command -v ccache &>/dev/null; then + echo "clearing ccache..." + ccache -C -z +fi + +export CCACHE_DISABLE=1 + +cmake --fresh \ + -S "${ROOT_DIR}" \ + -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_FLAGS="-ftime-trace" \ + -DCMAKE_CXX_FLAGS="-ftime-trace" \ + -DGGML_CCACHE=OFF \ + -DGGML_OPENMP=ON \ + -DGGML_NATIVE=OFF \ + -DLLAMA_BUILD_TESTS=$([ "${FULL}" -eq 1 ] && echo ON || echo OFF) \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=$([ "${FULL}" -eq 1 ] && echo ON || echo OFF) \ + -DLLAMA_BUILD_SERVER=$([ "${FULL}" -eq 1 ] && echo ON || echo OFF) \ + -DLLAMA_BUILD_APP=OFF + +echo + +echo "Initializing ClangBuildAnalyzer..." +ClangBuildAnalyzer --start "${BUILD_DIR}" +echo + +echo "building..." +echo + +START=$(date +%s) + +cmake --build "${BUILD_DIR}" --clean-first "${JOBS}" + +END=$(date +%s) +ELAPSED=$((END - START)) + +echo +printf "build time: %ds (%dm %ds)\n" "${ELAPSED}" "$((ELAPSED / 60))" "$((ELAPSED % 60))" +echo + +echo "Aggregating profile metrics..." +ClangBuildAnalyzer --stop "${BUILD_DIR}" "${OUTPUT_BIN}" > /dev/null + +echo +echo "================================================================================" +TUS=$(grep -oP "Compilation \(\K[0-9]+" "${REPORT}" 2>/dev/null || echo "?") +ClangBuildAnalyzer --analyze "${OUTPUT_BIN}" | tee "${REPORT}" + +echo +echo "translation units: ${TUS}" +echo +echo "largest trace files (top 20 by size):" +find "${BUILD_DIR}" -name "*.json" ! -name "compile_commands.json" \ + | xargs ls -l 2>/dev/null \ + | awk 'NF>5 {print $5, $NF}' \ + | sort -rn \ + | awk 'NR<=20 {printf "%8.1f KB %s\n", $1/1024, $2}' + +echo +echo "ClangBuildAnalyzer report was generated: ${REPORT}" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 221e14f7ff23..bc922b6a7bd6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,40 +8,44 @@ llama_add_compile_flags() file(GLOB LLAMA_MODELS_SOURCES "models/*.cpp") +set(LLAMA_CORE_SOURCES + llama.cpp + llama-adapter.cpp + llama-arch.cpp + llama-batch.cpp + llama-chat.cpp + llama-context.cpp + llama-cparams.cpp + llama-grammar.cpp + llama-graph.cpp + llama-hparams.cpp + llama-impl.cpp + llama-io.cpp + llama-kv-cache.cpp + llama-kv-cache-iswa.cpp + llama-kv-cache-dsa.cpp + llama-kv-cache-dsa-iswa.cpp + llama-kv-cache-msa.cpp + llama-kv-cache-dsv4.cpp + llama-memory.cpp + llama-memory-hybrid.cpp + llama-memory-hybrid-iswa.cpp + llama-memory-hybrid-idx.cpp + llama-memory-recurrent.cpp + llama-mmap.cpp + llama-model-loader.cpp + llama-model-saver.cpp + llama-model.cpp + llama-quant.cpp + llama-sampler.cpp + llama-vocab.cpp + unicode-data.cpp + unicode.cpp +) + add_library(llama ../include/llama.h - llama.cpp - llama-adapter.cpp - llama-arch.cpp - llama-batch.cpp - llama-chat.cpp - llama-context.cpp - llama-cparams.cpp - llama-grammar.cpp - llama-graph.cpp - llama-hparams.cpp - llama-impl.cpp - llama-io.cpp - llama-kv-cache.cpp - llama-kv-cache-iswa.cpp - llama-kv-cache-dsa.cpp - llama-kv-cache-dsa-iswa.cpp - llama-kv-cache-msa.cpp - llama-kv-cache-dsv4.cpp - llama-memory.cpp - llama-memory-hybrid.cpp - llama-memory-hybrid-iswa.cpp - llama-memory-hybrid-idx.cpp - llama-memory-recurrent.cpp - llama-mmap.cpp - llama-model-loader.cpp - llama-model-saver.cpp - llama-model.cpp - llama-quant.cpp - llama-sampler.cpp - llama-vocab.cpp - unicode-data.cpp - unicode.cpp + ${LLAMA_CORE_SOURCES} unicode.h ${LLAMA_MODELS_SOURCES} ) @@ -50,13 +54,20 @@ set_target_properties(llama PROPERTIES VERSION ${LLAMA_VERSION_BASE} SOVERSION ${LLAMA_VERSION_MAJOR} MACHO_CURRENT_VERSION 0 # keep macOS linker from seeing oversized version number + UNITY_BUILD ON + UNITY_BUILD_BATCH_SIZE 16 ) +# exclude non-model sources from unity build +set_source_files_properties(${LLAMA_CORE_SOURCES} ../include/llama.h unicode.h + PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) + configure_file(llama-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/llama-version.h @ONLY) target_include_directories(llama PRIVATE . ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(llama PUBLIC ../include) target_compile_features (llama PRIVATE cxx_std_17) # don't bump +target_precompile_headers (llama PRIVATE models/models.h) target_link_libraries(llama PUBLIC ggml) diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index ea616db3ba3c..bb628203aaaf 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -82,7 +82,7 @@ std::unique_ptr llama_model_gemma3n::build_arch_graph(const l } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim -static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { +static ggml_tensor * gemma3n_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { GGML_ASSERT(idx < (int) x->ne[2]); return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); @@ -139,7 +139,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * predictions = altup_predict(cur, il); // [n_embd, n_tokens, n_altup] // predicted value will go through self-attention and laurel - ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens] + ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act); // [n_embd, n_tokens] cur = active_prediction; cb(cur, "active_prediction", il); @@ -236,13 +236,13 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * first_prediction; // [n_embd, n_tokens] { - first_prediction = ggml_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens] + first_prediction = gemma3n_view_2d_slice(ctx0, corrected, i_altup_act); // [n_embd, n_tokens] first_prediction = ggml_mul(ctx0, first_prediction, model.layers[il].altup_correct_scale); first_prediction = build_lora_mm(model.layers[il].per_layer_inp_gate, first_prediction); first_prediction = ggml_gelu(ctx0, first_prediction); // [n_embd_altup, n_tokens] cb(first_prediction, "first_prediction_gated", il); - ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens] + ggml_tensor * inp_this_layer = gemma3n_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_altup, n_tokens] first_prediction = ggml_mul(ctx0, first_prediction, inp_this_layer); // [n_embd_altup, n_tokens] cb(first_prediction, "first_prediction_scaled", il); @@ -253,7 +253,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par } // equivalent to python code: corrected_predictions[1:] += first_prediction { - ggml_tensor * slice_first = ggml_view_2d_slice(ctx0, corrected, 0); + ggml_tensor * slice_first = gemma3n_view_2d_slice(ctx0, corrected, 0); ggml_tensor * slice_rest = ggml_view_3d( ctx0, corrected, n_embd, n_tokens, n_altup - 1, ggml_row_size(corrected->type, n_embd), ggml_row_size(corrected->type, n_embd * n_tokens), n_embd * n_tokens * ggml_element_size(corrected)); @@ -271,7 +271,7 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par // cur now has multiple altup(s), we want to merge them back to 1 altup { - ggml_tensor * target_magnitude = calc_magnitude(ggml_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens] + ggml_tensor * target_magnitude = calc_magnitude(gemma3n_view_2d_slice(ctx0, cur, i_altup_act)); // [n_embd, n_tokens] // do a view to skip the first slice (active altup) ggml_tensor * alt_slice = ggml_view_3d(ctx0, cur, n_embd, n_tokens, n_altup - 1, ggml_row_size(cur->type, n_embd), @@ -283,9 +283,9 @@ llama_model_gemma3n::graph::graph(const llama_model & model, const llm_graph_par cb(altup_unembd, "altup_unembd", -1); // equivalent to torch.mean(hidden_states, dim=0) - cur = ggml_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens] + cur = gemma3n_view_2d_slice(ctx0, cur, 0); // [n_embd, n_tokens] for (int i = 0; i < n_altup - 1; ++i) { - cur = ggml_add(ctx0, cur, ggml_view_2d_slice(ctx0, altup_unembd, i)); + cur = ggml_add(ctx0, cur, gemma3n_view_2d_slice(ctx0, altup_unembd, i)); } cur = ggml_scale(ctx0, cur, 1.0f / float(n_altup)); // [n_embd, n_tokens] cb(cur, "unembd_merged", -1); @@ -419,7 +419,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_compute_router_modalities(ggml_t // input cur shape: [n_embd, n_tokens, n_altup] // output shape: [n_embd, n_tokens, n_altup] ggml_tensor * llama_model_gemma3n::graph::altup_predict(ggml_tensor * cur, int il) { - ggml_tensor * activated = ggml_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens] + ggml_tensor * activated = gemma3n_view_2d_slice(ctx0, cur, i_altup_act); // [n_embd, n_tokens] ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens] cb(modalities, "modalities", il); @@ -447,7 +447,7 @@ ggml_tensor * llama_model_gemma3n::graph::altup_correct(ggml_tensor * prediction ggml_tensor * modalities = altup_compute_router_modalities(activated, il); // [n_altup, n_tokens] cb(modalities, "modalities", il); - ggml_tensor * active_prediction = ggml_view_2d_slice(ctx0, predictions, i_altup_act); + ggml_tensor * active_prediction = gemma3n_view_2d_slice(ctx0, predictions, i_altup_act); ggml_tensor * innovation = ggml_sub(ctx0, activated, active_prediction); // [n_embd, n_tokens] cb(innovation, "innovation", il); diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index 388126e26a6f..39e899aa6e9f 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -145,7 +145,7 @@ std::unique_ptr llama_model_gemma4::build_arch_graph(const ll } // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim -static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { +static ggml_tensor * gemma4_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { GGML_ASSERT(idx < (int) x->ne[2]); return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); @@ -372,7 +372,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_tokens] cur = ggml_gelu(ctx0, cur); - ggml_tensor * inp_this_layer = ggml_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] + ggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] // TODO @ngxson : improve this if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 920c58c738e4..0c4e4d5a9b52 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -278,6 +278,8 @@ llama_build_and_test( peg-parser/test-unicode.cpp peg-parser/tests.h ) +target_precompile_headers(test-peg-parser PRIVATE peg-parser/tests.h) + if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") set(MODEL_NAME "tinyllamas/stories15M-q4_0.gguf") diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 907468e87ec7..176eb1505740 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -84,6 +84,13 @@ target_link_libraries (mtmd PUBLIC ggml llama) target_link_libraries (mtmd PRIVATE Threads::Threads vendor::hash vendor::miniaudio vendor::stb vendor::sheredom) target_include_directories(mtmd PUBLIC .) target_compile_features (mtmd PRIVATE cxx_std_17) +target_precompile_headers (mtmd PRIVATE models/models.h) + +set_source_files_properties( + mtmd-helper.cpp + mtmd-helper-gen.cpp + PROPERTIES SKIP_PRECOMPILE_HEADERS ON +) if (MTMD_VIDEO) target_compile_definitions(mtmd PRIVATE MTMD_VIDEO) diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt index 43c2456333ec..f02a2ba3b17f 100644 --- a/tools/server/CMakeLists.txt +++ b/tools/server/CMakeLists.txt @@ -32,6 +32,7 @@ endif() target_include_directories(${TARGET} PRIVATE ../mtmd) target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT}) +target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) # llama-server-impl: server logic, reusable by app @@ -49,6 +50,7 @@ set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(${TARGET} PRIVATE ../mtmd ${CMAKE_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC server-context llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT}) +target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) add_dependencies(${TARGET} llama-ui-assets) From 5bda51bfbc62e64193221e639f6ad4e08767d760 Mon Sep 17 00:00:00 2001 From: Foad Abo Dahood <32059146+masterFoad@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:12:55 +0300 Subject: [PATCH 40/65] metal : skip the empty half of the mul_mm_id token tile (#28301) kernel_mul_mm_id splits its NR1 = 32 token tile into two 16-row halves and skips the upper half when the expert did not fill it, on both the tensor and simdgroup paths. The tB extents are corrected to (NK, NR1H) for the [NR1][NK] row-major tile. The B tile is staged unconditionally, as on master: rows past nr1 restage a clamped duplicate of a valid row, lie in the output-row dimension so they never contribute to a valid row, and are dropped by the final store loop. test-backend-ops: re-draw the expert ids between perf iterations of test_mul_mat_id so MoE perf numbers are not warm-cache, and add token-tile boundary coverage using n_used == n_mats, which routes every token to every expert so each expert receives exactly n rows; n = 32, 33, 47, 48, 49 reach mul_mm_id and leave a last tile of 32, 1, 15, 16 and 17 rows. --- ggml/src/ggml-metal/kernels/mul_mm.metal | 89 ++++++++++++++++-------- tests/test-backend-ops.cpp | 55 +++++++++++---- 2 files changed, 102 insertions(+), 42 deletions(-) diff --git a/ggml/src/ggml-metal/kernels/mul_mm.metal b/ggml/src/ggml-metal/kernels/mul_mm.metal index ee848eed6d6a..0a45bb1bbe57 100644 --- a/ggml/src/ggml-metal/kernels/mul_mm.metal +++ b/ggml/src/ggml-metal/kernels/mul_mm.metal @@ -496,6 +496,13 @@ kernel void kernel_mul_mm_id( + args.nb11*i11 + args.nb10*iy); + // skip the upper half of the token tile when the expert did not fill it + constexpr short NR1H = NR1/2; + + const bool has_hi = nr1 > NR1H; + + const short lb1 = (short) tiitg/NL1; // 0 .. NR1-1, this thread's row of the B tile + #ifndef GGML_METAL_HAS_TENSOR S0_8x8 ma[4]; S1_8x8 mb[2]; @@ -505,15 +512,22 @@ kernel void kernel_mul_mm_id( for (short i = 0; i < 8; i++){ mc[i] = make_filled_simdgroup_matrix(0.f); } + + // simdgroups 2,3 own rows NR1H..NR1-1 + const bool sg_active = has_hi || sgitg < 2; #else - auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0)); - auto tB = tensor, tensor_inline>(sb, dextents(NR1, NK )); + auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0)); + + // sb is [NR1][NK] row-major + auto tB0 = tensor, tensor_inline>(sb, dextents(NK, NR1H)); + auto tB1 = tensor, tensor_inline>(sb + NR1H*NK, dextents(NK, NR1H)); mpp::tensor_ops::matmul2d< - mpp::tensor_ops::matmul2d_descriptor(NR1, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), + mpp::tensor_ops::matmul2d_descriptor(NR1H, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), execution_simdgroups<4>> mm; - auto cT = mm.get_destination_cooperative_tensor(); + auto cT0 = mm.get_destination_cooperative_tensor(); + auto cT1 = mm.get_destination_cooperative_tensor(); #endif for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { @@ -656,37 +670,45 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); #ifndef GGML_METAL_HAS_TENSOR - // load matrices from threadgroup memory and conduct outer products - threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); - threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); + if (sg_active) { + // load matrices from threadgroup memory and conduct outer products + threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); + threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); - FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { - simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 4; i++) { - simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); - } + FOR_UNROLL (short i = 0; i < 4; i++) { + simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + } - simdgroup_barrier(mem_flags::mem_none); + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 2; i++) { - simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); - } + FOR_UNROLL (short i = 0; i < 2; i++) { + simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); + } - simdgroup_barrier(mem_flags::mem_none); + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 8; i++){ - simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); - } + FOR_UNROLL (short i = 0; i < 8; i++){ + simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); + } - lsma += 8*64; - lsmb += 4*64; + lsma += 8*64; + lsmb += 4*64; + } } #else - auto sA = tA.slice(0, 0); - auto sB = tB.slice(0, 0); + auto sA = tA.slice(0, 0); + auto sB0 = tB0.slice(0, 0); - mm.run(sB, sA, cT); + mm.run(sB0, sA, cT0); + + if (has_hi) { + auto sB1 = tB1.slice(0, 0); + + mm.run(sB1, sA, cT1); + } #endif } @@ -694,13 +716,20 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); #ifdef GGML_METAL_HAS_TENSOR - auto tC = tensor, tensor_inline>(sc, dextents(NR0, NR1)); - cT.store(tC); + auto tC0 = tensor, tensor_inline>(sc, dextents(NR0, NR1H)); + cT0.store(tC0); + + if (has_hi) { + auto tC1 = tensor, tensor_inline>(sc + NR1H*NR0, dextents(NR0, NR1H)); + cT1.store(tC1); + } #else - threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + if (sg_active) { + threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + } } #endif diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 15c42a10815f..b63b3773eef8 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1219,6 +1219,11 @@ struct test_case { } } + // re-draw data-dependent inputs between timed perf iterations + virtual void reinit_perf_iter(ggml_context * ctx) { + GGML_UNUSED(ctx); + } + virtual size_t op_size(ggml_tensor * t) { size_t size = ggml_nbytes(t); // add source tensors @@ -1653,6 +1658,9 @@ struct test_case { total_time_us += end_time - start_time; total_mem += mem; total_runs += n_runs; + + // re-draw any data-dependent inputs (expert ids) outside the timed region + reinit_perf_iter(ctx.get()); } while (total_time_us < 1000*1000); // run for at least 1 second // Create test result @@ -5000,25 +5008,31 @@ struct test_mul_mat_hadamard : public test_mul_mat { } }; -static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { +static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats) { std::random_device rd; std::default_random_engine rng(rd()); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { - if (t->type == GGML_TYPE_I32) { - if (ggml_is_view_op(t->op)) { continue; } - // ids - for (int64_t r = 0; r < ggml_nrows(t); r++) { - std::vector data(t->ne[0]); - for (int i = 0; i < t->ne[0]; i++) { - data[i] = i % n_mats; - } - std::shuffle(data.begin(), data.end(), rng); - ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + if (t->type != GGML_TYPE_I32 || ggml_is_view_op(t->op)) { + continue; + } + for (int64_t r = 0; r < ggml_nrows(t); r++) { + std::vector data(t->ne[0]); + for (int i = 0; i < t->ne[0]; i++) { + data[i] = i % n_mats; } - } else { + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); + } + } +} + +static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->type != GGML_TYPE_I32) { init_tensor_uniform(t); } } + init_mul_mat_id_ids(ctx, n_mats); } // GGML_OP_MUL_MAT_ID @@ -5085,6 +5099,10 @@ struct test_mul_mat_id : public test_case { void initialize_tensors(ggml_context * ctx) override { init_mul_mat_id_tensors(ctx, n_mats); } + + void reinit_perf_iter(ggml_context * ctx) override { + init_mul_mat_id_ids(ctx, n_mats); + } }; // GGML_OP_MUL_MAT_ID + GGML_OP_ADD or GGML_OP_MUL @@ -9890,6 +9908,19 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 1, 3, 2})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, 256, {2, 3}, {1, 1}, {0, 3, 2, 1})); + // token-tile boundary coverage. With n_used == n_mats every token routes to every expert, so + // each expert receives exactly n rows, with no dependence on the random draw. mul_mm_id is used + // from 32 tokens up: n = 32, 33, 47, 48, 49 reach it, leaving a last tile of 32, 1, 15, 16 and + // 17 rows - 16 and 17 straddle the point where the upper half stops being skipped. The smaller + // n cover the same row counts on the mat-vec path. + for (ggml_type type_a : {GGML_TYPE_Q4_K, GGML_TYPE_IQ2_XS, GGML_TYPE_F16}) { + for (int n : {1, 15, 16, 17, 31, 32, 33, 47, 48, 49}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 4, false, 512, n, 256)); + } + // experts that receive no rows at all + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 8, 1, false, 512, 1, 256)); + } + for (ggml_type type_a : other_types) { for (ggml_type type_b : {GGML_TYPE_F32}) { if (ggml_blck_size(type_a) != 256) { From 43f3dda6237a453a587a8f00230d52decfeaa8e5 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Fri, 11 Sep 2026 21:17:08 +0800 Subject: [PATCH 41/65] ggml: skip 0-sized ids tensor when offloading selected experts (#28739) --- ggml/src/ggml-backend.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 40e50c5c9dbd..6faa680474c4 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1705,6 +1705,10 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s ggml_tensor * ids_tensor = node->src[2]; ggml_backend_t ids_backend = split_backend; + if (ggml_nelements(ids_tensor) == 0) { + continue; + } + // if the ids tensor is also an input of the split, it may not have been copied yet to the split backend // in that case, we use the original ids tensor for (int i = input_id + 1; i < split->n_inputs; i++) { From 8172e6577ac2b35de1ec1e5d1c0aaad6c4a2129f Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 11 Sep 2026 15:50:12 +0200 Subject: [PATCH 42/65] tests: tolerate a shared pool abort in test_completion_unified (#28759) The expected success table holds when the four requests enter the shared pool together. On a loaded runner they are admitted tens of milliseconds apart, the slot lifetimes overlap differently and the pool overflows while a short request is still resident. The decode failure aborts every slot, so a request the table marks as successful comes back with the context error instead of its generation. Such a request now passes on that error too, while any other status, a different error or a truncated generation still fails the test. --- tools/server/tests/unit/test_completion.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/server/tests/unit/test_completion.py b/tools/server/tests/unit/test_completion.py index 9375e0110e53..01732eb16313 100644 --- a/tools/server/tests/unit/test_completion.py +++ b/tools/server/tests/unit/test_completion.py @@ -394,7 +394,12 @@ def test_completion_unified(n_ctx, n_slots, n_predict_vals, expected_success): results = parallel_function_calls(tasks) for res, n_predict, expect_ok in zip(results, n_predict_vals, expected_success): if expect_ok: - assert res.status_code == 200 + # the pool is aborted as a whole, so a request that fits on its own + # is still dropped when the slots overlap, and it says so explicitly + assert res.status_code == 200 or ( + res.status_code == 500 + and "context size has been exceeded" in res.body["error"]["message"].lower() + ) # note: https://github.com/ggml-org/llama.cpp/pull/18700#issuecomment-3728695581 if res.status_code == 200: From 982937a3337f7e97ef08fd5603f4157575ece7e1 Mon Sep 17 00:00:00 2001 From: Rohanjames1997 Date: Fri, 11 Sep 2026 13:19:37 -0500 Subject: [PATCH 43/65] tests: extend test-quantize-fns to test nrc=2 (i8mm) kernels (#16234) * Test for nrc=2 as well | i8mm kernels * Trigger only on supported HW * Remove trailing whitespace * Address review comment * test: properly prepare nrc=2 inputs with independent data per row * tests : make nrc=2 dot product inputs distinct Assisted-by: Kiro * tests : use non-trivial strides in nrc=2 dot product test * tests : fail nrc=2 dot product test on non-finite errors --- tests/test-quantize-fns.cpp | 69 ++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 9510ac14ce00..570fca89a876 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -5,6 +5,8 @@ #undef NDEBUG #include +#include +#include #include #include #include @@ -32,9 +34,9 @@ static const char* RESULT_STR[] = {"ok", "FAILED"}; // Generate synthetic data -static void generate_data(float offset, size_t n, float * dst) { +static void generate_data(float offset, size_t n, float * dst, float amplitude = 2.0f) { for (size_t i = 0; i < n; i++) { - dst[i] = 0.1 + 2*cosf(i + offset); + dst[i] = 0.1 + amplitude*cosf(i + offset); } } @@ -83,23 +85,50 @@ static float dot_product(const float * a1, const float * a2, size_t test_size) { } // Total dot product error -static float dot_product_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data1, const float * test_data2) { - GGML_UNUSED(qfns); - - std::vector tmp_q1(2*test_size); - std::vector tmp_q2(2*test_size); - +static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type src0_type, size_t test_size, + const float * test_data1, const float * test_data2, + const float * test_data3, const float * test_data4, + const int nrc) { const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type); + const size_t pad = 64; + const size_t bx = ggml_row_size(src0_type, test_size) + pad; + const size_t by = ggml_row_size(qfns_cpu->vec_dot_type, test_size) + pad; + + std::vector tmp_q1(bx * nrc); + std::vector tmp_q2(by * nrc); qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size); vdot->from_float(test_data2, tmp_q2.data(), test_size); - float result = INFINITY; - qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); + if (nrc == 1) { + float result = INFINITY; + qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); + + const float dot_ref = dot_product(test_data1, test_data2, test_size); + return fabsf(result - dot_ref) / test_size; + } + + // nrc == 2: kernel computes a 2x2 dot product matrix + // Output layout: s[0]=dot(vx0,vy0), s[1]=dot(vx1,vy0), s[bs]=dot(vx0,vy1), s[bs+1]=dot(vx1,vy1) + // row and output strides are padded, same as in the mul_mat path + qfns_cpu->from_float(test_data3, tmp_q1.data() + bx, test_size); + vdot->from_float(test_data4, tmp_q2.data() + by, test_size); + + const size_t bs = 16; + std::vector result(bs + 2, INFINITY); + qfns_cpu->vec_dot(test_size, result.data(), bs, tmp_q1.data(), bx, tmp_q2.data(), by, 2); + + const float ref00 = dot_product(test_data1, test_data2, test_size); + const float ref10 = dot_product(test_data3, test_data2, test_size); + const float ref01 = dot_product(test_data1, test_data4, test_size); + const float ref11 = dot_product(test_data3, test_data4, test_size); - const float dot_ref = dot_product(test_data1, test_data2, test_size); + const auto err = [test_size](float val, float ref) { + const float e = fabsf(val - ref) / test_size; + return std::isfinite(e) ? e : INFINITY; + }; - return fabsf(result - dot_ref) / test_size; + return std::max({err(result[0], ref00), err(result[1], ref10), err(result[bs], ref01), err(result[bs + 1], ref11)}); } static int test_vec_dot_f32(bool verbose) { @@ -133,9 +162,13 @@ static int test_vec_dot_q(bool verbose) { std::vector test_data(test_size); std::vector test_data2(test_size); + std::vector test_data3(test_size); + std::vector test_data4(test_size); generate_data(0.0, test_data.size(), test_data.data()); generate_data(1.0, test_data2.size(), test_data2.data()); + generate_data(3.0, test_data3.size(), test_data3.data(), 1.0f); + generate_data(4.0, test_data4.size(), test_data4.data(), 1.5f); for (int i = 0; i < GGML_TYPE_COUNT; i++) { ggml_type type = (ggml_type) i; @@ -178,7 +211,7 @@ static int test_vec_dot_q(bool verbose) { printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error); } - const float vec_dot_error = dot_product_error(qfns, qfns_cpu, test_size, test_data.data(), test_data2.data()); + const float vec_dot_error = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), nullptr, nullptr, 1); const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S ? MAX_DOT_PRODUCT_ERROR_LOWBIT @@ -194,6 +227,16 @@ static int test_vec_dot_q(bool verbose) { if (failed || verbose) { printf("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error); } + + // Test nrc=2 path for types that support it + if (qfns_cpu->nrows == 2) { + const float vec_dot_error_nrc2 = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), test_data3.data(), test_data4.data(), 2); + failed = !(vec_dot_error_nrc2 < max_allowed_error); + num_failed += failed; + if (failed || verbose) { + printf("%5s dot product error (nrc=2): %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2); + } + } } } From b78a39a2f93b13a79a3e01aff3f14274efb43afc Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 11 Sep 2026 22:00:57 +0300 Subject: [PATCH 44/65] ci : run test-backend-ops as a dedicated ci/run.sh test (#28740) * ci : run test-backend-ops as a dedicated gg test Run test-backend-ops as a separate gg test in ci/run.sh so it is executed outside ctest. With GG_BUILD_HIGH_PERF it keeps the existing CPU-only invocation (-b CPU); otherwise it runs all available backends without a backend filter. Remove the dedicated backend-ops workflow and keep test-backend-ops as a built target that is not registered with ctest to avoid duplicate runs. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * ci : run test-backend-ops earlier and enable high-perf on kleidiai Move the test-backend-ops gg test before test-llama-archs. Enable GG_BUILD_HIGH_PERF and LLAMA_ARG_THREADS on the Graviton4 KleidiAI job and use the standard self-hosted results/mnt paths. Add TODO markers for decoupling tests from libllama. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * ci : run test-backend-ops in parallel Pass -j $(nproc) to test-backend-ops in both high-perf and all-backend modes. Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp * ci : disable parallel tests for ROCm * cont : disable parallel tests with MoltenVK --- .github/workflows/build-openvino.yml | 2 +- .github/workflows/build-self-hosted.yml | 10 +++++-- .github/workflows/build-vulkan.yml | 4 +-- .github/workflows/build-webgpu.yml | 4 +-- ci/run.sh | 38 ++++++++++++++++++------- tests/CMakeLists.txt | 14 +++------ 6 files changed, 42 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml index 8879a6af16fa..86aba456ce39 100644 --- a/.github/workflows/build-openvino.yml +++ b/.github/workflows/build-openvino.yml @@ -33,7 +33,7 @@ env: LLAMA_ARG_LOG_PREFIX: 1 LLAMA_ARG_LOG_TIMESTAMPS: 1 # TODO: fix failing tests on OpenVINO backend - CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state" + CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state" jobs: ubuntu-24-openvino: diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index fda4879e2149..02a38466fa76 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -395,7 +395,11 @@ jobs: - name: Test id: ggml-ci run: | - LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp + LLAMA_ARG_THREADS=$(nproc) \ + GG_BUILD_HIGH_PERF=1 \ + GG_BUILD_NO_BF16=1 \ + GG_BUILD_EXTRA_TESTS_0=1 \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp cpu-arm64-graviton4-kleidiai: runs-on: ah-ubuntu_22_04-c8g_8x @@ -434,6 +438,8 @@ jobs: - name: Test id: ggml-ci run: | + LLAMA_ARG_THREADS=$(nproc) \ GG_BUILD_KLEIDIAI=1 \ GG_BUILD_EXTRA_TESTS_0=1 \ - bash ./ci/run.sh ./tmp/results ./tmp/mnt + GG_BUILD_HIGH_PERF=1 \ + bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml index 9de52e990a67..21d2a773531f 100644 --- a/.github/workflows/build-vulkan.yml +++ b/.github/workflows/build-vulkan.yml @@ -164,9 +164,7 @@ jobs: export GGML_VK_VISIBLE_DEVICES=0 export GGML_VK_DISABLE_F16=1 export GGML_VK_DISABLE_COOPMAT=1 - # This is using llvmpipe and runs slower than other backends - # test-backend-ops is too slow on llvmpipe, skip it - ctest -L main -E test-backend-ops --verbose --timeout 900 + ctest -L main --verbose --timeout 900 windows: runs-on: windows-2025 diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml index e624e3ba8016..8277abcc47c3 100644 --- a/.github/workflows/build-webgpu.yml +++ b/.github/workflows/build-webgpu.yml @@ -190,6 +190,4 @@ jobs: id: cmake_test run: | cd build - # This is using llvmpipe and runs slower than other backends - # test-backend-ops is too slow on llvmpipe, skip it - ctest -L main -E test-backend-ops --verbose --timeout 900 + ctest -L main --verbose --timeout 900 diff --git a/ci/run.sh b/ci/run.sh index 294cbe57bb42..1ceb19fd50aa 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -190,7 +190,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON" # TODO: fix failing tests on OpenVINO backend - CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state" + CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state" fi ## helpers @@ -250,7 +250,7 @@ function gg_run_ctest_debug { (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log (time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log - (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-backend-ops|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log + (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log set +e } @@ -768,25 +768,43 @@ function gg_check_build_requirements { fi } -function gg_run_test_backend_ops_cpu { +function gg_run_test_backend_ops { cd ${SRC} cd build-ci-release set -e - (time ./bin/test-backend-ops -b CPU ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops-cpu.log + local args_extra="-j $(nproc)" + + # TODO: fix multi-threaded for ROCm + # https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865 + if [ ! -z ${GG_BUILD_ROCM} ]; then + args_extra="" + fi + + # TODO: MoltenVK bug? + # https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897 + if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then + args_extra="" + fi + + if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then + (time ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log + else + (time ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log + fi set +e } -function gg_sum_test_backend_ops_cpu { +function gg_sum_test_backend_ops { gg_printf '### %s\n\n' "${ci}" - gg_printf 'Runs test-backend-ops for CPU backend\n' + gg_printf 'Runs test-backend-ops\n' gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" gg_printf '```\n' - gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops-cpu.log)" + gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops.log)" gg_printf '```\n' gg_printf '\n' } @@ -819,13 +837,11 @@ ret=0 test $ret -eq 0 && gg_run ctest_debug test $ret -eq 0 && gg_run ctest_release +test $ret -eq 0 && gg_run test_backend_ops + test $ret -eq 0 && gg_run test_llama_archs_models test $ret -eq 0 && gg_run test_llama_archs_tensor_split -if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then - test $ret -eq 0 && gg_run test_backend_ops_cpu -fi - if [ -z ${GG_BUILD_LOW_PERF} ]; then test $ret -eq 0 && gg_run embd_bge_small test $ret -eq 0 && gg_run rerank_tiny diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0c4e4d5a9b52..b3559a173f1e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,7 +10,7 @@ function(llama_build source) endif() add_executable(${TEST_TARGET} ${TEST_SOURCES}) - target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common) + target_link_libraries(${TEST_TARGET} PRIVATE llama llama-common) # TODO: [TAG_TESTS_LLAMA_LINK] if (LLAMA_TESTS_INSTALL) install(TARGETS ${TEST_TARGET} RUNTIME) endif() @@ -310,15 +310,9 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC) # TODO: repair known memory leaks llama_build_and_test(test-opt.cpp) endif() -llama_build_and_test(test-backend-ops.cpp) - -# the tensor API kernels come from a separate metallib - check they produce correct results -# ref: https://github.com/ggml-org/llama.cpp/issues/27473 -if (GGML_METAL AND NOT GGML_METAL_EMBED_LIBRARY) - llama_test(test-backend-ops NAME test-backend-ops-metallib-tensor - ARGS test -b MTL0 -o MUL_MAT -p type_a=q6_K) - set_tests_properties(test-backend-ops-metallib-tensor PROPERTIES ENVIRONMENT GGML_METAL_TENSOR_ENABLE=1) -endif() + +# TODO: make this test (and others) not link `libllama` as it is not needed [TAG_TESTS_LLAMA_LINK] +llama_build(test-backend-ops.cpp) llama_build_and_test(test-model-load-cancel.cpp LABEL "model") llama_build_and_test(test-autorelease.cpp LABEL "model") From 8ea290247c87ced2ab245b056ffe96dbcf90d36c Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Fri, 11 Sep 2026 21:36:52 +0200 Subject: [PATCH 45/65] cmake : skip PCH for llama-server PCH when using MSVC (#28763) This commit fixes an issue that I introduced when adding PCH (precompiled headers) in Commit 3bcfeb700fce9ff38a050dcd3f6a856319e948ba ("cmake : add PCH and unity build to improve build times (#28091)". See linked issue for details. Co-authored-by: mjungnickel18 Co-authored-by: Pascal Resolves: https://github.com/ggml-org/llama.cpp/issues/28758 Refs: https://github.com/ggml-org/llama.cpp/actions/runs/34592933983/job/103262608990#step:9:1284 --- tools/server/CMakeLists.txt | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt index f02a2ba3b17f..02607c838ca8 100644 --- a/tools/server/CMakeLists.txt +++ b/tools/server/CMakeLists.txt @@ -1,5 +1,13 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +# MSVC emits a PCH bookkeeping symbol that WINDOWS_EXPORT_ALL_SYMBOLS exports as an ambiguous "__" + +set(LLAMA_SERVER_PCH ON) + +if (BUILD_SHARED_LIBS AND CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + set(LLAMA_SERVER_PCH OFF) +endif() + # server-context containing the core server logic, used by llama-server and CLI set(TARGET server-context) @@ -32,7 +40,10 @@ endif() target_include_directories(${TARGET} PRIVATE ../mtmd) target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT}) -target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) + +if (LLAMA_SERVER_PCH) + target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) +endif() # llama-server-impl: server logic, reusable by app @@ -50,7 +61,10 @@ set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(${TARGET} PRIVATE ../mtmd ${CMAKE_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC server-context llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT}) -target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) + +if (LLAMA_SERVER_PCH) + target_precompile_headers(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/common/common.h) +endif() add_dependencies(${TARGET} llama-ui-assets) From 82d6bb284d1ff1c6ef37f29a4c3b63d1a8b11806 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sat, 12 Sep 2026 00:53:07 +0200 Subject: [PATCH 46/65] server: refactor subproc handling (#28555) * server: refactor subproc handling * fix Windows build * download: keep concurrent downloads of one blob apart Every process writes the same path + .downloadInProgress, so a second download of the same blob finds that file, takes it for its own partial transfer and asks for the bytes after it, which produces a corrupt result. The in-progress file now carries the pid of the process writing it. std::rename also replaces an existing destination on POSIX but fails on Windows, so a download whose blob appeared in the meantime is dropped after every retry and an etag rewrite silently keeps the old value. std::filesystem::rename has the POSIX behaviour everywhere, and the error now carries the reason reported by the system. * Revert "download: keep concurrent downloads of one blob apart" This reverts commit 917b83f149c625527f872fb2cf41289358fa5371. * tests: serialize the router tests that download the same model Parallel workers share one cache, so the two tests fetch the same blob into the same in-progress file and race to rename it. They now take a file lock around the download, like the session fixture does for the preset models. * Revert "tests: serialize the router tests that download the same model" This reverts commit c368a4a98c677938ca87002edb6186ff2c02fd83. --------- Co-authored-by: Pascal --- tools/server/server-common.cpp | 145 ++++++++++ tools/server/server-common.h | 38 +++ tools/server/server-models.cpp | 469 +++++++++++++++++++-------------- tools/server/server-models.h | 26 +- 4 files changed, 472 insertions(+), 206 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 2ac98b6fddbc..eade7db21256 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -16,6 +16,21 @@ #include #include +#ifdef _WIN32 +// windows.h defines min and max as macros, which breaks std::min and std::max +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +#include +#else +#include +#include +#include +#include +#endif + json format_error_response(const std::string & message, const enum error_type type) { std::string type_str; int code = 500; @@ -1832,3 +1847,133 @@ server_tokens format_prompt_rerank( return result; } + +// +// server_subproc +// + +bool server_subproc::has_output() { + if (out_handle >= 0) { + return true; + } + FILE * f = sproc.stdout_file(); // combined stdout/stderr + if (!f) { + return false; + } +#ifdef _WIN32 + HANDLE h = (HANDLE) _get_osfhandle(_fileno(f)); + if (h != INVALID_HANDLE_VALUE) { + out_handle = (intptr_t) h; + } +#else + int fd = fileno(f); + if (fd >= 0) { + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + out_handle = fd; + } +#endif + return out_handle >= 0; +} + +int server_subproc::read_output(char * buf, size_t len) { + if (!has_output()) { + return -1; + } +#ifdef _WIN32 + HANDLE h = (HANDLE) out_handle; + DWORD avail = 0; + if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) { + return -1; // pipe broken, child gone + } + if (avail == 0) { + return 0; + } + DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len; + DWORD got = 0; + if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) { + return -1; + } + return (int) got; +#else + while (true) { + ssize_t r = read((int) out_handle, buf, len); + if (r > 0) { + return (int) r; + } + if (r == 0) { + return -1; // EOF + } + if (errno == EINTR) { + continue; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return 0; + } + return -1; + } +#endif +} + +server_subproc::waiter::waiter() { +#ifndef _WIN32 + int fds[2]; + GGML_ASSERT(pipe(fds) == 0); + for (int fd : fds) { + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); + } + wake_fd[0] = fds[0]; + wake_fd[1] = fds[1]; +#endif +} + +server_subproc::waiter::~waiter() { +#ifndef _WIN32 + close((int) wake_fd[0]); + close((int) wake_fd[1]); +#endif +} + +void server_subproc::waiter::wake() { +#ifndef _WIN32 + char c = 1; + (void) !write((int) wake_fd[1], &c, 1); +#endif +} + +void server_subproc::waiter::wait(const std::vector & procs, std::vector & ready, int64_t timeout_ms) { + ready.assign(procs.size(), false); +#ifdef _WIN32 + // no waitable wait exists for anonymous pipes, so poll them in 50 ms steps + bool any = false; + for (size_t i = 0; i < procs.size(); i++) { + DWORD avail = 0; + if (!procs[i]->has_output() || !PeekNamedPipe((HANDLE) procs[i]->out_handle, NULL, 0, NULL, &avail, NULL) || avail > 0) { + ready[i] = true; // data or broken pipe, read_output() tells which + any = true; + } + } + if (!any) { + int64_t step = timeout_ms < 0 ? 50 : std::min(timeout_ms, 50); + std::this_thread::sleep_for(std::chrono::milliseconds(step)); + } +#else + std::vector pfds; + pfds.reserve(procs.size() + 1); + pfds.push_back({ (int) wake_fd[0], POLLIN, 0 }); + for (auto * p : procs) { + pfds.push_back({ p->has_output() ? (int) p->out_handle : -1, POLLIN, 0 }); // poll() skips negative fds + } + int timeout = timeout_ms < 0 ? -1 : (int) std::min(timeout_ms, std::numeric_limits::max()); + int r = poll(pfds.data(), pfds.size(), timeout); + if (r < 0 && errno != EINTR) { + LOG_ERR("%s: poll() failed: %s\n", __func__, strerror(errno)); + } + if (pfds[0].revents) { + char buf[64]; + while (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {} + } + for (size_t i = 0; i < procs.size(); i++) { + ready[i] = pfds[i + 1].fd < 0 || pfds[i + 1].revents != 0; + } +#endif +} diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 6c681a2cf56d..9894f5f06fb0 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -6,6 +6,7 @@ #include "chat.h" #include "mtmd.h" #include "mtmd-helper.h" +#include "subproc.h" #include "json.h" @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -611,3 +613,39 @@ struct server_pipe { return true; } }; + +// wrapper around common_subproc to manage a child server process +// mainly used by router mode +struct server_subproc { + common_subproc sproc; + std::atomic stopped{false}; // set by the monitor once the process exited and was reaped + + bool is_alive() { return sproc.alive(); } + void terminate() { sproc.terminate(); } + int join() { return sproc.join(); } + + // true if the child's combined stdout/stderr pipe is available (call after create()) + bool has_output(); + + // non-blocking read + // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken + int read_output(char * buf, size_t len); + + // wait until one of a set of children has output, wake() is called, or a timeout passes + struct waiter { + waiter(); + ~waiter(); + + // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway + void wake(); + + // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data (or a broken pipe) + void wait(const std::vector & procs, std::vector & ready, int64_t timeout_ms); + + private: + intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe + }; + +private: + intptr_t out_handle = -1; // fd on POSIX, HANDLE on Windows; taken lazily from sproc +}; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 4d2592b25964..f1783c083036 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -44,30 +44,215 @@ extern char **environ; #define CMD_ROUTER_TO_CHILD_EXIT "cmd_router_to_child:exit" #define CMD_CHILD_TO_ROUTER_STATE "cmd_child_to_router:state:" // followed by json string +// note: SIGPIPE is ignored by the server +static void request_child_exit(server_subproc & proc) { + FILE * stdin_file = proc.sproc.stdin_file(); + if (stdin_file) { + fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); + fflush(stdin_file); + } +} + // address for child process, this is needed because router may run on 0.0.0.0 // ref: https://github.com/ggml-org/llama.cpp/issues/17862 #define CHILD_ADDR "127.0.0.1" -struct server_subproc { - common_subproc sproc; // not yet spawned while in DOWNLOADING state - std::atomic stopped{false}; // set to cancel a download or signal child process exit +// single-threaded, watching all child processes at once +struct server_monitor { + server_monitor(server_models & models) : models(models) { + th = std::thread([this]() { run(); }); + } + + ~server_monitor() { + push({ cmd_t::QUIT, {}, "", 0, false }); + th.join(); + } + + // thread-safe + void watch(const std::string & name, std::shared_ptr proc, server_child_mode mode, int port) { + child_t c; + c.name = name; + c.proc = std::move(proc); + c.mode = mode; + c.port = port; + if (!c.proc->has_output()) { + SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str()); + c.eof = true; + } + push({ cmd_t::WATCH, std::move(c), "", 0, false }); + } + + // thread-safe + void stop(const std::string & name, int stop_timeout, bool send_exit) { + push({ cmd_t::STOP, {}, name, stop_timeout, send_exit }); + } + +private: + struct child_t { + std::string name; + std::shared_ptr proc; + server_child_mode mode = SERVER_CHILD_MODE_NORMAL; + int port = 0; + std::string buf; // partial line + bool eof = false; // output closed, waiting for the process to be reaped + int64_t deadline = 0; // force-kill time in ms, 0 when no stop is pending + }; + + struct cmd_t { + enum { WATCH, STOP, QUIT } type; + child_t child; + std::string name; + int stop_timeout; + bool send_exit; + }; + + void push(cmd_t && cmd) { + { + std::lock_guard lk(mu); + cmds.push_back(std::move(cmd)); + } + waiter.wake(); + } + + // returns true if the loop should exit + bool handle_commands() { + std::deque batch; + { + std::lock_guard lk(mu); + batch.swap(cmds); + } + for (auto & cmd : batch) { + switch (cmd.type) { + case cmd_t::WATCH: + children.push_back(std::move(cmd.child)); + break; + case cmd_t::STOP: + // the newest child with this name is the one the registry knows + for (auto it = children.rbegin(); it != children.rend(); ++it) { + if (it->name != cmd.name) { + continue; + } + if (cmd.send_exit && !it->eof) { + request_child_exit(*it->proc); + } + it->deadline = ggml_time_ms() + (int64_t) cmd.stop_timeout * 1000; + break; + } + break; + case cmd_t::QUIT: + return true; + } + } + return false; + } - bool is_alive() { - return sproc.alive(); + // read what the child wrote, forward complete lines + void read_output(child_t & c) { + char chunk[4096]; + while (!c.eof) { + int n = c.proc->read_output(chunk, sizeof(chunk)); + if (n < 0) { + c.eof = true; + break; + } + if (n == 0) { + break; + } + c.buf.append(chunk, (size_t) n); + size_t start = 0; + while (true) { + size_t nl = c.buf.find('\n', start); + if (nl == std::string::npos) { + break; + } + std::string line = c.buf.substr(start, nl + 1 - start); + start = nl + 1; + on_line(c, line); + } + c.buf.erase(0, start); + if (c.buf.size() > max_line) { + c.buf.clear(); // a child that never writes a newline must not grow this without bound + } + } + if (c.eof && !c.buf.empty()) { + on_line(c, c.buf); + c.buf.clear(); + } } - void request_exit() { - FILE * stdin_file = sproc.stdin_file(); - if (stdin_file) { - fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); - fflush(stdin_file); + void on_line(child_t & c, const std::string & line) { + if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) { + LOG_DBG("[%5d] %s", c.port, line.c_str()); // prevent spamming the log + models.handle_child_state(c.name, line); + } else { + LOG("[%5d] %s", c.port, line.c_str()); // forward log } - stopped.store(true, std::memory_order_relaxed); } - void terminate() { - sproc.terminate(); + void run() { + while (true) { + if (handle_commands()) { + return; + } + + // wait for output, a wakeup, or the next deadline; + // a child whose output closed is polled for its exit every 50 ms + int64_t now = ggml_time_ms(); + int64_t timeout = -1; + for (const auto & c : children) { + if (c.eof) { + timeout = timeout < 0 ? 50 : std::min(timeout, 50); + } + if (c.deadline) { + int64_t d = std::max(0, c.deadline - now); + timeout = timeout < 0 ? d : std::min(timeout, d); + } + } + std::vector procs; + std::vector owners; + for (auto & c : children) { + if (!c.eof) { + procs.push_back(c.proc.get()); + owners.push_back(&c); + } + } + std::vector ready; + waiter.wait(procs, ready, timeout); + for (size_t i = 0; i < owners.size(); i++) { + if (ready[i]) { + read_output(*owners[i]); + } + } + + // deadlines and exits + now = ggml_time_ms(); + for (auto it = children.begin(); it != children.end();) { + if (it->deadline && now >= it->deadline && !it->proc->stopped.load(std::memory_order_acquire)) { + SRV_WRN("force-killing model instance name=%s after timeout\n", it->name.c_str()); + it->proc->terminate(); + it->deadline = 0; + } + if (it->eof && !it->proc->is_alive()) { + int exit_code = it->proc->join(); + it->proc->stopped.store(true, std::memory_order_release); + models.on_child_exit(it->name, it->proc, it->mode, exit_code); + SRV_INF("instance name=%s exited with status %d\n", it->name.c_str(), exit_code); + it = children.erase(it); + } else { + ++it; + } + } + } } + + static constexpr size_t max_line = 1024 * 1024; + + server_models & models; + std::mutex mu; + std::deque cmds; + std::vector children; // monitor thread only + server_subproc::waiter waiter; + std::thread th; }; struct server_lru_sched { @@ -395,7 +580,8 @@ server_models::server_models( base_params(params), base_env(get_environment()), base_preset(ctx_preset.load_from_args(argc, argv)), - sched(std::make_unique(*this)) { + sched(std::make_unique(*this)), + monitor(std::make_unique(*this)) { // clean up base preset unset_reserved_args(base_preset, true); // set binary path @@ -412,6 +598,10 @@ server_models::server_models( server_models::~server_models() = default; +void server_models::instance_t::request_exit() const { + request_child_exit(*subproc); +} + void server_models::add_model(server_model_meta && meta) { if (mapping.find(meta.name) != mapping.end()) { throw std::runtime_error(string_format("model '%s' appears multiple times", meta.name.c_str())); @@ -466,7 +656,6 @@ void server_models::add_model(server_model_meta && meta) { std::string name = meta.name; mapping[name] = instance_t{ /* subproc */ std::make_shared(), - /* th */ std::thread(), /* meta */ std::move(meta) }; } @@ -621,9 +810,7 @@ void server_models::load_models() { }; // Phase 2: acquire the lock once for all mapping mutations. - // We temporarily release it only when calling functions that acquire it internally - // (unload, load) or when joining threads (the monitoring thread calls update_status - // which locks the mutex, so joining while holding it would deadlock). + // We temporarily release it only when calling functions that acquire it internally (unload) std::unique_lock lk(mutex); need_reload = false; @@ -708,49 +895,15 @@ void server_models::load_models() { return true; }); - // collect all threads to join in one pass while the lock is held: - // - monitoring threads from just-unloaded models (to_unload) - // - threads of finished downloads (DOWNLOADED), they acquire the mutex on exit - // - threads of already-UNLOADED models that are being removed from source - std::vector threads_to_join; - for (const auto & name : to_unload) { - auto it = mapping.find(name); - if (it != mapping.end() && it->second.th.joinable()) { - threads_to_join.push_back(std::move(it->second.th)); - } - } - for (auto & [name, inst] : mapping) { - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { - continue; // downloading models are not from config sources, leave them alone - } - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) { - // joining this thread under the lock deadlocks: it locks the mutex on its way out - if (inst.th.joinable()) { - threads_to_join.push_back(std::move(inst.th)); - } - continue; - } - if (final_presets.find(name) == final_presets.end() && !inst.meta.is_running() && inst.th.joinable()) { - threads_to_join.push_back(std::move(inst.th)); - } - } - - // join outside the lock - monitoring thread calls update_status (needs lock) - lk.unlock(); - for (auto & th : threads_to_join) th.join(); - lk.lock(); - // erase models no longer in any source for (auto it = mapping.begin(); it != mapping.end(); ) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { ++it; // download thread is still busy, skip } else if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) { - // download finished, thread is joined above, safe to erase - GGML_ASSERT(!it->second.th.joinable()); + // download finished, safe to erase it = mapping.erase(it); } else if (final_presets.find(it->first) == final_presets.end()) { SRV_INF("(reload) removing model name=%s (no longer in source)\n", it->first.c_str()); - GGML_ASSERT(!it->second.th.joinable()); // must have been joined above it = mapping.erase(it); } else { ++it; @@ -1030,117 +1183,12 @@ void server_models::load(const std::string & name, const load_options & opts) { } } - // start a thread to manage the child process - // captured variables are guaranteed to be destroyed only after the thread is joined - inst.th = std::thread([ - this, name, - child_proc = inst.subproc, - port = inst.meta.port, - stop_timeout = inst.meta.stop_timeout, - child_mode = opts.mode - ]() { - FILE * stdin_file = child_proc->sproc.stdin_file(); - FILE * stdout_file = child_proc->sproc.stdout_file(); // combined stdout/stderr - - std::thread log_thread([&]() { - // read stdout/stderr and forward to main server log - // also handle status report from child process - std::vector vec_buf(128 * 1024); // large buffer for storing info - char * buffer = vec_buf.data(); - if (stdout_file) { - while (fgets(buffer, vec_buf.size(), stdout_file) != nullptr) { - std::string str(buffer); - if (string_starts_with(buffer, CMD_CHILD_TO_ROUTER_STATE)) { - LOG_DBG("[%5d] %s", port, buffer); // prevent spamming the log - this->handle_child_state(name, str); - } else { - // forward log - LOG("[%5d] %s", port, buffer); - } - } - } else { - SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str()); - } - }); - - std::thread stopping_thread([&]() { - // thread to monitor explicit stop requests; child crash is signalled via child_proc->stopped - auto is_stopping = [this, &name]() { - return this->stopping_models.find(name) != this->stopping_models.end(); - }; - { - std::unique_lock lk(this->mutex); - this->cv_stop.wait(lk, [&]() { - return is_stopping() || child_proc->stopped.load(std::memory_order_acquire); - }); - } - // child crashed or finished on its own, skip graceful shutdown sequence - if (child_proc->stopped.load(std::memory_order_acquire)) { - return; - } - SRV_INF("stopping model instance name=%s\n", name.c_str()); - fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT); - fflush(stdin_file); - int64_t start_time = ggml_time_ms(); - while (true) { - std::unique_lock lk(this->mutex); - if (!is_stopping() || child_proc->stopped.load(std::memory_order_acquire)) { - return; - } - int64_t elapsed = ggml_time_ms() - start_time; - if (elapsed >= stop_timeout * 1000) { - lk.unlock(); - SRV_WRN("force-killing model instance name=%s after %d seconds timeout\n", name.c_str(), stop_timeout); - child_proc->terminate(); - return; - } - this->cv_stop.wait_for(lk, std::chrono::seconds(1), [&]() { - return !is_stopping() || child_proc->stopped.load(std::memory_order_acquire); - }); - } - }); - - // we reach here when the child process exits (stdout EOF) - // note: we cannot join() prior to this point because it will close stdin_file - if (log_thread.joinable()) { - log_thread.join(); - } - - child_proc->stopped.store(true, std::memory_order_release); - { - std::lock_guard lk(this->mutex); - stopping_models.erase(name); - cv_stop.notify_all(); - } - if (stopping_thread.joinable()) { - stopping_thread.join(); - } - - // get the exit code - int exit_code = child_proc->sproc.join(); - - // update status and exit code - if (child_mode == SERVER_CHILD_MODE_DOWNLOAD) { - // instance will be cleaned up on next load_models() call - } else { - this->update_status(name, { - SERVER_MODEL_STATUS_UNLOADED, - exit_code - }); - } - SRV_INF("instance name=%s exited with status %d\n", name.c_str(), exit_code); - }); - - // clean up old process/thread if exists + // old process should have exited already, but just in case, we clean it up here { - auto & old_instance = mapping[name]; - // old process should have exited already, but just in case, we clean it up here - if (old_instance.subproc && old_instance.subproc->is_alive()) { + auto it = mapping.find(name); + if (it != mapping.end() && it->second.subproc && it->second.subproc->is_alive()) { SRV_WRN("old process for model name=%s is still alive, this is unexpected\n", name.c_str()); - old_instance.subproc->terminate(); // force kill - } - if (old_instance.th.joinable()) { - old_instance.th.join(); + it->second.subproc->terminate(); // force kill } } @@ -1148,13 +1196,41 @@ void server_models::load(const std::string & name, const load_options & opts) { {"status", server_model_status_to_string(inst.meta.status)}, }); + auto proc = inst.subproc; + int port = inst.meta.port; mapping[name] = std::move(inst); + monitor->watch(name, proc, opts.mode, port); cv.notify_all(); } -void server_models::request_stop(const std::string & name) { +void server_models::request_stop(const std::string & name, bool send_exit) { + auto it = mapping.find(name); + if (it == mapping.end() || stopping_models.count(name)) { + return; + } stopping_models.insert(name); - cv_stop.notify_all(); + monitor->stop(name, it->second.meta.stop_timeout, send_exit); +} + +void server_models::on_child_exit(const std::string & name, const std::shared_ptr & proc, server_child_mode mode, int exit_code) { + { + std::lock_guard lk(mutex); + stopping_models.erase(name); + auto it = mapping.find(name); + if (it == mapping.end() || it->second.subproc != proc) { + return; // entry erased, or a newer instance took the name + } + } + if (mode == SERVER_CHILD_MODE_DOWNLOAD) { + // instance will be cleaned up on next load_models() call + std::lock_guard lk(mutex); + cv.notify_all(); + } else { + update_status(name, { + SERVER_MODEL_STATUS_UNLOADED, + exit_code + }); + } } void server_models::unload(const std::string & name) { @@ -1163,20 +1239,21 @@ void server_models::unload(const std::string & name) { if (it != mapping.end()) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { SRV_INF("cancelling download for model name=%s\n", name.c_str()); - it->second.subproc->request_exit(); + it->second.request_exit(); // for convenience, we wait the status change here wait(lk, name, [](const server_model_meta & new_meta) { return new_meta.status != SERVER_MODEL_STATUS_DOWNLOADING; }); } else if (it->second.meta.is_running()) { SRV_INF("stopping model instance name=%s\n", name.c_str()); - if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { // special case: if model is in loading state, unloading means force-killing it SRV_WRN("model name=%s is still loading, force-killing\n", name.c_str()); it->second.subproc->terminate(); } - request_stop(name); - // status change will be handled by the managing thread + request_stop(name, !loading); + // status change will be handled by the monitor } else { SRV_WRN("model instance name=%s is not running\n", name.c_str()); } @@ -1184,27 +1261,29 @@ void server_models::unload(const std::string & name) { } void server_models::unload_all() { - std::vector to_join; - { - std::lock_guard lk(mutex); - for (auto & [name, inst] : mapping) { - if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { - SRV_INF("cancelling download for model name=%s\n", name.c_str()); - inst.subproc->stopped.store(true, std::memory_order_relaxed); - } else if (inst.meta.is_running()) { - SRV_INF("stopping model instance name=%s\n", name.c_str()); - request_stop(name); - // status change will be handled by the managing thread + std::unique_lock lk(mutex); + for (auto & [name, inst] : mapping) { + if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + SRV_INF("cancelling download for model name=%s\n", name.c_str()); + inst.request_exit(); + } else if (inst.meta.is_running()) { + SRV_INF("stopping model instance name=%s\n", name.c_str()); + bool loading = inst.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { + inst.subproc->terminate(); } - // moving the thread to join list to avoid deadlock - to_join.push_back(std::move(inst.th)); + request_stop(name, !loading); } } - for (auto & th : to_join) { - if (th.joinable()) { - th.join(); + // wait for every child to exit, the monitor force-kills the ones that ignore the exit command + cv.wait(lk, [this]() { + for (const auto & [name, inst] : mapping) { + if (inst.meta.is_running() || inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + return false; + } } - } + return true; + }); } void server_models::update_status(const std::string & name, const update_status_args & args) { @@ -1291,18 +1370,18 @@ bool server_models::remove(const std::string & name) { if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { // cancel in-flight download SRV_INF("cancelling download for model name=%s\n", name.c_str()); - it->second.subproc->request_exit(); + it->second.request_exit(); } else if (it->second.meta.is_running()) { // stop running instance SRV_INF("stopping model instance name=%s\n", name.c_str()); - stopping_models.insert(name); - if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { + bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING; + if (loading) { it->second.subproc->terminate(); } - cv_stop.notify_all(); + request_stop(name, !loading); } - // wait until the monitoring thread finishes + // wait until the child is gone wait(lk, name, [](const server_model_meta & meta) { return meta.status == SERVER_MODEL_STATUS_UNLOADED || meta.status == SERVER_MODEL_STATUS_DOWNLOADED; @@ -1311,8 +1390,7 @@ bool server_models::remove(const std::string & name) { // re-find after wait - load_models() may have erased the entry during the wait it = mapping.find(name); if (it == mapping.end()) { - // load_models() already joined the thread and erased the entry; - // we just need to clean up the cached files on disk + // load_models() already erased the entry; we just need to clean up the cached files on disk lk.unlock(); bool ok = common_download_remove(name); SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial"); @@ -1320,11 +1398,6 @@ bool server_models::remove(const std::string & name) { return true; } - // join before erasing - thread no longer acquires this mutex - if (it->second.th.joinable()) { - it->second.th.join(); - } - // remove from disk (best-effort: cancelled downloads may have no cached files) bool ok = common_download_remove(name); mapping.erase(name); @@ -1539,7 +1612,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri std::lock_guard lk(mutex); auto it = mapping.find(name); if (it != mapping.end()) { - return it->second.subproc->request_exit(); + return it->second.request_exit(); } }; if (result == "download_finished") { diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 7f6c26b358b4..90161bf34ad6 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -107,27 +108,29 @@ struct server_model_meta { }; struct server_models_routes; -struct server_subproc; // defined in server-models.cpp struct server_lru_sched; // defined in server-models.cpp +struct server_monitor; // defined in server-models.cpp struct server_models { friend struct server_models_routes; friend struct server_lru_sched; + friend struct server_monitor; private: struct instance_t { - std::shared_ptr subproc; // shared between main thread and monitoring thread - std::thread th; + std::shared_ptr subproc; // shared with the monitor thread server_model_meta meta; int req_count = 0; // number of active proxy requests + + // ask the child to exit (it handles the command on its stdin, see server_child::setup) + void request_exit() const; }; std::mutex mutex; std::condition_variable cv; std::map mapping; - // for stopping models - std::condition_variable cv_stop; + // models asked to stop, still counted as running until the monitor records their exit std::set stopping_models; // set to true while load_models() is executing a reload; load() will wait until clear @@ -216,9 +219,12 @@ struct server_models { // not thread-safe, caller must hold mutex void add_model(server_model_meta && meta); - // ask the monitoring thread to stop a running instance + // ask the monitor to stop a running instance; send_exit is false for a child that was already force-killed // not thread-safe, caller must hold mutex - void request_stop(const std::string & name); + void request_stop(const std::string & name, bool send_exit = true); + + // called by the monitor once a child exited and was reaped + void on_child_exit(const std::string & name, const std::shared_ptr & proc, server_child_mode mode, int exit_code); // notify SSE clients void notify_sse(const std::string & event, const std::string & model_id, const json & data = nullptr); @@ -297,12 +303,16 @@ struct server_models { // handle message sent from server_child::notify_to_router() // raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string - // this function is not thread-safe, must be called from instance's monitoring thread + // called from the monitor thread // payload per state: // state = loading -> payload = {} (TODO: add progress info) // state = ready -> payload = model_info (json), or {} if wakeup from sleeping // state = sleeping -> payload = {} void handle_child_state(const std::string & name, const std::string & raw_input); + +private: + // one thread watching every child; keep last, the destructor joins the thread + std::unique_ptr monitor; }; struct server_child { From d3146f2b56c2db4711ac8391871c9e529d1946d7 Mon Sep 17 00:00:00 2001 From: Mendy Berger <12537668+MendyBerger@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:47:29 -0400 Subject: [PATCH 47/65] ggml-webgpu: Update to a recent version of Dawn (#28683) * ggml-webgpu: Update to a recent version of Dawn * No module scanning * Accept review suggestion to update comment Co-authored-by: Masashi Yoshimura --------- Co-authored-by: Masashi Yoshimura --- .github/workflows/build-self-hosted.yml | 8 ++++---- .github/workflows/build-wasm.yml | 2 +- .github/workflows/build-webgpu.yml | 8 ++++---- docs/build.md | 2 +- ggml/src/ggml-webgpu/CMakeLists.txt | 6 ++++++ ggml/src/ggml-webgpu/ggml-webgpu.cpp | 21 +++++++++++---------- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 02a38466fa76..c7f992540b12 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -160,10 +160,10 @@ jobs: - name: Dawn Dependency id: dawn-depends run: | - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" @@ -246,10 +246,10 @@ jobs: - name: Dawn Dependency id: dawn-depends run: | - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 81b943df7b65..5a3166ce6885 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -68,7 +68,7 @@ jobs: - name: Fetch emdawnwebgpu run: | - DAWN_TAG="v20260317.182325" + DAWN_TAG="v20260908.214631" EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip" echo "Downloading ${EMDAWN_PKG}" curl -L -o emdawn.zip \ diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml index 8277abcc47c3..ec582ff274b3 100644 --- a/.github/workflows/build-webgpu.yml +++ b/.github/workflows/build-webgpu.yml @@ -77,10 +77,10 @@ jobs: - name: Dawn Dependency id: dawn-depends run: | - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" @@ -147,10 +147,10 @@ jobs: id: dawn-depends run: | sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev - DAWN_VERSION="v20260317.182325" + DAWN_VERSION="v20260908.214631" DAWN_OWNER="google" DAWN_REPO="dawn" - DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release" + DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release" echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" curl -L -o artifact.tar.gz \ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz" diff --git a/docs/build.md b/docs/build.md index 28dcbc2e53ea..70fc17af2402 100644 --- a/docs/build.md +++ b/docs/build.md @@ -806,7 +806,7 @@ To read documentation for how to build on Android, [click here](./android.md) ## WebGPU -The WebGPU backend relies on [Dawn](https://dawn.googlesource.com/dawn). Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md) to install Dawn locally so that llama.cpp can find it using CMake. The current implementation is up-to-date with Dawn commit `18eb229`. +The WebGPU backend relies on [Dawn](https://dawn.googlesource.com/dawn). Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md) to install Dawn locally so that llama.cpp can find it using CMake. The current implementation is up-to-date with Dawn commit `94c3c9c`. In the llama.cpp directory, build with CMake: diff --git a/ggml/src/ggml-webgpu/CMakeLists.txt b/ggml/src/ggml-webgpu/CMakeLists.txt index 1503a1ef8ba6..2eacca7f2b3b 100644 --- a/ggml/src/ggml-webgpu/CMakeLists.txt +++ b/ggml/src/ggml-webgpu/CMakeLists.txt @@ -39,6 +39,12 @@ ggml_add_backend_library(ggml-webgpu add_dependencies(ggml-webgpu generate_shaders) +# Dawn needs C++20 (https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md#prerequisites) +target_compile_features(ggml-webgpu PRIVATE cxx_std_20) + +# Disable C++20 module scanning since emscan-deps fails to find webgpu_cpp.h +set_target_properties(ggml-webgpu PROPERTIES CXX_SCAN_FOR_MODULES OFF) + if(EMSCRIPTEN) set(EMDAWNWEBGPU_DIR "" CACHE PATH "Path to emdawnwebgpu_pkg") diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index f06a9c872db9..13db0b856f69 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -4006,16 +4006,17 @@ static void ggml_backend_webgpu_request_adapter(wgpu::Instance & instance, wgpu: options.nextInChain = &adapterTogglesDesc; #endif - instance.WaitAny(instance.RequestAdapter( - &options, wgpu::CallbackMode::AllowSpontaneous, - [&adapter](wgpu::RequestAdapterStatus status, wgpu::Adapter _adapter, const char * message) { - if (status != wgpu::RequestAdapterStatus::Success) { - GGML_LOG_ERROR("ggml_webgpu: Failed to get an adapter: %s\n", message); - return; - } - adapter = std::move(_adapter); - }), - UINT64_MAX); + instance.WaitAny( + instance.RequestAdapter( + &options, wgpu::CallbackMode::AllowSpontaneous, + [&adapter](wgpu::RequestAdapterStatus status, wgpu::Adapter _adapter, wgpu::StringView message) { + if (status != wgpu::RequestAdapterStatus::Success) { + GGML_LOG_ERROR("ggml_webgpu: Failed to get an adapter: %s\n", std::string(message).c_str()); + return; + } + adapter = std::move(_adapter); + }), + UINT64_MAX); } static void create_webgpu_device(ggml_backend_webgpu_reg_context * ctx) { From eafe15a5e3d87dd68ae33acf6a7cbd9415a0ac5e Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Fri, 11 Sep 2026 20:46:51 -0700 Subject: [PATCH 48/65] hexagon: support for multi-device model split (aka row-split) (#28589) * hex-row-split: add support for multi-device row spliting Co-authored-by: Max Krasnyansky * hex-mdev: add work splitting to fused kernels * hex-mdev: use mdev_ prefix for all multi-device state * hex-mdev: make device configuration more expressive to support device groups * hex-mdev: fix mdev session init * hex-mdev: fused nx (2x,3x) matmuls must update row counts for each w/o * hex-mdev: fix MUL_MAT work partitioning bugs introduced by mdev * hex-cont: fix crashes with new tests due to wrong striding * hex-mdev: move fences after l2flushes * hex-cont: fix work splitting for mnpu -- align chunks to cachelines * hex-mdev: fix CPY tests with multi-dev * hex-mmid: fix work partitioning with mnpu * hex-mm: fix test failures with mdev * hex-binary: fix work partitioning for mdev * hex-argsort: fix mdev partitioning * hex-mdev: fix work partitioning and general updates for all simple ops * hex-fa: fix mdev work splitting issues * hex-mdev: fixing more failing ops test * hex-mdev: update the rest of the ops * hex-mdev: refactor all mdev splitting logic to be contained within if (mdev_count > 1) {...} * hex-mdev: fix macros * hex-mdev: simplify session flush logic * hex-sync: fix recursion in session flush * hex-mdev: factor out fence buffer and allocator * hex-fence: make fence allocation more robust with reserved slots for mdev * hex-mdev: keep all mdev state in htp_mdev_group * hex-mdev: further cleanup mdev group handling at the host * hex-mdev: update group idx in the opbatch before serializing * hex-batch: remove separate op_pending and use batch_req/rsp_seq * hex-async: workaround another missing tensor_init in ggml-meta * hex-fence: cleanup and robustify fences and error handling in multi-device scenarios * hex-ar: improve ALLREDUCE error handling * hex-async: robust error handling for op_cpy_fence * hex-async: use seq0 from allreduce context to allocate fence_seq * hex-mdev: fix remaining issues with fence and barrier clearing in CPY_FENCE * hex-misc: realign macros and fix misplaces trace events * hex-misc: align macros * hex-mdev: fix unclone buffer re-entrancy * hex-glu: fix mdev partitioning logic * hex-mdev: make buffer uncloning/cleanup work with tensor-split scenarios * hex-mdev: tighten up the can_split check in act-ops * hex-mdev: factor out common bits of the partitioning logic * hex-mm: minor realignment of the macros * hex-bufs: fix incorrectly placed assert for MAX_BUFS * hex-pad: tighten up gating checks for PAD * hex-kparams: make sure all kernels properly use kparams->n_threads * hex-docs: update user and developer docs with new features and detailed guide for ops development * hex-scripts: update run script to properly parse dev groups * hex-misc: formatting * hex-sess: minor cleanup for session init * hex-ar: fix vtcm size calc in allreduce kparams * hex-scripts: fix flake8 warnings * hex-rope: update ROPE to support mdev work split * hex-ops: remove redunant checks and minor reformat * hex-dev-guide: update dev-guide to avoid redundant null checks * hex-async: improve event_wait, event_sync and fence implementations * hex-async: remove synchronous flush from event_sync * hex-async: symplify fence recovery protocol and make sync more robust * hex-async: futher simplify error recovery for fences * hex-err: return status instead of just -1 * hex-async: print all seq nums in hex * hex-async: make sure fences flush dirty ranges * hex-async: add dirty ranges merging to reduce fence flushes * hex-async: properly sync before freeing the event * hex-async: make sure fence owner session is not overriden * hex-async: more fence write order more robust * hex-async: make sure not to fuse ALLREDUCE+ADD if their dsts overlap * hex-fusion: cleanup redundant checks --------- Co-authored-by: Alexander Lu --- docs/backend/snapdragon/README.md | 141 +- docs/backend/snapdragon/developer.md | 360 ++++- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 1163 ++++++++++++----- ggml/src/ggml-hexagon/htp-opnode.h | 6 + ggml/src/ggml-hexagon/htp/act-ops.c | 233 ++-- ggml/src/ggml-hexagon/htp/allreduce-ops.c | 111 +- ggml/src/ggml-hexagon/htp/allreduce-ops.h | 11 + ggml/src/ggml-hexagon/htp/argsort-ops.c | 50 +- ggml/src/ggml-hexagon/htp/binary-ops.c | 228 ++-- ggml/src/ggml-hexagon/htp/concat-ops.c | 81 +- ggml/src/ggml-hexagon/htp/cpy-ops.c | 464 ++++--- ggml/src/ggml-hexagon/htp/cumsum-ops.c | 105 +- ggml/src/ggml-hexagon/htp/diag-ops.c | 101 +- ggml/src/ggml-hexagon/htp/fill-ops.c | 69 +- ggml/src/ggml-hexagon/htp/flash-attn-ops.c | 84 +- ggml/src/ggml-hexagon/htp/flash-attn-ops.h | 1 + .../ggml-hexagon/htp/gated-delta-net-ops.c | 74 +- ggml/src/ggml-hexagon/htp/get-rows-ops.c | 62 +- ggml/src/ggml-hexagon/htp/hex-common.h | 9 + ggml/src/ggml-hexagon/htp/hex-utils.h | 1 - ggml/src/ggml-hexagon/htp/hmx-utils.h | 14 +- ggml/src/ggml-hexagon/htp/htp-ctx.h | 51 +- ggml/src/ggml-hexagon/htp/htp-fence.h | 89 ++ ggml/src/ggml-hexagon/htp/htp-ops.h | 23 +- ggml/src/ggml-hexagon/htp/htp-tensor.c | 135 +- ggml/src/ggml-hexagon/htp/htp-tensor.h | 109 ++ ggml/src/ggml-hexagon/htp/hvx-arith.h | 274 ++-- ggml/src/ggml-hexagon/htp/hvx-div.h | 94 +- ggml/src/ggml-hexagon/htp/hvx-inverse.h | 46 +- ggml/src/ggml-hexagon/htp/hvx-scale.h | 44 +- ggml/src/ggml-hexagon/htp/hvx-sigmoid.h | 80 +- ggml/src/ggml-hexagon/htp/im2col-ops.c | 103 +- ggml/src/ggml-hexagon/htp/main.c | 103 +- ggml/src/ggml-hexagon/htp/matmul-ops.c | 371 ++++-- ggml/src/ggml-hexagon/htp/pad-ops.c | 163 +-- ggml/src/ggml-hexagon/htp/repeat-ops.c | 49 +- ggml/src/ggml-hexagon/htp/rope-ops.c | 41 +- ggml/src/ggml-hexagon/htp/set-rows-ops.c | 54 +- ggml/src/ggml-hexagon/htp/softmax-ops.c | 53 +- ggml/src/ggml-hexagon/htp/solve-tri-ops.c | 92 +- ggml/src/ggml-hexagon/htp/ssm-conv.c | 169 +-- ggml/src/ggml-hexagon/htp/sum-rows-ops.c | 80 +- ggml/src/ggml-hexagon/htp/unary-ops.c | 278 ++-- .../snapdragon/ggml-hexagon-align-macros.py | 296 +++++ scripts/snapdragon/run.py | 128 +- 45 files changed, 4345 insertions(+), 1948 deletions(-) create mode 100644 ggml/src/ggml-hexagon/htp/htp-fence.h create mode 100755 scripts/snapdragon/ggml-hexagon-align-macros.py diff --git a/docs/backend/snapdragon/README.md b/docs/backend/snapdragon/README.md index 391c8bf230f0..5d32a5877ad3 100644 --- a/docs/backend/snapdragon/README.md +++ b/docs/backend/snapdragon/README.md @@ -188,7 +188,7 @@ llama_memory_breakdown_print: | - Host | 439 = Op test for MUL_MAT: ``` -~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --hex-hostbuf 0 --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT +~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT ... Backend 2/3: HTP0:0 Device description: Hexagon @@ -213,14 +213,109 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v | llama 1B Q4_0 | 729.75 MiB | 1.24 B | HTP | 99 | 4 | 128 | 0 | tg64 | 51.54 ± 1.13 | ``` +## Multi-Device Execution Modes + +The Hexagon backend supports multiple execution and partitioning modes to accommodate different model sizes, memory +constraints, and single- or multi-NPU hardware topologies: + +### 1. Single-Device Mode with Dynamic Buffer Mapping + +Runs the model on a single NPU session (e.g. `HTP0` or `HTP0:0`). + +A single NPU session provides ~3.5GB of available virtual address space. For models larger than 3.5GB, the backend +automatically maps and unmaps weight buffers during graph execution. This allows large models to run on a single NPU +without manual configuration: + +```bash +./scripts/snapdragon/run.py --target adb --devices HTP0:0 -- \ + llama-cli -m models/Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "Hello" +``` + +### 2. Layer-Split Mode across Virtual Sessions (`HTP0,HTP1,...` or `HTP0:0,HTP0:1,...`) + +Partitions model layers at load time across multiple virtual sessions hosted on a single physical NPU. + +Each virtual session acts as an independent backend device from llama.cpp's perspective (similar to multiple GPUs). +Because layers are permanently distributed across sessions, each session's allocated weights remain within its private 3.5GB +address space window, eliminating runtime buffer re-mapping overhead. + +Here is an example of running the GPT-OSS-20B model on a Snapdragon device using 4 virtual sessions on a single NPU: + +```bash +./scripts/snapdragon/run.py --target adb \ + --devices HTP0:0,HTP0:1,HTP0:2,HTP0:3 -- \ + llama-cli --load-mode none -m /data/local/tmp/gguf/gpt-oss-20b-Q4_0.gguf -t 4 \ + --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -no-cnv -f surfing.txt +``` + +Log output snippet: + +``` +... +llama_model_loader: - type f32: 289 tensors +llama_model_loader: - type q4_0: 96 tensors +llama_model_loader: - type q8_0: 2 tensors +llama_model_loader: - type mxfp4: 72 tensors +... +load_tensors: offloaded 25/25 layers to GPU +load_tensors: CPU model buffer size = 1182.09 MiB +load_tensors: HTP0:1 model buffer size = 2512.58 MiB +load_tensors: HTP0:3 model buffer size = 2093.83 MiB +load_tensors: HTP0:0 model buffer size = 2931.34 MiB +load_tensors: HTP0:2 model buffer size = 2512.58 MiB +... +llama_perf_context_print: prompt eval time = 3843.67 ms / 197 tokens ( 19.51 ms per token, 51.25 tokens per second) +llama_perf_context_print: eval time = 1686.13 ms / 31 runs ( 54.39 ms per token, 18.39 tokens per second) +llama_perf_context_print: total time = 6266.30 ms / 228 tokens +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - HTP0:0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP0:3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - Host | 1476 = 1208 + 105 + 162 | +``` + +### 3. Tensor-Split Mode across Physical Devices (`HTP0:0,HTP1:0,...`) + +Distributes model tensors across distinct physical NPU hardware cores using llama.cpp's tensor parallelism +(`--split-mode tensor`). + +Tensors are partitioned across physical NPUs for parallel execution (proportions are distributed equally by default without +needing an explicit `--tensor-split` option): + +```bash +./scripts/snapdragon/run.py --target adb \ + --devices HTP0:0,HTP1:0 -- \ + llama-cli -m models/Llama-3.2-3B-Instruct-Q4_0.gguf --split-mode tensor -ngl 99 -p "Hello" +``` + +### 4. Row-Split Multi-Device Mode via Device Grouping (`HTP0[0-1]`) + +Groups multiple physical NPU cores into a single logical device using bracket notation (`HTP0[0-1]` or `HTP0[0,1]`). + +Unlike host-level tensor-splitting, row-splitting is executed entirely inside the Hexagon backend: + +```bash +./scripts/snapdragon/run.py --target adb \ + --devices 'HTP0[0-1]' -- \ + llama-cli -m models/Llama-3.2-3B-Instruct-Q4_0.gguf -ngl 99 -p "Hello" +``` + +You can also combine row-splitting with layer-splitting across multiple grouped devices (e.g. `--devices 'HTP0[0-1],HTP1[2-3]'` +on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs using virtual sessions 0 and 1). + ## Environment variables - `GGML_HEXAGON_DEVICES` (default: not set, defaults to HTP0 session) - Controls which NPU devices and sessions to allocate. Can be configured as: - - A single integer `N`: Allocates `N` sessions named `HTP0`, `HTP1`, ..., `HTP` (behaves identically to `GGML_HEXAGON_NDEV=N`). - - A comma-separated list of device names in `HTP:` format (or legacy `HTP` format). For example, `HTP0:0,HTP0:1` creates two virtual - sessions on the first physical NPU (useful for memory limits). `HTP0:0,HTP1:0` allocates one session on each of the two physical NPUs - on a dual-NPU device. + Controls which NPU devices and sessions to allocate. Configurable via `--devices` in `run.py`: + - `N` (single integer): Allocates `N` virtual sessions named `HTP0`, `HTP1`, ..., `HTP` on physical NPU 0. + - `HTP:,...`: Comma-separated list of individual devices specifying physical and virtual index: + - `HTP0:0,HTP0:1`: Two virtual sessions on physical NPU 0 (layer-split on single NPU). + - `HTP0:0,HTP1:0`: One session on physical NPU 0 and one on physical NPU 1 (tensor-split across physical cores). + - `HTP[]`: Device grouping syntax for row-split multi-device execution: + - `HTP0[0-1]`: A single logical device `HTP0` that groups physical cores 0 and 1. + - `HTP0[0-1],HTP1[2-3]`: Two layer-split devices across 4 physical NPUs (cores 0-1 and 2-3). + - `HTP0[0-1:0],HTP1[0-1:1]`: Two layer-split devices across 2 physical NPUs using virtual sessions 0 and 1. - `GGML_HEXAGON_NDEV` (deprecated) Replaced by `GGML_HEXAGON_DEVICES`. Controls the number of virtual sessions to allocate on physical NPU `0`. @@ -229,9 +324,8 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v - `GGML_HEXAGON_NHVX=0` Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version). -- `GGML_HEXAGON_HOSTBUF=1` - Controls whether the Hexagon backend allocates host buffers. By default, all buffers except for REPACK are host buffers. - This option is required for testing Ops that require REPACK buffers (MUL_MAT and MUL_MAT_ID). +- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled) + Enables allocating host buffers for debugging. By default, host buffers are disabled. - `GGML_HEXAGON_VERBOSE=1` Enables verbose logging of Ops from the backend. Example output: @@ -246,23 +340,26 @@ ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v ``` - `GGML_HEXAGON_PROFILE=1` - Enables Op profiling: + Enables Op profiling (configurable via `--hex-profile` in `run.py`): - - `1` Basic profile with per-op `usecs` and `cycles` counters - - `2` Extended profile with per-op `usecs`, `cycles` and default PMU counter data - - `0x1,...,0x8` Extended profile with per-op `usecs`, `cycles` and custom PMU counter data + - `1`: Basic profile with per-op `usecs` and `cycles` counters + - `2`: Extended profile with per-op `usecs`, `cycles` and default PMU counter data + - `0x1,...,0x8`: Extended profile with per-op `usecs`, `cycles` and custom PMU counter data - The logging output can be either saved into a file for post-processing or it can be piped directly into the post-processing tool - to generate the report. - Examples: + The logging output can be saved to a file or piped directly into the post-processing script: - `GGML_HEXAGON_PROFILE=1 ./scripts/snapdragon/run.py --target adb -- llama-cli ... |& ./scripts/snapdragon/ggml-hexagon-profile.py -` + ```bash + ./scripts/snapdragon/run.py --target adb --hex-profile 1 -- llama-cli ... |& \ + ./scripts/snapdragon/ggml-hexagon-profile.py - + ``` - `GGML_HEXAGON_OPFILTER=regex` - Allows filtering (disabling) Ops that match the regex pattern: + Filters (disables) Ops matching the regex pattern (configurable via `--hex-opfilter` in `run.py`): - Examples: - - `GGML_HEXAGON_OPFILTER="FLASH_ATTN_EXT" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable Flash Attention on Hexagon (falls back to CPU or GPU) - `GGML_HEXAGON_OPFILTER="ADD\|SUB" ./scripts/snapdragon/run.py --target adb -- llama-cli ...` - Disable ADD and SUB on Hexagon (fall back to CPU or GPU) + ```bash + # Disable Flash Attention on Hexagon (falls back to CPU or GPU) + ./scripts/snapdragon/run.py --target adb --hex-opfilter "FLASH_ATTN_EXT" -- llama-cli ... + # Disable ADD and SUB on Hexagon (fall back to CPU or GPU) + ./scripts/snapdragon/run.py --target adb --hex-opfilter "ADD|SUB" -- llama-cli ... + ``` diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md index d7d9f2a2790d..633643c16ddd 100644 --- a/docs/backend/snapdragon/developer.md +++ b/docs/backend/snapdragon/developer.md @@ -2,16 +2,16 @@ ## Backend libraries -The Hexagon backend consist of two parts: +The Hexagon backend consists of two parts: - `libggml-hexagon` - This is the regular CPU-side GGML backend library, either shared or statically linked + This is the regular CPU-side GGML backend library, either shared or statically linked. - `libggml-htp-vNN` This is the NPU-side (HTP stands for Hexagon Tensor Processor) shared library that contains the Op dispatcher and kernels. The correct library is selected automatically at runtime based on the HW version. -Here is an example of the build artifacts +Here is an example of the build artifacts: ``` ~/src/llama.cpp$ ls -l pkg-adb/llama.cpp/lib/libggml* @@ -26,75 +26,307 @@ pkg-adb/llama.cpp/lib/libggml-htp-v81.so ## Memory buffers -Hexagon NPU backend takes advantage of the Snapdragon's unified memory model where all buffers are fully accessible by the CPU and GPU. -The NPU does have a dedicated tightly-coupled memory called VTCM but that memory is used only for intermediate data (e.g. dynamically -quantized tensors) or temporary data (chunks of the weight tensors fetched via DMA). - -Please note that currently the Hexagon backend does not implement SET/GET_ROWS Ops because there is no advantage in offloading those -to the NPU at this point. - -The backend does allocates non-host buffers for the tensors with datatypes that require repacking: Q4_0, Q8_0, MXFP4. -From the MMU perspective these buffers are still regular buffers (normal access by the CPU) they are marked as non-host simply to force -the repacking. +The Hexagon NPU backend takes advantage of Snapdragon unified memory where all DDR buffers are accessible by CPU, GPU, and NPU. +The NPU has dedicated tightly-coupled memory called VTCM (Vector Tightly-Coupled Memory). VTCM is used for intermediate data (such as +dynamically quantized activations) and streaming buffers (chunks of weight and activation tensors fetched via DMA). ## Large model handling -Hexagon NPU sessions (aka Process Domains (PD) in the Hexagon SDK) are limited to a maximum memory mapping window of around 3.5GB. +Hexagon NPU sessions have a 32-bit virtual address space window of around 3.5GB. In llama.cpp/GGML, each Hexagon session is mapped to a single GGML backend device (e.g., `HTP0:0`, `HTP0:1`, etc. when using `GGML_HEXAGON_DEVICES`, or `HTP0`, `HTP1` in legacy mode). -To support running models larger than 3.5GB on a single device, the Hexagon backend dynamically maps and unmaps execution buffers -during the graph execution cycle to stay within the Process Domain window. This enables large models to run successfully on a single -NPU device. +To support running models larger than 3.5GB on a single device, the Hexagon backend dynamically maps and unmaps buffers: +- Buffers are allocated in shared DDR (RPCMEM) via file descriptors (`fastrpc_mmap` using `FASTRPC_MAP_FD_DELAYED`). +- Pinned buffers (such as KV cache and active compute buffers) remain mapped throughout execution. +- Inactive weight buffers are dynamically mapped into the NPU session via `HAP_mmap()` during batch buffer preparation + (`prep_op_bufs()` in `htp/main.c`) and unmapped via `htp_iface_munmap()` when no longer needed by the active batch. +- This dynamic sliding window allows a single NPU session to execute models that exceed the 3.5GB window. + +Alternatively, users can partition and split the model across multiple virtual sessions or physical NPUs using layer-splitting, +tensor-splitting, or row-splitting modes. For user-facing execution modes and examples, see the +[Snapdragon user guide](README.md#multi-device-execution-modes). + +## Op and Kernel Development Guidelines + +Writing high-performance operators for Hexagon requires following specific guidelines. + +### DDR -> DMA -> VTCM Execution Pipeline + +- Strongly prefer the `DDR -> DMA -> VTCM -> compute (HVX/HMX) -> VTCM -> DMA -> DDR` data flow. +- Direct HVX reads/writes from/to DDR are less efficient and should only be used as a fallback. +- The DMA queue is a strict FIFO where operations must be pushed and popped in strict order. +- Follow the pipelined multi-buffering sequence properly (typically 2x to 16x buffering) so every push has a corresponding pop: + + 1. In the prologue, push initial DDR -> VTCM transfers to prime the pipeline. + 2. In the loop body, wait for buffer N via DMA pop, launch HVX/HMX compute on buffer N, push VTCM -> DDR writeback of result N, + and push DDR -> VTCM prefetch of buffer N+2. + 3. In the epilogue, pop all remaining in-flight transfers to drain the pipeline. + +- Because every push must be matched by a pop, `dma_queue_flush()` is not required when the pipeline sequence is followed + properly. Flushing is only used in rare exceptions where a batch of operations is pushed without individual pops. +- Use the DMA queue interface from [`dma-queue.h`](../../../ggml/src/ggml-hexagon/htp/dma-queue.h) + (`dma_queue_push_ddr_to_vtcm`, `dma_queue_pop`, `dma_queue_push_vtcm_to_ddr`). + See [`cumsum-ops.c`](../../../ggml/src/ggml-hexagon/htp/cumsum-ops.c) and + [`act-ops.c`](../../../ggml/src/ggml-hexagon/htp/act-ops.c) for reference implementations. + +### Avoid Scalar Reads and Writes to VTCM + +- Access VTCM data using DMA transfers or HVX/HMX vector instructions rather than scalar reads and writes. + +### Avoid Scalar Division in Inner Loops + +- Hexagon cores do not have hardware division instructions. +- For recurring divisions across iterations or threads, use `fastdiv` from + [`hex-fastdiv.h`](../../../ggml/src/ggml-hexagon/htp/hex-fastdiv.h) with precomputed divisors (such as + `octx->ctx->mdev.count_div` or `octx->n_threads_div`). +- Do not call `init_fastdiv_values()` for single-use divisions; use standard compiler division (`/`) instead. + +### Host-Side Precomputation via `kernel_params` + +- Precompute tensor shapes, strides, scale conversions, tiling layouts, and validation checks on the host CPU during graph + preparation in [`ggml-hexagon.cpp`](../../../ggml/src/ggml-hexagon/ggml-hexagon.cpp). +- Pack precomputed parameters into the operator's fixed `kernel_params` structure in `htp_op_node` (such as + `htp_mm_kernel_params`, `htp_unary_kernel_params`, `htp_fa_kernel_params`, `htp_get_rows_kernel_params`). +- The NPU executes directly using `octx->kernel_params` without redundant runtime metadata extraction or validation. +- **Strict Host-Kernel Alignment**: + - Verify that parameters calculated by the host CPU are strictly honored by the NPU kernel. + - Ensure the kernel does not ignore host-computed fields (for example, falling back to `octx->n_threads` instead of + using `kparams->n_threads`, or ignoring precomputed `tasks_per_thread` and chunk counts). + - Both human developers and coding agents must audit both sides of the interface: ensure fields populated in `kernel_params` + in [`ggml-hexagon.cpp`](../../../ggml/src/ggml-hexagon/ggml-hexagon.cpp) are actively and consistently utilized by the + corresponding operator entry point and worker threads in `htp/*-ops.c`. + +### Tracing Instrumentation + +- All kernels must include trace events for performance profiling and timeline visualization in Perfetto + ([`hex-profile.h`](../../../ggml/src/ggml-hexagon/htp/hex-profile.h)). +- Surround compute sections with `htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)` and + `htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) info)`. +- Use specific event types for major phases: + - `HTP_TRACE_EVT_HVX_COMP`: Vector compute execution. + - `HTP_TRACE_EVT_DMA`: DMA transfer wait or poll cycles. + - `HTP_TRACE_EVT_FENCE`: Multi-device fence barrier synchronization. + - `HTP_TRACE_EVT_L2FLUSH`: L2 cache cleaning operations. +- Pass meaningful progress metrics (such as row index, chunk index, or token index) in the 16-bit `info` parameter. + +### Work Queue and Threading + +- Distribute parallel work across NPU worker threads using the thread pool work queue: + + ```c + work_queue_run(ctx->work_queue, worker_func, &op_ctx, n_threads); + ``` + +- Keep worker functions independent and re-entrant. Worker threads should only operate on their designated chunk of rows or elements. + +### Avoid Redundant Defensive NULL Checks + +- Do not add defensive NULL checks or assertions for internal framework pointers or required graph operands and outputs. + Internal pointers include `ctx`, `octx`, local context structs like `*ctx`, `kparams`, and worker callback `data`. +- These pointers are architectural invariants during kernel execution and host-side graph preparation. + Graph compute receives allocated nodes with valid required `node->src[N]` and `node->data` pointers. +- Do not turn an invariant violation into an unsupported operation or missed fusion. + Checks such as `if (!octx || !octx->ctx)` clutter the code, obscure intent, and hide upstream errors. +- **Distinction**: `octx->src[N]` pointers *can* be NULL by design and must be checked when optional. + Examples include attention masks, optional bias or weights in fused kernels, and frequency factors. + +### Multiline Macro Formatting + +- Keep trailing backslashes in multiline `#define` macros cleanly aligned to a consistent column. +- Avoid trailing whitespace after macro backslashes. +- Use [`scripts/snapdragon/ggml-hexagon-align-macros.py`](../../../scripts/snapdragon/ggml-hexagon-align-macros.py) to inspect, diff, + or automatically align macro definitions across Hexagon kernel sources: + + ```bash + # Check for misaligned macros + python3 scripts/snapdragon/ggml-hexagon-align-macros.py ggml/src/ggml-hexagon/htp/ + + # Fix misaligned macros in-place + python3 scripts/snapdragon/ggml-hexagon-align-macros.py --fix ggml/src/ggml-hexagon/htp/ + ``` + +## Multi-Device Partitioning (mdev) + +Multi-device (mdev) mode enables row-level tensor parallel execution across multiple physical NPU cores or virtual NPU +sessions. + +### 128-Byte Cache Line Alignment + +- Shared tensor buffers reside in DDR (RPCMEM) with a 128-byte cache line granularity + (`HEX_L2_LINE_SIZE` = 128 bytes, `HTP_TENSOR_MDEV_LINE_SIZE`). +- **Rule**: Multi-device work partitions must align destination write regions to 128-byte cache line boundaries so distinct + devices never share or overwrite the same cache line. + +### Partitioning Helpers in `htp-tensor.h` + +Common partitioning logic is factored into reusable inline helpers in +[`htp-tensor.h`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h): + +1. [`htp_tensor_mdev_rows_per_chunk`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L67): + Determines the minimum number of rows per chunk so that the chunk byte size is a multiple of 128 bytes: + + ``` + rows_per_chunk = 128 / hex_gcd_u32(row_size, 128) + ``` + + If row stride `nb[1]` is already a multiple of 128 bytes, `rows_per_chunk = 1`. + Returns `false` if the tensor cannot be safely row-partitioned (such as unaligned base pointer, permuted layout, + or non-128-byte aligned outer strides). -Alternatively, users can choose to use standard llama.cpp/GGML layer-splitting mode to partition and split the model across -multiple Hexagon devices or virtual sessions (which behave like multiple GPUs from the offload and splitting perspective). +2. [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L94): + Calculates the per-device work range `struct htp_tensor_mdev_range { uint32_t start; uint32_t count; }` given + `total_units`, `units_per_chunk`, `mdev_idx`, `mdev_count`, and the precomputed `mdev_count_div`. + Handles chunk distribution across devices, assigns remainder units to the last device, and automatically triggers + single-device fallback when partitioning is unsafe. -Here is an example of running GPT-OSS-20B model on a Snapdragon device using 4 virtual sessions on a single NPU (physical index 0). +### Row-Partitioned Operators +For row-wise operators +(such as activations in [`act-ops.c`](../../../ggml/src/ggml-hexagon/htp/act-ops.c), +binary ops in [`binary-ops.c`](../../../ggml/src/ggml-hexagon/htp/binary-ops.c), +unary ops in [`unary-ops.c`](../../../ggml/src/ggml-hexagon/htp/unary-ops.c), and +sameshape copies in [`cpy-ops.c`](../../../ggml/src/ggml-hexagon/htp/cpy-ops.c)): + +```c +const uint32_t total_rows = ne01 * ne02 * ne03; +const size_t dst_row_size = dst->ne[0] * elem_size; + +uint32_t row_start = 0; +uint32_t nrows = total_rows; + +if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; +} + +if (nrows == 0) { + return HTP_STATUS_OK; +} ``` -~/src/llama.cpp$ ./scripts/snapdragon/run.py --target adb --devices HTP0:0,HTP0:1,HTP0:2,HTP0:3 -- llama-cli --load-mode none -m /data/local/tmp/gguf/gpt-oss-20b-Q4_0.gguf -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -no-cnv -f surfing.txt -... -llama_model_loader: - type f32: 289 tensors -llama_model_loader: - type q4_0: 96 tensors -llama_model_loader: - type q8_0: 2 tensors -llama_model_loader: - type mxfp4: 72 tensors -... -load_tensors: offloaded 25/25 layers to GPU -load_tensors: CPU model buffer size = 1182.09 MiB -load_tensors: HTP0:1 model buffer size = 2512.58 MiB -load_tensors: HTP0:3 model buffer size = 2093.83 MiB -load_tensors: HTP0:0 model buffer size = 2931.34 MiB -load_tensors: HTP0:2 model buffer size = 2512.58 MiB -... -llama_context: n_ctx_per_seq (8192) < n_ctx_train (131072) -- the full capacity of the model will not be utilized -llama_context: CPU output buffer size = 0.77 MiB -llama_kv_cache_iswa: creating non-SWA KV cache, size = 8192 cells -llama_kv_cache: HTP0:1 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0:3 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0:0 KV buffer size = 25.50 MiB -llama_kv_cache: HTP0:2 KV buffer size = 25.50 MiB -llama_kv_cache: size = 102.00 MiB ( 8192 cells, 12 layers, 1/1 seqs), K (q8_0): 51.00 MiB, V (q8_0): 51.00 MiB -llama_kv_cache_iswa: creating SWA KV cache, size = 256 cells -llama_kv_cache: HTP0:1 KV buffer size = 0.80 MiB -llama_kv_cache: HTP0:3 KV buffer size = 0.53 MiB -llama_kv_cache: HTP0:0 KV buffer size = 1.06 MiB -llama_kv_cache: HTP0:2 KV buffer size = 0.80 MiB -llama_kv_cache: size = 3.19 MiB ( 256 cells, 12 layers, 1/1 seqs), K (q8_0): 1.59 MiB, V (q8_0): 1.59 MiB -llama_context: HTP0:0 compute buffer size = 16.06 MiB -llama_context: HTP0:1 compute buffer size = 16.06 MiB -llama_context: HTP0:2 compute buffer size = 16.06 MiB -llama_context: HTP0:3 compute buffer size = 16.06 MiB -llama_context: CPU compute buffer size = 98.19 MiB -... -llama_perf_context_print: prompt eval time = 3843.67 ms / 197 tokens ( 19.51 ms per token, 51.25 tokens per second) -llama_perf_context_print: eval time = 1686.13 ms / 31 runs ( 54.39 ms per token, 18.39 tokens per second) -llama_perf_context_print: total time = 6266.30 ms / 228 tokens -llama_perf_context_print: graphs reused = 30 -llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | -llama_memory_breakdown_print: | - HTP0:0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP0:1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP0:2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - HTP0:3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | -llama_memory_breakdown_print: | - Host | 1476 = 1208 + 105 + 162 | + +### Element-Partitioned Operators + +For flat element-wise operations (such as reshape copies in +[`cpy-ops.c`](../../../ggml/src/ggml-hexagon/htp/cpy-ops.c)): +- Partition total linear elements N = ne0 * ne1 * ne2 * ne3 in 128-byte cache line chunks (`elems_per_line = (elem_size == 4) ? 32 : 64`). +- Requires strict 1D contiguity: + [`htp_tensor_is_contiguous(dst, elem_size)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L28) + and 128-byte aligned destination pointer + [`htp_tensor_mdev_data_aligned(dst)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L47). +- If contiguous and aligned, pass `elems_per_line` to + [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L94); + otherwise pass 0 to trigger Device 0 fallback. + +### Single-Device Fallback (Device 0) + +- Fallback to Device 0 (`mdev.idx == 0`) when partitioning would cause cache line tearing or when work cannot be evenly distributed. +- Triggers: + 1. Destination tensor cannot be safely partitioned (`rows_per_chunk == 0` or non-contiguous/unaligned buffer). + 2. Total aligned chunks < `mdev_count`. +- Device 0 processes the entire tensor `[0, total_units)`. +- Devices 1 ... N-1 receive `count = 0` and return `HTP_STATUS_OK` immediately. + +### Flatten Outer Dimensions Globally + +- **Never partition solely on `ne01` (dimension 1).** +- Partitioning only on `ne01` repeats the device boundary across every 2D slice (`ne02`, `ne03`). If each 2D slice is small, + false sharing occurs repeatedly throughout the tensor. +- Always flatten outer dimensions globally: `total_rows = ne01 * ne02 * ne03` and partition once across the combined row space. + +### Stateless Starting Coordinates + +- Do not use incremental state variables across slices that assume the thread or device starts at index 0. +- Precompute starting multidimensional coordinates at `r = row_start` (or `e = elem_start`) once using `fastdiv`. +- In inner loops, step base pointers directly (`ptr += stride`) or reset/wrap coordinates explicitly (`if (++i01 == ne01) { ... }`). + +### Clean Range Encapsulation + +- Initialize single-device default ranges at declaration: + + ```c + uint32_t row_start = 0; + uint32_t nrows = total_rows; + ``` + +- Encapsulate all multi-device logic inside `if (octx->ctx->mdev.count > 1)`. If the block is omitted or compiled out, + the operator runs standard single-device execution untouched. +- Do not propagate `mdev_` prefixes to worker functions or context structs. Worker threads are device-agnostic and + should only receive standard range parameters (`ctx.row_start`, `ctx.nrows`). +- In worker threads, calculate row intervals using standard arithmetic: + + ```c + const uint32_t ir0 = ctx->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, ctx->row_start + ctx->nrows); + ``` + + In single-device mode (`row_start == 0`), this naturally simplifies to `dr * ith` and `MIN(ir0 + dr, ctx->nrows)` with zero overhead. + +## Multi-Device Synchronization + +Multi-device execution synchronizes worker sessions across devices using explicit barriers and tensor cache flushing. + +### Synchronization Fence Protocol + +Multi-device execution synchronizes worker sessions through atomic fence slots and barriers defined in +[`htp-fence.h`](../../../ggml/src/ggml-hexagon/htp/htp-fence.h): + ``` +[NPU Session 0] [NPU Session 1] + | | + (Input Prep) (Input Prep) + | | + Pre-Op Barrier ----------------------------- Pre-Op Barrier + (mdev_sync_fence) (mdev_sync_fence) + | | + Kernel Execution Kernel Execution + (Output Slice 0) (Output Slice 1) + | | + Tensor Cache Flush Tensor Cache Flush + (htp_tensor_flush_all) (htp_tensor_flush_all) + | | + Post-Op/Batch Barrier ---------------------- Post-Op/Batch Barrier + (htp_mdev_group_barrier) (htp_mdev_group_barrier) + | | + Return Response to Host Return Response to Host +``` + +### Atomic Fence Slots and Cache Invalidation + +- Fence synchronization operates on dedicated RPCMEM shared memory mapped across all participating sessions (`ctx->mdev.fence_base`). +- Each device owns a dedicated 128-byte cache-line aligned fence slot: + + ```c + atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx); + ``` + +- **Writing to fence ([`htp_fence_write`](../../../ggml/src/ggml-hexagon/htp/htp-fence.h#L18))**: + Stores `seq` and `status`, issues a `syncht` thread synchronization barrier, and flushes/invalidates the line + using `Q6_dccleaninva_A(fence)`. +- **Reading from peer fence ([`htp_fence_read`](../../../ggml/src/ggml-hexagon/htp/htp-fence.h#L26))**: + Executes `Q6_dccleaninva_A(fence)` and `syncht` before reading atomic values to ensure fresh data from DDR. + +### Deterministic Monotonic Sequence Numbers + +- Barrier fences use monotonically increasing sequence numbers: + + ```c + const uint32_t seq = ++ctx->mdev.fence_seq; + ``` + +- Comparing sequence numbers with signed arithmetic `(int32_t)(peer_seq - seq) >= 0` prevents race conditions or + misaligned barrier arrivals across iterations. +- If any peer reports an error status (`peer_status > HTP_STATUS_OK`), the barrier propagates the error and unblocks immediately. + +### Tensor Cache Flush and Pipeline Completion + +- In the kernel, ensure all pushed DMA operations have been popped in strict FIFO order to drain the queue. +- Use [`htp_tensor_flush_all()`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h) to flush specific dirty tensors back to DDR: + - [`htp_tensor_flush_all()`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h) flushes only modified tensor address ranges, + ensuring peer devices and the host CPU observe consistent data in DDR. +- Never signal completion before all DMA transfers are drained and dirty tensor flushes have completed. + diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 112e9bae6020..ec7801388689 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -66,7 +66,6 @@ using u32vec = std::vector; #define GGML_HEXAGON_MAX_SESSIONS 16 -#define GGML_HEXAGON_FENCE_BUFFER_SIZE 8192 #define GGML_HEXAGON_FENCE_SLOT_SIZE 128 struct ggml_hexagon_device_config { @@ -75,6 +74,8 @@ struct ggml_hexagon_device_config { int domain_id = 0; std::string domain_name; std::string name; + + std::vector mdev_group; }; static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS]; @@ -350,27 +351,48 @@ struct ggml_hexagon_tensor_extra { }; static inline bool ggml_hexagon_tensor_is_fuseable(const struct ggml_tensor * t) { - if (!t || !t->extra) return false; + if (!t->extra) return false; auto extra = (const struct ggml_hexagon_tensor_extra *) t->extra; return (extra->flags & GGML_HEXAGON_TENSOR_FUSEABLE) != 0; } +static inline bool ggml_hexagon_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) { + const uintptr_t a0 = (uintptr_t) a->data; + const uintptr_t b0 = (uintptr_t) b->data; + const uintptr_t a1 = a0 + ggml_nbytes(a); + const uintptr_t b1 = b0 + ggml_nbytes(b); + + return a0 < b1 && b0 < a1; +} + struct htp_opnode; struct ggml_hexagon_opbatch; struct ggml_hexagon_opqueue; struct ggml_hexagon_shared_buffer; +struct ggml_hexagon_fence_buffer; struct ggml_hexagon_session; +struct ggml_backend_hexagon_device_context; + +struct ggml_hexagon_mdev_group { + uint32_t idx = 0; + uint32_t count = 1; + std::vector> sessions; +}; struct ggml_backend_hexagon_comm_context { std::vector backends; size_t n_backends = 0; - uint32_t fence_seq = 0; + volatile uint32_t * fence_slots[GGML_HEXAGON_MAX_SESSIONS] = {}; + ggml_tensor fence_tensors[GGML_HEXAGON_MAX_SESSIONS] = {}; }; struct ggml_hexagon_event { - ggml_hexagon_session * sess = nullptr; - uint64_t seq = 0; + ggml_hexagon_session * sess = nullptr; + ggml_hexagon_session * fence_sess = nullptr; + volatile uint32_t * fence_slot = nullptr; + ggml_tensor fence_tensor = {}; + uint32_t seq = 0; }; struct ggml_hexagon_session { @@ -387,12 +409,12 @@ struct ggml_hexagon_session { bool valid_queue; bool valid_iface; - std::atomic op_pending; ggml_hexagon_opbatch* op_batch; ggml_hexagon_opqueue* op_queue; std::unordered_map> cloned_buffers; - std::unordered_set sync_peers; + std::unordered_set virt_peers; + std::unordered_set phys_peers; uint32_t n_threads = 0; uint32_t n_hvx = 0; @@ -400,14 +422,23 @@ struct ggml_hexagon_session { uint64_t vtcm_size = 0; size_t max_vmem = 0; size_t max_bufsize = 0; - uint32_t fence_seq; + uint32_t fence_seq = 0; + + std::atomic batch_req_seq{0}; + std::atomic batch_rsp_seq{0}; + std::atomic last_error{HTP_STATUS_OK}; uint64_t cached_uid = 0; std::vector cached_nodes; mutable std::unordered_set needs_repack; - ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr) noexcept(false); + ggml_hexagon_mdev_group mdev; + ggml_backend_dev_t dev = nullptr; + ggml_backend_hexagon_device_context * dev_ctx = nullptr; + ggml_hexagon_fence_buffer * fence_buf = nullptr; + + ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr, uint32_t mdev_idx = 0, uint32_t mdev_count = 0) noexcept(false); ~ggml_hexagon_session() noexcept(true); const char* c_name() const { return name.c_str(); } @@ -415,31 +446,36 @@ struct ggml_hexagon_session { void allocate(const ggml_hexagon_device_config & config) noexcept(false); void release() noexcept(true); + uint8_t * alloc_fence(uint32_t n_slots = 1); + void free_fence(void * ptr, uint32_t n_slots = 1); + + uint8_t * mdev_fence_slot = nullptr; + std::unordered_map cpy_fence_slots; + + void enqueue_mdev_group(); void enqueue_op(const htp_opnode & node); void enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor = nullptr, uint32_t fence_seq = 0); - void enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq = 0); - void enqueue_allreduce(const ggml_tensor * dst, const std::vector & src_tensors, const std::vector & sync_tensors, uint32_t rank, uint32_t n_ranks, uint32_t fence_seq_entry = 0, uint32_t fence_seq_exit = 0); + void enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq = 0, bool wait = true); + void enqueue_allreduce(const ggml_tensor * dst, const std::vector & src_tensors, + const std::vector & sync_tensors, uint32_t rank, uint32_t n_ranks, + uint32_t fence_seq_entry = 0, uint32_t fence_seq_exit = 0); - void flush(bool all = true); - void flush_pending(bool all = false); + void flush_sync(bool all = true); + void flush_async(); void flush_batch(size_t min_ops = 1); - - uint64_t record_event(); - void wait_event(uint64_t seq); + void flush_peers(); + void flush_pending(bool all = true); bool clone_buffer(const ggml_hexagon_shared_buffer*); + void release_buffer(const ggml_hexagon_shared_buffer*); + void unclone_buffer(const ggml_hexagon_shared_buffer*); - void add_sync_peer(ggml_hexagon_session * peer) { - sync_peers.insert(peer); - } - - void flush_sync_peers() { - if (sync_peers.empty()) return; - - for (auto * peer : sync_peers) { - peer->flush_batch(); + void add_peer(ggml_hexagon_session * peer) { + if (this->phys_idx == peer->phys_idx) { + virt_peers.insert(peer); + } else { + phys_peers.insert(peer); } - sync_peers.clear(); } }; @@ -451,8 +487,9 @@ struct ggml_backend_hexagon_device_context { ggml_backend_dev_t dev = nullptr; size_t max_bufsize = 0; - ggml_backend_buffer_type buffer_type = {}; - ggml_backend_buffer_type host_buffer_type = {}; + ggml_backend_buffer_type buffer_type = {}; + ggml_backend_buffer_type host_buffer_type = {}; + ggml_backend_buffer_type fence_buffer_type = {}; std::unique_ptr sess; @@ -484,6 +521,8 @@ struct ggml_hexagon_rpcmem_block { int fd = -1; size_t size = 0; + std::unordered_set mapped_clones; + ggml_hexagon_rpcmem_block(size_t size) { base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size); if (!base) { @@ -508,8 +547,6 @@ struct ggml_hexagon_shared_buffer { ggml_hexagon_session * sess; std::shared_ptr mem; std::vector tensor_extra; - uint32_t fence_head = 0; - size_t fences_size = 0; bool mapped; bool pinned; @@ -518,16 +555,6 @@ struct ggml_hexagon_shared_buffer { size_t size() const { return mem ? mem->size : 0; } int fd() const { return mem ? mem->fd : -1; } - uint8_t * alloc_fence() { - if (fences_size == 0) return nullptr; - int max_slots = fences_size / GGML_HEXAGON_FENCE_SLOT_SIZE; - uint32_t slot = (fence_head++) % max_slots; - - size_t guard_offset = size() - fences_size; - uint8_t * fence_ptr = base() + guard_offset + (size_t)slot * GGML_HEXAGON_FENCE_SLOT_SIZE; - return fence_ptr; - } - void mmap() { if (!this->mem) return; fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : FASTRPC_MAP_FD_DELAYED; @@ -581,29 +608,24 @@ struct ggml_hexagon_shared_buffer { this->mem = nullptr; } - ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false, size_t fence_size = 0) { - this->sess = sess; - this->mapped = false; - this->pinned = pinned; - this->fences_size = fence_size; + ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false) { + this->sess = sess; + this->mapped = false; + this->pinned = pinned; - // Size adjustment inside the buffer class + // Size adjustment inside the buffer class: 4K aligned data size + 4K guard page size_t guard_offset = (size + 4095) & ~4095; - size_t total_size = guard_offset; - if (fence_size > 0) { - total_size += 4096 + fence_size; - } + size_t total_size = guard_offset + 4096; alloc(total_size); } // Clone constructor for cross-session mapping ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) { - this->sess = sess; - this->mem = other.mem; - this->mapped = false; - this->pinned = other.pinned; - this->fences_size = other.fences_size; + this->sess = sess; + this->mem = other.mem; + this->mapped = false; + this->pinned = other.pinned; } ~ggml_hexagon_shared_buffer() { @@ -614,6 +636,59 @@ struct ggml_hexagon_shared_buffer { } }; +struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer { + uint32_t slot_count = 0; + uint32_t slot_head = 0; + std::vector free_slots; + ggml_backend_buffer backend_buffer{}; + + ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size) + : ggml_hexagon_shared_buffer(sess, size, false /* pinned */), + slot_count(size / GGML_HEXAGON_FENCE_SLOT_SIZE), + slot_head(0) { + backend_buffer.buft = buft; + backend_buffer.context = static_cast(this); + backend_buffer.size = size; + } + + uint8_t * alloc_slot(uint32_t n_slots = 1) { + uint8_t * ptr = nullptr; + if (n_slots == 1 && !free_slots.empty()) { + uint32_t slot = free_slots.back(); + free_slots.pop_back(); + ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE; + } else if (slot_head + n_slots <= slot_count) { + uint32_t slot = slot_head; + slot_head += n_slots; + ptr = base() + (size_t) slot * GGML_HEXAGON_FENCE_SLOT_SIZE; + } + if (ptr) { + memset(ptr, 0, (size_t) n_slots * GGML_HEXAGON_FENCE_SLOT_SIZE); + } + return ptr; + } + + void free_slot(void * ptr, uint32_t n_slots = 1) { + if (!ptr) return; + uint32_t slot = ((uint8_t *) ptr - base()) / GGML_HEXAGON_FENCE_SLOT_SIZE; + for (uint32_t i = 0; i < n_slots; i++) { + free_slots.push_back(slot + i); + } + } +}; + +inline uint8_t * ggml_hexagon_session::alloc_fence(uint32_t n_slots) { + uint8_t * ptr = fence_buf->alloc_slot(n_slots); + GGML_ASSERT(ptr); + return ptr; +} + +inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) { + if (fence_buf) { + fence_buf->free_slot(ptr, n_slots); + } +} + static ggml_hexagon_session * ggml_backend_hexagon_buffer_get_sess(ggml_backend_buffer_t buffer) { auto sbuf = static_cast(buffer->context); return sbuf->sess; @@ -621,6 +696,7 @@ static ggml_hexagon_session * ggml_backend_hexagon_buffer_get_sess(ggml_backend_ static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) { auto sbuf = static_cast(buffer->context); + sbuf->sess->unclone_buffer(sbuf); delete sbuf; } @@ -1537,7 +1613,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); } catch (const std::exception & exc) { GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", dev_ctx->c_name(), exc.what()); @@ -1550,7 +1626,7 @@ static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; auto sess = dev_ctx->session(); try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size); } catch (const std::exception & exc) { GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", dev_ctx->c_name(), exc.what()); @@ -1618,11 +1694,16 @@ ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev host_buffer_type.device = dev; host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-HOST", this); + + fence_buffer_type.device = dev; + fence_buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; + fence_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-FENCE", this); } ggml_backend_hexagon_device_context::~ggml_backend_hexagon_device_context() { delete static_cast(buffer_type.context); delete static_cast(host_buffer_type.context); + delete static_cast(fence_buffer_type.context); } static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) { @@ -1698,8 +1779,8 @@ struct ggml_hexagon_opbatch { if (it != b_map.end()) { return it->second; } // Add new buffer to the batch - int bi = n_bufs++; GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS); + int bi = n_bufs++; b_map.insert({sbuf->fd(), bi}); @@ -1902,6 +1983,12 @@ struct ggml_hexagon_opbatch { } } + void update_mdev_group(uint32_t mdev_idx) { + if (n_ops > 0 && h_ops[0].opcode == HTP_OP_MDEV_GROUP) { + h_ops[0].params[0] = (int32_t) mdev_idx; + } + } + bool try_fuse_allreduce_add(const htp_opnode & node) { if (n_ops == 0 || opt_ar_select != 2) return false; if (node.opcode != HTP_OP_ADD) return false; @@ -1910,15 +1997,16 @@ struct ggml_hexagon_opbatch { if (last_node.opcode != HTP_OP_ALLREDUCE) return false; auto * ar_kparams = (struct htp_allreduce_kernel_params *) last_node.kernel_params; - const uint32_t rank = (uint32_t) ar_kparams->rank; - const ggml_tensor * ar_local = (rank < last_node.inputs.size()) ? last_node.inputs[rank] : nullptr; + const uint32_t rank = (uint32_t) ar_kparams->rank; + const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; + const ggml_tensor * ar_local = last_node.inputs[rank]; const ggml_tensor * add_src0 = node.src0(); const ggml_tensor * add_src1 = node.src1(); + const ggml_tensor * add_dst = node.dst(); - if (!add_src0 || !add_src1 || !ar_local) return false; if (!ggml_hexagon_tensor_is_fuseable(ar_local)) return false; - const ggml_tensor * res_tensor = nullptr; + const ggml_tensor * res_tensor; if (add_src0 == ar_local || add_src0->data == ar_local->data) { res_tensor = add_src1; } else if (add_src1 == ar_local || add_src1->data == ar_local->data) { @@ -1927,14 +2015,12 @@ struct ggml_hexagon_opbatch { return false; } - if (!res_tensor || !res_tensor->data) return false; - if (ar_local->type != res_tensor->type) return false; const bool is_same_shape = (ar_local->ne[0] == res_tensor->ne[0] && ar_local->ne[1] == res_tensor->ne[1] && ar_local->ne[2] == res_tensor->ne[2] && ar_local->ne[3] == res_tensor->ne[3]); - const bool is_row_bcast = (ar_local->ne[0] == res_tensor->ne[0] && - res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); + const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] && res_tensor->ne[1] == 1 && + res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); if (!is_same_shape && !is_row_bcast) return false; @@ -1947,13 +2033,21 @@ struct ggml_hexagon_opbatch { return false; } } - if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(node.dst())) { + if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) { return false; } + for (uint32_t r = 0; r < n_ranks; r++) { + const ggml_tensor * ar_src = last_node.inputs[r]; + if (ggml_hexagon_tensors_overlap(add_dst, ar_src)) { + HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", sess->c_name(), r); + return false; + } + } + struct htp_allreduce_kernel_params new_kparams; if (!ggml_hexagon_precompute_allreduce_params( - sess, node.dst(), (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, &new_kparams + sess, add_dst, (uint32_t) ar_kparams->rank, (uint32_t) ar_kparams->n_ranks, true, is_row_bcast, &new_kparams )) { HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", sess->c_name()); return false; @@ -1961,7 +2055,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -1972,7 +2065,7 @@ struct ggml_hexagon_opbatch { } }; fit_t(res_tensor); - fit_t(node.dst()); + fit_t(add_dst); if ((extra_bufs + n_bufs) > n_bufs_max || (extra_tens + n_tens) > n_tens_max || (extra_vmem + b_vmem) > b_vmem_max) { return false; } @@ -1981,7 +2074,7 @@ struct ggml_hexagon_opbatch { last_node.name = "ALLREDUCE+ADD"; last_node.inputs.push_back(res_tensor); last_node.outputs.clear(); - last_node.outputs.push_back(node.dst()); + last_node.outputs.push_back(add_dst); last_node.fused.push_back(node.node); memcpy(last_node.kernel_params, &new_kparams, sizeof(new_kparams)); @@ -1989,9 +2082,8 @@ struct ggml_hexagon_opbatch { o.opcode = HTP_OP_ALLREDUCE_ADD; memcpy(o.kernel_params, &new_kparams, sizeof(new_kparams)); - const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; o.src[2 * n_ranks] = add_tensor(res_tensor); - o.dst[0] = add_tensor(node.dst()); + o.dst[0] = add_tensor(add_dst); for (uint32_t d = 1; d < HTP_OP_MAX_OUTPUTS; d++) { o.dst[d] = 0xffff; } @@ -2011,10 +2103,9 @@ struct ggml_hexagon_opbatch { const ggml_tensor * mul_src1 = node.src1(); const ggml_tensor * rms_out = last_node.dst(); - if (!mul_src0 || !mul_src1 || !rms_out) return false; if (!ggml_hexagon_tensor_is_fuseable(rms_out)) return false; - const ggml_tensor * weight = nullptr; + const ggml_tensor * weight; if (mul_src0 == rms_out || mul_src0->data == rms_out->data) { weight = mul_src1; } else if (mul_src1 == rms_out || mul_src1->data == rms_out->data) { @@ -2023,10 +2114,7 @@ struct ggml_hexagon_opbatch { return false; } - if (!weight || !weight->data) return false; - const ggml_tensor * src0 = last_node.src0(); - if (!src0 || !src0->data) return false; if (src0->ne[0] != weight->ne[0] || src0->ne[0] != node.dst()->ne[0]) { return false; @@ -2057,7 +2145,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2112,10 +2199,9 @@ struct ggml_hexagon_opbatch { const ggml_tensor * add_src1 = node.src1(); const ggml_tensor * mm_out = last_node.dst(); - if (!add_src0 || !add_src1 || !mm_out) return false; if (!ggml_hexagon_tensor_is_fuseable(mm_out)) return false; - const ggml_tensor * src2 = nullptr; + const ggml_tensor * src2; if (add_src0 == mm_out || add_src0->data == mm_out->data) { src2 = add_src1; } else if (add_src1 == mm_out || add_src1->data == mm_out->data) { @@ -2124,11 +2210,8 @@ struct ggml_hexagon_opbatch { return false; } - if (!src2 || !src2->data) return false; - const ggml_tensor * src0 = last_node.src0(); const ggml_tensor * src1 = last_node.src1(); - if (!src0 || !src1) return false; struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_fused_matmul_add_params(sess, src0, src1, src2, node.dst(), &kparams); @@ -2144,7 +2227,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2197,7 +2279,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * w_in = node.src0(); const ggml_tensor * x_in = node.src1(); const ggml_tensor * d_in = node.dst(); - if (!w_in || !x_in || !d_in) return false; htp_opnode & last_node = ops[n_ops - 1]; @@ -2231,7 +2312,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2282,7 +2362,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * w0 = last_node.src0(); const ggml_tensor * x = last_node.src1(); const ggml_tensor * w1 = node.src0(); - if (!w0 || !x || !w1) return false; struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_fused_mmnx_params(sess, w0, x, 2, &kparams); @@ -2297,7 +2376,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2359,7 +2437,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * x_in = node.src1(); const ggml_tensor * ids_in = node.node->src[2]; const ggml_tensor * d_in = node.dst(); - if (!w_in || !x_in || !ids_in || !d_in) return false; htp_opnode & last_node = ops[n_ops - 1]; @@ -2394,7 +2471,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2447,7 +2523,6 @@ struct ggml_hexagon_opbatch { const ggml_tensor * x = last_node.src1(); const ggml_tensor * ids = last_node.node->src[2]; const ggml_tensor * w1 = node.src0(); - if (!w0 || !x || !ids || !w1) return false; struct htp_mm_kernel_params kparams; ggml_hexagon_precompute_fused_mmidnx_params(sess, w0, x, node.dst(), 2, &kparams); @@ -2462,7 +2537,6 @@ struct ggml_hexagon_opbatch { size_t extra_bufs = 0, extra_vmem = 0, extra_tens = 0; auto fit_t = [&](const ggml_tensor * t) { - if (!t) return; if (!t_map.count(t)) { extra_tens++; auto sbuf = static_cast(t->buffer->context); @@ -2540,17 +2614,14 @@ struct ggml_hexagon_opqueue { // Shared buffer for storing batches ggml_hexagon_shared_buffer *shm_buf; size_t shm_blk_size; - - uint64_t req_seq = 0; - uint64_t rsp_seq = 0; + size_t depth; using opvec = std::vector; - std::queue done; // completed batch ids std::vector op_cache; // per batch op cache std::vector start_usec; // per batch start time - ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) { + ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) { size_t n_bufs = HTP_OP_MAX_BUFS; size_t n_ops = batch_size; size_t n_tensors = n_ops * HTP_OP_MAX_OUTPUTS + n_ops * HTP_OP_MAX_INPUTS; @@ -2571,9 +2642,6 @@ struct ggml_hexagon_opqueue { op_cache.resize(depth); start_usec.resize(depth, 0); - // init done queue - for (unsigned int i = 0; i < depth; i++) { done.push(i); } - if (opt_verbose) { GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n", sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size); @@ -2587,7 +2655,7 @@ struct ggml_hexagon_opqueue { size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; } // push new batch - bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, ggml_hexagon_opbatch* op_batch) { + bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) { static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8"); static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8"); static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8"); @@ -2595,16 +2663,17 @@ struct ggml_hexagon_opqueue { static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8"); static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8"); - if (done.empty()) { return false; } + if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; } - req.id = done.front(); done.pop(); // batch id + const uint32_t slot = (uint32_t) ((seq - 1) % depth); + + req.seq = seq; req.n_bufs = op_batch->n_bufs; req.n_tensors = op_batch->n_tens; req.n_ops = op_batch->n_ops; - req.seq = ++req_seq; - op_cache[req.id] = std::move(op_batch->ops); - start_usec[req.id] = ggml_time_us(); + op_cache[slot] = op_batch->ops; + start_usec[slot] = ggml_time_us(); const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs; const size_t t_size = sizeof(htp_tensor) * req.n_tensors; @@ -2619,7 +2688,7 @@ struct ggml_hexagon_opqueue { req.n_traces = 0; } - dbuf.ptr = shm_buf->base() + (req.id * shm_blk_size); + dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size); dbuf.fd = shm_buf->fd(); dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base(); @@ -2632,18 +2701,14 @@ struct ggml_hexagon_opqueue { uint8_t * t_ptr = m_ptr; m_ptr += t_size; uint8_t * o_ptr = m_ptr; - op_batch->sort_buffers(); - memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size); memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size); memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size); - HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%u : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", - shm_buf->sess->c_name(), req.id, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, + HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, b_size, t_size, o_size, (size_t) dbuf.size); - op_batch->reset(); - if (opt_verbose > 1) { htp_buf_desc *b = (htp_buf_desc*) b_ptr; for (unsigned int i=0; i < req.n_bufs; i++) { @@ -2662,9 +2727,7 @@ struct ggml_hexagon_opqueue { } void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) { - GGML_ASSERT(rsp.id < op_cache.size()); - - done.push(rsp.id); + const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth); const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs; const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors; @@ -2681,15 +2744,15 @@ struct ggml_hexagon_opqueue { const size_t m_size = b_size + t_size + o_size + p_size + tr_size; GGML_ASSERT(m_size <= shm_blk_size); - HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%u : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", - shm_buf->sess->c_name(), rsp.id, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, + HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, (size_t) dbuf.size, b_size, t_size, o_size); uint8_t * m_ptr = (uint8_t*) dbuf.ptr; uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size); if (rsp.n_ops > 0) { - auto & ops = op_cache[rsp.id]; + auto & ops = op_cache[slot]; GGML_ASSERT(rsp.n_ops <= ops.size()); const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr; @@ -2712,16 +2775,41 @@ struct ggml_hexagon_opqueue { ggml_hexagon_dump_trace_events(shm_buf->sess->name, rsp, trace_events, n_traces); } } - - if (rsp.seq > rsp_seq) { - rsp_seq = rsp.seq; - } } }; -// Flush HTP response queue i.e wait for all outstanding requests to complete +void ggml_hexagon_session::flush_peers() { + auto vpeers = std::move(virt_peers); + virt_peers.clear(); + for (auto * peer : vpeers) { + peer->flush_sync(); + } + + auto ppeers = std::move(phys_peers); + phys_peers.clear(); + for (auto * peer : ppeers) { + peer->flush_async(); + } + + for (auto & sub : this->mdev.sessions) { + sub->flush_peers(); + } +} + +void ggml_hexagon_session::flush_async() { + flush_peers(); + flush_batch(); +} + void ggml_hexagon_session::flush_pending(bool all) { - while (this->op_pending) { + for (auto & sub : this->mdev.sessions) { + sub->flush_pending(all); + if (sub->last_error > HTP_STATUS_OK) { + this->last_error = sub->last_error.load(); + } + } + + while (this->batch_rsp_seq < this->batch_req_seq) { struct htp_opbatch_rsp rsp; uint32_t rsp_size; uint32_t flags; @@ -2746,32 +2834,64 @@ void ggml_hexagon_session::flush_pending(bool all) { GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs); } - if (rsp.status != HTP_STATUS_OK) { - GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp: %s\n", this->c_name(), status_to_str(rsp.status)); - // TODO: handle errors + if (rsp.status > HTP_STATUS_OK) { + GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status)); + this->last_error = rsp.status; + for (auto & sub : this->mdev.sessions) { + sub->last_error = rsp.status; + } } op_queue->pop(rsp, dbuf); - this->op_pending--; // atomic dec + GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1); + this->batch_rsp_seq = rsp.seq; if (!all) break; } } +void ggml_hexagon_session::flush_sync(bool all) { + flush_async(); + flush_pending(all); +} + void ggml_hexagon_session::flush_batch(size_t min_ops) { if (op_batch->n_ops < min_ops) { return; } + op_batch->sort_buffers(); + htp_opbatch_req req {}; dspqueue_buffer dbuf{}; - if (!op_queue->push(req, dbuf, op_batch)) { + const uint64_t seq = ++this->batch_req_seq; + + op_batch->update_mdev_group(this->mdev.idx); + + if (!op_queue->push(req, dbuf, op_batch, seq)) { flush_pending(false); - op_queue->push(req, dbuf, op_batch); + op_queue->push(req, dbuf, op_batch, seq); } - // Bump pending flag (cleared in the session::flush once we get the response) - this->op_pending++; // atomic inc + for (auto & sub : this->mdev.sessions) { + htp_opbatch_req sub_req {}; + dspqueue_buffer sub_dbuf{}; + + sub->batch_req_seq = seq; + op_batch->update_mdev_group(sub->mdev.idx); + + if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) { + sub->flush_pending(false); + sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq); + } + + HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size); + + int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT); + if (err != 0) { + GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err); + } + } HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size); @@ -2779,28 +2899,28 @@ void ggml_hexagon_session::flush_batch(size_t min_ops) { if (err != 0) { GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err); } -} -void ggml_hexagon_session::flush(bool all) { - flush_sync_peers(); - flush_batch(); - flush_pending(all); + op_batch->reset(); } void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { - for (auto t : node.get_inputs()) { + auto clone_tensor_buffer = [this](const ggml_tensor * t) { if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) { + auto sbuf = static_cast(t->buffer->context); if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) { - this->clone_buffer(static_cast(t->buffer->context)); + this->clone_buffer(sbuf); + } + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(sbuf); } } + }; + + for (auto t : node.get_inputs()) { + clone_tensor_buffer(t); } for (auto t : node.get_outputs()) { - if (t && t->buffer && ggml_backend_buffer_is_hexagon(t->buffer)) { - if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != this) { - this->clone_buffer(static_cast(t->buffer->context)); - } - } + clone_tensor_buffer(t); } if (opt_opfusion && op_batch->try_fuse(node)) { @@ -2808,39 +2928,84 @@ void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { } if (!op_batch->fit_op(node)) { - flush_batch(); + flush_async(); } + + if (this->mdev.count > 1 && op_batch->n_ops == 0) { + enqueue_mdev_group(); + } + op_batch->add_op(node); } +void ggml_hexagon_session::enqueue_mdev_group() { + htp_opnode group_node(HTP_OP_MDEV_GROUP); + + uint8_t * fence_slot = this->mdev_fence_slot; + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + ggml_tensor dummy_t {}; + dummy_t.buffer = &this->fence_buf->backend_buffer; + dummy_t.extra = &fence_extra; + dummy_t.data = (void *) fence_slot; + dummy_t.type = GGML_TYPE_I8; + dummy_t.ne[0] = HTP_FENCE_SLOT_SIZE; + dummy_t.ne[1] = (int64_t) this->mdev.count; + dummy_t.ne[2] = 1; + dummy_t.ne[3] = 1; + dummy_t.nb[0] = 1; + dummy_t.nb[1] = HTP_FENCE_SLOT_SIZE; + dummy_t.nb[2] = dummy_t.nb[1] * dummy_t.ne[1]; + dummy_t.nb[3] = dummy_t.nb[2]; + dummy_t.op = GGML_OP_NONE; + dummy_t.op_params[0] = (int32_t) this->mdev.idx; + + ggml_tensor * node = group_node.add_dummy(dummy_t); + node->src[0] = node; + group_node.init(node); + group_node.outputs.clear(); + group_node.name = "MDEV_GROUP"; + + if (this->fence_buf->sess != this) { + this->clone_buffer(this->fence_buf); + } + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(this->fence_buf); + } + + op_batch->add_op(group_node); +} + void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) { - htp_opnode cpy_node(HTP_OP_CPY); + const bool with_fence = sync_tensor != nullptr; + htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY); ggml_tensor* node = cpy_node.add_dummy(*dst); node->op = GGML_OP_CPY; node->src[0] = const_cast(src); - node->src[1] = sync_tensor ? cpy_node.add_dummy(*sync_tensor) : nullptr; - if (sync_tensor) { + node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr; + if (with_fence) { node->op_params[0] = (int32_t) fence_seq; } cpy_node.init(node); - if (sync_tensor) { + if (with_fence) { cpy_node.name = "CPY+FENCE"; } this->enqueue_op(cpy_node); } -void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq) { +void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) { htp_opnode sync_node(HTP_OP_FENCE); ggml_tensor* node = sync_node.add_dummy(*sync_tensor); node->op = GGML_OP_NONE; node->src[0] = node; node->op_params[0] = (int32_t) fence_seq; + node->op_params[1] = wait ? 0 : 1; sync_node.init(node); - sync_node.name = "FENCE"; + sync_node.name = wait ? "FENCE_WAIT" : "FENCE_SIGNAL"; this->enqueue_op(sync_node); } @@ -2858,7 +3023,6 @@ static bool ggml_hexagon_precompute_allreduce_params( kparams->n_ranks = (int32_t) n_ranks; kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0; - const uint32_t n_bufs = n_ranks + 1 + (has_add ? 1 : 0); const uint32_t nelem = (uint32_t) ggml_nelements(dst); const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); const bool is_contiguous = ggml_is_contiguous(dst); @@ -2902,6 +3066,7 @@ static bool ggml_hexagon_precompute_allreduce_params( const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem; const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nelem / 128)); kparams->n_threads = n_threads; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); uint32_t block_elems = 65536; if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) { @@ -2911,15 +3076,15 @@ static bool ggml_hexagon_precompute_allreduce_params( kparams->block_elems = block_elems; kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_elems > 128) { - const size_t max_bytes_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2); + const size_t max_bytes_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2); block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128); if (block_elems < 128) break; kparams->block_elems = block_elems; kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; } if (sess->vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) { @@ -2935,6 +3100,7 @@ static bool ggml_hexagon_precompute_allreduce_params( const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem; const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, (std::max)(1u, rank_nrows)); kparams->n_threads = n_threads; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); const uint32_t row_bytes = ne0 * elem_size; const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128); @@ -2946,14 +3112,14 @@ static bool ggml_hexagon_precompute_allreduce_params( kparams->block_elems = block_rows; kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; while ((size_t) kparams->vtcm_size > sess->vtcm_size && block_rows > 1) { - const size_t max_rows_per_buf = sess->vtcm_size / (n_threads * n_bufs * 2 * row_size_aligned); + const size_t max_rows_per_buf = sess->vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned); block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf); kparams->block_elems = block_rows; kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); - kparams->vtcm_size = n_threads * n_bufs * kparams->vtcm_size_per_thread; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; if (max_rows_per_buf == 0) break; } @@ -3009,28 +3175,20 @@ void ggml_hexagon_session::enqueue_allreduce( this->enqueue_op(ar_node); } -void ggml_hexagon_session::wait_event(uint64_t seq) { - flush_sync_peers(); - HEX_VERBOSE("ggml-hex: %s opqueue-wait start: seq %llu, current rsp-seq %llu, pending %d\n", - this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending); - while (op_queue->rsp_seq < seq && this->op_pending > 0) { - this->flush_pending(false); - } - HEX_VERBOSE("ggml-hex: %s opqueue-wait end: seq %llu, current rsp-seq %llu, pending %d\n", - this->name.c_str(), (unsigned long long)seq, (unsigned long long)op_queue->rsp_seq, (int)this->op_pending); -} - -uint64_t ggml_hexagon_session::record_event() { - flush_batch(); - return op_queue->req_seq; -} - bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer *sbuf) { - if (this->cloned_buffers.find(sbuf->fd()) != this->cloned_buffers.end()) return true; + GGML_ASSERT(sbuf && sbuf->mem); + if (sbuf->sess == this) return true; + + auto mem = sbuf->mem; + int fd = mem->fd; + + GGML_ASSERT(fd >= 0); + + if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true; HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(), - sbuf->c_name(), sbuf->base(), sbuf->size(), sbuf->fd()); + sbuf->c_name(), sbuf->base(), sbuf->size(), fd); auto clone = std::make_unique(this, *sbuf); try { @@ -3040,10 +3198,38 @@ bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer *sbuf) return false; } - this->cloned_buffers[sbuf->fd()] = std::move(clone); + this->cloned_buffers[fd] = std::move(clone); + mem->mapped_clones.insert(this); return true; } +void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); + + auto mem = sbuf->mem; + int fd = mem->fd; + + GGML_ASSERT(fd >= 0); + + auto it = this->cloned_buffers.find(fd); + if (it != this->cloned_buffers.end()) { + auto clone = std::move(it->second); + this->cloned_buffers.erase(it); + } + mem->mapped_clones.erase(this); +} + +void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); + + auto mem = sbuf->mem; + std::vector sessions(mem->mapped_clones.begin(), mem->mapped_clones.end()); + + for (auto * sess : sessions) { + sess->release_buffer(sbuf); + } +} + static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) { // Allocate a bunch pinned buffers till failure. // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device. @@ -3082,14 +3268,16 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->valid_queue = false; this->valid_iface = false; - this->phys_idx = phys_idx; - this->virt_idx = virt_idx; - this->domain_id = config.domain_id; - this->session_id = 0; - this->name = config.name; - this->op_pending = 0; + this->name = config.name; + this->phys_idx = phys_idx; + this->virt_idx = virt_idx; + this->domain_id = config.domain_id; + this->session_id = 0; + this->batch_req_seq = 0; + this->batch_rsp_seq = 0; + this->last_error = HTP_STATUS_OK; - GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str()); + GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx); if (config.domain_id < 0 || config.domain_name.empty()) { GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx); @@ -3098,25 +3286,14 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n const std::string & dom_name = config.domain_name; - // Enable Unsigned PD for all domains - { - struct remote_rpc_control_unsigned_module u; - u.domain = -1; - u.enable = 1; - int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); - if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); - throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); - } - } - // Create new session if virtual_idx > 0 if (virt_idx > 0) { - struct remote_rpc_reserve_new_session n; + struct remote_rpc_reserve_new_session n {}; n.domain_name_len = dom_name.size(); n.domain_name = const_cast(dom_name.c_str()); n.session_name = const_cast(this->name.c_str()); n.session_name_len = this->name.size(); + n.session_id = virt_idx; int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n)); if (err != AEE_SUCCESS) { @@ -3130,7 +3307,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->domain_id = n.effective_domain_id; this->valid_session = true; } else { - struct remote_rpc_effective_domain_id eff = {}; + struct remote_rpc_effective_domain_id eff {}; eff.domain_name = const_cast(dom_name.c_str()); eff.domain_name_len = dom_name.size(); eff.session_id = 0; @@ -3144,6 +3321,18 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n } } + // Enable unsigned modules + { + struct remote_rpc_control_unsigned_module u; + u.domain = this->domain_id; + u.enable = 1; + int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); + if (err != AEE_SUCCESS) { + GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); + throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); + } + } + char session_uri[256]; { char htp_uri[256]; @@ -3171,7 +3360,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n // Open session int err = htp_iface_open(session_uri, &this->handle); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: %s failed to open session : error 0x%x\n", this->c_name(), err); + GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err); throw std::runtime_error("ggml-hex: failed to open session (see log for details)"); } @@ -3186,8 +3375,9 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n unsigned long long hw_vtcm_size = 0; int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size); if (hw_err == 0) { - this->n_threads = opt_nhvx > 0 ? (uint32_t)opt_nhvx : (uint32_t)hw_n_threads; - this->n_hvx = opt_nhvx > 0 ? (uint32_t)opt_nhvx : (uint32_t)hw_n_hvx; + const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads); + this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads; + this->n_hvx = this->n_threads; this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; this->vtcm_size = (uint64_t)hw_vtcm_size; GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n", @@ -3195,8 +3385,9 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n (unsigned long long)(this->vtcm_size / (1024 * 1024))); } else { GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err); - this->n_threads = opt_nhvx > 0 ? (uint32_t)opt_nhvx : 8; - this->n_hvx = opt_nhvx > 0 ? (uint32_t)opt_nhvx : 8; + const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS); + this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads; + this->n_hvx = this->n_threads; this->n_hmx = (opt_nhmx != 0) ? 1 : 0; this->vtcm_size = 8 * 1024 * 1024; } @@ -3252,6 +3443,11 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n // Allocate buffers and state for op batching this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue); + this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024); + if (this->mdev.count > 1) { + this->mdev_fence_slot = this->alloc_fence(this->mdev.count); + } + if (!opt_vmem) { opt_vmem = ggml_hexagon_measure_max_vmem(this); GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem); @@ -3262,7 +3458,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); // Start dspqueue/opbatch processing - err = htp_iface_start(this->handle, this->session_id, this->queue_id, opt_nhvx, opt_nhmx, this->max_vmem); + err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem); if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); @@ -3283,6 +3479,8 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n void ggml_hexagon_session::release() noexcept(true) { GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str()); + this->mdev.sessions.clear(); + int err; if (this->valid_iface) { @@ -3295,6 +3493,19 @@ void ggml_hexagon_session::release() noexcept(true) { delete this->op_batch; delete this->op_queue; + for (auto & it : this->cpy_fence_slots) { + free_fence((void *) it.second, 1); + } + this->cpy_fence_slots.clear(); + + if (this->fence_buf) { + unclone_buffer(this->fence_buf); + delete this->fence_buf; + this->fence_buf = nullptr; + } + while (!this->cloned_buffers.empty()) { + release_buffer(this->cloned_buffers.begin()->second.get()); + } if (opt_etm) { err = htp_iface_etm(this->handle, 0); @@ -3321,23 +3532,30 @@ void ggml_hexagon_session::release() noexcept(true) { if (this->valid_handle) { htp_iface_close(this->handle); } - - this->cloned_buffers.clear(); } -ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) noexcept(false) { - op_batch = nullptr; - op_queue = nullptr; - fence_seq = ((uintptr_t)this) & 0xFFFF; +ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) { + this->dev = dev; + this->dev_ctx = static_cast(dev->context); + this->mdev.idx = mdev_idx; + this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size()); + op_batch = nullptr; + op_queue = nullptr; + fence_buf = nullptr; + fence_seq = ((uintptr_t)this) & 0xFFFF; try { allocate(config); + if (this->mdev.idx == 0 && !config.mdev_group.empty()) { + for (size_t i = 0; i < config.mdev_group.size(); i++) { + this->mdev.sessions.push_back(std::make_unique( + config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count)); + } + } } catch (const std::exception & exc) { release(); throw; } - - GGML_UNUSED(dev); } ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { @@ -3563,10 +3781,6 @@ static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_ses const struct ggml_tensor * state = op->src[5]; const struct ggml_tensor * dst = op; - if (!q || !k || !v || !g || !beta || !state) { - return false; - } - if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 || g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { @@ -3754,6 +3968,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( struct htp_mm_kernel_params * kparams ) { kparams->n_hmx = 0; + kparams->n_threads = sess->n_threads; const bool is_quant = (wtype != GGML_TYPE_F16 && wtype != GGML_TYPE_F32); const int src1_nrows = ne11 * ne12 * ne13; @@ -4193,6 +4408,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params( struct htp_mm_kernel_params * kparams ) { memset(kparams, 0, sizeof(*kparams)); + kparams->n_threads = sess->n_threads; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; @@ -4921,6 +5137,14 @@ static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, return false; } + const int32_t lp0 = ((const int32_t *) op->op_params)[0]; + const int32_t rp0 = ((const int32_t *) op->op_params)[1]; + const int32_t circular = ((const int32_t *) op->op_params)[8]; + + if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) { + return false; + } + return true; GGML_UNUSED(sess); @@ -4972,10 +5196,6 @@ static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * const struct ggml_tensor * src1 = op->src[1]; // B const struct ggml_tensor * dst = op; // X - if (!src0 || !src1) { - return false; - } - if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { return false; } @@ -5145,7 +5365,7 @@ static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const st } static bool is_mergeable_mul_mat(const ggml_tensor * t) { - if (!t || t->op != GGML_OP_MUL_MAT) return false; + if (t->op != GGML_OP_MUL_MAT) return false; const ggml_tensor * src0 = t->src[0]; const ggml_tensor * src1 = t->src[1]; @@ -5179,7 +5399,7 @@ static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor } static bool is_mergeable_mul_mat_id(const ggml_tensor * t) { - if (!t || t->op != GGML_OP_MUL_MAT_ID) return false; + if (t->op != GGML_OP_MUL_MAT_ID) return false; const ggml_tensor * src0 = t->src[0]; return ggml_hexagon_is_repack_type(src0->type); @@ -5213,6 +5433,10 @@ static bool is_mergeable_mul_mat_id_pair(const ggml_tensor * n1, const ggml_tens static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { auto sess = static_cast(backend->context); + if (sess->last_error > HTP_STATUS_OK) { + return GGML_STATUS_FAILED; + } + HEX_VERBOSE("ggml-hex: %s graph-compute n_nodes %d\n", sess->c_name(), graph->n_nodes); const std::vector * nodes_ptr = nullptr; @@ -5228,6 +5452,8 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg auto * extra = (ggml_hexagon_tensor_extra *) graph->nodes[i]->extra; if (!extra) continue; + extra->flags &= ~GGML_HEXAGON_TENSOR_FUSEABLE; + if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE; } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT || graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { @@ -5299,6 +5525,10 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg sess->enqueue_op(node); } + if (sess->last_error > HTP_STATUS_OK) { + return GGML_STATUS_FAILED; + } + return GGML_STATUS_SUCCESS; } @@ -5308,7 +5538,10 @@ static void ggml_backend_hexagon_synchronize(ggml_backend_t backend) { HEX_VERBOSE("ggml-hex: %s synchronize\n", sess->c_name()); // Wait until all pending ops complete - sess->flush(); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s synchronize failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } } enum ggml_hexagon_mem_range_type { @@ -5543,27 +5776,38 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr GGML_UNUSED(backend); } +static uint64_t ggml_hexagon_session_key(const ggml_hexagon_session * sess) { + return ((uint64_t) (uint32_t) sess->phys_idx << 32) | (uint32_t) sess->virt_idx; +} + static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context; - if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; - uint32_t fence_seq = sess_dst->fence_seq++; - if (sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; + if (!sess_src->clone_buffer(sbuf_dst)) { return false; } - volatile uint32_t * fence = (volatile uint32_t *) sbuf_dst->alloc_fence(); + const uint64_t src_key = ggml_hexagon_session_key(sess_src); + auto & fence_slot = sess_dst->cpy_fence_slots[src_key]; + if (!fence_slot) { + fence_slot = (volatile uint32_t *) sess_dst->alloc_fence(1); + } + + if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; } + + if (++sess_dst->fence_seq == 0) sess_dst->fence_seq = 1; + uint32_t fence_seq = sess_dst->fence_seq; - HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq %u\n", + HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src), fence_seq); - // dummy extra (must be static) + // dummy fence extra (must be static) static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; ggml_tensor fence_tensor {}; - fence_tensor.buffer = dst->buffer; + fence_tensor.buffer = &sess_dst->fence_buf->backend_buffer; fence_tensor.extra = &fence_extra; - fence_tensor.data = (void *) fence; + fence_tensor.data = (void *) fence_slot; fence_tensor.type = GGML_TYPE_I32; fence_tensor.ne[0] = 1; fence_tensor.ne[1] = 1; @@ -5576,9 +5820,9 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_ fence_tensor.op = GGML_OP_NONE; sess_src->enqueue_cpy(src, dst, &fence_tensor, fence_seq); - sess_dst->enqueue_fence(&fence_tensor, fence_seq); + sess_dst->enqueue_fence(&fence_tensor, fence_seq, /* wait = */ true); - sess_dst->add_sync_peer(sess_src); + sess_dst->add_peer(sess_src); return true; } @@ -5586,15 +5830,15 @@ static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_ static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); - auto sbuf_dst = (ggml_hexagon_shared_buffer *) dst->buffer->context; + auto sbuf_src = (ggml_hexagon_shared_buffer *) src->buffer->context; - if (!sess_src->clone_buffer(sbuf_dst)) { return false; } + if (!sess_dst->clone_buffer(sbuf_src)) { return false; } HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); - sess_src->enqueue_cpy(src, dst); - sess_src->flush(true); + sess_dst->enqueue_cpy(src, dst); + sess_dst->add_peer(sess_src); return true; } @@ -5604,7 +5848,14 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg return false; } - *(ggml_hexagon_tensor_extra *) dst->extra = *(const ggml_hexagon_tensor_extra *) src->extra; + // FIXME: ggml-meta needs to call init_tensor on auxiliary tensors + if (!dst->extra) { + ggml_backend_buffer_init_tensor(dst->buffer, dst); + } + + auto * dst_extra = static_cast(dst->extra); + const auto * src_extra = static_cast(src->extra); + dst_extra->flags = src_extra->flags & ~GGML_HEXAGON_TENSOR_FUSEABLE; auto sess_src = static_cast(backend_src->context); auto sess_dst = static_cast(backend_dst->context); @@ -5612,7 +5863,6 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg if (sess_src == sess_dst) { HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); sess_src->enqueue_cpy(src, dst); - sess_src->flush_batch(); return true; } @@ -5623,8 +5873,30 @@ static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, gg } static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_dev_t dev) { + auto dev_ctx = static_cast(dev->context); + auto sess = dev_ctx->session(); + ggml_hexagon_event * hex_event = new ggml_hexagon_event(); - HEX_VERBOSE("ggml-hex: %s event-new : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event); + hex_event->fence_sess = sess; + hex_event->sess = sess; + hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1); + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + hex_event->fence_tensor.buffer = &sess->fence_buf->backend_buffer; + hex_event->fence_tensor.extra = &fence_extra; + hex_event->fence_tensor.data = (void *) hex_event->fence_slot; + hex_event->fence_tensor.type = GGML_TYPE_I32; + hex_event->fence_tensor.ne[0] = 1; + hex_event->fence_tensor.ne[1] = 1; + hex_event->fence_tensor.ne[2] = 1; + hex_event->fence_tensor.ne[3] = 1; + hex_event->fence_tensor.nb[0] = sizeof(int32_t); + hex_event->fence_tensor.nb[1] = sizeof(int32_t); + hex_event->fence_tensor.nb[2] = sizeof(int32_t); + hex_event->fence_tensor.nb[3] = sizeof(int32_t); + hex_event->fence_tensor.op = GGML_OP_NONE; + + HEX_VERBOSE("ggml-hex: %s event-new : event %p fence %p\n", ggml_backend_dev_name(dev), (void *)hex_event, (void *)hex_event->fence_slot); return new ggml_backend_event { /* .device = */ dev, @@ -5632,49 +5904,83 @@ static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_d }; } -static void ggml_backend_hexagon_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { - GGML_UNUSED(dev); - - if (event == nullptr) { +static void ggml_hexagon_event_synchronize(ggml_backend_dev_t dev, ggml_hexagon_event * hex_event) { + if (hex_event->seq == 0) { return; } - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + HEX_VERBOSE("ggml-hex: %s event-synchronize : event %p seq 0x%x fence %p\n", + ggml_backend_dev_name(dev), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); + + auto * fence = reinterpret_cast *>(hex_event->fence_slot); + + if ((int32_t)(fence[0].load(std::memory_order_relaxed) - hex_event->seq) < 0) { + hex_event->sess->flush_async(); + } + + while (true) { + if ((int32_t)(fence[0].load(std::memory_order_acquire) - hex_event->seq) >= 0) { + uint32_t status = fence[1].load(std::memory_order_acquire); + if (status > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s event-synchronize failed : dsp-error %s\n", + hex_event->sess->c_name(), status_to_str(status)); + } + break; + } + std::this_thread::yield(); + } +} + +static void ggml_backend_hexagon_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { + auto * hex_event = static_cast(event->context); + ggml_hexagon_event_synchronize(dev, hex_event); HEX_VERBOSE("ggml-hex: %s event-free : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event); + hex_event->fence_sess->free_fence((void *) hex_event->fence_slot, 1); delete hex_event; delete event; } static void ggml_backend_hexagon_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) { - GGML_UNUSED(dev); - - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; - HEX_VERBOSE("ggml-hex: %s event-synchronize : event %p seq %llu\n", - ggml_backend_dev_name(dev), (void *)hex_event, (unsigned long long)hex_event->seq); - if (hex_event->sess != nullptr) { - hex_event->sess->wait_event(hex_event->seq); - } + auto * hex_event = static_cast(event->context); + ggml_hexagon_event_synchronize(dev, hex_event); } static void ggml_backend_hexagon_event_record(ggml_backend_t backend, ggml_backend_event_t event) { auto sess = static_cast(backend->context); - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; + auto hex_event = static_cast(event->context); + if (++sess->fence_seq == 0) sess->fence_seq = 1; hex_event->sess = sess; - hex_event->seq = sess->record_event(); - HEX_VERBOSE("ggml-hex: %s event-record : event %p seq %llu\n", - sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq); + hex_event->seq = sess->fence_seq; + + sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ false); + + HEX_VERBOSE("ggml-hex: %s event-record : event %p seq 0x%x fence %p\n", + sess->c_name(), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); } static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { - GGML_UNUSED(backend); + auto sess = static_cast(backend->context); + auto hex_event = static_cast(event->context); + + if (hex_event->seq == 0) { + return; + } + + HEX_VERBOSE("ggml-hex: %s event-wait : event %p seq 0x%x fence %p\n", + sess->c_name(), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); - ggml_hexagon_event * hex_event = (ggml_hexagon_event *)event->context; - if (hex_event->sess != nullptr) { - HEX_VERBOSE("ggml-hex: %s event-wait : event %p seq %llu\n", - hex_event->sess->c_name(), (void *)hex_event, (unsigned long long)hex_event->seq); - hex_event->sess->wait_event(hex_event->seq); + // same physical NPU runs sequentially in FIFO order + if (sess->phys_idx == hex_event->sess->phys_idx) { + if (sess != hex_event->sess) { + sess->add_peer(hex_event->sess); + } + return; } + + sess->clone_buffer(hex_event->fence_sess->fence_buf); + sess->add_peer(hex_event->sess); + sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ true); } static void ggml_backend_hexagon_set_tensor_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { @@ -5688,7 +5994,10 @@ static void ggml_backend_hexagon_get_tensor_async(ggml_backend_t backend, const auto sess = static_cast(backend->context); HEX_VERBOSE("ggml-hex: %s get-tensor-async %s : data %p offset %zu size %zu usage %d\n", sess->c_name(), tensor->name, data, offset, size, tensor->buffer ? (int) tensor->buffer->usage : -1); - sess->flush(true); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s get-tensor-async failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } ggml_backend_tensor_get(tensor, data, offset, size); } @@ -5717,7 +6026,10 @@ static void ggml_backend_hexagon_get_tensor_2d_async(ggml_backend_t backend, auto sess = static_cast(backend->context); HEX_VERBOSE("ggml-hex: %s get-tensor-2d-async %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, tensor->buffer ? (int) tensor->buffer->usage : -1); - sess->flush(true); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s get-tensor-2d-async failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } ggml_backend_tensor_get_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); } @@ -6083,13 +6395,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons static bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { auto dev_ctx = static_cast(dev->context); - // Technically we can clone hexagon buffers from any session but for some reason the output is garbled with layer-split, - // tensor-split works correctly, so it needs mode debugging and investigation. For now accept only our own buffers. -#if 0 - bool supp = (buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment); -#else bool supp = (buft == &dev_ctx->host_buffer_type) || (buft == &dev_ctx->buffer_type); -#endif HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", dev_ctx->c_name(), ggml_backend_buft_name(buft), supp ? "yes" : "no"); return supp; @@ -6122,6 +6428,19 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) { // Create devices for (size_t i = 0; i < opt_ndev; i++) { + const auto & cfg = opt_device_configs[i]; + if (cfg.mdev_group.empty()) { + GGML_LOG_INFO("ggml-hex: device %zu: %s (phys=%d, virt=%d, domain=%s:%d)\n", + i, cfg.name.c_str(), cfg.physical_idx, cfg.virtual_idx, cfg.domain_name.c_str(), cfg.domain_id); + } else { + std::string peers_str; + for (const auto & p : cfg.mdev_group) { + if (!peers_str.empty()) peers_str += ", "; + peers_str += p.name + " (phys=" + std::to_string(p.physical_idx) + ")"; + } + GGML_LOG_INFO("ggml-hex: device %zu: %s (phys=%d, virt=%d, domain=%s:%d) [mdev peers: %s]\n", + i, cfg.name.c_str(), cfg.physical_idx, cfg.virtual_idx, cfg.domain_name.c_str(), cfg.domain_id, peers_str.c_str()); + } devices[i].iface = ggml_backend_hexagon_device_i; devices[i].reg = reg; devices[i].context = new ggml_backend_hexagon_device_context(i, opt_device_configs[i], &devices[i]); @@ -6172,17 +6491,51 @@ static void * ggml_backend_hexagon_comm_init(ggml_backend_t * backends, size_t n } } + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(backends[i]->context); + for (size_t j = i + 1; j < n_backends; j++) { + auto sess_j = static_cast(backends[j]->context); + if (sess_i->phys_idx == sess_j->phys_idx) { + return nullptr; + } + } + } + auto * ctx = new ggml_backend_hexagon_comm_context(); ctx->backends.assign(backends, backends + n_backends); ctx->n_backends = n_backends; - ctx->fence_seq = (((uintptr_t) ctx) & 0xFFFF) | 1; + + static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(backends[i]->context); + ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1); + ctx->fence_tensors[i] = {}; + ctx->fence_tensors[i].buffer = &sess_i->fence_buf->backend_buffer; + ctx->fence_tensors[i].extra = &fence_extra; + ctx->fence_tensors[i].data = (void *) ctx->fence_slots[i]; + ctx->fence_tensors[i].type = GGML_TYPE_I32; + ctx->fence_tensors[i].ne[0] = 4; + ctx->fence_tensors[i].ne[1] = 1; + ctx->fence_tensors[i].ne[2] = 1; + ctx->fence_tensors[i].ne[3] = 1; + ctx->fence_tensors[i].nb[0] = sizeof(int32_t); + ctx->fence_tensors[i].nb[1] = sizeof(int32_t); + ctx->fence_tensors[i].nb[2] = sizeof(int32_t); + ctx->fence_tensors[i].nb[3] = sizeof(int32_t); + ctx->fence_tensors[i].op = GGML_OP_NONE; + } return ctx; } static void ggml_backend_hexagon_comm_free(void * comm_ctx_v) { if (!comm_ctx_v) return; - delete static_cast(comm_ctx_v); + auto * ctx = static_cast(comm_ctx_v); + for (size_t i = 0; i < ctx->n_backends; i++) { + auto sess_i = static_cast(ctx->backends[i]->context); + sess_i->free_fence((void *) ctx->fence_slots[i], 1); + } + delete ctx; } static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { @@ -6192,6 +6545,16 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct if (n_backends < 2 || n_backends > 4) return false; + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(comm_ctx->backends[i]->context); + for (size_t j = i + 1; j < n_backends; j++) { + auto sess_j = static_cast(comm_ctx->backends[j]->context); + if (sess_i->phys_idx == sess_j->phys_idx) { + return false; + } + } + } + for (size_t i = 0; i < n_backends; i++) { if (!tensors[i] || !tensors[i]->buffer || !ggml_backend_buffer_is_hexagon(tensors[i]->buffer)) { return false; @@ -6219,42 +6582,28 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct } } - if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; - uint32_t fence_seq_entry = comm_ctx->fence_seq++; - if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; - uint32_t fence_seq_exit = comm_ctx->fence_seq++; - if (comm_ctx->fence_seq == 0) comm_ctx->fence_seq = 1; - - volatile uint32_t * fences[GGML_HEXAGON_MAX_SESSIONS]; - for (size_t i = 0; i < n_backends; i++) { - auto sbuf = (ggml_hexagon_shared_buffer *) tensors[i]->buffer->context; - fences[i] = (volatile uint32_t *) sbuf->alloc_fence(); + uint32_t max_seq = static_cast(comm_ctx->backends[0]->context)->fence_seq; + for (size_t i = 1; i < n_backends; i++) { + auto sess_i = static_cast(comm_ctx->backends[i]->context); + if ((int32_t)(sess_i->fence_seq - max_seq) > 0) { + max_seq = sess_i->fence_seq; + } } + if (++max_seq == 0) max_seq = 1; + uint32_t fence_seq_entry = max_seq; + if (++max_seq == 0) max_seq = 1; + uint32_t fence_seq_exit = max_seq; - static ggml_hexagon_tensor_extra fence_extra { {}, 0, GGML_HEXAGON_TENSOR_FENCE }; - ggml_tensor fence_tensors[GGML_HEXAGON_MAX_SESSIONS]; for (size_t i = 0; i < n_backends; i++) { - fence_tensors[i] = {}; - fence_tensors[i].buffer = tensors[i]->buffer; - fence_tensors[i].extra = &fence_extra; - fence_tensors[i].data = (void *) fences[i]; - fence_tensors[i].type = GGML_TYPE_I32; - fence_tensors[i].ne[0] = 4; - fence_tensors[i].ne[1] = 1; - fence_tensors[i].ne[2] = 1; - fence_tensors[i].ne[3] = 1; - fence_tensors[i].nb[0] = sizeof(int32_t); - fence_tensors[i].nb[1] = sizeof(int32_t); - fence_tensors[i].nb[2] = sizeof(int32_t); - fence_tensors[i].nb[3] = sizeof(int32_t); - fence_tensors[i].op = GGML_OP_NONE; + auto sess_i = static_cast(comm_ctx->backends[i]->context); + sess_i->fence_seq = max_seq; } std::vector data_tensors(n_backends); std::vector sync_tensors(n_backends); for (size_t i = 0; i < n_backends; i++) { data_tensors[i] = tensors[i]; - sync_tensors[i] = &fence_tensors[i]; + sync_tensors[i] = &comm_ctx->fence_tensors[i]; } for (size_t r = 0; r < n_backends; r++) { @@ -6262,7 +6611,7 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct sess->enqueue_allreduce(tensors[r], data_tensors, sync_tensors, (uint32_t) r, (uint32_t) n_backends, fence_seq_entry, fence_seq_exit); for (size_t j = 0; j < n_backends; j++) { if (r != j) { - sess->add_sync_peer(static_cast(comm_ctx->backends[j]->context)); + sess->add_peer(static_cast(comm_ctx->backends[j]->context)); } } } @@ -6270,8 +6619,23 @@ static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct return true; } +static ggml_backend_buffer_type_t ggml_backend_hexagon_split_buffer_type(int main_device, const float * tensor_split) { + GGML_UNUSED(tensor_split); + auto reg = ggml_backend_hexagon_reg(); + auto dev = ggml_backend_reg_dev_get(reg, main_device); + if (!dev) { + dev = ggml_backend_reg_dev_get(reg, 0); + } + if (!dev) return nullptr; + auto dev_ctx = static_cast(dev->context); + return &dev_ctx->buffer_type; +} + static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) { GGML_UNUSED(reg); + if (strcmp(name, "ggml_backend_split_buffer_type") == 0) { + return (void *) ggml_backend_hexagon_split_buffer_type; + } if (strcmp(name, "ggml_backend_comm_init") == 0) { return (void *) ggml_backend_hexagon_comm_init; } @@ -6304,6 +6668,41 @@ template std::string vec_to_str(std::vector v) { return str; } +static void ggml_hexagon_resolve_device_domain(ggml_hexagon_device_config & cfg, bool discovery_supported, const std::unordered_map & cdsp_map) { + if (discovery_supported) { + auto it = cdsp_map.find(cfg.physical_idx); + if (it != cdsp_map.end()) { + cfg.domain_id = it->second.id; + cfg.domain_name = it->second.name; + } else { + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not found on device (%zu CDSP core(s) available)\n", + cfg.physical_idx, cdsp_map.size()); + cfg.domain_id = -1; + cfg.domain_name = ""; + } + } else { + switch (cfg.physical_idx) { + case 0: + cfg.domain_id = 3; + cfg.domain_name = CDSP_DOMAIN_NAME; + break; + case 1: + cfg.domain_id = 4; + cfg.domain_name = "cdsp1"; + break; + default: + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not supported without dynamic discovery\n", + cfg.physical_idx); + cfg.domain_id = -1; + cfg.domain_name = ""; + break; + } + } + for (auto & sub_cfg : cfg.mdev_group) { + ggml_hexagon_resolve_device_domain(sub_cfg, discovery_supported, cdsp_map); + } +} + // Enumerate NPU (aka CDSP) domains via FASTRPC_GET_DOMAINS if supported, // and populate domain_id and domain_name for all configured devices. static void ggml_hexagon_discover_devices() { @@ -6350,36 +6749,7 @@ static void ggml_hexagon_discover_devices() { // Populate domain IDs and names for all configured devices for (size_t i = 0; i < opt_ndev; i++) { - auto & cfg = opt_device_configs[i]; - if (discovery_supported) { - auto it = cdsp_map.find(cfg.physical_idx); - if (it != cdsp_map.end()) { - cfg.domain_id = it->second.id; - cfg.domain_name = it->second.name; - } else { - GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not found on device (%zu CDSP core(s) available)\n", - cfg.physical_idx, cdsp_map.size()); - cfg.domain_id = -1; - cfg.domain_name = ""; - } - } else { - switch (cfg.physical_idx) { - case 0: - cfg.domain_id = 3; - cfg.domain_name = CDSP_DOMAIN_NAME; - break; - case 1: - cfg.domain_id = 4; - cfg.domain_name = "cdsp1"; - break; - default: - GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not supported without dynamic discovery\n", - cfg.physical_idx); - cfg.domain_id = -1; - cfg.domain_name = ""; - break; - } - } + ggml_hexagon_resolve_device_domain(opt_device_configs[i], discovery_supported, cdsp_map); } } @@ -6487,21 +6857,126 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_device_configs[i].physical_idx = 0; opt_device_configs[i].virtual_idx = (int)i; opt_device_configs[i].name = "HTP" + std::to_string(i); + opt_device_configs[i].mdev_group.clear(); } } else { std::string s_devices(str_devices); - std::stringstream ss(s_devices); - std::string item; - opt_ndev = 0; - while (std::getline(ss, item, ',')) { - size_t start = item.find_first_not_of(" \t\r\n"); - size_t end = item.find_last_not_of(" \t\r\n"); - if (start == std::string::npos) { - continue; + std::vector items; + std::string curr_item; + int bracket_depth = 0; + for (char ch : s_devices) { + if (ch == '[') { + bracket_depth++; + curr_item += ch; + } else if (ch == ']') { + if (bracket_depth > 0) bracket_depth--; + curr_item += ch; + } else if (ch == ',' && bracket_depth == 0) { + size_t s = curr_item.find_first_not_of(" \t\r\n"); + size_t e = curr_item.find_last_not_of(" \t\r\n"); + if (s != std::string::npos) { + items.push_back(curr_item.substr(s, e - s + 1)); + } + curr_item.clear(); + } else { + curr_item += ch; } - item = item.substr(start, end - start + 1); + } + size_t s = curr_item.find_first_not_of(" \t\r\n"); + size_t e = curr_item.find_last_not_of(" \t\r\n"); + if (s != std::string::npos) { + items.push_back(curr_item.substr(s, e - s + 1)); + } + + opt_ndev = 0; + for (const auto & item : items) { + size_t b_open = item.find('['); + size_t b_close = item.rfind(']'); + + if (b_open != std::string::npos && b_close != std::string::npos && b_close > b_open) { + // Grouped / composite syntax: Name[phys_spec:virt] or Name[phys_spec] + std::string dev_name = item.substr(0, b_open); + std::string content = item.substr(b_open + 1, b_close - b_open - 1); - if (item.rfind("HTP", 0) == 0) { + int virt = 0; + std::string phys_spec = content; + size_t colon_pos = content.find(':'); + if (colon_pos != std::string::npos) { + phys_spec = content.substr(0, colon_pos); + try { + virt = std::stoi(content.substr(colon_pos + 1)); + } catch (...) { + virt = 0; + } + } else { + size_t dev_colon = dev_name.find(':'); + if (dev_colon != std::string::npos) { + try { + virt = std::stoi(dev_name.substr(dev_colon + 1)); + } catch (...) { + virt = 0; + } + } + } + + // Parse physical indices from phys_spec (e.g. 0-1, 0,1, 0-3, etc.) + std::vector phys_list; + std::stringstream pss(phys_spec); + std::string p_part; + while (std::getline(pss, p_part, ',')) { + size_t ps = p_part.find_first_not_of(" \t\r\n"); + size_t pe = p_part.find_last_not_of(" \t\r\n"); + if (ps == std::string::npos) continue; + p_part = p_part.substr(ps, pe - ps + 1); + + size_t dash_pos = p_part.find('-'); + if (dash_pos != std::string::npos) { + try { + int p_start = std::stoi(p_part.substr(0, dash_pos)); + int p_end = std::stoi(p_part.substr(dash_pos + 1)); + for (int p = p_start; p <= p_end; p++) { + if (std::find(phys_list.begin(), phys_list.end(), p) == phys_list.end()) { + phys_list.push_back(p); + } + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse physical range in '%s'\n", p_part.c_str()); + } + } else { + try { + int p = std::stoi(p_part); + if (std::find(phys_list.begin(), phys_list.end(), p) == phys_list.end()) { + phys_list.push_back(p); + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse physical index in '%s'\n", p_part.c_str()); + } + } + } + + if (phys_list.empty()) { + phys_list.push_back(0); + } + + if (opt_ndev < GGML_HEXAGON_MAX_SESSIONS) { + auto & cfg = opt_device_configs[opt_ndev]; + cfg.name = dev_name; + cfg.physical_idx = phys_list[0]; + cfg.virtual_idx = virt; + cfg.mdev_group.clear(); + + for (size_t k = 1; k < phys_list.size(); k++) { + ggml_hexagon_device_config sub_cfg; + sub_cfg.physical_idx = phys_list[k]; + sub_cfg.virtual_idx = virt; + sub_cfg.name = "HTP" + std::to_string(phys_list[k]) + ":" + std::to_string(virt); + cfg.mdev_group.push_back(sub_cfg); + } + opt_ndev++; + } else { + GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str()); + } + } else if (item.rfind("HTP", 0) == 0) { std::string rest = item.substr(3); size_t colon_pos = rest.find(':'); int phys = 0; @@ -6525,6 +7000,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_device_configs[opt_ndev].name = colon_pos == std::string::npos ? "HTP" + std::to_string(phys) : "HTP" + std::to_string(phys) + ":" + std::to_string(virt); + opt_device_configs[opt_ndev].mdev_group.clear(); opt_ndev++; } else { GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str()); @@ -6539,6 +7015,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_device_configs[0].physical_idx = 0; opt_device_configs[0].virtual_idx = 0; opt_device_configs[0].name = "HTP0"; + opt_device_configs[0].mdev_group.clear(); } #if defined(__ANDROID__) diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index b083e26718bd..ef7b5184fc70 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -344,6 +344,12 @@ struct htp_opformat { } else if (htp_op_is_unary(node.opcode)) { const auto * kparams = (const struct htp_unary_kernel_params *) node.kernel_params; snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) { + snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]); + } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) { + snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]); + } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) { + snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]); } else { snprintf(str, max_size, "----"); } diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c index ac00b447d989..5fff372f2817 100644 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ b/ggml/src/ggml-hexagon/htp/act-ops.c @@ -3,7 +3,6 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include -#include #include #include @@ -15,7 +14,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "hex-common.h" #include "htp-tensor.h" #include "htp-vtcm.h" @@ -80,6 +79,7 @@ struct htp_act_context { uint32_t block; uint32_t src0_nrows; uint32_t src0_nrows_per_thread; + uint32_t row_start; int nc; uint8_t * vtcm_src0; @@ -329,104 +329,104 @@ static void geglu_f32(const float * restrict src0, } } -#define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \ - static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_act_context * actx = (struct htp_act_context *) data; \ - htp_act_preamble; \ - \ - struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \ - \ - size_t src0_row_size = actx->src0_row_size; \ - size_t src1_row_size = actx->src1_row_size; \ - size_t dst_row_size = actx->dst_row_size; \ - \ - size_t src0_row_stride = actx->src0_row_stride; \ - size_t src1_row_stride = actx->src1_row_stride; \ - \ - const uint32_t src0_nrows = actx->src0_nrows; \ - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \ - \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ - \ - /* no work for this thread */ \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - const uint8_t * restrict data_src0 = actx->data_src0; \ - const uint8_t * restrict data_src1 = actx->data_src1; \ - uint8_t * restrict data_dst = actx->data_dst; \ - \ - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ - \ - uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \ - uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \ - uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \ - \ - size_t src0_spad_half_size = actx->src0_spad_half_size; \ - size_t src1_spad_half_size = actx->src1_spad_half_size; \ - size_t dst_spad_half_size = actx->dst_spad_half_size; \ - \ - const int BLOCK = actx->block; \ - if (BLOCK == 0) { \ - FARF(ERROR, \ - OP_STR \ - " : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", \ - actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ - return; \ - } \ - \ - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; \ - \ - /* See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 */ \ - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ - \ - /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \ - dma_queue_push_vtcm_to_ddr(dma_queue, \ - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \ - dst_row_size, dst_row_size_aligned, 0); \ - \ - dma_queue_push( \ - dma_queue, \ - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \ - src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \ - dma_queue_push( \ - dma_queue, \ - dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \ - src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \ - } \ - \ - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \ - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ - \ - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; \ - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; \ - float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ - CORE_EXPR; \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ - \ - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), \ - dst_row_size, dst_row_size_aligned, block_size); \ - \ - /* prefetch N+2 loop iteration if any */ \ - const uint32_t pref_block = (ir + BLOCK * 2); \ - if (pref_block < src0_end_row) { \ - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \ - dma_queue_push(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_stride)), \ - src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \ - dma_queue_push(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_stride)), \ - src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \ - } \ - } \ - \ - dma_queue_flush(dma_queue); \ - \ +#define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \ + static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_act_context * actx = (struct htp_act_context *) data; \ + htp_act_preamble; \ + \ + struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \ + \ + size_t src0_row_size = actx->src0_row_size; \ + size_t src1_row_size = actx->src1_row_size; \ + size_t dst_row_size = actx->dst_row_size; \ + \ + size_t src0_row_stride = actx->src0_row_stride; \ + size_t src1_row_stride = actx->src1_row_stride; \ + \ + const uint32_t src0_nrows = actx->src0_nrows; \ + const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \ + \ + const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \ + \ + /* no work for this thread */ \ + if (src0_start_row >= src0_end_row) { \ + return; \ + } \ + \ + const uint8_t * restrict data_src0 = actx->data_src0; \ + const uint8_t * restrict data_src1 = actx->data_src1; \ + uint8_t * restrict data_dst = actx->data_dst; \ + \ + const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ + const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ + const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ + \ + uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \ + uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \ + uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \ + \ + size_t src0_spad_half_size = actx->src0_spad_half_size; \ + size_t src1_spad_half_size = actx->src1_spad_half_size; \ + size_t dst_spad_half_size = actx->dst_spad_half_size; \ + \ + const int BLOCK = actx->block; \ + if (BLOCK == 0) { \ + FARF(ERROR, \ + OP_STR \ + " : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", \ + actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ + return; \ + } \ + \ + dma_queue * dma_queue = actx->octx->ctx->dma[ith]; \ + \ + /* See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 */ \ + for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \ + dma_queue_push_vtcm_to_ddr(dma_queue, \ + dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \ + dst_row_size, dst_row_size_aligned, 0); \ + \ + dma_queue_push( \ + dma_queue, \ + dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \ + dma_queue_push( \ + dma_queue, \ + dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \ + } \ + \ + for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + float * dst_spad = (float *) dma_queue_pop(dma_queue).src; \ + float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; \ + float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + CORE_EXPR; \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + \ + dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), \ + dst_row_size, dst_row_size_aligned, block_size); \ + \ + /* prefetch N+2 loop iteration if any */ \ + const uint32_t pref_block = (ir + BLOCK * 2); \ + if (pref_block < src0_end_row) { \ + const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \ + dma_queue_push(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \ + dma_queue_push(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \ + } \ + } \ + \ + dma_queue_flush(dma_queue); \ + \ } DEFINE_GLU_PER_THREAD(swiglu, "swiglu-f32", swiglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) @@ -473,14 +473,30 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { } const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); + const size_t dst_row_size = dst->ne[0] * SIZEOF_FP32; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). - const size_t nc_bytes = dst->ne[0] * SIZEOF_FP32; - const size_t src0_row_size = nc_bytes; - const size_t src1_row_size = nc_bytes; - const size_t dst_row_size = nc_bytes; + const size_t nc_bytes = dst_row_size; + const size_t src0_row_size = nc_bytes; + const size_t src1_row_size = nc_bytes; const size_t src0_row_stride = src0->nb[1]; const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; @@ -518,7 +534,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { struct htp_act_context actx; actx.octx = octx; - actx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); actx.src0_row_size = src0_row_size; actx.src1_row_size = src1_row_size; @@ -545,7 +561,8 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { actx.dst_spad_half_size = L.dst_bytes_per_thread / 2; actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned; - actx.src0_nrows = src0_nrows; + actx.src0_nrows = nrows; + actx.row_start = row_start; actx.nc = dst->ne[0]; @@ -570,7 +587,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { actx.data_src1 = data_src1; actx.data_dst = (uint8_t *) dst->data; - worker_pool_run_func(octx->ctx->worker_pool, act_op_func, &actx, n_threads); + work_queue_run(octx->ctx->work_queue, act_op_func, &actx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c index d35f685a6dc0..d6e7f0d10c85 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.c +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c @@ -17,6 +17,7 @@ #include "hex-dma.h" #include "hex-profile.h" #include "allreduce-ops.h" +#include "htp-fence.h" struct htp_allreduce_context { struct htp_ops_context * octx; @@ -242,7 +243,42 @@ DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f32, float, hvx_add_f32_aaa, 1, 0) DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f16, __fp16, hvx_add_f16_aaa, 1, 1) DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f32, float, hvx_add_f32_aaa, 1, 1) +static int validate_allreduce( + struct htp_ops_context * octx, + const struct htp_allreduce_kernel_params * kparams, + uint32_t n_ranks +) { + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) { + return HTP_STATUS_INVAL_PARAMS; + } + + const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count( + n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0); + const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread; + if (vtcm_size != (size_t) kparams->vtcm_size) { + return HTP_STATUS_INVAL_PARAMS; + } + if (vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_F32) { + return HTP_STATUS_NO_SUPPORT; + } + + return HTP_STATUS_OK; +} + int op_allreduce(struct htp_ops_context * octx) { + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + const struct htp_allreduce_kernel_params * kparams = (const struct htp_allreduce_kernel_params *) octx->kernel_params; const struct htp_tensor * dst = octx->dst; @@ -253,38 +289,53 @@ int op_allreduce(struct htp_ops_context * octx) { return HTP_STATUS_INVAL_PARAMS; } - if (dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_F32) { - return HTP_STATUS_NO_SUPPORT; + const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0]; + const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1]; + + const struct htp_tensor * my_sync = octx->src[n_ranks + rank]; + atomic_uint * my_fence = (atomic_uint *) (uintptr_t) my_sync->data; + + const int status = validate_allreduce(octx, kparams, n_ranks); + if (status != HTP_STATUS_OK) { + if (status == HTP_STATUS_NO_SUPPORT) { + FARF(ERROR, "ggml-hex: allreduce unsupported type %d : rank %u\n", dst->type, rank); + } + htp_fence_write(my_fence, fence_seq_exit, status); + return status; } + const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; - const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0]; - const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1]; // 1. Entry Barrier: Synchronize all ranks before reading struct htp_thread_trace * tr0 = &octx->ctx->trace[0]; htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); - const struct htp_tensor * my_sync = octx->src[n_ranks + rank]; - atomic_uint * my_fence = (atomic_uint *) my_sync->data; - - atomic_store(&my_fence[0], fence_seq_entry); - asm volatile ("syncht" : : : "memory"); - Q6_dccleaninva_A((void *) my_fence); + htp_fence_write(my_fence, fence_seq_entry, octx->status); for (uint32_t j = 0; j < n_ranks; j++) { if (j == rank) continue; const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; - atomic_uint * peer_fence = (atomic_uint *) peer_sync->data; + atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data; uint64_t spins = 0; while (1) { - Q6_dccleaninva_A((void *) peer_fence); - uint32_t val = atomic_load(&peer_fence[0]); - if (val == fence_seq_entry || val == fence_seq_exit) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - fence_seq_entry) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: allreduce entry peer %u failed with status %u\n", j, peer_status); + htp_fence_write(my_fence, fence_seq_exit, peer_status); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); + return peer_status; + } break; } if (++spins > HTP_FENCE_TIMEOUT) { - FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_entry); + FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n", + rank, j, peer_fence, fence_seq_entry, peer_seq); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); return HTP_STATUS_INTERNAL_ERR; } hex_pause(); @@ -301,8 +352,6 @@ int op_allreduce(struct htp_ops_context * octx) { const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread; const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread; - const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); - struct htp_allreduce_context actx; actx.octx = octx; actx.n_ranks = n_ranks; @@ -339,6 +388,8 @@ int op_allreduce(struct htp_ops_context * octx) { } break; default: + FARF(ERROR, "ggml-hex: allreduce unsupported kernel %d : rank %u\n", kparams->kernel_type, rank); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_NO_SUPPORT); return HTP_STATUS_NO_SUPPORT; } @@ -368,23 +419,31 @@ int op_allreduce(struct htp_ops_context * octx) { // 4. Exit Barrier: Synchronize all ranks after writing htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); - atomic_store(&my_fence[0], fence_seq_exit); - asm volatile ("syncht" : : : "memory"); - Q6_dccleaninva_A((void *) my_fence); + htp_fence_write(my_fence, fence_seq_exit, octx->status); for (uint32_t j = 0; j < n_ranks; j++) { if (j == rank) continue; const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; - atomic_uint * peer_fence = (atomic_uint *) peer_sync->data; + atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data; uint64_t spins = 0; while (1) { - Q6_dccleaninva_A((void *) peer_fence); - uint32_t val = atomic_load(&peer_fence[0]); - if (val == fence_seq_exit) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - fence_seq_exit) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: allreduce exit peer %u failed with status %u\n", j, peer_status); + htp_fence_write(my_fence, fence_seq_exit, peer_status); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); + return peer_status; + } break; } if (++spins > HTP_FENCE_TIMEOUT) { - FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_exit); + FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n", + rank, j, peer_fence, fence_seq_exit, peer_seq); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); return HTP_STATUS_INTERNAL_ERR; } hex_pause(); @@ -394,5 +453,5 @@ int op_allreduce(struct htp_ops_context * octx) { htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); - return HTP_STATUS_OK; + return octx->status; } diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h index de447d87e912..0aed2b8b7e67 100644 --- a/ggml/src/ggml-hexagon/htp/allreduce-ops.h +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h @@ -2,6 +2,8 @@ #define ALLREDUCE_OPS_H #include +#include +#include #define HTP_ALLREDUCE_MAX_RANKS 4 @@ -15,6 +17,15 @@ enum htp_allreduce_kernel_type { HTP_ALLREDUCE_KERNEL_DMA_2D, }; +static inline size_t htp_allreduce_vtcm_buffer_count( + uint32_t n_ranks, + uint32_t n_threads, + bool has_add, + bool is_row_bcast +) { + return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0); +} + struct htp_allreduce_kernel_params { int32_t rank; int32_t n_ranks; diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c index 774faef5f388..e3c49e763d41 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.c +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c @@ -11,9 +11,10 @@ #include "hvx-utils.h" #include "hex-dma.h" +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -22,6 +23,9 @@ struct htp_argsort_context { struct htp_ops_context * octx; uint32_t nrows_per_thread; + uint32_t total_rows; + uint32_t row_start; + uint32_t row_end; uint8_t * vtcm_base; size_t vtcm_per_thread; }; @@ -336,10 +340,9 @@ static void htp_argsort_f32_##ne00##_##order_name(unsigned int n, unsigned int i const struct htp_tensor * src0 = octx->src[0]; \ const struct htp_tensor * dst = octx->dst; \ uint8_t * spad = actx->vtcm_base + actx->vtcm_per_thread * i; \ - uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; \ uint32_t rows_per_thread = actx->nrows_per_thread; \ - uint32_t start_row = rows_per_thread * i; \ - uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); \ + uint32_t start_row = actx->row_start + rows_per_thread * i; \ + uint32_t end_row = MIN(start_row + rows_per_thread, actx->row_end); \ size_t values_size = hex_round_up(ne00 * sizeof(float), 128); \ float * values_buf = (float *) spad; \ int32_t * indices_buf = (int32_t *) (spad + values_size); \ @@ -386,9 +389,6 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data // Dimensions uint32_t ne00 = src0->ne[0]; - uint32_t ne01 = src0->ne[1]; - uint32_t ne02 = src0->ne[2]; - uint32_t ne03 = src0->ne[3]; uint32_t nb01 = src0->nb[1]; @@ -398,10 +398,9 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data enum ggml_sort_order order = (enum ggml_sort_order) octx->op_params[0]; // Rows to process - uint32_t total_rows = ne01 * ne02 * ne03; uint32_t rows_per_thread = actx->nrows_per_thread; - uint32_t start_row = rows_per_thread * i; - uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); + uint32_t start_row = actx->row_start + rows_per_thread * i; + uint32_t end_row = MIN(start_row + rows_per_thread, actx->row_end); size_t values_size = hex_round_up(ne00 * sizeof(float), 128); uint32_t num_vec_ind_values = hmx_ceil_div(ne00, VLEN/(sizeof(int32_t))); @@ -451,8 +450,28 @@ int op_argsort(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t total_rows = octx->src[0]->ne[1] * octx->src[0]->ne[2] * octx->src[0]->ne[3]; - const uint32_t n_threads = MIN(total_rows, octx->n_threads); + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_row_size = dst->ne[0] * sizeof(int32_t); + + uint32_t row_start = 0; + uint32_t row_end = total_rows; + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(int32_t), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + row_end = range.start + range.count; + } + + const uint32_t nrows = row_end - row_start; + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; // Allocate scratchpad // We need 1 row of float + 1 row of int32 per thread. @@ -478,7 +497,10 @@ int op_argsort(struct htp_ops_context * octx) { struct htp_argsort_context actx; actx.octx = octx; - actx.nrows_per_thread = (total_rows + n_threads - 1) / n_threads; + actx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + actx.total_rows = nrows; + actx.row_start = row_start; + actx.row_end = row_end; actx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; actx.vtcm_per_thread = spad_per_thread; @@ -508,7 +530,7 @@ int op_argsort(struct htp_ops_context * octx) { } // Run jobs - worker_pool_run_func(octx->ctx->worker_pool, job_func, &actx, n_threads); + work_queue_run(octx->ctx->work_queue, job_func, &actx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c index db6177963541..bfa849e0edbf 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.c +++ b/ggml/src/ggml-hexagon/htp/binary-ops.c @@ -13,9 +13,10 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" #include "htp-tensor.h" #ifndef MIN @@ -36,6 +37,8 @@ struct htp_binary_context { uint32_t block_max; uint32_t nrows_per_thread; + uint32_t total_rows; + uint32_t row_start; size_t src0_row_size_aligned; size_t src1_row_size_aligned; size_t dst_row_size_aligned; @@ -48,27 +51,27 @@ struct htp_binary_context { const struct htp_tensor * src0 = octx->src[0]; \ const struct htp_tensor * src1 = octx->src[1]; \ const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne10 = src1->ne[0]; \ - const uint32_t ne11 = src1->ne[1]; \ - const uint32_t ne12 = src1->ne[2]; \ - const uint32_t ne13 = src1->ne[3]; \ - \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb11 = src1->nb[1]; \ - const uint32_t nb12 = src1->nb[2]; \ - const uint32_t nb13 = src1->nb[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t ne10 = src1->ne[0]; \ + const uint32_t ne11 = src1->ne[1]; \ + const uint32_t ne12 = src1->ne[2]; \ + const uint32_t ne13 = src1->ne[3]; \ + \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb11 = src1->nb[1]; \ + const uint32_t nb12 = src1->nb[2]; \ + const uint32_t nb13 = src1->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_t ir, uint32_t end_row, uint32_t ne01, uint32_t ne02) { @@ -93,87 +96,87 @@ static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_ } // Macro for scalar op switch -#define COMPUTE_SCALAR_OP(DST, SRC, VAL, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_SUB: hvx_sub_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_MUL: hvx_mul_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ +#define COMPUTE_SCALAR_OP(DST, SRC, VAL, TYPE, N) \ + if(TYPE == HTP_TYPE_F32) { \ + switch (octx->op) { \ + case HTP_OP_ADD: hvx_add_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ + case HTP_OP_SUB: hvx_sub_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ + case HTP_OP_MUL: hvx_mul_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ case HTP_OP_DIV: hvx_mul_scalar_f32_aa(DST, SRC, 1.0f / (*(float *)VAL), N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_SUB: hvx_sub_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_MUL: hvx_mul_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_DIV: hvx_div_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - default: break; \ - } \ + default: break; \ + } \ + } \ + else { \ + switch (octx->op) { \ + case HTP_OP_ADD: hvx_add_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ + case HTP_OP_SUB: hvx_sub_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ + case HTP_OP_MUL: hvx_mul_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ + case HTP_OP_DIV: hvx_div_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ + default: break; \ + } \ } // Macro for vector op switch (All Aligned) -#define COMPUTE_VECTOR_OP_AAA(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ +#define COMPUTE_VECTOR_OP_AAA(DST, SRC0, SRC1, TYPE, N) \ + if(TYPE == HTP_TYPE_F32) { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f32_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f32_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f32_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f32_aaa(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ + default: break; \ + } \ + } \ + else { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f16_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f16_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f16_aaa(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f16_aaa(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ + default: break; \ + } \ } // Macro for vector op switch (Dst Aligned, Src0 Aligned, Src1 Unaligned) -#define COMPUTE_VECTOR_OP_AAU(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ +#define COMPUTE_VECTOR_OP_AAU(DST, SRC0, SRC1, TYPE, N) \ + if(TYPE == HTP_TYPE_F32) { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f32_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f32_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f32_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f32_aau(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ + default: break; \ + } \ + } \ + else { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f16_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f16_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f16_aau(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f16_aau(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ + default: break; \ + } \ } // Macro for vector op switch (All Unaligned - generic loop used in element repeat) -#define COMPUTE_VECTOR_OP_UUU(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ +#define COMPUTE_VECTOR_OP_UUU(DST, SRC0, SRC1, TYPE, N) \ + if(TYPE == HTP_TYPE_F32) { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f32_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f32_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f32_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f32_uuu(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ + default: break; \ + } \ + } \ + else { \ + switch (octx->op) { \ case HTP_OP_ADD: hvx_add_f16_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_SUB: hvx_sub_f16_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_MUL: hvx_mul_f16_uuu(DST, SRC0, SRC1, N); break; \ case HTP_OP_DIV: hvx_div_f16_uuu(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ + default: break; \ + } \ } // 1. Scalar src1 (ne10 == 1) @@ -184,9 +187,8 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_type = octx->src[0]->type; const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); @@ -222,6 +224,8 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { } // Main loop + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); @@ -242,12 +246,14 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { uint8_t * src1_ptr = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; uint32_t s1_stride = (ne11 == 1) ? 0 : nb11; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; COMPUTE_SCALAR_OP(r_dst, r_src0, src1_ptr, src0_type, ne00); src1_ptr += s1_stride; } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); @@ -266,6 +272,7 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { } ir += current_block_size; } + dma_queue_flush(q); } @@ -277,9 +284,8 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi const uint32_t src0_type = octx->src[0]->type; const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); @@ -323,18 +329,22 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; uint8_t * s1_spad = (uint8_t *) dma_queue_pop(q).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; uint8_t * r_src1 = s1_spad + r * bctx->src1_row_size_aligned; uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint32_t i03, i02, i01, rem; i03 = fastdiv(ir, &bctx->src0_dim12_div); @@ -366,6 +376,7 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi } ir += current_block_size; } + dma_queue_flush(q); } @@ -377,9 +388,8 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, const uint32_t src0_type = octx->src[0]->type; const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); @@ -416,17 +426,21 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; uint8_t * r_src1 = (uint8_t *)s1_ptr; // Constant uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); @@ -447,6 +461,7 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, } ir += current_block_size; } + dma_queue_flush(q); } @@ -458,9 +473,8 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * const uint32_t src0_type = octx->src[0]->type; const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); @@ -493,6 +507,8 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; @@ -503,6 +519,7 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint32_t r_i01 = i01 + r; uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); @@ -516,6 +533,7 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * // Read src1 from DDR (unaligned) COMPUTE_VECTOR_OP_AAU(r_dst, r_src0, r_src1, src0_type, ne00); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); @@ -532,6 +550,7 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * } ir += current_block_size; } + dma_queue_flush(q); } @@ -544,9 +563,8 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * const uint32_t src0_type = octx->src[0]->type; const uint32_t elem_size_bytes = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); const uint32_t row_size_bytes = ne00 * elem_size_bytes;; - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); @@ -579,6 +597,8 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; @@ -589,6 +609,7 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint32_t r_i01 = i01 + r; uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); @@ -606,6 +627,7 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * COMPUTE_VECTOR_OP_UUU(r_dst + c * elem_size_bytes, r_src0 + c * elem_size_bytes, r_src1_row, src0_type, len); } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); @@ -622,6 +644,7 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * } ir += current_block_size; } + dma_queue_flush(q); } @@ -650,9 +673,8 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { const uint32_t nb2 = dst->nb[2]; const uint32_t nb3 = dst->nb[3]; - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); @@ -683,6 +705,8 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; @@ -693,6 +717,7 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); for (uint32_t r = 0; r < current_block_size; r++) { uint32_t r_i01 = i01 + r; // linear within block since we split at ne01 @@ -704,6 +729,7 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { hvx_add_f32_aau(r_dst, r_src0, r_src1, ne00); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, ne00 * sizeof(float), current_block_size); @@ -720,6 +746,7 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { } ir += current_block_size; } + dma_queue_flush(q); } @@ -729,15 +756,31 @@ static int execute_op_binary(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - // Use packed row sizes for VTCM allocation + // Use packed row sizes for VTCM allocation and alignment const uint32_t src0_type = octx->src[0]->type; const size_t elem_size = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); const size_t src0_row_size = src0->ne[0] * elem_size; const size_t src1_row_size = src1->ne[0] * elem_size; const size_t dst_row_size = dst->ne[0] * elem_size; + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN); size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); @@ -815,7 +858,9 @@ static int execute_op_binary(struct htp_ops_context * octx) { struct htp_binary_context bctx; bctx.octx = octx; - bctx.nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + bctx.total_rows = nrows; + bctx.row_start = row_start; bctx.block_max = rows_per_buffer; bctx.src0_row_size_aligned = src0_row_size_aligned; bctx.src1_row_size_aligned = src1_row_size_aligned; @@ -850,7 +895,7 @@ static int execute_op_binary(struct htp_ops_context * octx) { dma_queue_pop(q); } - worker_pool_run_func(octx->ctx->worker_pool, worker_func, &bctx, n_threads); + work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads); return HTP_STATUS_OK; } @@ -870,4 +915,3 @@ int op_binary(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c index 51d39e8d98f5..966e867b3976 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.c +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -1,5 +1,8 @@ +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hexagon_types.h" #include "hexagon_protos.h" #include "hvx_hexagon_protos.h" @@ -13,6 +16,10 @@ struct htp_concat_context { struct htp_ops_context * octx; uint32_t dim; uint32_t nrows_per_thread; + uint32_t row_start; + uint32_t nrows; + uint32_t elem_start; + uint32_t nelems; struct fastdiv_values div_ne0; struct fastdiv_values div_ne1; struct fastdiv_values div_ne2; @@ -28,10 +35,10 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src0_ne0 = src0->ne[0]; const uint32_t src1_ne0 = src1->ne[0]; - const uint32_t ne1 = dst->ne[1]; - const uint32_t start_i = ith * cctx->nrows_per_thread; - const uint32_t end_i = (start_i + cctx->nrows_per_thread < ne1) ? (start_i + cctx->nrows_per_thread) : ne1; + const uint32_t row_end = cctx->row_start + cctx->nrows; + const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; + const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; if (start_i >= end_i) return; dma_queue * q = octx->ctx->dma[ith]; @@ -51,6 +58,8 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(float), VLEN); uint32_t mu = src1_ne0_padded * spad1_stride; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t i = start_i; i < end_i; i += block_i) { uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i; @@ -66,6 +75,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); for (uint32_t j = 0; j < src1_ne0_padded; j += 32) { #pragma unroll(4) for (uint32_t ii = 0; ii < current_block_i; ii++) { @@ -75,6 +85,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * hvx_vmemu(dst_ptr) = vtcm_tmp[ii]; } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); dma_queue_pop(q); // src0 @@ -95,10 +106,10 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t src0_ne0 = src0->ne[0]; const uint32_t src1_ne0 = src1->ne[0]; - const uint32_t ne1 = dst->ne[1]; - const uint32_t start_i = ith * cctx->nrows_per_thread; - const uint32_t end_i = (start_i + cctx->nrows_per_thread < ne1) ? (start_i + cctx->nrows_per_thread) : ne1; + const uint32_t row_end = cctx->row_start + cctx->nrows; + const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; + const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; if (start_i >= end_i) return; dma_queue * q = octx->ctx->dma[ith]; @@ -118,6 +129,8 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(__fp16), VLEN); uint32_t mu = src1_ne0_padded * spad1_stride; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t i = start_i; i < end_i; i += block_i) { uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i; @@ -133,6 +146,7 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); for (uint32_t j = 0; j < src1_ne0_padded; j += 64) { #pragma unroll(4) for (uint32_t ii = 0; ii < current_block_i; ii++) { @@ -142,6 +156,7 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * hvx_vmemu(dst_ptr) = vtcm_tmp[ii]; } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); dma_queue_pop(q); // src0 @@ -164,11 +179,14 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) { const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2; const uint32_t ne[4] = {dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]}; - const uint32_t total_elements = ne[0] * ne[1] * ne[2] * ne[3]; - const uint32_t chunk_size = (total_elements + nth - 1) / nth; - const uint32_t start_idx = MIN(ith * chunk_size, total_elements); - const uint32_t end_idx = MIN(start_idx + chunk_size, total_elements); + // Per-device element range aligned to prevent false sharing + const uint32_t elem_start = cctx->elem_start; + const uint32_t nelems = cctx->nelems; + const uint32_t chunk_size = (nelems + nth - 1) / nth; + + const uint32_t start_idx = MIN(elem_start + ith * chunk_size, elem_start + nelems); + const uint32_t end_idx = MIN(start_idx + chunk_size, elem_start + nelems); // Naive scalar element-wise copy for (uint32_t idx = start_idx; idx < end_idx; idx++) { @@ -236,13 +254,28 @@ int op_concat(struct htp_ops_context * octx) { void (*worker_func)(unsigned int, unsigned int, void *) = concat_generic; if (dim == 0 && is_2d && is_src1_transposed && !is_src0_transposed) { - n_threads = MIN(dst->ne[1], n_threads); - if (n_threads < 1) { - n_threads = 1; + const uint32_t total_rows = dst->ne[1]; + const size_t dst_data_row_size = dst->ne[0] * type_size; + uint32_t row_start = 0; + uint32_t nrows = total_rows; + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + cctx.row_start = row_start; + cctx.nrows = nrows; + uint32_t block_i = (type_size == 4) ? 32 : 64; - cctx.nrows_per_thread = hmx_ceil_div(dst->ne[1], n_threads); + cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); // Allocate VTCM uint32_t spad1_stride = block_i * type_size; @@ -270,8 +303,26 @@ int op_concat(struct htp_ops_context * octx) { } else { worker_func = concat_2d_f16_transposed; } + } else { + const uint32_t total_elements = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; + uint32_t elem_start = 0; + uint32_t nelems = total_elements; + if (octx->ctx->mdev.count > 1) { + const uint32_t elems_per_chunk = HEX_L2_LINE_SIZE / type_size; + const bool can_split = htp_tensor_mdev_data_aligned(dst) && htp_tensor_is_contiguous(dst, type_size) && !htp_tensor_is_permuted(dst); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elements, can_split ? elems_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelems = range.count; + } + + if (nelems == 0) { + return HTP_STATUS_OK; + } + + cctx.elem_start = elem_start; + cctx.nelems = nelems; } - worker_pool_run_func(octx->ctx->worker_pool, worker_func, &cctx, n_threads); + work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index b151b757f413..7f01a8c1e043 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -16,6 +16,7 @@ #include "htp-ops.h" #include "hvx-utils.h" #include "htp-tensor.h" +#include "htp-fence.h" struct htp_copy_context { struct htp_ops_context * octx; @@ -29,7 +30,23 @@ struct htp_copy_context { uint32_t src0_blocks_per_row; uint32_t dst_blocks_per_row; + uint32_t elem_start; + uint32_t nelem; + uint32_t elem_per_thread; + uint32_t src0_nrows_per_thread; + uint32_t row_start; + uint32_t nrows; + + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02_ne01; + + struct fastdiv_values div_ne0; + struct fastdiv_values div_ne1_ne0; + struct fastdiv_values div_ne2_ne1_ne0; + struct fastdiv_values div_ne00; + struct fastdiv_values div_ne01_ne00; + struct fastdiv_values div_ne02_ne01_ne00; }; #define cpy_preamble \ @@ -54,131 +71,113 @@ struct htp_copy_context { const uint32_t nb0 = dst->nb[0]; \ const uint32_t nb1 = dst->nb[1]; \ const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ - const uint32_t nr = ne01; - -#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_copy_context * ct = (struct htp_copy_context *) data; \ - struct htp_ops_context * octx = ct->octx; \ - cpy_preamble; \ - const uint32_t dr = ct->src0_nrows_per_thread; \ - const uint32_t ir0 = dr * ith; \ - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \ - if (ir0 >= nr) return; \ - for (uint32_t i03 = 0; i03 < ne03; i03++) { \ - for (uint32_t i02 = 0; i02 < ne02; i02++) { \ - _Pragma("unroll(4)") \ - for (uint32_t i01 = ir0; i01 < ir1; i01++) { \ - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \ - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ - hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \ - hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ - } \ - } \ - } \ + const uint32_t nb3 = dst->nb[3]; + +#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ +static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_copy_context * ct = (struct htp_copy_context *) data; \ + struct htp_ops_context * octx = ct->octx; \ + cpy_preamble; \ + const uint32_t dr = ct->src0_nrows_per_thread; \ + const uint32_t ir0 = ct->row_start + dr * ith; \ + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \ + if (ir0 >= ir1) return; \ + const bool contiguous = (nb01 == ne00 * ELEM_SIZE) && (nb1 == nb01) && \ + (nb02 == ne01 * nb01) && (nb2 == nb02) && \ + (nb03 == ne02 * nb02) && (nb3 == nb03); \ + const uint32_t ne02_ne01 = ne02 * ne01; \ + uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); \ + uint32_t rem = ir0 - i03 * ne02_ne01; \ + uint32_t i02 = fastdiv(rem, &ct->div_ne01); \ + uint32_t i01 = rem - i02 * ne01; \ + uint8_t * dst_ptr = (uint8_t *) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \ + uint8_t * src0_ptr = (uint8_t *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ + if (contiguous) { \ + hvx_copy_uu(dst_ptr, src0_ptr, (ir1 - ir0) * ne00, ELEM_SIZE); \ + return; \ + } \ + for (uint32_t r = ir0; r < ir1; r++) { \ + hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \ + hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ + dst_ptr += nb1; \ + src0_ptr += nb01; \ + if (++i01 == ne01) { \ + i01 = 0; \ + if (++i02 == ne02) { \ + i02 = 0; \ + i03++; \ + } \ + dst_ptr = (uint8_t *) dst->data + i02*nb2 + i03*nb3; \ + src0_ptr = (uint8_t *) src0->data + i02*nb02 + i03*nb03; \ + } \ + } \ } DEFINE_CPY_SAMESHAPE(f32, float, 4) DEFINE_CPY_SAMESHAPE(f16, __fp16, 2) -#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ -static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ - struct htp_copy_context * ct = (struct htp_copy_context *) data; \ - struct htp_ops_context * octx = ct->octx; \ - cpy_preamble; \ - const uint32_t dr = ct->src0_nrows_per_thread; \ - const uint32_t ir0 = dr * ith; \ - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \ - if (ir0 >= nr) return; \ - const bool src0_contig = (nb00 == ELEM_SIZE) && \ - (nb01 == ne00 * nb00) && \ - (nb02 == ne01 * nb01) && \ - (nb03 == ne02 * nb02); \ - const bool dst_contig = (nb0 == ELEM_SIZE) && \ - (nb1 == ne0 * nb0) && \ - (nb2 == ne1 * nb1) && \ - (nb3 == ne2 * nb2); \ - if (src0_contig && dst_contig) { \ - for (int64_t i03 = 0; i03 < ne03; i03++) { \ - for (int64_t i02 = 0; i02 < ne02; i02++) { \ - uint8_t * src_ptr = (uint8_t *) src0->data + i03*nb03 + i02*nb02 + ir0*nb01; \ - uint32_t flat = ((i03*ne02 + i02)*ne01 + ir0) * ne00; \ - uint8_t * dst_ptr = (uint8_t *) dst->data + flat * ELEM_SIZE; \ - hvx_copy_uu(dst_ptr, src_ptr, (ir1 - ir0) * ne00, ELEM_SIZE); \ - } \ - } \ - return; \ - } \ - const bool reshape_flat_fast = (ne03 == 1 && ne2 == 1 && ne3 == 1) && \ - (ne0 == ne00 * ne01) && (ne1 == ne02) && \ - (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \ - if (reshape_flat_fast) { \ - for (uint32_t i02 = 0; i02 < ne02; i02++) { \ - for (uint32_t i01 = ir0; i01 < ir1; i01++) { \ - uint8_t * src0_ptr = (uint8_t *) src0->data + i01 * nb01 + i02 * nb02; \ - uint8_t * dst_ptr = (uint8_t *) dst->data + i01 * ne00 * ELEM_SIZE + i02 * nb1; \ - hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \ - } \ - } \ - return; \ - } \ - int64_t k10 = 0; \ - int64_t i11 = 0; \ - int64_t i12 = 0; \ - int64_t i13 = 0; \ - const int64_t nk00 = ct->src0_blocks_per_row; \ - const int64_t nk0 = ct->dst_blocks_per_row; \ - for (int64_t i03 = 0; i03 < ne03; i03++) { \ - for (int64_t i02 = 0; i02 < ne02; i02++) { \ - k10 += nk00 * ir0; \ - while (k10 >= nk0) { \ - k10 -= nk0; \ - if (++i11 == ne1) { \ - i11 = 0; \ - if (++i12 == ne2) { \ - i12 = 0; \ - if (++i13 == ne3) { \ - i13 = 0; \ - } \ - } \ - } \ - } \ - for (int64_t i01 = ir0; i01 < ir1; i01++) { \ - for (int64_t k00 = 0; k00 < nk00; k00++) { \ - const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); \ - char * dst_ptr = ((char *) dst->data + k10*nb0 + i11*nb1 + i12*nb2 + i13*nb3); \ - memcpy(dst_ptr, src0_ptr, ELEM_SIZE); \ - if (++k10 == nk0) { \ - k10 = 0; \ - if (++i11 == ne1) { \ - i11 = 0; \ - if (++i12 == ne2) { \ - i12 = 0; \ - if (++i13 == ne3) { \ - i13 = 0; \ - } \ - } \ - } \ - } \ - } \ - } \ - k10 += nk00 * (ne01 - ir1); \ - while (k10 >= nk0) { \ - k10 -= nk0; \ - if (++i11 == ne1) { \ - i11 = 0; \ - if (++i12 == ne2) { \ - i12 = 0; \ - if (++i13 == ne3) { \ - i13 = 0; \ - } \ - } \ - } \ - } \ - } \ - } \ +#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ +static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_copy_context * ct = (struct htp_copy_context *) data; \ + struct htp_ops_context * octx = ct->octx; \ + cpy_preamble; \ + const uint32_t th_nelem = ct->elem_per_thread; \ + const uint32_t th_start = ct->elem_start + ith * th_nelem; \ + const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \ + if (th_start >= th_end) return; \ + \ + const uint32_t ne01_ne00 = ne01 * ne00; \ + const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \ + const uint32_t ne1_ne0 = ne1 * ne0; \ + const uint32_t ne2_ne1_ne0 = ne2 * ne1_ne0; \ + \ + uint32_t e = th_start; \ + uint32_t i13 = fastdiv(e, &ct->div_ne2_ne1_ne0); \ + uint32_t rem = e - i13 * ne2_ne1_ne0; \ + uint32_t i12 = fastdiv(rem, &ct->div_ne1_ne0); \ + uint32_t rem2 = rem - i12 * ne1_ne0; \ + uint32_t i11 = fastdiv(rem2, &ct->div_ne0); \ + uint32_t i10 = rem2 - i11 * ne0; \ + \ + uint32_t i03 = fastdiv(e, &ct->div_ne02_ne01_ne00); \ + uint32_t rem_s = e - i03 * ne02_ne01_ne00; \ + uint32_t i02 = fastdiv(rem_s, &ct->div_ne01_ne00); \ + uint32_t rem2_s = rem_s - i02 * ne01_ne00; \ + uint32_t i01 = fastdiv(rem2_s, &ct->div_ne00); \ + uint32_t i00 = rem2_s - i01 * ne00; \ + \ + char * dst_ptr = (char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \ + const char * src0_ptr = (const char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \ + \ + for (; e < th_end; e++) { \ + *((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \ + \ + dst_ptr += nb0; \ + if (++i10 == ne0) { \ + i10 = 0; \ + if (++i11 == ne1) { \ + i11 = 0; \ + if (++i12 == ne2) { \ + i12 = 0; \ + i13++; \ + } \ + } \ + dst_ptr = (char *) dst->data + i11*nb1 + i12*nb2 + i13*nb3; \ + } \ + \ + src0_ptr += nb00; \ + if (++i00 == ne00) { \ + i00 = 0; \ + if (++i01 == ne01) { \ + i01 = 0; \ + if (++i02 == ne02) { \ + i02 = 0; \ + i03++; \ + } \ + } \ + src0_ptr = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ + } \ + } \ } DEFINE_CPY_RESHAPE(f32, float, 4) @@ -189,22 +188,33 @@ static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, voi struct htp_ops_context * octx = ct->octx; cpy_preamble; - // parallelize by src0 rows const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - if (ir0 >= nr) return; - - // copy by rows - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - #pragma unroll(2) - for (uint32_t i01 = ir0; i01 < ir1; i01++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2); - hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); + const uint32_t ir0 = ct->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); + if (ir0 >= ir1) return; + + const uint32_t ne02_ne01 = ne02 * ne01; + uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); + uint32_t rem = ir0 - i03 * ne02_ne01; + uint32_t i02 = fastdiv(rem, &ct->div_ne01); + uint32_t i01 = rem - i02 * ne01; + + uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; + uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + + for (uint32_t r = ir0; r < ir1; r++) { + hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2); + hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); + dst_ptr += nb1; + src0_ptr += nb01; + if (++i01 == ne01) { + i01 = 0; + if (++i02 == ne02) { + i02 = 0; + i03++; } + dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; + src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; } } } @@ -214,22 +224,33 @@ static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, voi struct htp_ops_context * octx = ct->octx; cpy_preamble; - // parallelize by src0 rows const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - if (ir0 >= nr) return; - - // copy by rows - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - #pragma unroll(2) - for (uint32_t i01 = ir0; i01 < ir1; i01++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2); - hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00); + const uint32_t ir0 = ct->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); + if (ir0 >= ir1) return; + + const uint32_t ne02_ne01 = ne02 * ne01; + uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); + uint32_t rem = ir0 - i03 * ne02_ne01; + uint32_t i02 = fastdiv(rem, &ct->div_ne01); + uint32_t i01 = rem - i02 * ne01; + + uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; + uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + + for (uint32_t r = ir0; r < ir1; r++) { + hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2); + hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00); + dst_ptr += nb1; + src0_ptr += nb01; + if (++i01 == ne01) { + i01 = 0; + if (++i02 == ne02) { + i02 = 0; + i03++; } + dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; + src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; } } } @@ -250,15 +271,19 @@ static inline void cpy_dma_sametype_sameshape( dma_queue * q = octx->ctx->dma[0]; if (contiguous_outer) { - dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03); - dma_queue_pop(q); + if (!dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03)) { + dma_queue_flush(q); + dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03); + } + dma_queue_flush(q); return; } for (uint32_t i03 = 0; i03 < ne03; i03++) { for (uint32_t i02 = 0; i02 < ne02; i02++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03; + uint8_t * dst_ptr = (uint8_t *) dst->data + i02 * nb2 + i03 * nb3; + uint8_t * src0_ptr = (uint8_t *) src0->data + i02 * nb02 + i03 * nb03; + if (!dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01)) { dma_queue_flush(q); dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01); @@ -269,10 +294,9 @@ static inline void cpy_dma_sametype_sameshape( dma_queue_flush(q); } -int op_cpy(struct htp_ops_context * octx) { +static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { cpy_preamble; - - const uint32_t n_threads = MIN(nr, octx->n_threads); + *use_dma = false; struct htp_copy_context ct; ct.octx = octx; @@ -296,59 +320,117 @@ int op_cpy(struct htp_ops_context * octx) { } const bool sametype = (src0->type == dst->type); - const bool transposed = (nb00 > nb01) || (nb0 > nb1); + const bool transposed = (nb00 > nb01) || (nb0 > nb1) || + (nb00 != ct.src0_type_size) || (nb0 != ct.dst_type_size) || + (nb01 < ne00 * ct.src0_type_size) || (nb1 < ne0 * ct.dst_type_size); const bool sameshape = !transposed && (ne00 == ne0 && ne01 == ne1 && ne02 == ne2 && ne03 == ne3); - ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads; + const uint32_t n_threads = octx->n_threads; - worker_callback_t copy_fun = NULL; - bool use_dma = false; + const bool dst_is_contiguous = htp_tensor_is_contiguous(dst, ct.dst_type_size); - if (sametype && sameshape) { - use_dma = true; - } else if (sameshape) { - /**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) - copy_fun = cpy_thread_f16_f32_sameshape; - else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) - copy_fun = cpy_thread_f32_f16_sameshape; - else - return HTP_STATUS_NO_SUPPORT; - } else if (sametype) { - if (src0->type == HTP_TYPE_F32) { - copy_fun = cpy_thread_f32_reshape; + if (sameshape) { + const uint32_t total_rows = ne01 * ne02 * ne03; + const uint32_t row_size = ne00 * ct.dst_type_size; + + ct.div_ne01 = init_fastdiv_values(ne01); + ct.div_ne02_ne01 = init_fastdiv_values(ne02 * ne01); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, can_split ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + ct.row_start = row_start; + ct.nrows = nrows; + ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + + if (sametype && octx->ctx->mdev.count <= 1) { + *use_dma = true; + cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3); } else { - copy_fun = cpy_thread_f16_reshape; + work_queue_func_t copy_fun = NULL; + if (sametype) { + copy_fun = (src0->type == HTP_TYPE_F32) ? cpy_thread_f32_sameshape : cpy_thread_f16_sameshape; + } else if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) { + copy_fun = cpy_thread_f16_f32_sameshape; + } else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) { + copy_fun = cpy_thread_f32_f16_sameshape; + } else { + return HTP_STATUS_NO_SUPPORT; + } + work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); + } + } else if (sametype) { + const uint32_t total_elems = ne0 * ne1 * ne2 * ne3; + const uint32_t elems_per_line = (ct.dst_type_size == 4) ? 32 : 64; + + ct.div_ne0 = init_fastdiv_values(ne0); + ct.div_ne1_ne0 = init_fastdiv_values(ne1 * ne0); + ct.div_ne2_ne1_ne0 = init_fastdiv_values(ne2 * ne1 * ne0); + ct.div_ne00 = init_fastdiv_values(ne00); + ct.div_ne01_ne00 = init_fastdiv_values(ne01 * ne00); + ct.div_ne02_ne01_ne00 = init_fastdiv_values(ne02 * ne01 * ne00); + + uint32_t elem_start = 0; + uint32_t nelem = total_elems; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elems, can_split ? elems_per_line : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelem = range.count; + } + + if (nelem == 0) { + return HTP_STATUS_OK; } + + ct.elem_start = elem_start; + ct.nelem = nelem; + ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div); + + work_queue_func_t copy_fun = (src0->type == HTP_TYPE_F32) ? cpy_thread_f32_reshape : cpy_thread_f16_reshape; + work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); } else { return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "cpy-%s-%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_dma=%d n_threads %u\n", - src0->type == HTP_TYPE_F32 ? "f32" : "f16", dst->type == HTP_TYPE_F32 ? "f32" : "f16", - ne00, ne01, ne02, ne03, ne0, ne1, ne2, ne3, use_dma, n_threads); + return HTP_STATUS_OK; +} - if (use_dma) { - cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3); - } else { - worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads); - } +int op_cpy(struct htp_ops_context * octx) { + bool use_dma = false; + int status = exec_cpy(octx, &use_dma); + + htp_ops_context_set_status(octx, status); - const struct htp_tensor *sync = octx->src[1]; - if (sync && (sync->flags & HTP_TENSOR_FENCE)) { + if (octx->op == HTP_OP_CPY_FENCE) { if (!use_dma) { - // htp_tensor_flush_all(octx->ctx, octx->dsts, 1); - qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + htp_flush_dirty_ranges(octx->ctx); } - atomic_uint * sync_fence = (atomic_uint *) sync->data; - const uint32_t seq = (uint32_t) octx->op_params[0]; + htp_mdev_group_barrier(octx); - atomic_store(&sync_fence[0], seq); - asm volatile ("syncht" : : : "memory"); - Q6_dccleaninva_A((void *) sync_fence); + if (octx->ctx->mdev.idx == 0) { + const struct htp_tensor * sync = octx->src[1]; + const uint32_t seq = (uint32_t) octx->op_params[0]; + atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; + htp_fence_write(sync_fence, seq, octx->status); - FARF(HIGH, "ggml-hex: sync-release : fence %p seq %u\n", sync_fence, seq); + FARF(HIGH, "ggml-hex: sync-release : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status); + } } - return HTP_STATUS_OK; + return octx->status; } diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c index 2d45c39f23b5..971fa3bccb3a 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c @@ -7,6 +7,8 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -17,25 +19,25 @@ #define htp_cumsum_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ const struct htp_tensor * restrict dst = octx->dst; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; struct htp_cumsum_context { @@ -46,6 +48,7 @@ struct htp_cumsum_context { size_t dst_row_size_aligned; uint32_t rows_per_thread; uint32_t total_rows; + uint32_t row_start; }; #define htp_cumsum_preamble \ @@ -116,11 +119,8 @@ static inline void hvx_cumsum_row_f32(const float * restrict src, float * restri static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t ir0 = cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); if (ir0 >= ir1) { return; @@ -149,11 +149,15 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat src_row_size_aligned, src_row_size, 1); } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = ir0; ir < ir1; ir++) { float * dst_spad_row = (float *) dma_queue_pop(dma_queue).src; float * src_spad_row = (float *) dma_queue_pop(dma_queue).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); hvx_cumsum_row_f32(src_spad_row, dst_spad_row, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(dst_data + (ir * dst_row_size), (uint8_t *) dst_spad_row), @@ -168,12 +172,10 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat } dma_queue_flush(dma_queue); - t2 = HAP_perf_get_qtimer_count(); - FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } // --------------------------------------------------------------------------- @@ -183,14 +185,14 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ir0 = cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); for (uint32_t ir = ir0; ir < ir1; ir++) { const float * restrict src_row = (const float *) (src_data + ir * cctx->src_row_size); @@ -198,12 +200,11 @@ static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { hvx_cumsum_row_f32(src_row, dst_row, ne00); } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); - FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } int op_cumsum_f32(struct htp_ops_context * octx) { @@ -214,8 +215,25 @@ int op_cumsum_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_rows); + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; const size_t src_row_size = src0->nb[1]; const size_t dst_row_size = dst->nb[1]; @@ -240,14 +258,15 @@ int op_cumsum_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_row_size_aligned = src_row_size_aligned, .dst_row_size_aligned = dst_row_size_aligned, - .rows_per_thread = (total_rows + n_threads - 1) / n_threads, - .total_rows = total_rows, + .rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_rows = nrows, + .row_start = row_start, }; if (octx->ctx->vtcm_size < spad_per_thread * n_threads) { - worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32, &cctx, n_threads); + work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32_dma, &cctx, n_threads); + work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.c index 9b3194d90846..a69fd89d38b3 100644 --- a/ggml/src/ggml-hexagon/htp/diag-ops.c +++ b/ggml/src/ggml-hexagon/htp/diag-ops.c @@ -5,8 +5,11 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-types.h" #include "hex-utils.h" #include "hvx-copy.h" @@ -15,17 +18,17 @@ #define htp_diag_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ const struct htp_tensor * restrict dst = octx->dst; \ - \ - const uint32_t ne02 = src0->ne[2]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne02 = src0->ne[2]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; struct htp_diag_context { @@ -36,6 +39,7 @@ struct htp_diag_context { size_t dst_row_size_aligned; uint32_t batches_per_thread; uint32_t total_batches; + uint32_t batch_start; }; #define htp_diag_preamble \ @@ -57,11 +61,8 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) htp_diag_preamble; dma_queue * dma_queue = octx->ctx->dma[ith]; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t ib0 = dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); if (ib0 >= ib1) { return; @@ -79,6 +80,8 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned); uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ib = ib0; ib < ib1; ib++) { const uint32_t i3 = ib / ne02; const uint32_t i2 = ib % ne02; @@ -96,7 +99,9 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) for (uint32_t i1 = 0; i1 < ne1; i1++) { // Compute row in VTCM + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1)); hvx_diag_row_f32(src_spad_f32, dst_spad_f32, i1, ne0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1)); // Write completed row back to DDR uint8_t * dst_row = dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1; @@ -107,12 +112,9 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) } } - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } // --------------------------------------------------------------------------- @@ -122,14 +124,14 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { htp_diag_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ib0 = dctx->batches_per_thread * ith; - const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->total_batches); + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); for (uint32_t ib = ib0; ib < ib1; ib++) { const uint32_t i3 = ib / ne02; @@ -143,12 +145,11 @@ static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); - FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } int op_diag_f32(struct htp_ops_context * octx) { @@ -160,7 +161,36 @@ int op_diag_f32(struct htp_ops_context * octx) { } const uint32_t total_batches = src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_batches); + const size_t dst_batch_size = dst->ne[1] * dst->nb[1]; + + uint32_t batch_start = 0; + uint32_t nbatches = total_batches; + + if (octx->ctx->mdev.count > 1) { + bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !htp_tensor_is_permuted(dst); + uint32_t batches_per_chunk = 1; + if (can_split) { + if (dst->ne[2] > 1 && (dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && + (dst->ne[3] <= 1 || (dst->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) { + batches_per_chunk = 1; + } else if (dst->nb[2] == dst_batch_size && + (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) { + batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1; + } else { + can_split = false; + } + } + + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + batch_start = range.start; + nbatches = range.count; + } + + if (nbatches == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; const size_t src_batch_size = src0->ne[0] * sizeof(float); const size_t dst_row_size = dst->ne[0] * sizeof(float); @@ -185,14 +215,15 @@ int op_diag_f32(struct htp_ops_context * octx) { .dst_row_size = dst_row_size, .src_batch_size_aligned = src_batch_size_aligned, .dst_row_size_aligned = dst_row_size_aligned, - .batches_per_thread = (total_batches + n_threads - 1) / n_threads, - .total_batches = total_batches, + .batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div), + .total_batches = nbatches, + .batch_start = batch_start, }; if (octx->ctx->vtcm_size < spad_per_thread * n_threads) { - worker_pool_run_func(octx->ctx->worker_pool, diag_thread_f32, &dctx, n_threads); + work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, diag_thread_f32_dma, &dctx, n_threads); + work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.c index 3ccfbe74ee45..1f6eaafada93 100644 --- a/ggml/src/ggml-hexagon/htp/fill-ops.c +++ b/ggml/src/ggml-hexagon/htp/fill-ops.c @@ -3,10 +3,11 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include -#include - #include +#include "hex-common.h" +#include "hex-profile.h" + #include "hvx-copy.h" #include "hvx-utils.h" @@ -14,28 +15,30 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" // ggml op_params layout for FILL: // op_params[0] (as float) - the scalar fill value -#define fill_preamble \ +#define fill_preamble \ const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ const uint32_t nr = ne1 * ne2 * ne3; struct htp_fill_context { struct htp_ops_context * octx; uint32_t nrows_per_thread; uint32_t total_rows; // ne1 * ne2 * ne3 + uint32_t row_start; bool opt_path; HVX_Vector splat_vec; uint32_t elem_size; @@ -47,10 +50,15 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) { fill_preamble; // Parallelise over the flat row index spanning ne1*ne2*ne3 - const uint32_t ir0 = fctx->nrows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->total_rows); + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); - uint64_t t1 = HAP_perf_get_qtimer_count(); + if (ir0 >= ir1) { + return; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); if (fctx->opt_path) { // Opt path: tensor is fully contiguous, treat as flat array @@ -69,9 +77,8 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) { } } - uint64_t t2 = HAP_perf_get_qtimer_count(); - FARF(HIGH, "fill %u/%u: rows %u:%u usec %u\n", - ith, nth, ir0, ir1, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1); + FARF(HIGH, "fill %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); } int op_fill(struct htp_ops_context * octx) { @@ -85,8 +92,23 @@ int op_fill(struct htp_ops_context * octx) { return HTP_STATUS_OK; } + uint32_t row_start = 0; + uint32_t nrows = nr; + + if (octx->ctx->mdev.count > 1) { + const uint32_t row_size = nb1; + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(nr, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + // nr = ne1*ne2*ne3 (flat row count across all outer dims); parallelise over it. - const uint32_t n_threads = MIN(nr, octx->n_threads); + const uint32_t n_threads = octx->n_threads; // Optimize if fully contiguous: skip stride arithmetic, treat as flat array const bool opt_path = (nb2 == nb1 * ne1) && (nb3 == nb2 * ne2); @@ -99,8 +121,9 @@ int op_fill(struct htp_ops_context * octx) { struct htp_fill_context fctx = { .octx = octx, - .nrows_per_thread = (nr + n_threads - 1) / n_threads, - .total_rows = nr, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_rows = nrows, + .row_start = row_start, .opt_path = opt_path, }; @@ -117,7 +140,7 @@ int op_fill(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - worker_pool_run_func(octx->ctx->worker_pool, fill_thread, &fctx, n_threads); + work_queue_run(octx->ctx->work_queue, fill_thread, &fctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index c76b4d3a3ac6..8a1caba22b79 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -75,6 +74,7 @@ struct htp_fa_context { uint32_t qrows; uint32_t qrows_per_thread; + uint32_t qrow_start; bool is_q_fp32; @@ -89,8 +89,6 @@ struct htp_fa_context { const struct htp_tensor * k; const struct htp_tensor * v; - - uint64_t t_start; }; struct hmx_fa_context { @@ -206,10 +204,9 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; // total rows in q - const uint32_t nr = factx->qrows; - const uint32_t dr = factx->qrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, nr); + const uint32_t dr = factx->qrows_per_thread; + const uint32_t ir0 = factx->qrow_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, factx->qrow_start + factx->qrows); if (ir0 >= ir1) return; @@ -1888,6 +1885,24 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const uint32_t n_threads = factx.n_threads; const uint32_t G = factx.G; + // Multi-device: split Q blocks across devices + const uint32_t n_q_blocks = (neq1 + Br - 1) / Br; + uint32_t q_start_min = 0; + uint32_t q_start_max = neq1; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_q_blocks, htp_tensor_mdev_data_aligned(dst) ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + const uint32_t block_start = range.start; + const uint32_t block_end = range.start + range.count; + + if (block_start >= block_end) { + return HTP_STATUS_OK; + } + + q_start_min = block_start * Br; + q_start_max = MIN(block_end * Br, neq1); + } + // ======== VTCM allocation (GQA-aware) ======== // K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used // throughout the KV loop below. @@ -1977,7 +1992,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // ======== Main loop ======== for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) { const uint32_t im3 = mask ? fastmodulo(ib3, mask->ne[3], &factx.src3_div3) : 0; - for (uint32_t q_start = 0; q_start < neq1; q_start += Br) { + for (uint32_t q_start = q_start_min; q_start < q_start_max; q_start += Br) { const uint32_t n_rows_q = hex_smin(Br, neq1 - q_start); const size_t n_rows_g = n_rows_q * G; const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS); @@ -1991,8 +2006,9 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { // 1. Push Q and KV DMAs for the very first iteration. // Subsequent iterations are enqueued early at the end of the previous iteration. - if (ib3 == 0 && q_start == 0 && kv_head == 0) { - const uint8_t * q_ptr = (const uint8_t *) q->data; + if (ib3 == 0 && q_start == q_start_min && kv_head == 0) { + const uint8_t * q_ptr = (const uint8_t *) q->data + q_start * q->nb[1] + + (kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3]; const size_t q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed; const size_t n_rows = q_transposed ? factx.G : n_rows_q; dma_queue_push(dma, dma_make_ptr(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows); @@ -2311,8 +2327,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { if (next_kv_head >= n_kv_heads) { next_kv_head = 0; next_q_start = q_start + Br; - if (next_q_start >= neq1) { - next_q_start = 0; + if (next_q_start >= q_start_max) { + next_q_start = q_start_min; next_ib3 = ib3 + 1; } } @@ -2398,6 +2414,10 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + if (kparams->kernel_type == HTP_FA_KERNEL_HMX) { return hmx_flash_attn_ext(octx); } @@ -2407,8 +2427,6 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { factx.k = k; factx.v = v; - factx.t_start = HAP_perf_get_qtimer_count(); - factx.src0_div21 = kparams->u.hvx.src0_div21; factx.src0_div1 = kparams->u.hvx.src0_div1; @@ -2451,8 +2469,30 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { } // total rows in q - factx.qrows = kparams->qrows; - factx.qrows_per_thread = kparams->qrows_per_thread; + const uint32_t neq1 = q->ne[1]; + const uint32_t neq2 = q->ne[2]; + const uint32_t neq3 = q->ne[3]; + const uint32_t total_qrows = neq1 * neq2 * neq3; + + uint32_t qrow_start = 0; + uint32_t qrows = total_qrows; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_qrows, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + qrow_start = range.start; + qrows = range.count; + } + + if (qrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + factx.qrows = qrows; + factx.qrow_start = qrow_start; + factx.qrows_per_thread = fastdiv(qrows + n_threads - 1, &octx->n_threads_div); size_t size_vkq_acc = hex_round_up(v->ne[0] * sizeof(float), 128); // VKQ32 @@ -2461,18 +2501,18 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { uint8_t * vtcm_cur = octx->ctx->vtcm_base; - factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * octx->n_threads); - factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * octx->n_threads); - factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * octx->n_threads); - factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * octx->n_threads); - factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * octx->n_threads); + factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * n_threads); + factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * n_threads); + factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * n_threads); + factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * n_threads); + factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * n_threads); if ((size_t) (vtcm_cur - octx->ctx->vtcm_base) > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; } if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h index c4d19063169b..0278454114ec 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h @@ -51,6 +51,7 @@ struct htp_fa_kernel_params { uint32_t qrows; uint32_t qrows_per_thread; + uint32_t qrow_start; float m0; float m1; uint32_t n_head_log2; diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c index 96655215298a..0b6529571d15 100644 --- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c @@ -4,10 +4,13 @@ #include "hvx-utils.h" #include "hex-fastdiv.h" +#include "hex-common.h" +#include "hex-profile.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" +#include "htp-tensor.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -22,6 +25,8 @@ struct htp_gdn_context { size_t state_bytes; uint8_t * vtcm_base; size_t vtcm_per_thread; + uint32_t row_start; + uint32_t nrows; }; static inline HVX_Vector gdn_mul_dot_f32(float * restrict dst, const float * restrict mul, const float * restrict dot, uint32_t n) { @@ -586,8 +591,9 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo const uint32_t n_seqs = v->ne[3]; const uint32_t K = octx->op_params[0]; - const uint32_t total_rows = H * n_seqs; - if (ith >= total_rows) { + const uint32_t row_end = gctx->row_start + gctx->nrows; + + if (ith >= gctx->nrows) { return; } @@ -621,11 +627,11 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo const uint64_t state_seq_stride = state->nb[3] / sizeof(float); const uint64_t state_size_per_snap = (uint64_t) S_v * S_v * H * n_seqs; - uint32_t ir_prefetch = ith; + uint32_t ir_prefetch = gctx->row_start + ith; int spad_idx = 0; // Prefetch preamble (up to 2 steps) - for (int k = 0; k < 2 && ir_prefetch < total_rows; k++) { + for (int k = 0; k < 2 && ir_prefetch < row_end; k++) { const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; @@ -646,8 +652,11 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (gctx->row_start + ith)); + int curr_spad_idx = 0; - for (uint32_t ir = ith; ir < total_rows; ir += nth) { + for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { dma_queue_pop(dma); dma_queue_pop(dma); @@ -812,7 +821,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo S_v * sizeof(float), S_v); // Prefetch next block (if any) - if (ir_prefetch < total_rows) { + if (ir_prefetch < row_end) { const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; @@ -828,6 +837,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo curr_spad_idx ^= 1; } dma_queue_flush(dma); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row_end); } @@ -847,8 +857,9 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo const uint32_t H = v->ne[1]; const uint32_t n_seqs = v->ne[3]; - const uint32_t total_rows = H * n_seqs; - if (ith >= total_rows) { + const uint32_t row_end = gctx->row_start + gctx->nrows; + + if (ith >= gctx->nrows) { return; } @@ -881,11 +892,11 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo const uint64_t state_seq_stride = state->nb[3] / sizeof(float); - uint32_t ir_prefetch = ith; + uint32_t ir_prefetch = gctx->row_start + ith; int spad_idx = 0; // Prefetch preamble (up to 2 steps) - for (int k = 0; k < 2 && ir_prefetch < total_rows; k++) { + for (int k = 0; k < 2 && ir_prefetch < row_end; k++) { const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; @@ -906,8 +917,11 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (gctx->row_start + ith)); + int curr_spad_idx = 0; - for (uint32_t ir = ith; ir < total_rows; ir += nth) { + for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { dma_queue_pop(dma); dma_queue_pop(dma); @@ -1057,7 +1071,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo S_v * sizeof(float), S_v); // Prefetch next block (if any) - if (ir_prefetch < total_rows) { + if (ir_prefetch < row_end) { const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H); const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H); const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v; @@ -1073,6 +1087,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo curr_spad_idx ^= 1; } dma_queue_flush(dma); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row_end); } @@ -1085,10 +1100,6 @@ int op_gated_delta_net(struct htp_ops_context * octx) { const struct htp_tensor * state = octx->src[5]; const struct htp_tensor * dst = octx->dst; - if (!q || !k || !v || !g || !beta || !state || !dst) { - return HTP_STATUS_INVAL_PARAMS; - } - if (q->type != HTP_TYPE_F32 || k->type != HTP_TYPE_F32 || v->type != HTP_TYPE_F32 || g->type != HTP_TYPE_F32 || beta->type != HTP_TYPE_F32 || state->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { @@ -1124,16 +1135,37 @@ int op_gated_delta_net(struct htp_ops_context * octx) { return HTP_STATUS_OK; } + const uint32_t total_rows = H * n_seqs; + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + const uint32_t head_bytes = S_v * sizeof(float); + const uint32_t rows_per_chunk = (head_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(head_bytes, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + struct htp_gdn_context gctx; gctx.octx = octx; - gctx.rows_per_thread = (H * n_seqs + octx->n_threads - 1) / octx->n_threads; + gctx.row_start = row_start; + gctx.nrows = nrows; + gctx.rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); gctx.state_bytes = (size_t) S_v * S_v * sizeof(float); size_t state_aligned = (size_t) S_v * S_v * sizeof(float); state_aligned = (state_aligned + 127) & ~(size_t)127; - assert(octx->ctx->vtcm_base != NULL); - assert(octx->ctx->vtcm_size >= 2 * state_aligned * octx->n_threads); + assert(octx->ctx->vtcm_size >= 2 * state_aligned * n_threads); gctx.vtcm_base = octx->ctx->vtcm_base; gctx.vtcm_per_thread = 2 * state_aligned; @@ -1148,9 +1180,9 @@ int op_gated_delta_net(struct htp_ops_context * octx) { gctx.vtcm_per_thread * octx->n_threads, octx->n_threads); if (n_tokens == 1) { - worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_tg_thread, &gctx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_tg_thread, &gctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_pp_thread, &gctx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_pp_thread, &gctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index a87962d22910..d294ba57a042 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -10,6 +10,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -23,9 +24,12 @@ struct get_rows_context { const struct htp_get_rows_kernel_params * kparams; struct htp_get_rows_vtcm_layout vtcm_layout; uint8_t * vtcm_base; + uint32_t task_start; + uint32_t tasks; + uint32_t tasks_per_thread; }; -#define get_rows_preamble \ +#define get_rows_preamble \ const uint32_t ne00 = octx->src[0]->ne[0]; \ const uint32_t ne01 = octx->src[0]->ne[1]; \ const uint32_t ne02 = octx->src[0]->ne[2]; \ @@ -61,12 +65,12 @@ static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, vo struct htp_ops_context * octx = grctx->octx; \ const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ get_rows_preamble; \ - const uint32_t dr = kparams->tasks_per_thread; \ - const uint32_t ir0 = dr * ith; \ - if (ir0 >= kparams->total_tasks) { \ + const uint32_t dr = grctx->tasks_per_thread; \ + const uint32_t ir0 = grctx->task_start + dr * ith; \ + if (ir0 >= grctx->task_start + grctx->tasks) { \ return; \ } \ - const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \ dma_queue * dma_queue = octx->ctx->dma[ith]; \ for (uint32_t i = ir0; i < ir1; ++i) { \ @@ -101,12 +105,12 @@ static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \ get_rows_preamble; \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = kparams->tasks_per_thread; \ - const uint32_t ir0 = dr * ith; \ - if (ir0 >= kparams->total_tasks) { \ + const uint32_t dr = grctx->tasks_per_thread; \ + const uint32_t ir0 = grctx->task_start + dr * ith; \ + if (ir0 >= grctx->task_start + grctx->tasks) { \ return; \ } \ - const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \ const uint32_t chunks_per_row = kparams->chunks_per_row; \ const uint32_t chunk_size = kparams->chunk_size; \ dma_queue * dma_queue = octx->ctx->dma[ith]; \ @@ -225,13 +229,41 @@ int op_get_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } + const struct htp_tensor * dst = octx->dst; + const uint32_t total_tasks = kparams->total_tasks; + const size_t dst_row_size = htp_tensor_get_row_size(dst->type, dst->ne[0]); + + uint32_t task_start = 0; + uint32_t tasks = total_tasks; + + if (octx->ctx->mdev.count > 1) { + uint32_t tasks_per_chunk = 1; + htp_tensor_mdev_rows_per_chunk(dst, dst_row_size / dst->ne[0], (uint32_t) dst_row_size, &tasks_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + task_start = range.start; + tasks = range.count; + } + + if (tasks == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + struct get_rows_context grctx; grctx.octx = octx; grctx.kparams = kparams; grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; + grctx.task_start = task_start; + grctx.tasks = tasks; + grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); const uint32_t ne00 = octx->src[0]->ne[0]; - htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, kparams->n_threads); + htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, n_threads); const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); @@ -247,14 +279,14 @@ int op_get_rows(struct htp_ops_context * octx) { } } - FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use_dma=%d n_threads %d\n", + FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use-dma %d n-threads %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - grctx.vtcm_layout.src0_bytes_per_thread * kparams->n_threads, - grctx.vtcm_layout.dst_bytes_per_thread * kparams->n_threads, - kparams->use_dma, kparams->n_threads); + grctx.vtcm_layout.src0_bytes_per_thread * n_threads, + grctx.vtcm_layout.dst_bytes_per_thread * n_threads, + kparams->use_dma, n_threads); - work_queue_run(octx->ctx->work_queue, q_func, &grctx, kparams->n_threads); + work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/hex-common.h b/ggml/src/ggml-hexagon/htp/hex-common.h index 4714486a042f..e6a52540d58b 100644 --- a/ggml/src/ggml-hexagon/htp/hex-common.h +++ b/ggml/src/ggml-hexagon/htp/hex-common.h @@ -77,4 +77,13 @@ static inline bool hex_add_overflow(size_t a, size_t b, size_t *out) { return false; } +static inline uint32_t hex_gcd_u32(uint32_t a, uint32_t b) { + while (b != 0) { + uint32_t t = b; + b = a % b; + a = t; + } + return a; +} + #endif // HEX_COMMON_H diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h index 1b3965030009..853f1c1b2d8d 100644 --- a/ggml/src/ggml-hexagon/htp/hex-utils.h +++ b/ggml/src/ggml-hexagon/htp/hex-utils.h @@ -39,7 +39,6 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) { #define HEX_L2_LINE_SIZE 128 #define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration) -#define HEX_L2_FLUSH_IL_THRESHOLD 1024 // inline flush threshold #define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) #define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) diff --git a/ggml/src/ggml-hexagon/htp/hmx-utils.h b/ggml/src/ggml-hexagon/htp/hmx-utils.h index 2a61ca7349df..ad295cb7df71 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-utils.h +++ b/ggml/src/ggml-hexagon/htp/hmx-utils.h @@ -27,7 +27,7 @@ static inline void hmx_init_column_scales(void *out_scales, HVX_Vector v_scale) // vscatter offsets for fused dequant+transpose: write K-values directly to [K][N] tile. // word[i] = i*128 maps K-row-pair i to byte offset i*128. // Column offset (n*4) is added at runtime. Entries 0..15 cover one tile (region 2047); -// entries 16..31 cover the next adjacent tile (region 4095) — pick region size at the +// entries 16..31 cover the next adjacent tile (region 4095) - pick region size at the // call site to scatter into one tile (masked) or two contiguous tiles (unmasked). static const int32_t hmx_transpose_scatter_offsets[32] __attribute__((aligned(VLEN))) = { 0 * 128, 1 * 128, 2 * 128, 3 * 128, 4 * 128, 5 * 128, 6 * 128, 7 * 128, 8 * 128, 9 * 128, 10 * 128, @@ -198,16 +198,16 @@ static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out, } // --- HMX inline asm macros for load-store packetization --- -#define HMX_LOAD_MPY_F16(act, wt, range) \ - "{\n" \ +#define HMX_LOAD_MPY_F16(act, wt, range) \ + "{\n" \ " activation.hf = mxmem(" act ", " range ")\n" \ - " weight.hf = mxmem(" wt ", " range ")\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ "}\n" -#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \ - "{\n" \ +#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \ + "{\n" \ " activation.hf = mxmem(" act ", " range "):deep\n" \ - " weight.hf = mxmem(" wt ", " range ")\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ "}\n" #define HMX_STORE_AFTER_F16(out, scale_reg) \ diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index c8a909d61907..3b60c8bdb08c 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -19,7 +19,7 @@ #endif #define HTP_MAX_MMAPS 16 -#define HTP_MAX_DIRTY_RANGES 16 +#define HTP_MAX_DIRTY_RANGES 32 // Memory mapping struct htp_mmap { @@ -29,6 +29,11 @@ struct htp_mmap { uint32_t reserved; }; +struct htp_dirty_range { + uint32_t start; + uint32_t end; +}; + // Scratchpad state struct htp_spad { const struct htp_tensor * src; // original src of the data (for reuse) @@ -38,6 +43,14 @@ struct htp_spad { uint32_t size_per_thread; // size per thread }; +struct htp_mdev_group { + uint16_t idx; + uint16_t count; + struct fastdiv_values count_div; + uint8_t * fence_base; + uint32_t fence_seq; +}; + struct htp_context; // Context while processing an Op @@ -65,8 +78,10 @@ struct htp_ops_context { struct htp_spad src3_spad; struct htp_spad dst_spad; - uint32_t n_threads; - uint32_t flags; + uint32_t flags; + uint32_t n_threads; + struct fastdiv_values n_threads_div; + int status; }; // Main context for htp DSP backend @@ -76,6 +91,7 @@ struct htp_context { struct htp_mmap mmap[HTP_MAX_MMAPS]; dma_queue_t dma[HTP_MAX_NTHREADS]; dma_queue_t dma_cached[HTP_MAX_NTHREADS]; + struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; work_queue_t work_queue; hmx_queue_t hmx_queue; @@ -88,7 +104,6 @@ struct htp_context { bool hmx_enabled; bool etm; uint32_t profiler; - struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; uint8_t * vtcm_base; size_t vtcm_size; @@ -97,16 +112,13 @@ struct htp_context { atomic_bool vtcm_needs_release; uint64_t max_vmem; - struct htp_dirty_range { - uint32_t start; - uint32_t end; - uint32_t bi; - } dirty_ranges[HTP_MAX_DIRTY_RANGES]; + struct htp_dirty_range dirty_ranges[HTP_MAX_DIRTY_RANGES]; // Persistent DDR scratchpad for MUL_MAT_ID mappings void * ddr_spad_base; size_t ddr_spad_size; + struct htp_mdev_group mdev; struct htp_ops_context octx; qurt_thread_t main_thread; @@ -115,6 +127,27 @@ struct htp_context { size_t footprint; }; +static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) { + if (n_threads == 0 || n_threads > octx->ctx->n_threads) { + return false; + } + + if (n_threads != octx->n_threads) { + octx->n_threads = n_threads; + octx->n_threads_div = n_threads == octx->ctx->n_threads + ? octx->ctx->n_threads_div + : init_fastdiv_values(n_threads); + } + + return true; +} + +static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) { + if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) { + octx->status = status; + } +} + int op_matmul(struct htp_ops_context * octx); int op_matmul_id(struct htp_ops_context * octx); int op_matmul_nx(struct htp_ops_context * octx); diff --git a/ggml/src/ggml-hexagon/htp/htp-fence.h b/ggml/src/ggml-hexagon/htp/htp-fence.h new file mode 100644 index 000000000000..7450b5de5363 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-fence.h @@ -0,0 +1,89 @@ +#ifndef HTP_FENCE_H +#define HTP_FENCE_H + +#include +#include + +#include + +#include "hex-utils.h" +#include "htp-ops.h" +#include "htp-ctx.h" + +static inline atomic_uint * htp_mdev_fence_slot(const void * fence_base, uint32_t idx) { + return (atomic_uint *) ((const uint8_t *) fence_base + (size_t) idx * HTP_FENCE_SLOT_SIZE); +} + +static inline void htp_fence_write(void * fence_ptr, uint32_t seq, uint32_t status) { + atomic_uint * fence = (atomic_uint *) fence_ptr; + atomic_store(&fence[1], status); + atomic_store(&fence[0], seq); + asm volatile ("syncht" : : : "memory"); + Q6_dccleaninva_A((void *) fence); +} + +static inline void htp_fence_read(const void * fence_ptr, uint32_t * seq, uint32_t * status) { + const atomic_uint * fence = (const atomic_uint *) fence_ptr; + Q6_dccleaninva_A((void *) fence); + asm volatile ("syncht" : : : "memory"); + *seq = atomic_load(&fence[0]); + *status = atomic_load(&fence[1]); +} + +static inline void htp_mdev_group_barrier(struct htp_ops_context * octx) { + struct htp_context * ctx = octx->ctx; + if (ctx->mdev.count <= 1) { + return; + } + + const uint32_t seq = ++ctx->mdev.fence_seq; + + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + + const uint32_t mdev_idx = ctx->mdev.idx; + const uint32_t mdev_count = ctx->mdev.count; + + uint8_t * fence_base = ctx->mdev.fence_base; + atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx); + htp_fence_write(my_fence, seq, octx->status); + + for (uint32_t d = 0; d < mdev_count; d++) { + if (d == mdev_idx) continue; + atomic_uint * peer_fence = htp_mdev_fence_slot(fence_base, d); + uint64_t spins = 0; + while (1) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - seq) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: mdev %u peer %u failed with status %u : seq 0x%08x\n", + mdev_idx, d, peer_status, seq); + htp_ops_context_set_status(octx, peer_status); + } + break; + } + if (++spins == 10000) { + FARF(ALWAYS, "ggml-hex: mdev %u waiting for mdev %u : seq 0x%08x (b %u op %u) my-fence %p peer-fence %p peer-seq 0x%08x (diff %d)\n", + mdev_idx, d, seq, seq >> 12, seq & 0xfff, my_fence, peer_fence, peer_seq, (int32_t)(peer_seq - seq)); + } + if (spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: mdev %u timeout waiting for mdev %u : seq 0x%08x (b %u op %u) peer-fence %p peer-seq 0x%08x\n", + mdev_idx, d, seq, seq >> 12, seq & 0xfff, peer_fence, peer_seq); + htp_ops_context_set_status(octx, HTP_STATUS_INTERNAL_ERR); + break; + } + hex_pause(); + } + } + asm volatile ("syncht" : : : "memory"); + + if (octx->status > HTP_STATUS_OK) { + htp_fence_write(my_fence, seq, octx->status); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); +} + +#endif // HTP_FENCE_H diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index 12a61b67f261..869b19b8c2de 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -77,6 +77,7 @@ enum htp_op_code { HTP_OP_GET_ROWS, HTP_OP_SCALE, HTP_OP_CPY, + HTP_OP_CPY_FENCE, HTP_OP_ARGSORT, HTP_OP_SQR, HTP_OP_SQRT, @@ -100,6 +101,7 @@ enum htp_op_code { HTP_OP_ALLREDUCE, HTP_OP_ALLREDUCE_ADD, HTP_OP_GLU_SWIGLU_CLAMP, + HTP_OP_MDEV_GROUP, HTP_OP_INVALID }; @@ -114,6 +116,7 @@ enum htp_op_code { #define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16) #define HTP_FENCE_TIMEOUT (1000000000ULL) +#define HTP_FENCE_SLOT_SIZE 128 #define HTP_OP_MAX_VMEM_DEFAULT (3355443200u) @@ -214,30 +217,26 @@ struct htp_prof_desc { }; struct htp_opbatch_req { - uint32_t id; // Batch id + uint64_t seq; // Sequence number uint32_t n_bufs; // Number of buffers uint32_t n_tensors; // Number of tensors uint32_t n_ops; // Number of ops uint32_t n_traces; // Number of trace descriptors per thread - uint32_t pad; // unused - uint64_t seq; // Sequence number // struct htp_buf_desc bufs[]; -- dspqueue buf 0 // struct htp_tensor tensors[]; -- dspqueue buf 0 // struct htp_op_desc ops[]; -- dspqueue buf 0 }; struct htp_opbatch_rsp { - uint32_t id; // Batch id - uint32_t status; // HTP_STATUS_... - uint32_t n_bufs; // Number of buffers - uint32_t n_tensors; // Number of tensors - uint32_t n_ops; // Number of op profile descriptors - uint32_t n_traces[HTP_MAX_NTHREADS + 1]; - uint32_t usecs; // Number of usec - uint32_t pad; // align to 8 bytes + uint64_t seq; // Sequence number uint64_t cycles_start; // Start cycle counter uint64_t cycles_stop; // Stop cycle counter - uint64_t seq; // Sequence number + uint32_t status; // HTP_STATUS_... + uint32_t n_bufs; // Number of buffers + uint32_t n_tensors; // Number of tensors + uint32_t n_ops; // Number of op profile descriptors + uint32_t usecs; // Number of usec + uint32_t n_traces[HTP_MAX_NTHREADS + 1]; // struct htp_prof_desc profs[]; -- dspqueue buf 0 }; diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.c b/ggml/src/ggml-hexagon/htp/htp-tensor.c index ae377c9221ff..760ccd8313a4 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.c +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.c @@ -20,7 +20,7 @@ struct l2flush_range { struct l2flush_multi_task { struct htp_thread_trace * trace; - struct l2flush_range ranges[HTP_OP_MAX_INPUTS]; + struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES]; uint32_t n_ranges; uint32_t total_blocks; uint32_t blocks_per_thread; @@ -73,6 +73,27 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); } +static void merge_dirty_ranges(struct htp_context * ctx) { + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) continue; + + for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES;) { + struct htp_dirty_range * s = &ctx->dirty_ranges[j]; + if (i == j || !s->start || r->end < s->start || s->end < r->start) { + j++; + continue; + } + + r->start = MIN(r->start, s->start); + r->end = MAX(r->end, s->end); + s->start = 0; + s->end = 0; + j = 0; + } + } +} + void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { const struct htp_tensor * pending[HTP_OP_MAX_OUTPUTS]; uint32_t n_pending = 0; @@ -83,11 +104,6 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co continue; } - if (t->size <= HEX_L2_FLUSH_IL_THRESHOLD) { - hex_l2flush((void *) (uintptr_t) t->data, t->size); - continue; - } - uint32_t t_start = t->data; uint32_t t_end = t_start + t->size; @@ -110,6 +126,8 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co } } + merge_dirty_ranges(ctx); + if (n_pending == 0) { return; } @@ -132,8 +150,8 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; r->start = pending[i]->data; r->end = pending[i]->data + pending[i]->size; - r->bi = pending[i]->bi; } + merge_dirty_ranges(ctx); return; } @@ -151,12 +169,12 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[i]; r->start = pending[i]->data; r->end = pending[i]->data + pending[i]->size; - r->bi = pending[i]->bi; } + merge_dirty_ranges(ctx); return; } - if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_OP_MAX_INPUTS) { + if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { struct l2flush_multi_task task; task.trace = ctx->trace; task.n_ranges = n_evict; @@ -195,7 +213,6 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; r->start = pending[i]->data; r->end = pending[i]->data + pending[i]->size; - r->bi = pending[i]->bi; } for (uint32_t i = 0; i < n_empty; i++) { @@ -203,8 +220,9 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; r->start = pending[n_evict + i]->data; r->end = pending[n_evict + i]->data + pending[n_evict + i]->size; - r->bi = pending[n_evict + i]->bi; } + + merge_dirty_ranges(ctx); } static void make_tensor_clean(struct htp_context * ctx, const struct htp_tensor * t) { @@ -242,17 +260,50 @@ static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_te return false; } -void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { - const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS]; - uint32_t n_dirty = 0; +static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) { + if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + struct l2flush_multi_task task; + task.trace = ctx->trace; + task.n_ranges = n_ranges; + + uint32_t block_acc = 0; + for (uint32_t i = 0; i < n_ranges; i++) { + const struct htp_dirty_range * r = &ranges[i]; + struct l2flush_range * rg = &task.ranges[i]; + rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); + rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); + rg->block_first = block_acc; + rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + block_acc += rg->n_blocks; + } + + task.total_blocks = block_acc; + task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + + work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + } else { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); + for (uint32_t i = 0; i < n_ranges; i++) { + const struct htp_dirty_range * r = &ranges[i]; + hex_l2flush((void *) (uintptr_t) r->start, r->end - r->start); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0); + } +} + +void htp_flush_dirty_ranges(struct htp_context * ctx) { + struct htp_dirty_range ranges[HTP_MAX_DIRTY_RANGES]; + uint32_t n_ranges = 0; uint64_t total_dirty = 0; - for (uint32_t i = 0; i < n; i++) { - const struct htp_tensor * t = tensors[i]; - if (t && !(t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE)) && is_tensor_dirty(ctx, t)) { - dirty_tensors[n_dirty++] = t; - total_dirty += t->size; + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + const struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) { + continue; } + ranges[n_ranges++] = *r; + total_dirty += r->end - r->start; } if (total_dirty == 0) { @@ -264,37 +315,37 @@ void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * co return; } - if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { - struct l2flush_multi_task task; - task.trace = ctx->trace; - task.n_ranges = 0; + flush_dirty_ranges(ctx, ranges, n_ranges, total_dirty); + memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); +} - uint32_t block_acc = 0; - for (uint32_t i = 0; i < n_dirty; i++) { - const struct htp_tensor * t = dirty_tensors[i]; - make_tensor_clean(ctx, t); +void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { + const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS]; + struct htp_dirty_range ranges[HTP_OP_MAX_INPUTS]; + uint32_t n_dirty = 0; + uint64_t total_dirty = 0; - struct l2flush_range * rg = &task.ranges[task.n_ranges++]; - rg->start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE); - rg->end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE); - rg->block_first = block_acc; - rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; - block_acc += rg->n_blocks; + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (t && is_tensor_dirty(ctx, t)) { + dirty_tensors[n_dirty++] = t; + ranges[n_dirty - 1].start = t->data; + ranges[n_dirty - 1].end = t->data + t->size; + total_dirty += t->size; } + } - task.total_blocks = block_acc; - task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + if (total_dirty == 0) { + return; + } - work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); return; } - struct htp_thread_trace * tr = &ctx->trace[0]; + flush_dirty_ranges(ctx, ranges, n_dirty, total_dirty); for (uint32_t i = 0; i < n_dirty; i++) { - const struct htp_tensor * t = dirty_tensors[i]; - htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); - hex_l2flush((void *) (uintptr_t) t->data, t->size); - htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); - make_tensor_clean(ctx, t); + make_tensor_clean(ctx, dirty_tensors[i]); } } diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h index c9cadbae3f23..3afff6917000 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.h +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h @@ -2,8 +2,20 @@ #define HTP_TENSOR_H #include +#include #include "htp-ops.h" #include "hex-bitmap.h" +#include "hex-common.h" +#include "hex-fastdiv.h" + +enum { + HTP_TENSOR_MDEV_LINE_SIZE = 128, +}; + +struct htp_tensor_mdev_range { + uint32_t start; + uint32_t count; +}; static inline void * htp_tensor_data(const struct htp_tensor * t) { return (void *) (uintptr_t) t->data; @@ -13,6 +25,102 @@ static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) { return (uint32_t *) &t->flags; } +static inline bool htp_tensor_is_contiguous(const struct htp_tensor * t, uint32_t type_size) { + uint32_t next_nb = type_size; + if (t->ne[0] != 1 && t->nb[0] != next_nb) { + return false; + } + next_nb *= t->ne[0]; + for (int i = 1; i < HTP_OP_MAX_DIMS; i++) { + if (t->ne[i] != 1 && t->nb[i] != next_nb) { + return false; + } + next_nb *= t->ne[i]; + } + return true; +} + +static inline bool htp_tensor_is_permuted(const struct htp_tensor * t) { + return t->nb[0] > t->nb[1] || t->nb[1] > t->nb[2] || t->nb[2] > t->nb[3]; +} + +static inline bool htp_tensor_mdev_data_aligned(const struct htp_tensor * t) { + return ((uintptr_t) t->data & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0; +} + +static inline bool htp_tensor_can_row_partition(const struct htp_tensor * t, uint32_t elem_size) { + if (!htp_tensor_mdev_data_aligned(t)) { + return false; + } + if (t->ne[0] != 1 && t->nb[0] != elem_size) { + return false; + } + if (htp_tensor_is_permuted(t)) { + return false; + } + if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + if (t->ne[2] > 1 && (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + if (t->ne[3] > 1 && (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + return true; +} + +static inline bool htp_tensor_mdev_rows_per_chunk(const struct htp_tensor * t, uint32_t elem_size, uint32_t row_size, uint32_t * rows_per_chunk) { + *rows_per_chunk = 0; + + if (!htp_tensor_mdev_data_aligned(t)) { + return false; + } + if (t->ne[0] != 1 && t->nb[0] != elem_size) { + return false; + } + if (htp_tensor_is_permuted(t)) { + return false; + } + if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && + (t->ne[2] <= 1 || (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0) && + (t->ne[3] <= 1 || (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) { + *rows_per_chunk = 1; + return true; + } + if (t->nb[1] == row_size && + (t->ne[2] <= 1 || t->nb[2] == t->nb[1] * t->ne[1]) && + (t->ne[3] <= 1 || t->nb[3] == t->nb[2] * t->ne[2])) { + *rows_per_chunk = (row_size > 0) ? (HTP_TENSOR_MDEV_LINE_SIZE / hex_gcd_u32(row_size, HTP_TENSOR_MDEV_LINE_SIZE)) : 1; + return true; + } + return false; +} + +static inline struct htp_tensor_mdev_range htp_tensor_mdev_partition(uint32_t total_units, uint32_t units_per_chunk, uint32_t mdev_idx, uint32_t mdev_count, const struct fastdiv_values * mdev_count_div) { + struct htp_tensor_mdev_range range = { 0, total_units }; + + if (mdev_count <= 1) { + return range; + } + + if (units_per_chunk == 0) { + range.start = (mdev_idx == 0) ? 0 : total_units; + range.count = (mdev_idx == 0) ? total_units : 0; + return range; + } + + const uint32_t total_chunks = total_units / units_per_chunk; + if (total_chunks < mdev_count) { + range.start = (mdev_idx == 0) ? 0 : total_units; + range.count = (mdev_idx == 0) ? total_units : 0; + return range; + } + + const uint32_t chunks_per_mdev = fastdiv(total_chunks + mdev_count - 1, mdev_count_div); + range.start = MIN(mdev_idx * chunks_per_mdev * units_per_chunk, total_units); + if (mdev_idx == mdev_count - 1) { + range.count = total_units - range.start; + } else { + range.count = MIN(chunks_per_mdev * units_per_chunk, total_units - range.start); + } + return range; +} + static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { switch (type) { case HTP_TYPE_F32: return ne00 * 4; @@ -23,6 +131,7 @@ static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { } struct htp_context; +void htp_flush_dirty_ranges(struct htp_context * ctx); void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index fe5477c1be48..6cbead74c70e 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -16,25 +16,25 @@ #define UNUSED(x) (void)(x) #define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \ - do { \ - dst_type * vdst = (dst_type *) dst; \ - src0_type * vsrc0 = (src0_type *) src0; \ - src1_type * vsrc1 = (src1_type *) src1; \ - \ - const uint32_t epv = 128 / (elem_size); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \ - } \ - if (nloe) { \ - HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \ - vec_store((void *) &vdst[i], nloe * (elem_size), v); \ - } \ + do { \ + dst_type * vdst = (dst_type *) dst; \ + src0_type * vsrc0 = (src0_type *) src0; \ + src1_type * vsrc1 = (src1_type *) src1; \ + \ + const uint32_t epv = 128 / (elem_size); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \ + vec_store((void *) &vdst[i], nloe * (elem_size), v); \ + } \ } while(0) #if __HVX_ARCH__ < 79 @@ -56,43 +56,43 @@ #define HVX_OP_MUL_F16(a, b) hvx_vec_mul_f16_f16(a, b) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \ -static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ +#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \ +static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src0 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ +} \ DEFINE_HVX_BINARY_OP_VARIANTS(hvx_add_f32, HVX_OP_ADD_F32, float) DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f32, HVX_OP_SUB_F32, float) @@ -103,25 +103,25 @@ DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f16, HVX_OP_SUB_F16, _Float16) DEFINE_HVX_BINARY_OP_VARIANTS(hvx_mul_f16, HVX_OP_MUL_F16, _Float16) // Dispatcher logic -#define HVX_BINARY_DISPATCHER(OP_NAME) \ +#define HVX_BINARY_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128)) { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ - else OP_NAME##_aau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ - else OP_NAME##_auu(dst, src0, src1, num_elems); \ - } \ - } else { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ - else OP_NAME##_uau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ - else OP_NAME##_uuu(dst, src0, src1, num_elems); \ - } \ - } \ + if (hex_is_aligned((void *) dst, 128)) { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ + else OP_NAME##_aau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ + else OP_NAME##_auu(dst, src0, src1, num_elems); \ + } \ + } else { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ + else OP_NAME##_uau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ + else OP_NAME##_uuu(dst, src0, src1, num_elems); \ + } \ + } \ } HVX_BINARY_DISPATCHER(hvx_add_f32) @@ -166,44 +166,44 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re // Scalar Operations -#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const uint32_t epv = 128 / (elem_size); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - HVX_Vector v = vsrc[i]; \ - vdst[i] = scalar_op_macro(v); \ - } \ - if (nloe) { \ - HVX_Vector v = vsrc[i]; \ - v = scalar_op_macro(v); \ - vec_store((void *) &vdst[i], nloe * (elem_size), v); \ - } \ +#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / (elem_size); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + HVX_Vector v = vsrc[i]; \ + vdst[i] = scalar_op_macro(v); \ + } \ + if (nloe) { \ + HVX_Vector v = vsrc[i]; \ + v = scalar_op_macro(v); \ + vec_store((void *) &vdst[i], nloe * (elem_size), v); \ + } \ } while(0) -#define HVX_OP_ADD_SCALAR_F32(v) \ - ({ \ +#define HVX_OP_ADD_SCALAR_F32(v) \ + ({ \ const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VwVw(inf, v); \ - HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \ - Q6_V_vmux_QVV(pred_inf, inf, out); \ + HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \ + Q6_V_vmux_QVV(pred_inf, inf, out); \ }) #define HVX_OP_MUL_SCALAR_F32(v) HVX_OP_MUL_F32(v, val_vec) #define HVX_OP_SUB_SCALAR_F32(v) HVX_OP_SUB_F32(v, val_vec) -#define HVX_OP_ADD_SCALAR_F16(v) \ - ({ \ +#define HVX_OP_ADD_SCALAR_F16(v) \ + ({ \ const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VhVh(inf, v); \ - HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \ - Q6_V_vmux_QVV(pred_inf, inf, out); \ + HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \ + Q6_V_vmux_QVV(pred_inf, inf, out); \ }) #define HVX_OP_MUL_SCALAR_F16(v) HVX_OP_MUL_F16(v, val_vec) @@ -212,31 +212,31 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re // Scalar Variants // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \ +#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \ static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src % 128 == 0); \ - hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src % 128 == 0); \ + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) dst % 128 == 0); \ - hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) dst % 128 == 0); \ + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) src % 128 == 0); \ - hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) src % 128 == 0); \ + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_add_scalar_f32, HVX_OP_ADD_SCALAR_F32, hvx_vec_splat_f32, float) DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f32, HVX_OP_SUB_SCALAR_F32, hvx_vec_splat_f32, float) @@ -247,17 +247,17 @@ DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f16, HVX_OP_SUB_SCALAR_F16, DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_mul_scalar_f16, HVX_OP_MUL_SCALAR_F16, hvx_vec_splat_f16, _Float16) // Dispatcher logic -#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \ +#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_aa(dst, src, val, num_elems); \ - } else if (hex_is_aligned((void *) dst, 128)) { \ - OP_NAME##_au(dst, src, val, num_elems); \ - } else if (hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_ua(dst, src, val, num_elems); \ - } else { \ - OP_NAME##_uu(dst, src, val, num_elems); \ - } \ + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_aa(dst, src, val, num_elems); \ + } else if (hex_is_aligned((void *) dst, 128)) { \ + OP_NAME##_au(dst, src, val, num_elems); \ + } else if (hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_ua(dst, src, val, num_elems); \ + } else { \ + OP_NAME##_uu(dst, src, val, num_elems); \ + } \ } HVX_BINARY_SCALAR_DISPATCHER(hvx_add_scalar_f32, float) @@ -350,12 +350,12 @@ static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * re // CLAMP Scalar variants -#define HVX_OP_CLAMP_SCALAR(v) \ - ({ \ +#define HVX_OP_CLAMP_SCALAR(v) \ + ({ \ HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VsfVsf(v, max_vec); \ HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VsfVsf(min_vec, v); \ - HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ - Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ + HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ + Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ }) static inline void hvx_clamp_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) { diff --git a/ggml/src/ggml-hexagon/htp/hvx-div.h b/ggml/src/ggml-hexagon/htp/hvx-div.h index 53ee304e749b..bb7ab0519dae 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-div.h +++ b/ggml/src/ggml-hexagon/htp/hvx-div.h @@ -219,64 +219,64 @@ static inline HVX_Vector hvx_vec_hybrid_div_f16(HVX_Vector vec1, HVX_Vector vec2 } while(0) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ +#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src0 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \ -} \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \ +} \ // Dispatcher logic -#define HVX_DIV_DISPATCHER(OP_NAME) \ +#define HVX_DIV_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128)) { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ - else OP_NAME##_aau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ - else OP_NAME##_auu(dst, src0, src1, num_elems); \ - } \ - } else { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ - else OP_NAME##_uau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ - else OP_NAME##_uuu(dst, src0, src1, num_elems); \ - } \ - } \ + if (hex_is_aligned((void *) dst, 128)) { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ + else OP_NAME##_aau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ + else OP_NAME##_auu(dst, src0, src1, num_elems); \ + } \ + } else { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ + else OP_NAME##_uau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ + else OP_NAME##_uuu(dst, src0, src1, num_elems); \ + } \ + } \ } DEFINE_HVX_DIV_OP_VARIANTS(hvx_div_f32, hvx_div_f32_loop_body) diff --git a/ggml/src/ggml-hexagon/htp/hvx-inverse.h b/ggml/src/ggml-hexagon/htp/hvx-inverse.h index f2054f45baca..256a8843ba1b 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-inverse.h +++ b/ggml/src/ggml-hexagon/htp/hvx-inverse.h @@ -169,36 +169,36 @@ static inline HVX_Vector hvx_vec_inverse_f16_guard(HVX_Vector v_sf, HVX_Vector n } while(0) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ +#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) src % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \ -} \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \ +} \ // Dispatcher logic -#define HVX_INV_DISPATCHER(OP_NAME) \ +#define HVX_INV_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_aa(dst, src, num_elems); \ - } else if (hex_is_aligned((void *) dst, 128)) { \ - OP_NAME##_au(dst, src, num_elems); \ - } else if (hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_ua(dst, src, num_elems); \ - } else { \ - OP_NAME##_uu(dst, src, num_elems); \ - } \ + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_aa(dst, src, num_elems); \ + } else if (hex_is_aligned((void *) dst, 128)) { \ + OP_NAME##_au(dst, src, num_elems); \ + } else if (hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_ua(dst, src, num_elems); \ + } else { \ + OP_NAME##_uu(dst, src, num_elems); \ + } \ } DEFINE_HVX_INV_OP_VARIANTS(hvx_inverse_f32, hvx_inverse_f32_loop_body) diff --git a/ggml/src/ggml-hexagon/htp/hvx-scale.h b/ggml/src/ggml-hexagon/htp/hvx-scale.h index 9b1a28f529a2..5d0650307ef8 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-scale.h +++ b/ggml/src/ggml-hexagon/htp/hvx-scale.h @@ -68,30 +68,30 @@ static inline void hvx_scale_f32(uint8_t * restrict dst, const uint8_t * restric } } -#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - HVX_Vector vs = hvx_vec_splat_f32(scale); \ - HVX_Vector vo = hvx_vec_splat_f32(offset); \ - \ - const uint32_t elem_size = sizeof(float); \ - const uint32_t epv = 128 / elem_size; \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; ++i) { \ +#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + HVX_Vector vs = hvx_vec_splat_f32(scale); \ + HVX_Vector vo = hvx_vec_splat_f32(offset); \ + \ + const uint32_t elem_size = sizeof(float); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; ++i) { \ HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \ - vdst[i] = Q6_Vsf_equals_Vqf32(v); \ - } \ - if (nloe) { \ + vdst[i] = Q6_Vsf_equals_Vqf32(v); \ + } \ + if (nloe) { \ HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \ - vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \ - } \ + vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \ + } \ } while(0) static inline void hvx_scale_offset_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h index dd66dd84c95a..552017309d19 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h @@ -68,50 +68,50 @@ static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) { return Q6_Vsf_equals_Vqf32(res); } -#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const HVX_Vector one = hvx_vec_splat_f32(1.f); \ - const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \ - const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \ - \ - const uint32_t epv = 128 / sizeof(float); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ - } \ - if (nloe) { \ +#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const HVX_Vector one = hvx_vec_splat_f32(1.f); \ + const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \ + const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \ + \ + const uint32_t epv = 128 / sizeof(float); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ + } \ + if (nloe) { \ HVX_Vector tmp = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ - vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ - } \ + vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ + } \ } while(0) -#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const uint32_t epv = 128 / sizeof(float); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \ - } \ - if (nloe) { \ - HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \ +#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / sizeof(float); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \ vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ - } \ + } \ } while(0) static inline void hvx_sigmoid_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c index 35fc103df8fe..52bbc37d1b0a 100644 --- a/ggml/src/ggml-hexagon/htp/im2col-ops.c +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c @@ -3,11 +3,12 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include -#include #include #include #include +#include "hex-common.h" + #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" @@ -16,14 +17,19 @@ #include "hex-dma.h" #include "hex-profile.h" #include "htp-vtcm.h" +#include "htp-tensor.h" struct htp_im2col_context { struct htp_ops_context * octx; + uint32_t patch_base; // first patch index assigned to this dev + uint32_t npatches; // number of patches assigned to this dev uint32_t npatches_per_thread; // patches = N*OH*OW (pure-DDR kernel) - uint32_t pe_rows_per_thread; // N*OH rows per worker - uint32_t pe_src_row_bytes; // one output row's source: IC*KH*IW*4, rounded 256 - uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256 + uint32_t pe_row_base; // first N*OH row index assigned to this dev (DMA path) + uint32_t pe_nrows; // number of N*OH rows assigned to this dev (DMA path) + uint32_t pe_rows_per_thread; // N*OH rows per worker + uint32_t pe_src_row_bytes; // one output row's source: IC*KH*IW*4, rounded 256 + uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256 // Patch-embed DMA path VTCM ping-pong. uint8_t * pe_vtcm_src; // base of the 2x src buffers region @@ -58,33 +64,27 @@ static inline void htp_im2col_vtcm_layout_build(struct htp_im2col_vtcm_layout * struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \ struct htp_ops_context * octx = ictx->octx; \ struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \ + const struct htp_tensor * restrict src0 = octx->src[0]; \ const struct htp_tensor * restrict src1 = octx->src[1]; \ const struct htp_tensor * restrict dst = octx->dst; \ - const int32_t s0 = octx->op_params[0]; \ - const int32_t s1 = octx->op_params[1]; \ - const int32_t p0 = octx->op_params[2]; \ - const int32_t p1 = octx->op_params[3]; \ - const int32_t d0 = octx->op_params[4]; \ - const int32_t d1 = octx->op_params[5]; \ - const uint32_t N = src1->ne[3]; \ - const uint32_t IC = src1->ne[2]; \ - const uint32_t IH = src1->ne[1]; \ - const uint32_t IW = src1->ne[0]; \ - const uint32_t KH = octx->src[0]->ne[1]; \ - const uint32_t KW = octx->src[0]->ne[0]; \ + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; \ + const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; \ + const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; \ + const uint32_t N = src1->ne[3], IC = src1->ne[2], IH = src1->ne[1], IW = src1->ne[0]; \ + const uint32_t KH = src0->ne[1], KW = src0->ne[0]; \ const uint32_t OH = dst->ne[2]; \ const uint32_t OW = dst->ne[1]; \ const uint32_t patch_stride = IC * KH * KW; \ const float * restrict src_data = (const float *) src1->data; \ DST_CTYPE * restrict dst_data = (DST_CTYPE *) dst->data; \ - const uint32_t npatches = N * OH * OW; \ - const uint32_t patch_start = ictx->npatches_per_thread * ith; \ - const uint32_t patch_end = MIN(patch_start + ictx->npatches_per_thread, npatches); \ - if (patch_start >= patch_end) { \ + const uint32_t patch_end = ictx->patch_base + ictx->npatches; \ + const uint32_t patch_start = ictx->patch_base + ictx->npatches_per_thread * ith; \ + const uint32_t patch_stop = MIN(patch_start + ictx->npatches_per_thread, patch_end);\ + if (patch_start >= patch_stop) { \ return; \ } \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, patch_start); \ - for (uint32_t p = patch_start; p < patch_end; p++) { \ + for (uint32_t p = patch_start; p < patch_stop; p++) { \ const uint32_t iow = p % OW; \ const uint32_t ioh = (p / OW) % OH; \ const uint32_t in = p / (OW * OH); \ @@ -154,10 +154,10 @@ IM2COL_PATCHEMBED_BODY(im2col_patchembed_f32_thread, float, hvx_copy_f32_uu, hvx uint8_t * dst_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; \ float * srcb = (float *) src_base; \ DST_CTYPE * dstb = (DST_CTYPE *) dst_base; \ - const uint32_t nrows = N * OH; \ + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; \ const uint32_t per_thread = ictx->pe_rows_per_thread; \ - const uint32_t row_start = per_thread * ith; \ - const uint32_t row_end = MIN(row_start + per_thread, nrows); \ + const uint32_t row_start = ictx->pe_row_base + per_thread * ith; \ + const uint32_t row_end = MIN(row_start + per_thread, row_end_max); \ if (row_start >= row_end) \ return; \ for (uint32_t r = row_start; r < row_end; r++) { \ @@ -266,26 +266,55 @@ int op_im2col(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t N = src1->ne[3]; - const uint32_t OH = dst->ne[2]; - const uint32_t OW = dst->ne[1]; - const uint32_t npatches = N * OH * OW; - const uint32_t n_threads = MIN(octx->n_threads, npatches); + if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + return HTP_STATUS_OK; + } + + const uint32_t N = src1->ne[3]; + const uint32_t OH = dst->ne[2]; + const uint32_t OW = dst->ne[1]; + const uint32_t total_patches = N * OH * OW; + const uint32_t total_rows = N * OH; + + uint32_t patch_base = 0; + uint32_t npatches = total_patches; + if (octx->ctx->mdev.count > 1) { + const uint32_t patch_size = dst->nb[1]; + const uint32_t patches_per_chunk = (patch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(patch_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_patches, htp_tensor_mdev_data_aligned(dst) ? patches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + patch_base = range.start; + npatches = range.count; + } + + uint32_t row_base = 0; + uint32_t nrows = total_rows; + if (octx->ctx->mdev.count > 1) { + const uint32_t row_size = dst->nb[2]; + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_base = range.start; + nrows = range.count; + } - if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) || n_threads == 0) { + if (npatches == 0 && nrows == 0) { return HTP_STATUS_OK; } + const uint32_t n_threads = MIN(octx->n_threads, MAX(npatches, 1)); + struct htp_im2col_context ictx = { 0 }; - ictx.octx = octx; - ictx.npatches_per_thread = (npatches + n_threads - 1) / n_threads; + ictx.octx = octx; + ictx.patch_base = patch_base; + ictx.npatches = npatches; + ictx.npatches_per_thread = (npatches + n_threads - 1) / n_threads; // Clean non-overlapping patch-embed -> DMA kernel (if it fits VTCM); // everything else (padding/dilation/stride edges) -> pure-DDR kernel. - if (im2col_use_patchembed_dma(octx)) { - const uint32_t nrows = N * OH; - const uint32_t pth = MIN(octx->n_threads, nrows); + if (im2col_use_patchembed_dma(octx) && nrows > 0) { + const uint32_t pth = MIN(octx->n_threads, nrows); if (pth > 0 && im2col_patchembed_dma_fits(octx, &ictx, pth)) { + ictx.pe_row_base = row_base; + ictx.pe_nrows = nrows; ictx.pe_rows_per_thread = (nrows + pth - 1) / pth; if (dst->type == HTP_TYPE_F16) { work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth); @@ -297,6 +326,10 @@ int op_im2col(struct htp_ops_context * octx) { // else: doesn't fit -> fall through to the pure-DDR kernel below. } + if (npatches == 0) { + return HTP_STATUS_OK; + } + if (dst->type == HTP_TYPE_F16) { work_queue_run(octx->ctx->work_queue, im2col_patchembed_thread, &ictx, n_threads); } else { diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index be54d4fe911c..1d291e16b463 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -34,6 +34,7 @@ #include "work-queue.h" #include "hex-profile.h" #include "allreduce-ops.h" +#include "htp-fence.h" #define HMX_QUEUE_CAPACITY 16 #define HMX_QUEUE_STACK_SIZE 16384 @@ -710,22 +711,43 @@ static inline void profile_stop(uint32_t mode, struct profile_data * d) { static int op_fence(struct htp_ops_context * octx) { struct htp_context *ctx = octx->ctx; struct htp_thread_trace * tr = &ctx->trace[0]; - const uint32_t seq = (uint32_t) octx->op_params[0]; + const uint32_t seq = (uint32_t) octx->op_params[0]; + const uint32_t mode = (uint32_t) octx->op_params[1]; htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); const struct htp_tensor * sync = octx->src[0]; - atomic_uint * sync_fence = (atomic_uint *) sync->data; + atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; + + if (mode == 1) { + htp_flush_dirty_ranges(ctx); + + htp_mdev_group_barrier(octx); + + if (ctx->mdev.idx == 0) { + htp_fence_write(sync_fence, seq, octx->status); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + FARF(HIGH, "ggml-hex: sync-signal : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status); + return octx->status; + } + + int status = HTP_STATUS_OK; uint64_t spins = 0; while (1) { - Q6_dccleaninva_A((void *) sync_fence); - asm volatile ("syncht" : : : "memory"); - uint32_t val = atomic_load(&sync_fence[0]); - if ((int32_t)(val - seq) >= 0) { + uint32_t sync_seq; + uint32_t sync_status; + htp_fence_read(sync_fence, &sync_seq, &sync_status); + if ((int32_t)(sync_seq - seq) >= 0) { + if (sync_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: sync-wait peer failed with status %u : fence %p seq 0x%x\n", sync_status, sync_fence, seq); + status = sync_status; + } break; } if (++spins > HTP_FENCE_TIMEOUT) { - FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq %u\n", sync_fence, spins, seq); + FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq); + status = HTP_STATUS_INTERNAL_ERR; break; } hex_pause(); @@ -733,12 +755,27 @@ static int op_fence(struct htp_ops_context * octx) { htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); - FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq %u\n", sync_fence, spins, seq); + FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq); + return status; +} + +static int op_mdev_group(struct htp_ops_context * octx) { + struct htp_context * ctx = octx->ctx; + const struct htp_tensor * sync = octx->src[0]; + ctx->mdev.idx = (uint16_t) octx->op_params[0]; + ctx->mdev.count = (uint16_t) sync->ne[1]; + if (ctx->mdev.count > 1) { + ctx->mdev.count_div = init_fastdiv_values(ctx->mdev.count); + ctx->mdev.fence_base = (uint8_t *) sync->data; + } return HTP_STATUS_OK; } static int execute_op(struct htp_ops_context * octx) { switch (octx->op) { + case HTP_OP_MDEV_GROUP: + return op_mdev_group(octx); + case HTP_OP_FENCE: return op_fence(octx); @@ -812,6 +849,7 @@ static int execute_op(struct htp_ops_context * octx) { return op_sum_rows(octx); case HTP_OP_CPY: + case HTP_OP_CPY_FENCE: return op_cpy(octx); case HTP_OP_REPEAT: @@ -855,7 +893,7 @@ static int execute_op(struct htp_ops_context * octx) { } FARF(ERROR, "Unknown Op %u", octx->op); - return -1; + return HTP_STATUS_NO_SUPPORT; } static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) { @@ -984,11 +1022,19 @@ static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, str } } -static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, uint32_t idx, struct htp_op_desc * op) { - memcpy(octx->op_params, op->params, sizeof(octx->op_params)); +static void mdev_group_init(struct htp_context * ctx, const struct htp_opbatch_req * req) { + memset(&ctx->mdev, 0, sizeof(ctx->mdev)); + ctx->mdev.fence_seq = (uint32_t)((req->seq & 0xfffff) << 12); +} + +static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs, uint32_t n_bufs, + struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) { + memcpy(octx->op_params, op->params, sizeof(octx->op_params)); memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params)); - octx->flags = op->flags; - octx->op = op->opcode; + octx->flags = op->flags; + octx->op = op->opcode; + octx->n_threads = octx->ctx->n_threads; + octx->n_threads_div = octx->ctx->n_threads_div; FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags); @@ -1027,9 +1073,13 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } + htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS); + + htp_mdev_group_barrier(octx); + int status = execute_op(octx); - htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS); + htp_ops_context_set_status(octx, status); octx->src0_spad.src = NULL; octx->src1_spad.src = NULL; @@ -1037,7 +1087,7 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u octx->src3_spad.src = NULL; octx->dst_spad.src = NULL; - return status; + return octx->status; } static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) { @@ -1059,7 +1109,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r return; } - FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req->id, + FARF(HIGH, "processing opbatch #%llu: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", (unsigned long long) req->seq, n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size); // Setup descriptor pointers @@ -1096,8 +1146,11 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r struct htp_ops_context *octx = &ctx->octx; memset(octx, 0, sizeof(*octx)); - octx->n_threads = ctx->n_threads; - octx->ctx = ctx; + octx->n_threads = ctx->n_threads; + octx->n_threads_div = ctx->n_threads_div; + octx->ctx = ctx; + + mdev_group_init(ctx, req); work_queue_wakeup(ctx->work_queue); if (ctx->hmx_queue) { @@ -1105,15 +1158,18 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r } int op_status = HTP_STATUS_OK; - for (uint32_t i = 0; i < n_ops && op_status == HTP_STATUS_OK; i++) { + octx->status = HTP_STATUS_OK; + for (uint32_t i = 0; i < n_ops; i++) { struct profile_data prof; profile_start(ctx->profiler, &prof); - op_status = proc_op_req(octx, tens, i, &ops[i]); + op_status = proc_op_req(octx, bufs, n_bufs, tens, i, &ops[i]); profile_stop(ctx->profiler, &prof); + htp_ops_context_set_status(octx, op_status); + if (ctx->profiler) { pds[i].opcode = ops[i].opcode; pds[i].usecs = prof.usecs; @@ -1136,19 +1192,20 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + htp_mdev_group_barrier(octx); + profile_stop(HTP_PROF_BASIC, &batch_prof); struct htp_opbatch_rsp rsp; memset(&rsp, 0, sizeof(rsp)); - rsp.id = req->id; - rsp.status = op_status; + rsp.seq = req->seq; + rsp.status = octx->status; rsp.n_bufs = n_bufs; rsp.n_tensors = n_tens; rsp.n_ops = n_ops; rsp.usecs = batch_prof.usecs; rsp.cycles_start = batch_prof.cycles_start; rsp.cycles_stop = batch_prof.cycles_stop; - rsp.seq = req->seq; if (ctx->profiler == HTP_PROF_TRACE) { for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 2a87dd19ee8c..1b597dcd9f20 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -21,6 +21,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "matmul-ops.h" #include "htp-vtcm.h" @@ -89,6 +90,8 @@ struct htp_mm_context { // Precomputed values uint32_t src0_nrows_per_thread; + uint32_t src0_row_start; + uint32_t src0_row_end; uint32_t src0_row_size_padded; uint32_t src1_nrows; @@ -135,6 +138,23 @@ struct htp_mm_context { uint32_t vtcm_dst_size_per_thread; }; +static int htp_mm_init_context( + struct htp_ops_context * octx, + const struct htp_mm_kernel_params * kparams +) { + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->n_hmx) { + if (kparams->n_act_threads <= 0 || kparams->n_act_threads > (int32_t) octx->n_threads) { + return HTP_STATUS_INVAL_PARAMS; + } + } + + return HTP_STATUS_OK; +} + // vdelta control to expand first 32 e8m0 values into 32 uint32 elements static const uint8_t __attribute__((aligned(128))) expand_x32_e8m0[128] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x02, 0x00, 0x08, 0x08, 0x01, 0x02, 0x00, 0x04, 0x04, 0x00, 0x00, @@ -238,22 +258,24 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { // This is the size of the rest of the dimensions of the result const uint32_t nr1 = ne1 * ne2 * ne3; + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; + // distribute the thread work across the inner or outer loop based on which one is larger uint32_t dr0, dr1, ith0, ith1; if (nr0 > nr1) { - dr0 = fastdiv(nr0 + nth - 1, &octx->ctx->n_threads_div); + dr0 = fastdiv(src0_nrows + nth - 1, &octx->n_threads_div); dr1 = nr1; ith0 = ith; ith1 = 0; } else { - dr0 = nr0; - dr1 = fastdiv(nr1 + nth - 1, &octx->ctx->n_threads_div); + dr0 = src0_nrows; + dr1 = fastdiv(nr1 + nth - 1, &octx->n_threads_div); ith0 = 0; ith1 = ith; } - const uint32_t ir0_start = dr0 * ith0; - const uint32_t ir0_end = MIN(ir0_start + dr0, nr0); + const uint32_t ir0_start = mmctx->src0_row_start + dr0 * ith0; + const uint32_t ir0_end = MIN(ir0_start + dr0, mmctx->src0_row_end); const uint32_t ir1_start = dr1 * ith1; const uint32_t ir1_end = MIN(ir1_start + dr1, nr1); @@ -312,11 +334,11 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ htp_matmul_preamble; \ \ - const uint32_t src0_nrows = ne01 * ne02 * ne03; \ + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ const uint32_t src1_nrows = ne11 * ne12 * ne13; \ \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ @@ -414,10 +436,10 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ htp_matmul_preamble; \ \ - const uint32_t src0_nrows = ne01; \ + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \ \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \ \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ @@ -549,12 +571,22 @@ static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, v uint32_t n_k_tiles_w = ne00 / 32; \ uint32_t tile_row_stride = n_k_tiles_w * tile_size; \ \ - const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; \ - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); \ + uint32_t src0_start_row = 0; \ + uint32_t src0_end_row = ne01; \ + if (octx->ctx->mdev.count > 1) { \ + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); \ + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, \ + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); \ + src0_start_row = range.start; \ + src0_end_row = range.start + range.count; \ + } \ + \ + const uint32_t nrows = src0_end_row - src0_start_row; \ + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); \ src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); \ \ - const uint32_t start_row = src0_nrows_per_thread * ith; \ - const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_nrows); \ + const uint32_t start_row = src0_start_row + src0_nrows_per_thread * ith; \ + const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_end_row); \ if (start_row >= end_row) continue; \ \ uint32_t ct_start = start_row / 32; \ @@ -735,11 +767,11 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); const uint32_t prefetch_mask = n_prefetch - 1; - const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows - const uint32_t src1_nrows = ne11 * ne12 * ne13; // src1 rows + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows + const uint32_t src1_nrows = ne11 * ne12 * ne13; // src1 rows - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); struct htp_thread_trace * tr = &octx->ctx->trace[ith]; @@ -781,7 +813,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - // Process src1 columns in pairs (2×2 tiling) + // Process src1 columns in pairs (2x2 tiling) uint32_t ir1 = 0; for (; ir1 + 1 < src1_nrows; ir1 += 2) { const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); @@ -791,7 +823,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); } - // Handle remaining src1 rows (fallback to 2×1) + // Handle remaining src1 rows (fallback to 2x1) for (; ir1 < src1_nrows; ++ir1) { const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); @@ -833,10 +865,10 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { htp_matmul_preamble; - const uint32_t src0_nrows = ne01; + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); struct htp_thread_trace * tr = &octx->ctx->trace[ith]; @@ -943,13 +975,10 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict ids = octx->src[2]; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t src0_nrows = ne01; // src0 rows per expert + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert const uint32_t src1_nrows = ne11; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); hvx_mm_run_quant_task(mmctx, ith); @@ -1036,9 +1065,9 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict ids = octx->src[2]; - const uint32_t src0_nrows = ne01; // src0 rows per expert - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); hvx_mm_run_quant_task(mmctx, ith); @@ -1143,12 +1172,22 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - const uint32_t src0_nrows = src_w->ne[1]; - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); + const uint32_t ne01 = src_w->ne[1]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } + + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); if (src0_start_row >= src0_end_row) continue; const uint8_t * restrict src0_row = (const uint8_t *) src_w->data + eid * src_w->nb[2]; @@ -1227,12 +1266,22 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) { const struct htp_tensor * restrict dst = octx->dsts[p]; if (!src_w || !dst) continue; - const uint32_t src0_nrows = src_w->ne[1]; - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); + const uint32_t ne01 = src_w->ne[1]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } + + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); if (src0_start_row >= src0_end_row) continue; const uint8_t * src0_row = (const uint8_t *) src_w->data + cur_a * src_w->nb[2]; @@ -1323,15 +1372,33 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - const uint32_t src0_nrows = ne01 * ne02 * ne03; + const uint32_t src0_nrows = ne01; const uint32_t src1_nrows = ne11 * ne12 * ne13; + uint32_t src0_row_start = 0; + uint32_t src0_row_end = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_row_start = range.start; + src0_row_end = range.start + range.count; + } + + if (src0_row_start >= src0_row_end) { + return HTP_STATUS_OK; + } + + const uint32_t nrows = src0_row_end - src0_row_start; + mmctx->src0_row_start = src0_row_start; + mmctx->src0_row_end = src0_row_end; + bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL || src0->type == HTP_TYPE_MXFP4); // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div); + mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); if (is_repacked) { mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); } else { @@ -1503,13 +1570,13 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { mmctx->vtcm_src1_size_per_thread = L.src1_bytes; } else { - mmctx->vtcm_src1_size_per_thread = fastdiv(L.src1_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src1_size_per_thread = fastdiv(L.src1_bytes, &octx->n_threads_div); } - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; FARF(HIGH, "matmul-%s : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu (%zu)\n", mmctx->type, L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); @@ -1583,13 +1650,21 @@ static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t ne00 = src_w->ne[0]; const uint32_t ne01 = src_w->ne[1]; - const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } - uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); src0_nrows_per_thread += (src0_nrows_per_thread & 1); - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); if (src0_start_row >= src0_end_row) continue; @@ -2638,10 +2713,6 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const struct htp_tensor * restrict src0 = octx->src[0]; const struct htp_tensor * restrict act = octx->src[n_weights]; - if (!src0 || !act) { - return HTP_STATUS_INVAL_PARAMS; - } - const int weight_type = (int) src0->type; const int k = (int) act->ne[0]; const int k_valid = (int) act->ne[0]; @@ -2714,16 +2785,31 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 - FARF(HIGH, "hmx-mm-nx-2d: n_weights %u m %d k %d wtype %d mc %d nc %d vtcm %zu/%zu", - n_weights, m, k, weight_type, m_chunk_n_rows, n_chunk_n_cols, L.total_bytes, vtcm_budget); + int m_start = 0; + int m_rows = m; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(octx->dsts[0], sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_rows = (int) range.count; + } + + if (m_rows == 0) { + return HTP_STATUS_OK; + } + + FARF(HIGH, "hmx-mm-nx-2d: n_weights %u m %d (%d..%d) k %d wtype %d mc %d nc %d vtcm %zu/%zu", + n_weights, m, m_start, m_start + m_rows, k, weight_type, m_chunk_n_rows, n_chunk_n_cols, L.total_bytes, vtcm_budget); htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + const size_t mr_end = (size_t)(m_start + m_rows); + if (pipeline) { hmx_matmul_job_t job_slots[2]; - for (size_t mr = 0; mr < (size_t) m; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 }; void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; @@ -2822,8 +2908,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k } } else { hmx_matmul_job_t job; - for (size_t mr = 0; mr < (size_t) m; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); struct activation_transfer_params act_params = { .ctx = ctx, @@ -3095,7 +3181,7 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ int chunk_dst_cols = params->n - (int)nc; if (chunk_dst_cols > 0) { transfer_output_chunk_threaded(ctx, output, src2_chunk, vtcm_output, (int) n_rows, (int) n_cols, - params->dst_stride, params->src2_stride, chunk_dst_cols, ctx->n_threads); + params->dst_stride, params->src2_stride, chunk_dst_cols, n_threads); } } } @@ -3216,7 +3302,10 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, int weight_type, const struct mmid_row_mapping *matrix_rows, int cur_a, - int mapping_stride) { + int mapping_stride, + int m_start, + int m_end, + int n_threads) { struct htp_thread_trace * tr = &ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); @@ -3247,7 +3336,6 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - const int n_threads = ctx->n_threads; const bool is_quant = (weight_type != HTP_TYPE_F16 && weight_type != HTP_TYPE_F32); const size_t vec_dot_size = k * sizeof(__fp16); @@ -3303,8 +3391,8 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, hmx_matmul_job_t job; - for (size_t mr = 0; mr < (size_t) m_padded; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m_padded - mr, m_chunk_n_rows); + for (size_t mr = (size_t) m_start; mr < (size_t) m_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin((size_t) m_end - mr, m_chunk_n_rows); const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); transfer_activation_chunk_gathered_threaded( @@ -3368,31 +3456,48 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k const int act_stride = (int)(src1->nb[1] / sizeof(float)); const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16)); + int m_start = 0; + int m_rows = m_total; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_total, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_rows = (int) range.count; + } + + if (m_rows == 0) { + return HTP_STATUS_OK; + } + const float * src2_ptr = NULL; uint32_t src2_stride = 0; size_t src2_nb2 = 0; size_t src2_nb3 = 0; if (src2) { - src2_ptr = (const float *) src2->data; src2_stride = (src2->ne[1] == 1) ? 0 : (uint32_t) (src2->nb[1] / sizeof(float)); + src2_ptr = (const float *) src2->data + m_start * src2_stride; src2_nb2 = (src2->ne[2] == 1) ? 0 : src2->nb[2]; src2_nb3 = (src2->ne[3] == 1) ? 0 : src2->nb[3]; } + const int dst_stride = (int)(dst->nb[1] / sizeof(float)); + float * dst_ptr = (float *) dst->data + m_start * dst_stride; + const float * act_ptr = (const float *) src1->data + m_start * act_stride; + int ret = -1; - const int n_threads = MIN(kparams->n_threads, (int) octx->n_threads); + const int n_threads = kparams->n_threads; if (kparams->kernel_type == HTP_MM_KERNEL_HMX_F16_BATCHED) { hmx_mm_f16_f32_batched_params_t batch_params = { - .dst = (float *) dst->data, + .dst = dst_ptr, .src2 = src2_ptr, - .activation = (float *) src1->data, + .activation = act_ptr, .weight = (const __fp16 *) src0->data, - .m = m_total, + .m = m_rows, .k = k, .n = n, .act_stride = act_stride, .weight_stride = wgt_stride, - .dst_stride = (int) (dst->nb[1] / sizeof(float)), + .dst_stride = dst_stride, .src2_stride = src2_stride, .ne02 = ne02, .ne03 = ne03, @@ -3420,9 +3525,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k kparams->vtcm_size); } else { ret = hmx_mm_2d_f32( - octx->ctx, (float*) dst->data, src2_ptr, (float*) src1->data, (const uint8_t *) src0->data, - m_total, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0], - (int)(dst->nb[1] / sizeof(float)), src2_stride, (int)dst->ne[0], + octx->ctx, dst_ptr, src2_ptr, act_ptr, (const uint8_t *) src0->data, + m_rows, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0], + dst_stride, src2_stride, (int)dst->ne[0], kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, kparams->n_act_threads, &kparams->div_n_act_threads, @@ -3441,6 +3546,11 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k int op_matmul(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + if (kparams->n_hmx) { return hmx_mm_op_matmul(octx, kparams); } @@ -3463,6 +3573,16 @@ static int hmx_mm_op_matmul_id( const int32_t cne1 = matrix_row_counts[cur_a]; if (cne1 == 0) continue; + const int m_padded = hex_align_up(cne1, 32); + int m_start = 0, m_end = m_padded; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && (uint32_t) cne1 >= octx->ctx->mdev.count; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_end = (int) (range.start + range.count); + } + if (m_start >= m_end) continue; + int ret = hmx_mm_id_2d_f32(octx->ctx, (float*) dst->data, (float*) src1->data, (const uint8_t *) src0->data + cur_a * nb02, cne1, ne00, ne01, @@ -3471,7 +3591,8 @@ static int hmx_mm_op_matmul_id( nb11, nb12, nb1, nb2, (int) src0->nb[1], (int) src0->type, - matrix_rows, cur_a, mmctx->mapping_stride); + matrix_rows, cur_a, mmctx->mapping_stride, + m_start, m_end, (int) octx->n_threads); if (ret != 0) { FARF(ERROR, "HMX matmul failed for expert %u, error %d\n", cur_a, ret); return HTP_STATUS_NO_SUPPORT; @@ -3524,7 +3645,7 @@ static int hvx_mm_matmul_id( htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads, 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; FARF(HIGH, "matmul-id-%s : src0-spad-size %zu src1-spad-size %zu src2-spad-size 0 dst-spad-size %zu (%zu)\n", mmctx->type, L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); @@ -3554,10 +3675,10 @@ static int hvx_mm_matmul_id( mmctx->vtcm_src0_stride = src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); mmctx->vtcm_src1_size_per_thread = L.src1_bytes; mmctx->vtcm_src2_size_per_thread = 0; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; @@ -3587,6 +3708,20 @@ static int hmx_mm_op_matmul_id_nx( const int32_t cne1 = matrix_row_counts[cur_a]; if (cne1 == 0) continue; + const int m_padded = hex_align_up(cne1, 32); + int m_start = 0, m_end = m_padded; + if (octx->ctx->mdev.count > 1) { + bool can_split = (uint32_t) cne1 >= octx->ctx->mdev.count; + for (uint32_t p = 0; p < n_weights && can_split; ++p) { + const struct htp_tensor * restrict dst = octx->dsts[p]; + can_split = !dst || htp_tensor_mdev_data_aligned(dst); + } + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_end = (int) (range.start + range.count); + } + if (m_start >= m_end) continue; + for (uint32_t p = 0; p < n_weights; ++p) { const struct htp_tensor * restrict src_w = octx->src[p]; const struct htp_tensor * restrict dst = octx->dsts[p]; @@ -3600,7 +3735,8 @@ static int hmx_mm_op_matmul_id_nx( act->nb[1], act->nb[2], dst->nb[1], dst->nb[2], (int) src_w->nb[1], (int) src_w->type, - matrix_rows, cur_a, mmctx->mapping_stride); + matrix_rows, cur_a, mmctx->mapping_stride, + m_start, m_end, (int) octx->n_threads); if (ret != 0) { FARF(ERROR, "HMX matmul ID NX failed for expert %u weight %u, error %d\n", cur_a, p, ret); return HTP_STATUS_NO_SUPPORT; @@ -3656,7 +3792,7 @@ static int hvx_mm_matmul_id_nx( htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads, 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; if (octx->ctx->vtcm_size < vtcm_size) { FARF(ERROR, "matmul-id-nx: current VTCM reservation %zu is too small, needed %zu\n", @@ -3678,9 +3814,9 @@ static int hvx_mm_matmul_id_nx( mmctx->vtcm_src0_stride = 0; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; @@ -3769,16 +3905,21 @@ static inline void scan_expert_ids( int op_matmul_id(struct htp_ops_context * octx) { htp_matmul_tensors_preamble; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - struct htp_mm_context mmctx_struct = {0}; - struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; mmctx->act = src1; - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - const struct htp_tensor * restrict ids = octx->src[2]; const size_t src0_row_size = nb01; @@ -3789,9 +3930,6 @@ int op_matmul_id(struct htp_ops_context * octx) { const uint32_t src0_nrows = ne01; // per expert const uint32_t src1_nrows = ne11 * ne12 * ne13; - mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div); - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); - // row groups const int n_ids = ids->ne[0]; // n_expert_used const int n_as = ne02; // n_expert @@ -3843,6 +3981,29 @@ int op_matmul_id(struct htp_ops_context * octx) { if (kparams->n_hmx) { s = hmx_mm_op_matmul_id(octx, mmctx); } else { + uint32_t src0_row_start = 0; + uint32_t src0_row_end = src0_nrows; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_row_start = range.start; + src0_row_end = range.start + range.count; + } + + if (src0_row_start >= src0_row_end) { + if (mapping_buf != octx->ctx->ddr_spad_base) { + free(mapping_buf); + } + return HTP_STATUS_OK; + } + + const uint32_t nrows = src0_row_end - src0_row_start; + mmctx->src0_row_start = src0_row_start; + mmctx->src0_row_end = src0_row_end; + + mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + if (hvx_mm_init_vec_dot(mmctx, src0->type) == 0) { s = hvx_mm_matmul_id(octx, mmctx, src1_nrows > 1 ? hvx_mm_id : hvx_mv_id); } else { @@ -3858,29 +4019,31 @@ int op_matmul_id(struct htp_ops_context * octx) { } int op_matmul_id_nx(struct htp_ops_context * octx) { + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + struct htp_thread_trace * tr = &octx->ctx->trace[0]; htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + mmctx->octx = octx; const uint32_t n_weights = kparams->n_weights; const struct htp_tensor * restrict src0 = octx->src[0]; const struct htp_tensor * restrict act = octx->src[n_weights]; const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; - struct htp_mm_context mmctx_struct = {0}; - struct htp_mm_context * mmctx = &mmctx_struct; - mmctx->octx = octx; mmctx->act = act; const size_t src0_row_size = src0->nb[1]; const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - const uint32_t src0_nrows = src0->ne[1]; const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3]; - mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div); - mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); - const int n_ids = ids->ne[0]; const int n_as = src0->ne[2]; @@ -3946,6 +4109,12 @@ int op_matmul_id_nx(struct htp_ops_context * octx) { } int op_matmul_nx(struct htp_ops_context * octx) { const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } + if (kparams->n_hmx) { return hmx_mm_nx_2d_f32(octx, kparams); } @@ -4012,7 +4181,7 @@ int op_matmul_nx(struct htp_ops_context * octx) { htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads, 0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, true); - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; + const size_t vtcm_size = L.total_bytes; if (octx->ctx->vtcm_size < vtcm_size) { FARF(ERROR, "matmul-nx: current VTCM reservation %zu is too small, needed %zu\n", @@ -4034,9 +4203,9 @@ int op_matmul_nx(struct htp_ops_context * octx) { mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); mmctx->vtcm_src1_size_per_thread = L.src1_bytes; - mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.c index aaa72b31590c..0222f24dcb59 100644 --- a/ggml/src/ggml-hexagon/htp/pad-ops.c +++ b/ggml/src/ggml-hexagon/htp/pad-ops.c @@ -12,8 +12,11 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" /* Circular wrap: maps any integer x into [0, n) */ static inline uint32_t wrap_around(int32_t x, uint32_t n) { @@ -68,6 +71,7 @@ struct htp_pad_context { uint32_t nrows_per_thread; uint32_t total_dst_rows; + uint32_t row_start; size_t type_size; @@ -78,39 +82,39 @@ struct htp_pad_context { size_t dst_row_size_aligned; }; -#define htp_pad_preamble \ - const struct htp_tensor * src = octx->src[0]; \ - const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne00 = src->ne[0]; \ - const uint32_t nb00 = src->nb[0]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ - const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \ - const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \ - const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \ - const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \ - \ - const size_t type_size = pctx->type_size; \ - \ - const uint32_t row_start = pctx->nrows_per_thread * ith; \ - const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->total_dst_rows); - - -#define htp_pad_dma_preamble \ - const size_t src_row_size = pctx->src_row_size; \ - const size_t src_row_size_aligned = pctx->src_row_size_aligned; \ - const size_t dst_row_size = pctx->dst_row_size; \ - const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ - \ +#define htp_pad_preamble \ + const struct htp_tensor * src = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + \ + const uint32_t ne00 = src->ne[0]; \ + const uint32_t nb00 = src->nb[0]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ + const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \ + const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \ + const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \ + const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \ + \ + const size_t type_size = pctx->type_size; \ + \ + const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \ + const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows); + + +#define htp_pad_dma_preamble \ + const size_t src_row_size = pctx->src_row_size; \ + const size_t src_row_size_aligned = pctx->src_row_size_aligned; \ + const size_t dst_row_size = pctx->dst_row_size; \ + const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ + \ uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \ uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \ \ @@ -125,8 +129,8 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { uint32_t i1, i2, i3; @@ -165,18 +169,17 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } // --------------------------------------------------------------------------- -// HVX + DMA PAD kernel — aligned, double-buffered +// HVX + DMA PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) { @@ -185,9 +188,6 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void htp_pad_preamble; htp_pad_dma_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - // ----------------------------------------------------------------------- // Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the // double-buffer pipeline before the main loop begins. @@ -222,6 +222,8 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void // Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops, // push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src; uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst; @@ -236,6 +238,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void lp2, rp2, ne2, lp3, rp3, ne3); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); if (!interior) { hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0); } else { @@ -249,6 +252,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void hvx_copy_f32_ua(dst_interior, src_spad_cur, ne00); } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); dma_queue_push_vtcm_to_ddr(dma, dma_make_ptr(dst_ptr, dst_spad_cur), @@ -274,14 +278,11 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void dma_queue_flush(dma); - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } // --------------------------------------------------------------------------- @@ -293,8 +294,8 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, struct htp_ops_context * octx = pctx->octx; htp_pad_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { uint32_t i1, i2, i3; @@ -344,18 +345,17 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } // --------------------------------------------------------------------------- -// HVX + DMA circular PAD kernel — aligned, double-buffered +// HVX + DMA circular PAD kernel - aligned, double-buffered // --------------------------------------------------------------------------- static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) { @@ -364,9 +364,6 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i htp_pad_preamble; htp_pad_dma_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - // ----------------------------------------------------------------------- // Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the // double-buffer pipeline. Every row is a real src DMA (no null DMAs). @@ -390,6 +387,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i // Main loop: pop completed DMAs, assemble circular row in VTCM with // aligned HVX ops, push dst DMA and prefetch src for the next+1 row. // ----------------------------------------------------------------------- + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = row_start; ir < row_end; ir++) { uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src; uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst; @@ -398,7 +397,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3); uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; - + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); if (lp0 > 0) { uint8_t * dst_left = dst_spad_cur; const uint8_t * src_left = src_spad_cur + (size_t)(ne00 - (uint32_t)lp0) * type_size; @@ -430,6 +429,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i } } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); dma_queue_push_vtcm_to_ddr(dma, dma_make_ptr(dst_ptr, dst_spad_cur), @@ -448,14 +448,11 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i dma_queue_flush(dma); - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } int op_pad(struct htp_ops_context * octx) { @@ -489,19 +486,33 @@ int op_pad(struct htp_ops_context * octx) { const uint32_t ne00 = src0->ne[0]; const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows > 0 ? total_dst_rows : 1); + const size_t dst_row_size = (size_t)ne0 * type_size; + + uint32_t row_start = 0; + uint32_t nrows = total_dst_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; const size_t src_row_size = (size_t)ne00 * type_size; - const size_t dst_row_size = (size_t)ne0 * type_size; const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN); const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); // Total VTCM needed: 2 buffers (ping+pong) for src and dst, per thread const size_t vtcm_needed = (size_t)n_threads * 2 * (src_row_size_aligned + dst_row_size_aligned); - const int use_dma = (src0->nb[0] == (uint32_t)type_size) && - (ne00 >= 512) && - (octx->ctx->vtcm_base != NULL) && + const int use_dma = (src0->nb[0] == (uint32_t)type_size) && (ne00 >= 512) && (octx->ctx->vtcm_size >= vtcm_needed); if (use_dma) { @@ -521,8 +532,9 @@ int op_pad(struct htp_ops_context * octx) { .lp1 = lp1, .rp1 = rp1, .lp2 = lp2, .rp2 = rp2, .lp3 = lp3, .rp3 = rp3, - .nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads, - .total_dst_rows = total_dst_rows, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_dst_rows = nrows, + .row_start = row_start, .type_size = type_size, .src_row_size = src_row_size, .src_row_size_aligned = src_row_size_aligned, @@ -537,11 +549,10 @@ int op_pad(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3); - if (circular && use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); } - else if (circular) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular, &pctx, n_threads); } - else if (use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_dma, &pctx, n_threads); } - else { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx, &pctx, n_threads); } + if (circular && use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); } + else if (circular) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); } + else if (use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); } + else { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); } return HTP_STATUS_OK; } - diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.c index a6f2f0ed5f3f..530279d6503b 100644 --- a/ggml/src/ggml-hexagon/htp/repeat-ops.c +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c @@ -12,8 +12,10 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" struct htp_repeat_context { struct htp_ops_context * octx; @@ -25,6 +27,7 @@ struct htp_repeat_context { uint32_t nrows_per_thread; uint32_t total_dst_rows; // ne1 * ne2 * ne3 + uint32_t row_start; size_t type_size; }; @@ -62,11 +65,11 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat const size_t row_bytes = ne00 * rctx->type_size; - const uint32_t row_start = rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->total_dst_rows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { // Decompose flat dst row index into (i1, i2, i3) @@ -89,12 +92,12 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } int op_repeat(struct htp_ops_context * octx) { @@ -119,21 +122,39 @@ int op_repeat(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows); - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { return HTP_STATUS_OK; } + const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; + const size_t dst_row_size = dst->ne[0] * type_size; + + uint32_t row_start = 0; + uint32_t nrows = total_dst_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + struct htp_repeat_context rctx = { .octx = octx, .nr0 = dst->ne[0] / src0->ne[0], .nr1 = dst->ne[1] / src0->ne[1], .nr2 = dst->ne[2] / src0->ne[2], .nr3 = dst->ne[3] / src0->ne[3], - .nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads, - .total_dst_rows = total_dst_rows, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_dst_rows = nrows, + .row_start = row_start, .type_size = type_size, }; @@ -142,7 +163,7 @@ int op_repeat(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], rctx.nr0, rctx.nr1, rctx.nr2, rctx.nr3); - worker_pool_run_func(octx->ctx->worker_pool, repeat_job_per_thread, &rctx, n_threads); + work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 0a4b31ccb1d8..c36976ed03fa 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -80,6 +80,8 @@ struct htp_rope_context { size_t dst_row_stride; size_t src0_row_size_aligned; uint32_t src0_nrows; + uint32_t row_start; + uint32_t nrows; struct fastdiv_values div_ne2_ne1; struct fastdiv_values div_ne1; @@ -539,11 +541,11 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_rope_preamble; - const uint32_t src0_nrows = rctx->src0_nrows; + const uint32_t src0_nrows = rctx->nrows; const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, rctx->row_start + src0_nrows); // no work for this thread if (src0_start_row >= src0_end_row) { @@ -706,9 +708,32 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { } const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params; - assert(kparams->n_threads > 0); + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } assert(octx->ctx->vtcm_size >= kparams->vtcm_size); + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const uint32_t ne0 = dst->ne[0]; const size_t src0_row_size = src0->ne[0] * sizeof(float); const size_t src0_row_stride = src0->nb[1]; @@ -752,15 +777,17 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rctx.dst_row_stride = dst_row_stride; rctx.src0_row_size_aligned = kparams->src0_row_size_aligned; - rctx.src0_nrows = kparams->src0_nrows; - rctx.src0_nrows_per_thread = kparams->src0_nrows_per_thread; + rctx.src0_nrows = nrows; + rctx.nrows = nrows; + rctx.row_start = row_start; + rctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); rctx.div_ne2_ne1 = kparams->div_ne2_ne1; rctx.div_ne1 = kparams->div_ne1; FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, kparams->n_threads); + work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads); return err; } diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c index 340a497f7a2c..fbd5162a7c00 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c @@ -18,6 +18,7 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" @@ -58,6 +59,9 @@ struct set_rows_context { const struct htp_set_rows_kernel_params * kparams; struct htp_set_rows_vtcm_layout vtcm_layout; uint8_t * vtcm_base; + uint32_t task_start; + uint32_t tasks; + uint32_t tasks_per_thread; }; #define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \ @@ -67,12 +71,12 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \ set_rows_preamble; \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - const uint32_t dr = kparams->tasks_per_thread; \ - const uint32_t ir0 = dr * ith; \ - if (ir0 >= kparams->total_tasks) { \ + const uint32_t dr = srctx->tasks_per_thread; \ + const uint32_t ir0 = srctx->task_start + dr * ith; \ + if (ir0 >= srctx->task_start + srctx->tasks) { \ return; \ } \ - const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \ + const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \ dma_queue * dma_queue = octx->ctx->dma[ith]; \ const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \ uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \ @@ -192,18 +196,44 @@ int op_set_rows(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) { - return HTP_STATUS_NO_SUPPORT; + if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + return HTP_STATUS_OK; + } + + const struct htp_tensor * dst = octx->dst; + const uint32_t total_tasks = kparams->total_tasks; + + uint32_t task_start = 0; + uint32_t tasks = total_tasks; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && !htp_tensor_is_permuted(dst); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + task_start = range.start; + tasks = range.count; } + if (tasks == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + // l2fetch the src1 (indices) tensor in the main thread hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]); struct set_rows_context srctx; srctx.octx = octx; srctx.kparams = kparams; + srctx.task_start = task_start; + srctx.tasks = tasks; + srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); - htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, kparams->n_threads); + htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads); srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; work_queue_func_t q_func = NULL; @@ -216,15 +246,15 @@ int op_set_rows(struct htp_ops_context * octx) { default: return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n_threads %d\n", + FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - srctx.vtcm_layout.src0_bytes_per_thread * kparams->n_threads, - srctx.vtcm_layout.dst_bytes_per_thread * kparams->n_threads, - kparams->n_threads); + srctx.vtcm_layout.src0_bytes_per_thread * n_threads, + srctx.vtcm_layout.dst_bytes_per_thread * n_threads, + n_threads); - work_queue_run(octx->ctx->work_queue, q_func, &srctx, kparams->n_threads); + work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.c index d78bcc0eb24e..2497ec76320c 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.c +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c @@ -14,9 +14,11 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #define htp_softmax_preamble3 \ const uint32_t ne00 = src0->ne[0]; \ @@ -69,6 +71,8 @@ struct htp_softmax_context { struct fastdiv_values fastdiv_ne13; // For mask broadcasting uint32_t src0_nrows_per_thread; + uint32_t row_start; + uint32_t nrows; }; static void apply_mask(float * restrict wp0, @@ -223,19 +227,17 @@ static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_softmax_preamble3; - const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows + const uint32_t src0_nrows = smctx->nrows; const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); // no work for this thread if (src0_start_row >= src0_end_row) { return; } - uint64_t qt = HAP_perf_get_qtimer_count(); - int is_aligned = 1; int opt_path = 0; @@ -262,6 +264,9 @@ static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) { uint32_t prev_i2 = (uint32_t)-1; float slope = 1.0f; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, src0_start_row); + for (uint32_t r = src0_start_row; r < src0_end_row; ++r) { uint32_t i1 = fastmodulo(r, ne01, &smctx->fastdiv_ne01); uint32_t r_div_ne01 = fastdiv(r, &smctx->fastdiv_ne01); @@ -323,10 +328,11 @@ static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) { } } - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "softmax-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u : opt %u f16 %u usec %u\n", ith, nth, + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, src0_start_row); + + FARF(HIGH, "softmax-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u : opt %u f16 %u\n", ith, nth, ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, - ne0, ne1, ne2, ne3, opt_path, smctx->use_f16, (unsigned) qt); + ne0, ne1, ne2, ne3, opt_path, smctx->use_f16); } static int execute_op_softmax_f32(struct htp_ops_context * octx) { @@ -342,13 +348,32 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { init_softmax_ctx(&smctx, octx); const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); + const size_t elem_size = sizeof(float); + const size_t dst_row_size = dst->nb[1]; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } - smctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + smctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + smctx.row_start = row_start; + smctx.nrows = nrows; const size_t src0_row_size = src0->nb[1]; const size_t src1_row_size = src0_row_size; - const size_t dst_row_size = dst->nb[1]; // VTCM scratchpads for all tensors // 4 rows per thread, padded to HVX vector size @@ -383,9 +408,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) { octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL; octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->dst_spad.src = NULL; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) return err; - - worker_pool_run_func(octx->ctx->worker_pool, softmax_job_f32, &smctx, n_threads); + work_queue_run(octx->ctx->work_queue, softmax_job_f32, &smctx, n_threads); return err; } diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c index ae8e1a50495f..847a78712de4 100644 --- a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c @@ -1,13 +1,16 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include -#include #include +#include "hex-common.h" +#include "hex-profile.h" + #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-types.h" #include "hvx-utils.h" @@ -15,6 +18,7 @@ struct htp_solve_tri_context { struct htp_ops_context * octx; uint32_t jobs_per_thread; uint32_t total_jobs; + uint32_t job_start; uint32_t k_chunks; uint32_t col_block; }; @@ -89,11 +93,11 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t col_block = VLEN_FP32; const uint32_t k_full = (k / col_block) * col_block; - const uint32_t start_batch = sctx->jobs_per_thread * ith; - const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->total_jobs); + const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith; + const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch); for (uint32_t batch = start_batch; batch < end_batch; ++batch) { const uint32_t i03 = batch / ne02; @@ -127,11 +131,10 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_batch); - FARF(HIGH, "solve-tri-batch %d/%d: A=(%ux%u) B=(%ux%u) batch %u:%u usec %u\n", - ith, nth, n, n, k, n, start_batch, end_batch, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + FARF(HIGH, "solve-tri-batch %d/%d: A=(%ux%u) B=(%ux%u) batch %u:%u\n", + ith, nth, n, n, k, n, start_batch, end_batch); } // Chunk-level thread: each job is one (batch, col_chunk) pair. @@ -148,11 +151,11 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t ne02 = src0->ne[2]; - const uint32_t start_job = sctx->jobs_per_thread * ith; - const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->total_jobs); + const uint32_t start_job = sctx->job_start + sctx->jobs_per_thread * ith; + const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job); for (uint32_t job = start_job; job < end_job; ++job) { const uint32_t batch = job / sctx->k_chunks; @@ -161,16 +164,14 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const uint32_t i03 = batch / ne02; const uint32_t i02 = batch - i03 * ne02; - const uint32_t col0 = chunk * sctx->col_block; - const uint32_t coln = MIN(sctx->col_block, k - col0); - const float * A_batch = (const float *) ((const uint8_t *) (uintptr_t) src0->data + i02 * src0->nb[2] + i03 * src0->nb[3]); const float * B_batch = (const float *) ((const uint8_t *) (uintptr_t) src1->data + i02 * src1->nb[2] + i03 * src1->nb[3]); float * X_batch = (float *) ((uint8_t *) (uintptr_t) dst->data + i02 * dst->nb[2] + i03 * dst->nb[3]); - const bool use_hvx = (coln >= 8); + const uint32_t col0 = chunk * sctx->col_block; + const uint32_t coln = MIN(sctx->col_block, k - col0); for (uint32_t row = 0; row < n; ++row) { const float diag = A_batch[row * n + row]; @@ -179,7 +180,7 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void const float * A_row = A_batch + row * n; const float * B_row = B_batch + row * k; - if (use_hvx) { + if (coln >= 8) { solve_tri_row_hvx(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); } else { solve_tri_row_scalar(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); @@ -187,11 +188,10 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_job); - FARF(HIGH, "solve-tri-chunk %d/%d: A=(%ux%u) B=(%ux%u) job %u:%u usec %u\n", - ith, nth, n, n, k, n, start_job, end_job, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + FARF(HIGH, "solve-tri-chunk %d/%d: A=(%ux%u) B=(%ux%u) jobs %u:%u\n", + ith, nth, n, n, k, n, start_job, end_job); } int op_solve_tri(struct htp_ops_context * octx) { @@ -235,32 +235,64 @@ int op_solve_tri(struct htp_ops_context * octx) { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], batched); if (batched) { + uint32_t job_start = 0; + uint32_t njobs = total_batches; + + if (octx->ctx->mdev.count > 1) { + const uint32_t batch_size = dst->nb[2]; + const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + job_start = range.start; + njobs = range.count; + } + + if (njobs == 0) { + return HTP_STATUS_OK; + } + // Batch-level parallelism - const uint32_t n_threads = MIN((uint32_t) octx->n_threads, total_batches); + const uint32_t n_threads = octx->n_threads; struct htp_solve_tri_context sctx = { .octx = octx, - .jobs_per_thread = (total_batches + n_threads - 1) / n_threads, - .total_jobs = total_batches, + .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .total_jobs = njobs, + .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - worker_pool_run_func(octx->ctx->worker_pool, solve_tri_batch_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads); } else { // Chunk-level parallelism const uint32_t total_jobs = total_batches * k_chunks; - const uint32_t n_threads = MIN((uint32_t) octx->n_threads, MAX(total_jobs, 1)); + + uint32_t job_start = 0; + uint32_t njobs = total_jobs; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_jobs, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + job_start = range.start; + njobs = range.count; + } + + if (njobs == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; struct htp_solve_tri_context sctx = { .octx = octx, - .jobs_per_thread = (total_jobs + n_threads - 1) / n_threads, - .total_jobs = total_jobs, + .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .total_jobs = njobs, + .job_start = job_start, .k_chunks = k_chunks, .col_block = col_block, }; - worker_pool_run_func(octx->ctx->worker_pool, solve_tri_chunk_thread_f32, &sctx, n_threads); + work_queue_run(octx->ctx->work_queue, solve_tri_chunk_thread_f32, &sctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c index a48bc9ed86b2..bef1425368e1 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.c +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -16,8 +15,9 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "hex-dma.h" +#include "hex-profile.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-utils.h" #define htp_ssm_conv_tensors_preamble \ @@ -63,6 +63,8 @@ struct htp_ssm_conv_context { uint32_t nrows_per_thread; uint32_t d_inner_tile; uint64_t t_start; + uint32_t row_start; + uint32_t nrows; }; #define htp_ssm_conv_preamble \ @@ -75,9 +77,6 @@ struct htp_ssm_conv_context { static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) { htp_ssm_conv_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint32_t d_conv = src1->ne[0]; const uint32_t d_inner = src0->ne[1]; const uint32_t n_t = dst->ne[1]; @@ -95,14 +94,17 @@ static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *da // Calculate row range for this thread const uint32_t d_inner_per_thread = scctx->nrows_per_thread; - const uint32_t d_inner_start = d_inner_per_thread * ith; - const uint32_t d_inner_end = MIN(d_inner_start + d_inner_per_thread, d_inner); + const uint32_t d_inner_start = scctx->row_start + d_inner_per_thread * ith; + const uint32_t d_inner_end = MIN(d_inner_start + d_inner_per_thread, scctx->row_start + scctx->nrows); // No work for this thread if (d_inner_start >= d_inner_end) { return; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) d_inner_start); + for (uint32_t i3 = 0; i3 < n_s; ++i3) { for (uint32_t i2 = 0; i2 < n_t; ++i2) { for (uint32_t i1 = d_inner_start; i1 < d_inner_end; ++i1) { @@ -121,12 +123,12 @@ static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *da } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) d_inner_end); - FARF(HIGH, "ssm-conv-f32 %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "ssm-conv-f32 %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], d_inner_start, d_inner_end, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], - dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[2], dst->ne[3]); } @@ -257,9 +259,6 @@ static inline void transpose_src0_block(const float * src0_block, static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) { htp_ssm_conv_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint32_t d_conv = src1->ne[0]; const uint32_t d_inner = src0->ne[1]; const uint32_t n_t = dst->ne[1]; @@ -273,13 +272,16 @@ static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void const uint32_t dst_stride_seq = dst->nb[2] / sizeof(float); const uint32_t dr = scctx->nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, d_inner); + const uint32_t ir0 = scctx->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); if (ir0 >= ir1) { return; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); + const uint32_t d_inner_per_thread = ir1 - ir0; const uint32_t d_inner_stride = scctx->nrows_per_thread; const uint32_t d_inner_tile = scctx->d_inner_tile; @@ -319,97 +321,118 @@ static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void HVX_Vector w = *(const HVX_Vector *) (src1_T + j * d_inner_stride + tile_off + cb); acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w)); } - HVX_Vector res = Q6_Vsf_equals_Vqf32(acc); - float * dst_ptr = dst_data + i3 * dst_stride_seq + t * dst_stride_token + (ir0 + tile_off + cb); + HVX_Vector y = Q6_Vsf_equals_Vqf32(acc); + + float * dst_ptr = dst_data + (ir0 + tile_off + cb) + t * dst_stride_token + i3 * dst_stride_seq; if (cb_n == C_TILE) { - *(HVX_UVector *) dst_ptr = res; + *(HVX_UVector *) dst_ptr = y; } else { - hvx_vec_store_u(dst_ptr, cb_n * sizeof(float), res); + hvx_vec_store_u(dst_ptr, cb_n * sizeof(float), y); } } } } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1); - FARF(HIGH, "ssm-conv-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) tile=%u * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", - ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, d_inner_tile, + FARF(HIGH, "ssm-conv-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", + ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], - dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[2], dst->ne[3]); } int op_ssm_conv_f32(struct htp_ops_context * octx) { - htp_ssm_conv_tensors_preamble; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { - FARF(ERROR, "ssm_conv: only (F32 x F32 -> F32) OPs supported"); return HTP_STATUS_NO_SUPPORT; } - struct htp_ssm_conv_context scctx = { 0 }; - scctx.octx = octx; - const uint32_t d_conv = src1->ne[0]; const uint32_t d_inner = src0->ne[1]; const uint32_t n_t = dst->ne[1]; // tokens per sequence const uint32_t n_s = dst->ne[2]; // number of sequences in the batch - const uint32_t n_threads = MIN(octx->n_threads, d_inner); + if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + return HTP_STATUS_OK; + } - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - uint32_t use_hvx = 0; - if (d_inner >= VLEN_FP32 && n_t >= VLEN_FP32) { - use_hvx = 1; - } + uint32_t row_start = 0; + uint32_t nrows = d_inner; + + if (octx->ctx->mdev.count > 1) { + const uint32_t elems_per_chunk = VLEN_FP32; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(d_inner, htp_tensor_mdev_data_aligned(dst) ? elems_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + struct htp_ssm_conv_context scctx = { 0 }; + scctx.octx = octx; + scctx.row_start = row_start; + scctx.nrows = nrows; + + uint32_t use_hvx = 0; + if (nrows >= VLEN_FP32 && n_t >= VLEN_FP32) { + use_hvx = 1; + } + + const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + scctx.nrows_per_thread = hex_round_up(raw_rpt, VLEN_FP32); - scctx.nrows_per_thread = hex_round_up((d_inner + n_threads - 1) / n_threads, VLEN_FP32); + const uint32_t d_inner_per_thread = scctx.nrows_per_thread; + const uint32_t ncs = src0->ne[0]; - const uint32_t d_inner_per_thread = scctx.nrows_per_thread; - const uint32_t ncs = src0->ne[0]; + const uint32_t src1_T_size = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 256); + const uint32_t src0_T_max = HTP_SSM_CONV_VTCM_BUDGET > src1_T_size ? HTP_SSM_CONV_VTCM_BUDGET - src1_T_size : 0; - const uint32_t src1_T_size = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 256); - const uint32_t src0_T_max = HTP_SSM_CONV_VTCM_BUDGET > src1_T_size ? HTP_SSM_CONV_VTCM_BUDGET - src1_T_size : 0; + uint32_t d_inner_tile = (src0_T_max / sizeof(float)) / ncs; + d_inner_tile -= (d_inner_tile % VLEN_FP32); + if (d_inner_tile == 0) { + FARF(HIGH, "ssm_conv-f32: inner tile rounds to 0 (ncs=%u), falling back to scalar\n", ncs); + use_hvx = 0; + } else { + scctx.d_inner_tile = d_inner_tile; - uint32_t d_inner_tile = (src0_T_max / sizeof(float)) / ncs; - d_inner_tile -= (d_inner_tile % VLEN_FP32); - if (d_inner_tile == 0) { - FARF(HIGH, "ssm_conv-f32: inner tile rounds to 0 (ncs=%u), falling back to scalar\n", ncs); + octx->src0_spad.size_per_thread = hex_round_up(d_inner_tile * ncs * sizeof(float), 256); + octx->src1_spad.size_per_thread = src1_T_size; + octx->dst_spad.size_per_thread = 0; + + octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads; + octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads; + octx->dst_spad.size = 0; + + octx->src0_spad.data = octx->ctx->vtcm_base; + octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; + octx->src0_spad.src = NULL; + octx->src1_spad.src = NULL; + + const size_t total_spad = octx->src0_spad.size + octx->src1_spad.size; + if (total_spad > octx->ctx->vtcm_size) { + FARF(HIGH, "ssm_conv-f32: scratchpad %zu exceeds VTCM %zu, falling back to scalar\n", + total_spad, octx->ctx->vtcm_size); use_hvx = 0; - } else { - scctx.d_inner_tile = d_inner_tile; - - octx->src0_spad.size_per_thread = hex_round_up(d_inner_tile * ncs * sizeof(float), 256); - octx->src1_spad.size_per_thread = src1_T_size; - octx->dst_spad.size_per_thread = 0; - - octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads; - octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads; - octx->dst_spad.size = 0; - - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; - octx->src0_spad.src = NULL; - octx->src1_spad.src = NULL; - - const size_t total_spad = octx->src0_spad.size + octx->src1_spad.size; - if (total_spad > octx->ctx->vtcm_size) { - FARF(HIGH, "ssm_conv-f32: scratchpad %zu exceeds VTCM %zu, falling back to scalar\n", - total_spad, octx->ctx->vtcm_size); - use_hvx = 0; - } } + } - FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_hvx %d\n", src0->ne[0], - src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], - dst->ne[1], dst->ne[2], dst->ne[3], use_hvx); + FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_hvx %d\n", src0->ne[0], + src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], + dst->ne[1], dst->ne[2], dst->ne[3], use_hvx); - if (use_hvx) { - worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32_hvx, &scctx, n_threads); - } else { - worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32, &scctx, n_threads); - } + if (use_hvx) { + work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_f32_hvx, &scctx, n_threads); + } else { + work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_f32, &scctx, n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c index 874c41ab2ac7..faf716b4bc18 100644 --- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c @@ -13,35 +13,38 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #define sum_rows_preamble \ const struct htp_tensor *src0 = octx->src[0]; \ const struct htp_tensor *dst = octx->dst; \ \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ struct sum_rows_context { + struct htp_ops_context * octx; const uint8_t * src_data; uint8_t * dst_data; uint32_t ne00; @@ -76,6 +79,9 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) // Calculate actual number of rows for this thread const uint32_t n_rows = end_row - start_row; + struct htp_thread_trace * tr = &smctx->octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); + for (uint32_t ir = 0; ir < n_rows; ir++) { const float * restrict src_local = src_th + (ir * (src_stride / sizeof(float))); @@ -89,6 +95,8 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) dst_th[ir] = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00); } } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); } int op_sum_rows(struct htp_ops_context * octx) { @@ -102,9 +110,26 @@ int op_sum_rows(struct htp_ops_context * octx) { return HTP_STATUS_OK; } - const uint32_t src0_nrows = ne01 * ne02 * ne03; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - const uint32_t rows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + const uint32_t src0_nrows = ne01 * ne02 * ne03; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); bool opt_path = false; if ((0 == hex_is_aligned((void *) src0->data, VLEN)) && !(nb01 & (VLEN - 1))) { @@ -112,17 +137,18 @@ int op_sum_rows(struct htp_ops_context * octx) { } struct sum_rows_context smctx = { - .src_data = (const uint8_t *) src0->data, - .dst_data = (uint8_t *) dst->data, + .octx = octx, + .src_data = (const uint8_t *) src0->data + row_start * nb01, + .dst_data = (uint8_t *) dst->data + row_start * nb1, .ne00 = ne00, .src_stride = nb01, .dst_stride = nb1, .rows_per_thread = rows_per_thread, - .total_rows = src0_nrows, + .total_rows = nrows, .opt_path = opt_path, }; - worker_pool_run_func(octx->ctx->worker_pool, sum_rows_thread_f32, &smctx, n_threads); + work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 7850ab27e00a..cb82bfa3c2d9 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -46,6 +46,7 @@ struct htp_unary_context { uint32_t block; uint32_t src0_nrows; uint32_t src0_nrows_per_thread; + uint32_t row_start; uint32_t nc; uint32_t col_tile; // tiled mode bool broadcast_weight; @@ -496,7 +497,7 @@ static void tri_f32(const float * restrict src, } if (boundary > ne0) boundary = ne0; - // Full HVX vectors — each starts at a 128-byte aligned offset + // Full HVX vectors - each starts at a 128-byte aligned offset for (uint32_t i = 0; i < nvec; i++) { const uint32_t vec_start = i * VLEN_FP32; const uint32_t vec_end = vec_start + VLEN_FP32; @@ -563,7 +564,7 @@ static void softplus_f32(const float * restrict src, for (uint32_t i = 0; i < ne0; i++) { float x = src_f[i]; - // For x > 20: softplus(x) ≈ x (avoids exp overflow) + // For x > 20: softplus(x) ~ x (avoids exp overflow) dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x)); } } @@ -661,8 +662,8 @@ static void unary_task_##SUFFIX##_##NAME(unsigned int nth, unsigned int ith, voi const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; \ \ const uint32_t src0_nrows = uctx->src0_nrows; \ - const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); \ \ if (src0_start_row >= src0_end_row) { \ return; \ @@ -833,124 +834,126 @@ DEFINE_UNARY_TASK_IMPL(unary_abs, _Float16, f16, false, false, abs_f16(src0_vtcm DEFINE_UNARY_TASK_IMPL(unary_log, _Float16, f16, false, false, log_f16(src0_vtcm, dst_vtcm, block_size, uctx)) // Apply a pointwise unary op to one column tile that is already in VTCM. -#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \ -static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \ - const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \ - struct htp_ops_context * octx = uctx->octx; \ - const struct htp_tensor * src = octx->src[0]; \ - const struct htp_tensor * dst = octx->dst; \ - struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ - \ - htp_unary_preamble; \ - \ - int32_t * op_params = octx->op_params; \ - const uint32_t col_tile = uctx->col_tile; \ - \ - const uint32_t src0_nrows = uctx->src0_nrows; \ - const uint32_t src0_start_row = uctx->src0_nrows_per_thread * ith; \ - const uint32_t src0_end_row = MIN(src0_start_row + uctx->src0_nrows_per_thread, src0_nrows); \ - \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ - const uint8_t * restrict data_src = uctx->data_src0; \ - uint8_t * restrict data_dst = uctx->data_dst; \ - \ - uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \ - uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \ - \ - const size_t src0_half = uctx->src0_vtcm_half_size; \ - const size_t dst_half = uctx->dst_vtcm_half_size; \ - \ - dma_queue * dmaq = octx->ctx->dma[ith]; \ - \ - const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \ - const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \ - const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \ - const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; \ - \ - const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; \ - const int32_t tri_ttype = (IS_TRI) ? op_params[0] : 0; \ - \ - const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \ - (nb03 == (size_t)ne02 * nb02); \ - const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \ - (nb3 == (size_t)ne2 * nb2); \ - \ - const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; \ - \ - for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { \ - const uint32_t row = src0_start_row + t / tiles_per_row; \ - const uint32_t col = (t % tiles_per_row) * col_tile; \ - const uint32_t tw = MIN(col_tile, ne0 - col); \ - const size_t tb = (size_t) tw * sizeof(float); \ - const size_t soff = (src0_contig ? (row * nb01) : \ - unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +\ - (size_t) col * sizeof(float); \ - \ - dma_queue_push(dmaq, dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); \ - dma_queue_push(dmaq, dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1);\ - } \ - \ - uint32_t row = src0_start_row; \ - uint32_t col = 0; \ - uint32_t tile_in_row = 0; \ - uint32_t i01 = fastmodulo(row, ne01, div_ne01); \ - \ - uint32_t prow = src0_start_row + fastdiv(2, div_tpr); \ - uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; \ - uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); \ - \ - for (uint32_t t = 0; t < total_tiles; t++) { \ - uint8_t * dst_vtcm = (uint8_t *) dma_queue_pop(dmaq).src; \ - uint8_t * src_vtcm = (uint8_t *) dma_queue_pop(dmaq).dst; \ - \ - const uint32_t tw = MIN(col_tile, ne0 - col); \ - \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); \ - CORE_TILE_EXPR; \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); \ - \ - const size_t doff = (dst_contig ? (row * nb1) : \ - unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + \ - (size_t) col * sizeof(float); \ - const size_t tb = (size_t) tw * sizeof(float); \ - dma_queue_push(dmaq, dma_make_ptr(data_dst + doff, dst_vtcm), tb, tb, tb, 1); \ - \ - const uint32_t pt = t + 2; \ - if (pt < total_tiles) { \ - const uint32_t ptw = MIN(col_tile, ne0 - pcol); \ - const size_t ptb = (size_t) ptw * sizeof(float); \ - const size_t psoff = (src0_contig ? (prow * nb01) : \ - unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, \ - nb03)) + \ - (size_t) pcol * sizeof(float); \ - dma_queue_push(dmaq, dma_make_ptr(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); \ - } \ - \ - tile_in_row++; \ - col += col_tile; \ - if (tile_in_row == tiles_per_row) { \ - tile_in_row = 0; \ - col = 0; \ - row++; \ - i01++; \ - if (i01 == ne01) { \ - i01 = 0; \ - } \ - } \ - \ - ptile_in_row++; \ - pcol += col_tile; \ - if (ptile_in_row == tiles_per_row) { \ - ptile_in_row = 0; \ - pcol = 0; \ - prow++; \ - } \ - } \ - \ - dma_queue_flush(dmaq); \ +#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \ +static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \ + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \ + struct htp_ops_context * octx = uctx->octx; \ + const struct htp_tensor * src = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + \ + htp_unary_preamble; \ + \ + uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; \ + \ + int32_t * op_params = octx->op_params; \ + const uint32_t col_tile = uctx->col_tile; \ + \ + const uint32_t src0_nrows = uctx->src0_nrows; \ + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); \ + \ + if (src0_start_row >= src0_end_row) { \ + return; \ + } \ + \ + const uint8_t * restrict data_src = uctx->data_src0; \ + uint8_t * restrict data_dst = uctx->data_dst; \ + \ + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \ + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \ + \ + const size_t src0_half = uctx->src0_vtcm_half_size; \ + const size_t dst_half = uctx->dst_vtcm_half_size; \ + \ + dma_queue * dmaq = octx->ctx->dma[ith]; \ + \ + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \ + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \ + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \ + const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; \ + \ + const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; \ + const int32_t tri_ttype = (IS_TRI) ? op_params[0] : 0; \ + \ + const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \ + (nb03 == (size_t)ne02 * nb02); \ + const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \ + (nb3 == (size_t)ne2 * nb2); \ + \ + const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; \ + \ + for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { \ + const uint32_t row = src0_start_row + t / tiles_per_row; \ + const uint32_t col = (t % tiles_per_row) * col_tile; \ + const uint32_t tw = MIN(col_tile, ne0 - col); \ + const size_t tb = (size_t) tw * sizeof(float); \ + const size_t soff = (src0_contig ? (row * nb01) : \ + unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) + \ + (size_t) col * sizeof(float); \ + \ + dma_queue_push(dmaq, dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); \ + dma_queue_push(dmaq, dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1); \ + } \ + \ + uint32_t row = src0_start_row; \ + uint32_t col = 0; \ + uint32_t tile_in_row = 0; \ + uint32_t i01 = fastmodulo(row, ne01, div_ne01); \ + \ + uint32_t prow = src0_start_row + fastdiv(2, div_tpr); \ + uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; \ + uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); \ + \ + for (uint32_t t = 0; t < total_tiles; t++) { \ + uint8_t * dst_vtcm = (uint8_t *) dma_queue_pop(dmaq).src; \ + uint8_t * src_vtcm = (uint8_t *) dma_queue_pop(dmaq).dst; \ + \ + const uint32_t tw = MIN(col_tile, ne0 - col); \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); \ + CORE_TILE_EXPR; \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); \ + \ + const size_t doff = (dst_contig ? (row * nb1) : \ + unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + \ + (size_t) col * sizeof(float); \ + const size_t tb = (size_t) tw * sizeof(float); \ + dma_queue_push(dmaq, dma_make_ptr(data_dst + doff, dst_vtcm), tb, tb, tb, 1); \ + \ + const uint32_t pt = t + 2; \ + if (pt < total_tiles) { \ + const uint32_t ptw = MIN(col_tile, ne0 - pcol); \ + const size_t ptb = (size_t) ptw * sizeof(float); \ + const size_t psoff = (src0_contig ? (prow * nb01) : \ + unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, \ + nb03)) + \ + (size_t) pcol * sizeof(float); \ + dma_queue_push(dmaq, dma_make_ptr(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); \ + } \ + \ + tile_in_row++; \ + col += col_tile; \ + if (tile_in_row == tiles_per_row) { \ + tile_in_row = 0; \ + col = 0; \ + row++; \ + i01++; \ + if (i01 == ne01) { \ + i01 = 0; \ + } \ + } \ + \ + ptile_in_row++; \ + pcol += col_tile; \ + if (ptile_in_row == tiles_per_row) { \ + ptile_in_row = 0; \ + pcol = 0; \ + prow++; \ + } \ + } \ + \ + dma_queue_flush(dmaq); \ } static inline void tile_scale_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) { @@ -1146,14 +1149,32 @@ static int execute_op_unary(struct htp_ops_context * octx) { const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = kparams->n_threads; + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); - const size_t src0_data_row_size = src0->ne[0] * elem_size; const size_t dst_data_row_size = dst->ne[0] * elem_size; + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; @@ -1191,8 +1212,9 @@ static int execute_op_unary(struct htp_ops_context * octx) { struct htp_unary_context uctx = { .octx = octx, .kparams = kparams, - .src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads, - .src0_nrows = src0_nrows, + .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows = nrows, + .row_start = row_start, .data_src0 = (const uint8_t *)src0->data, .data_src1 = (octx->op == HTP_OP_RMS_NORM_MUL) ? (const uint8_t *)src1->data : NULL, @@ -1287,7 +1309,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { } if (task_func) { - worker_pool_run_func(octx->ctx->worker_pool, task_func, &uctx, n_threads); + work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads); } else { FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op); err = HTP_STATUS_NO_SUPPORT; diff --git a/scripts/snapdragon/ggml-hexagon-align-macros.py b/scripts/snapdragon/ggml-hexagon-align-macros.py new file mode 100755 index 000000000000..b64db3e654a8 --- /dev/null +++ b/scripts/snapdragon/ggml-hexagon-align-macros.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +""" +align-macros.py - Inspect and align trailing backslashes in multiline C/C++ macros. + +Usage: + align-macros.py [paths...] # Check and report misaligned macros + align-macros.py --diff [paths...] # Show unified diff of fixes + align-macros.py --fix [paths...] # Fix misaligned macros in-place + align-macros.py --fix --mode majority ... # Align to the dominant column + align-macros.py --fix --pad 2 ... # Align to (max_content_len + pad) + +Safety rules: + - Macros that are ALREADY aligned are NEVER touched (unless --all is given). + - Whitespace after trailing backslashes is flagged and cleaned. +""" + +import argparse +import difflib +import logging +import os +import re +import sys +from collections import Counter +from typing import List, Optional, Tuple, NamedTuple + +logger = logging.getLogger("ggml-hexagon-align-macros") + + +class MacroLine(NamedTuple): + line_num: int # 1-indexed + raw: str # Original line including newline + content: str # Line content before trailing backslash (stripped of trailing whitespace) + bs_col: Optional[int] # 1-indexed column of backslash, or None if last line has no backslash + trailing_ws: bool # True if whitespace existed after the backslash + + +class MacroDef(NamedTuple): + name: str + filepath: str + start_line: int + end_line: int + lines: List[MacroLine] + + +def parse_macros(filepath: str) -> List[MacroDef]: + """Extract all multiline macros from a C/C++ source file.""" + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + lines = f.readlines() + except Exception as e: + logger.error(f"Error reading {filepath}: {e}") + return [] + + macros: List[MacroDef] = [] + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + m = re.match(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z0-9_]*)", line) + if m: + macro_name = m.group(1) + macro_start = i + 1 + macro_lines: List[MacroLine] = [] + cur = i + + while cur < n: + l_raw = lines[cur] + l_rstrip = l_raw.rstrip("\r\n") + + # Check if line has a trailing backslash + # Note: handle possible accidental spaces after backslash + match_bs = re.search(r"\\([ \t]*)$", l_rstrip) + if match_bs: + has_trailing_ws = len(match_bs.group(1)) > 0 + bs_index = match_bs.start() + content = l_rstrip[:bs_index].rstrip() + # 1-indexed column of the backslash + bs_col = bs_index + 1 + macro_lines.append(MacroLine( + line_num=cur + 1, + raw=l_raw, + content=content, + bs_col=bs_col, + trailing_ws=has_trailing_ws + )) + cur += 1 + else: + # Line does not end with backslash + if cur == i: + # Single-line macro, not multiline + break + else: + # Final line of a multiline macro + macro_lines.append(MacroLine( + line_num=cur + 1, + raw=l_raw, + content=l_rstrip.rstrip(), + bs_col=None, + trailing_ws=False + )) + break + + # Only record if it is a multiline macro (has at least one continuation line) + continuation_lines = [ml for ml in macro_lines if ml.bs_col is not None] + if continuation_lines: + macro_end = macro_lines[-1].line_num + macros.append(MacroDef( + name=macro_name, + filepath=filepath, + start_line=macro_start, + end_line=macro_end, + lines=macro_lines + )) + i = cur + i += 1 + + return macros + + +def is_macro_aligned(macro: MacroDef) -> bool: + """A macro is aligned if all continuation lines have backslashes at the same column.""" + bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] + if not bs_cols: + return True + has_trailing_ws = any(ml.trailing_ws for ml in macro.lines) + return len(set(bs_cols)) == 1 and not has_trailing_ws + + +def compute_target_column(macro: MacroDef, mode: str, pad: int, target_col: Optional[int]) -> int: + """Determine the column where backslashes should be aligned.""" + max_content_len = max(len(ml.content) for ml in macro.lines) + min_needed = max_content_len + pad + + if target_col is not None: + return max(target_col, min_needed) + + bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] + if not bs_cols: + return min_needed + + if mode == "min": + return min_needed + elif mode == "max": + return max(max(bs_cols), min_needed) + elif mode == "majority": + counts = Counter(bs_cols) + # Sort by frequency descending, then by column descending + majority_col = sorted(counts.items(), key=lambda x: (-x[1], -x[0]))[0][0] + return max(majority_col, min_needed) + else: + return min_needed + + +def realign_macro_lines(macro: MacroDef, target_col: int) -> List[str]: + """Format macro lines with backslashes aligned at target_col.""" + new_lines: List[str] = [] + for ml in macro.lines: + nl = "\r\n" if ml.raw.endswith("\r\n") else "\n" + if ml.bs_col is None: + # Last line without backslash + new_lines.append(ml.raw) + else: + if not ml.content: + spaces = " " * (target_col - 1) + new_lines.append(f"{spaces}\\{nl}") + else: + spaces_needed = max(1, target_col - len(ml.content) - 1) + new_lines.append(f"{ml.content}{' ' * spaces_needed}\\{nl}") + return new_lines + + +def process_file(filepath: str, args: argparse.Namespace) -> Tuple[int, int, Optional[str]]: + macros = parse_macros(filepath) + if not macros: + return 0, 0, None + + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + file_lines = f.readlines() + + misaligned_count = 0 + modified = False + new_file_lines = list(file_lines) + + for macro in macros: + aligned = is_macro_aligned(macro) + if not aligned or args.all: + if not aligned: + misaligned_count += 1 + + bs_cols = [ml.bs_col for ml in macro.lines if ml.bs_col is not None] + max_content = max(len(ml.content) for ml in macro.lines) + col_counts = Counter(bs_cols) + + if not args.quiet: + logger.info(f"{filepath}:{macro.start_line}-{macro.end_line} [{macro.name}]") + logger.info(f" Max content width: {max_content}, Min needed column (+{args.pad}): {max_content + args.pad}") + logger.info(f" Current backslash columns: {dict(sorted(col_counts.items()))}") + trailing_ws_lines = [ml.line_num for ml in macro.lines if ml.trailing_ws] + if trailing_ws_lines: + logger.warning(f" Warning: Trailing whitespace after backslash on line(s): {trailing_ws_lines}") + + target_col = compute_target_column(macro, args.mode, args.pad, args.target_col) + if not args.quiet: + logger.info(f" -> Target alignment column: {target_col}") + + realigned = realign_macro_lines(macro, target_col) + + start_idx = macro.start_line - 1 + end_idx = start_idx + len(macro.lines) + if new_file_lines[start_idx:end_idx] != realigned: + new_file_lines[start_idx:end_idx] = realigned + modified = True + + diff_text = None + if modified: + diff = difflib.unified_diff( + file_lines, + new_file_lines, + fromfile=f"a/{filepath}", + tofile=f"b/{filepath}", + lineterm="" + ) + diff_text = "\n".join(diff) + + if args.fix: + with open(filepath, "w", encoding="utf-8") as f: + f.writelines(new_file_lines) + if not args.quiet: + logger.info(f" [FIXED] Updated {filepath}") + + return len(macros), misaligned_count, diff_text + + +def find_source_files(paths: List[str]) -> List[str]: + extensions = {".c", ".cpp", ".cc", ".cxx", ".h", ".hpp", ".inl"} + result: List[str] = [] + for p in paths: + if os.path.isfile(p): + result.append(p) + elif os.path.isdir(p): + for root, _, files in os.walk(p): + for file in sorted(files): + _, ext = os.path.splitext(file) + if ext.lower() in extensions: + result.append(os.path.join(root, file)) + return sorted(result) + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(message)s") + parser = argparse.ArgumentParser( + description="Inspect and align backslashes in multiline C/C++ macros." + ) + parser.add_argument("paths", nargs="*", default=["."], help="Files or directories to scan (default: current dir)") + parser.add_argument("--fix", action="store_true", help="Fix misaligned macros in-place") + parser.add_argument("--diff", action="store_true", help="Display unified diff of suggested fixes") + parser.add_argument("--check", action="store_true", help="Exit with code 1 if misaligned macros exist") + parser.add_argument("--mode", choices=["min", "max", "majority"], default="min", + help="Alignment mode: 'min' (max_len + pad), 'max' (max existing col), 'majority' (dominant col)") + parser.add_argument("--pad", type=int, default=2, help="Spaces between longest line and backslash (default: 2)") + parser.add_argument("--target-col", type=int, default=None, help="Force alignment to an exact column") + parser.add_argument("--all", action="store_true", help="Realign all macros even if already aligned (default: only misaligned)") + parser.add_argument("-q", "--quiet", action="store_true", help="Only output errors and diffs/summary") + + args = parser.parse_args() + + files = find_source_files(args.paths) + if not files: + logger.error("No C/C++ source files found.") + sys.exit(0) + + total_macros = 0 + total_misaligned = 0 + diffs: List[str] = [] + + for filepath in files: + num_macros, num_misaligned, diff_text = process_file(filepath, args) + total_macros += num_macros + total_misaligned += num_misaligned + if diff_text: + diffs.append(diff_text) + + if args.diff and diffs: + logger.info("\n--- Proposed Changes ---\n") + for d in diffs: + logger.info(d) + + logger.info(f"\nSummary: scanned {len(files)} files, {total_macros} multiline macros, {total_misaligned} misaligned.") + + if args.check and total_misaligned > 0: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.py index 81eecd2e0ceb..dc71d4a3219d 100755 --- a/scripts/snapdragon/run.py +++ b/scripts/snapdragon/run.py @@ -14,6 +14,42 @@ logger = logging.getLogger("run") +MANAGED_ENV_NAMES = ( + "GGML_HEXAGON_DEVICES", + "GGML_HEXAGON_VERBOSE", + "GGML_HEXAGON_PROFILE", + "GGML_HEXAGON_NHVX", + "GGML_HEXAGON_NHMX", + "GGML_HEXAGON_HOSTBUF", + "GGML_HEXAGON_OPBATCH", + "GGML_HEXAGON_OPQUEUE", + "GGML_HEXAGON_OPPOLL", + "GGML_HEXAGON_OPFILTER", + "GGML_HEXAGON_OPFUSION", + "GGML_HEXAGON_VMEM", + "GGML_HEXAGON_MBUF", + "GGML_HEXAGON_MM_SELECT", + "GGML_HEXAGON_FA_SELECT", + "GGML_HEXAGON_AR_SELECT", + "GGML_HEXAGON_ETM", + "GGML_HEXAGON_ARCH", + "GGML_HEXAGON_OPTRACE", + "GGML_OPENCL_PLATFORM", + "GGML_OPENCL_DEVICE", + "GGML_OPENCL_OPFILTER", + "GGML_OPENCL_KERNEL_CACHE_DIR", + "GGML_OPENCL_KERNEL_CACHE_DEBUG", + "GGML_OPENCL_FA_TUNE", + "GGML_OPENCL_DISABLE_FUSION", + "GGML_OPENCL_ADRENO_XMEM_GEMM", + "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER", + "GGML_SCHED_DEBUG", + "MTMD_BACKEND_DEVICE", + "D", + "DEVICE", +) + + def parse_target(target_str): if not target_str: return None, None @@ -38,6 +74,57 @@ def shlex_join(args_list): return " ".join(pipes.quote(x) for x in args_list) +def split_device_list(devices): + parts = [] + curr = [] + bracket_depth = 0 + + for ch in devices: + if ch == '[': + bracket_depth += 1 + curr.append(ch) + elif ch == ']': + if bracket_depth > 0: + bracket_depth -= 1 + curr.append(ch) + elif ch == ',' and bracket_depth == 0: + part = "".join(curr).strip() + if part: + parts.append(part) + curr = [] + else: + curr.append(ch) + + part = "".join(curr).strip() + if part: + parts.append(part) + + return parts + + +def device_arg_from_devices(devices): + if devices.isdigit(): + n = int(devices) + return ",".join(f"HTP{i}" for i in range(n)) + + names = [] + for part in split_device_list(devices): + if "[" in part: + part = part.split("[", 1)[0].strip() + if part: + names.append(part) + + return ",".join(names) + + +def normalize_cmd_device_args(cmd_args): + for i, arg in enumerate(cmd_args): + if arg == "--device" and i + 1 < len(cmd_args): + cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1]) + elif arg.startswith("--device="): + cmd_args[i] = "--device=" + device_arg_from_devices(arg.split("=", 1)[1]) + + def main(): logging.basicConfig(level=logging.INFO, format='%(message)s') # Split arguments at '--' @@ -142,8 +229,6 @@ def main(): def set_env(env_name, opt_val): if opt_val is not None: env_vars[env_name] = str(opt_val) - elif env_name in os.environ: - env_vars[env_name] = os.environ[env_name] # Resolve and filter devices (HTP vs OpenCL) device_in_cmd = None @@ -166,7 +251,7 @@ def set_env(env_name, opt_val): hex_devices = devices_val cl_device = "" else: - parts = [p.strip() for p in devices_val.split(",")] + parts = split_device_list(devices_val) # Any device containing "htp" is Hexagon, rest is OpenCL hex_parts = [p for p in parts if "htp" in p.lower()] cl_parts = [ @@ -181,15 +266,13 @@ def set_env(env_name, opt_val): # Set Hexagon devices if hex_devices: env_vars["GGML_HEXAGON_DEVICES"] = hex_devices - elif "GGML_HEXAGON_DEVICES" in os.environ: - env_vars["GGML_HEXAGON_DEVICES"] = os.environ["GGML_HEXAGON_DEVICES"] + + normalize_cmd_device_args(cmd_args) # Set OpenCL device (unless overridden by --cl-device) final_cl_device = args.cl_device if args.cl_device is not None else cl_device if final_cl_device: env_vars["GGML_OPENCL_DEVICE"] = final_cl_device - elif "GGML_OPENCL_DEVICE" in os.environ: - env_vars["GGML_OPENCL_DEVICE"] = os.environ["GGML_OPENCL_DEVICE"] # Map shared & backend-specific parameters with correct overrides @@ -206,8 +289,6 @@ def set_env(env_name, opt_val): if args.cl_fa_tune or args.profile is not None: env_vars["GGML_OPENCL_FA_TUNE"] = "1" - elif "GGML_OPENCL_FA_TUNE" in os.environ: - env_vars["GGML_OPENCL_FA_TUNE"] = os.environ["GGML_OPENCL_FA_TUNE"] # Other Hexagon environment variables set_env("GGML_HEXAGON_NHVX", args.hex_nhvx) @@ -235,18 +316,12 @@ def set_env(env_name, opt_val): if args.cl_disable_fusion: env_vars["GGML_OPENCL_DISABLE_FUSION"] = "1" - elif "GGML_OPENCL_DISABLE_FUSION" in os.environ: - env_vars["GGML_OPENCL_DISABLE_FUSION"] = os.environ["GGML_OPENCL_DISABLE_FUSION"] if args.cl_adreno_xmem: env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = "1" - elif "GGML_OPENCL_ADRENO_XMEM_GEMM" in os.environ: - env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = os.environ["GGML_OPENCL_ADRENO_XMEM_GEMM"] if args.cl_adreno_large_buffer: env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = "1" - elif "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER" in os.environ: - env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = os.environ["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] if args.sched_debug: env_vars["GGML_SCHED_DEBUG"] = "2" @@ -288,15 +363,7 @@ def set_env(env_name, opt_val): has_b = any(arg == "-b" for arg in cmd_args) if not has_b: if args.devices: - if args.devices.isdigit(): - n = int(args.devices) - device_val = ",".join(f"HTP{i}" for i in range(n)) - else: - device_val = args.devices - elif "D" in os.environ: - device_val = os.environ["D"] - elif "DEVICE" in os.environ: - device_val = os.environ["DEVICE"] + device_val = device_arg_from_devices(args.devices) else: device_val = "HTP0" if device_val: @@ -305,17 +372,10 @@ def set_env(env_name, opt_val): has_device = any(arg.startswith("--device") for arg in cmd_args) if not has_device: if args.devices: - if args.devices.isdigit(): - n = int(args.devices) - device_val = ",".join(f"HTP{i}" for i in range(n)) - else: - device_val = args.devices - elif "D" in os.environ: - device_val = os.environ["D"] - elif "DEVICE" in os.environ: - device_val = os.environ["DEVICE"] + device_val = device_arg_from_devices(args.devices) else: device_val = "HTP0" + if device_val: cmd_args += ["--device", device_val] @@ -415,6 +475,8 @@ def set_env(env_name, opt_val): else: local_env["LD_LIBRARY_PATH"] = lib_dir + os.path.pathsep + local_env.get("LD_LIBRARY_PATH", "") + for k in MANAGED_ENV_NAMES: + local_env.pop(k, None) for k, v in env_vars.items(): local_env[k] = v From 3f5e94d7c2ab2267fe39852051777fe30c1f49ef Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 12 Sep 2026 06:40:21 +0200 Subject: [PATCH 49/65] webgpu: align tensor bindings to the type block size (#28382) Walk the binding offset back until the distance to the tensor is a whole number of blocks, so block quantized views get a valid element offset in the shader. --- ggml/src/ggml-webgpu/ggml-webgpu.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 13db0b856f69..8b060c41a1c6 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -374,20 +374,28 @@ static wgpu::Buffer ggml_webgpu_tensor_buf(const ggml_tensor * tensor) { return ctx->buffer; } +// Binding offset for a tensor: the largest aligned offset at or before the tensor whose +// distance to the tensor is a whole number of type blocks, so shaders can index the +// misalignment in elements even for block quantized types. +static size_t ggml_webgpu_tensor_align_offset(const ggml_tensor * t, size_t alignment) { + const size_t offset = ggml_webgpu_tensor_offset(t); + const size_t type_size = ggml_type_size(t->type); + size_t aligned = offset & ~(alignment - 1); + while ((offset - aligned) % type_size != 0) { + GGML_ASSERT(aligned >= alignment); + aligned -= alignment; + } + return aligned; +} + static size_t ggml_webgpu_tensor_misalignment(const ggml_tensor * t, size_t alignment) { - size_t offset = ggml_webgpu_tensor_offset(t); - return offset & (alignment - 1); + return ggml_webgpu_tensor_offset(t) - ggml_webgpu_tensor_align_offset(t, alignment); } static size_t ggml_webgpu_tensor_misalignment(webgpu_context & ctx, const ggml_tensor * t) { return ggml_webgpu_tensor_misalignment(t, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); } -static size_t ggml_webgpu_tensor_align_offset(const ggml_tensor * t, size_t alignment) { - size_t offset = ggml_webgpu_tensor_offset(t); - return offset & ~(alignment - 1); -} - static size_t ggml_webgpu_tensor_align_offset(webgpu_context & ctx, const ggml_tensor * t) { return ggml_webgpu_tensor_align_offset(t, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); } From 07fc97716fa3dab457797fcfe51705034ce8957a Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Fri, 11 Sep 2026 22:10:08 -0700 Subject: [PATCH 50/65] opencl: add bin kernel `kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin` (#28677) * opencl: add A8 Q4_K non-MoE binary kernel * opencl: fix layout compatibility * opencl: rename binary kernel selection helpers --------- Co-authored-by: Li He --- ggml/src/ggml-opencl/CMakeLists.txt | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 249 +++++++++++++++++- .../gemv_noshuffle_q4_k_f32_32b_trans.cl | 134 ++++++++++ 3 files changed, 376 insertions(+), 8 deletions(-) create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 716577bb77e0..45a7075b291f 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -185,6 +185,7 @@ set(GGML_OPENCL_KERNELS gemv_noshuffle_q4_k_f32_o4 gemv_noshuffle_q4_k_f32_tiled gemm_noshuffle_q4_k_f32 + gemv_noshuffle_q4_k_f32_32b_trans gemv_noshuffle_q6_k_f32 gemv_noshuffle_q6_k_f32_o4 gemv_noshuffle_q6_k_f32_tiled diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 231be2cf3ac4..d6820b37d75f 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1185,6 +1185,8 @@ struct ggml_backend_opencl_context { cl_kernel kernel_convert_block_q4_k_tiled_ns; // tiled-wide convert (opt-in) cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q4_k_f32; + cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM @@ -4260,6 +4262,43 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr; + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = + clCreateKernel(prog, "gemv_noshuffle_q4_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + } + } + std::string CL_moe_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -cl-fast-relaxed-math"; @@ -7722,6 +7761,7 @@ static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor } inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below (used by the GLU-subgraph fuse check) +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { @@ -7776,6 +7816,10 @@ static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx if (use_q4k_tiled(backend_ctx, gate->src[0]) || use_q4k_tiled(backend_ctx, up->src[0])) { return false; } + // q4_K bin kernel requires 32b transposed layout, not compatible with the fused gemv + if (use_q4_k_bin_kernels(backend_ctx, gate->src[0]) || use_q4_k_bin_kernels(backend_ctx, up->src[0])) { + return false; + } // that noshuffle layout is only produced at set_tensor time when // use_adreno_kernels() accepts the weight (ne0 >= 512 && ne1 >= 512). // Smaller weights stay in the plain q4_K layout, which this kernel would @@ -8349,7 +8393,7 @@ inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *b qh_img_width <= backend_ctx->image_max_buffer_size; } -inline bool use_q4_0_ila_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { @@ -8442,6 +8486,21 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_cont && tensor->ne[2] == 1 && tensor->ne[3] == 1; } +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && + !use_q4k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *)dev->context; ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; @@ -9625,7 +9684,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(K % 32 == 0); - if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { cl_int err; cl_image_format wimg_fmt; cl_image_desc wimg_desc; @@ -10576,8 +10635,25 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(K % 32 == 0); - // Transpose q, d, dm as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + } transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); @@ -11180,7 +11256,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); - if (use_q4_0_ila_kernels(backend_ctx, tensor)) { + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); } else { transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); @@ -11855,7 +11931,11 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, buf_trans_s.allocate(backend_ctx->context, size_s); // Transpose q, d, dm, s back - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); @@ -18639,9 +18719,9 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); - const bool use_ila = use_q4_0_ila_kernels(backend_ctx, src0); + const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0); - if (use_ila) { + if (use_bin) { if (use_q40_mc3) { static bool warned = false; if (!warned) { @@ -20196,6 +20276,145 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + int M = ne01; + int N = ne1; + int K = ne00; + + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + + size_t local_work_size[3] = { 64, 8, 1 }; + size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + const int gemm_tile_n = 64; + int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + cl_mem b_buf = nullptr; + if (N_pad == N) { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + b_buf = b_sub_buf; + } else { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_padded; + } + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -20248,6 +20467,20 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t // unified routes batched Q6_K lm_head to CPU). Per-layer mc3 is byte-identical. const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768); + const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4K_MC3 is bypassed by Q4_K binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q4_k_f32_adreno_ila(backend, src0, src1, dst); + return; + } + if (ne1 == 1 || use_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl new file mode 100644 index 000000000000..2dbd943fda29 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define K_SCALE_SIZE 12 +#define N_SIMDGROUP 8 +#define SIMDGROUP_WIDTH 64 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uint stride, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j*stride] & 63; + *m = q[(j+4)*stride] & 63; + } else { + *d = (q[(j+4)*stride] & 0x0F) | ((q[(j-4)*stride] & 0xC0) >> 2); + *m = ((q[(j+4)*stride] >> 4) & 0x0F) | ((q[j*stride] & 0xC0) >> 2); + } +} + +static inline float8 q4_k_to_fp32_packed8(ushort2 q4x8, float scale, float minv) { + float8 fp32x8; + fp32x8.s0 = (q4x8.s0 & 0x000F) * scale - minv; + fp32x8.s1 = ((q4x8.s0 & 0x00F0) >> 4) * scale - minv; + fp32x8.s2 = ((q4x8.s0 & 0x0F00) >> 8) * scale - minv; + fp32x8.s3 = ((q4x8.s0 & 0xF000) >> 12) * scale - minv; + fp32x8.s4 = (q4x8.s1 & 0x000F) * scale - minv; + fp32x8.s5 = ((q4x8.s1 & 0x00F0) >> 4) * scale - minv; + fp32x8.s6 = ((q4x8.s1 & 0x0F00) >> 8) * scale - minv; + fp32x8.s7 = ((q4x8.s1 & 0xF000) >> 12) * scale - minv; + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void gemv_noshuffle_q4_k_f32_32b_trans( + read_only image1d_buffer_t src0_q, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + uint i01 = get_global_id(0); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + int num_subblocks = ne00 / 32; + + __private float sum = 0.0f; + + // Loop over sub-blocks of 32 elements, N_SIMDGROUP sub-blocks per iter + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; + uint j = ib % 8; + + // Load d and dmin for this super-block + half d_val = src0_d[sb * ne01 + i01]; + half dm_val = src0_dm[sb * ne01 + i01]; + + // Load sub-block scale and min. s is transposed [nb][12][M]; stride ne01 per code. + global const uchar * sc = src0_s + sb * K_SCALE_SIZE * ne01 + i01; + uchar sv, mn; + get_scale_min_k4(j, sc, ne01, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = (float)dm_val * (float)mn; + + // Load 4 uints of quants (32 nibbles = 32 elements), column-major stride ne01 + uint q_base = ib * ne01 * 4 + i01; + + uint4 regQ; + regQ.s0 = read_imageui(src0_q, q_base).x; + regQ.s1 = read_imageui(src0_q, q_base + ne01).x; + regQ.s2 = read_imageui(src0_q, q_base + ne01 * 2).x; + regQ.s3 = read_imageui(src0_q, q_base + ne01 * 3).x; + + // Load activations: 32 floats = 8 float4s + uint y_offset = ib * 8; + + float4 y_local = (slid < 8) ? read_imagef(src1, (y_offset + slid)) : (float4)0.0f; + float4 y0 = sub_group_broadcast(y_local, 0); + float4 y1 = sub_group_broadcast(y_local, 1); + float4 y2 = sub_group_broadcast(y_local, 2); + float4 y3 = sub_group_broadcast(y_local, 3); + float4 y4 = sub_group_broadcast(y_local, 4); + float4 y5 = sub_group_broadcast(y_local, 5); + float4 y6 = sub_group_broadcast(y_local, 6); + float4 y7 = sub_group_broadcast(y_local, 7); + + float8 fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s0), scale, minv); + float4 acc = y0 * fp32x8.lo; + acc += y1 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s1), scale, minv); + acc += y2 * fp32x8.lo; + acc += y3 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s2), scale, minv); + acc += y4 * fp32x8.lo; + acc += y5 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s3), scale, minv); + acc += y6 * fp32x8.lo; + acc += y7 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory over N_SIMDGROUP subgroups + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid > 0) { + reduceLM[SIMDGROUP_WIDTH * (sgid - 1) + slid] = sum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) { + for (uint i = 0; i < N_SIMDGROUP - 1; ++i) { + sum += reduceLM[SIMDGROUP_WIDTH * i + slid]; + } + } + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01] = sum; + } +} From 8a56aedd6143a014e25ec9f4295164f2f66ce30f Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Fri, 11 Sep 2026 22:11:12 -0700 Subject: [PATCH 51/65] opencl: fix several bugs where the backend aborts (#27630) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 352 +++++++++++++++++---------- 1 file changed, 224 insertions(+), 128 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index d6820b37d75f..c107281a2160 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -203,39 +203,67 @@ static ggml_cl_version get_opencl_platform_version(cl_platform_id platform) { return parse_cl_version(param_value); } +// Returns the DEVICE's OpenCL version. On an error returns ggml_cl_version with all zeroes. +static ggml_cl_version get_opencl_device_version(cl_device_id device) { + size_t param_size; + if (clGetDeviceInfo(device, CL_DEVICE_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { + return {}; + } + std::unique_ptr param_storage(new char[param_size]); + if (clGetDeviceInfo(device, CL_DEVICE_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { + return {}; + } + + auto param_value = std::string_view(param_storage.get(), param_size); + const std::string version_prefix = "OpenCL "; // "OpenCL . " + if (param_value.find(version_prefix) != 0) { + return {}; + } + param_value.remove_prefix(version_prefix.length()); + return parse_cl_version(param_value); +} + // Return a version to use in OpenCL C compilation. On an error returns ggml_cl_version with all zeroes. static ggml_cl_version get_opencl_c_version(ggml_cl_version platform_version, cl_device_id device) { size_t param_size; #if CL_TARGET_OPENCL_VERSION >= 300 - if (platform_version.major >= 3) { - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size)); - if (!param_size) { - return {}; - } + // CL_DEVICE_OPENCL_C_ALL_VERSIONS is an OpenCL 3.0 *device* query, so gating it on the + // *platform* version is not enough: a 3.0 platform can expose 2.0 devices, where the + // query returns CL_INVALID_VALUE and the old CL_CHECK aborted during backend init. + // Gate on the device version, and treat a failure as "fall back to the legacy query" + // rather than fatal -- a device may advertise 3.0 and still refuse the property. + const ggml_cl_version device_version = get_opencl_device_version(device); + if (platform_version.major >= 3 && device_version.major >= 3) { + cl_int err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size); + if (err == CL_SUCCESS && param_size) { + std::unique_ptr versions(new cl_name_version[param_size]); + err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr); + if (err == CL_SUCCESS) { + unsigned versions_count = param_size / sizeof(cl_name_version); - std::unique_ptr versions(new cl_name_version[param_size]); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr)); - unsigned versions_count = param_size / sizeof(cl_name_version); + cl_version version_max = 0; + for (unsigned i = 0; i < versions_count; i++) { + version_max = std::max(versions[i].version, version_max); + } - cl_version version_max = 0; - for (unsigned i = 0; i < versions_count; i++) { - version_max = std::max(versions[i].version, version_max); + return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; + } } - - return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; + // fall through to CL_DEVICE_OPENCL_C_VERSION below } #else GGML_UNUSED(platform_version); #endif // CL_TARGET_OPENCL_VERSION >= 300 - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size)); - if (!param_size) { + if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { return {}; } std::unique_ptr param_storage(new char[param_size]); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr)); + if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { + return {}; + } auto param_value = std::string_view(param_storage.get(), param_size); const std::string version_prefix = "OpenCL C "; // Suffix: "XX.YY " @@ -1115,6 +1143,18 @@ struct ggml_backend_opencl_context { } void enqueue_ndrange_kernel(cl_kernel kernel, cl_uint work_dim, size_t *global_work_size, size_t *local_work_size, const ggml_tensor * tensor) { + // From the spec on clEnqueueNDRangeKernel: + // If the device associated with command_queue is an OpenCL 2.1 or newer device, + // and global_work_size is NULL or the value in any passed dimension is zero, + // then the kernel command will trivially succeed after its event dependencies + // are satisfied and subsequently update its completion event. + // So this ensures such cases always return trivially without causing errors in + // case of an older device. + for (cl_uint i = 0; i < work_dim; i++) { + if (global_work_size[i] == 0) { + return; + } + } #ifdef GGML_OPENCL_PROFILING cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, work_dim, NULL, global_work_size, local_work_size, 0, NULL, &evt)); @@ -9459,6 +9499,96 @@ static enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buff return GGML_STATUS_SUCCESS; } +// Allocate a temporary upload buffer of `nbytes` and populate it with `data` +// from host. On Adreno X1-85 the device-only pool intermittently fails to +// allocate at hundreds of MB once model weights fragment the heap (observed +// on Qwen3.5-9B output.weight Q6_K at 834 MB). Three-step retry: +// 1. CL_MEM_READ_WRITE alloc + clEnqueueWriteBuffer (normal fast path). +// 2. clFinish + retry (drains in-flight allocs that may be holding heap; +// mirrors the proven pattern at the FD-split partial buffer alloc). +// 3. CL_MEM_ALLOC_HOST_PTR + map(WRITE_INVALIDATE) + memcpy + unmap — +// different memory pool (host-pinned); true zero-copy on Adreno per +// QCOM guidance. (CL_MEM_USE_HOST_PTR is NOT zero-copy on Adreno: the +// driver triggers an internal copy because arbitrary host pages aren't +// guaranteed mappable/coherent, AND it draws from the same exhausted +// device pool — so it doesn't solve the problem.) +// Returns the ready-to-read buffer (caller must clReleaseMemObject) or NULL +// if all three strategies fail. The buffer is opaque to the caller — it can +// be passed as a kernel argument like any normal cl_mem. +static cl_mem ggml_cl_create_temp_upload_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const void * data, + const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + const cl_int werr = clEnqueueWriteBuffer(queue, buf, CL_TRUE, 0, nbytes, data, 0, NULL, NULL); + if (werr == CL_SUCCESS) { + return buf; + } + clReleaseMemObject(buf); + } + buf = clCreateBuffer(context, + CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_WRITE_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; + } + void * mapped = clEnqueueMapBuffer(queue, buf, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, nbytes, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + memcpy(mapped, data, nbytes); + const cl_int uerr = clEnqueueUnmapMemObject(queue, buf, mapped, 0, NULL, NULL); + if (uerr != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); + } + return buf; +} + +// Allocate a temporary download buffer of `nbytes`. The caller runs a kernel +// that writes into it, then reads it back to host via clEnqueueReadBuffer (or +// equivalent). Mirrors ggml_cl_create_temp_upload_buffer; the host-pinned +// fallback flags are flipped (CL_MEM_WRITE_ONLY | HOST_READ_ONLY) and the +// helper doesn't populate the buffer. +static cl_mem ggml_cl_create_temp_download_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + return buf; + } + buf = clCreateBuffer(context, + CL_MEM_WRITE_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_READ_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; + } + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s download (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); + } + return buf; +} + static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; @@ -9567,12 +9697,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // We consider the specified offset arg as always, although For weights // the offset arg should be 0 (we do not assert this). @@ -9730,12 +9856,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_m + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -9862,12 +9984,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10026,12 +10144,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_m + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10179,12 +10293,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. @@ -10290,12 +10400,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. @@ -10394,12 +10500,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10478,12 +10580,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_ASSERT(size_d + size_dm + size_s + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q4_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10680,9 +10778,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, "Incorrect tensor size"); cl_int err; - cl_mem data_device; - CL_CHECK((data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err), err)); - CL_CHECK(clEnqueueWriteBuffer(queue, data_device, CL_TRUE, 0, ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q5_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -10868,9 +10965,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, "Incorrect tensor size"); cl_int err; - cl_mem data_device; - CL_CHECK((data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err), err)); - CL_CHECK(clEnqueueWriteBuffer(queue, data_device, CL_TRUE, 0, ggml_nbytes(tensor), data, 0, NULL, NULL)); + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q6_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; @@ -11211,9 +11307,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, cl_int err; cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_trans4_ns; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; @@ -11282,10 +11377,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, } #endif - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); @@ -11310,10 +11403,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_trans4_ns; int ne00 = tensor->ne[0]; @@ -11385,10 +11476,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, } #endif - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); @@ -11416,9 +11505,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_int err; // TODO: use ggml_cl_buffer to manage this temporary buffer - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_trans4_ns; @@ -11520,9 +11608,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_int err; // TODO: use ggml_cl_buffer to manage this temporary buffer - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_trans4_ns; @@ -11627,10 +11714,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_MXFP4) { ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { @@ -11692,10 +11777,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)extra_src->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (enable_adreno_trans_weight(backend_ctx, tensor)) { @@ -11748,10 +11831,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_IQ4_NL) { ggml_tensor_extra_cl_iq4_nl * extra = (ggml_tensor_extra_cl_iq4_nl *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor)) { @@ -11820,10 +11901,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; @@ -11878,10 +11957,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_k_trans4_ns; @@ -11986,20 +12063,16 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, if (tensor->type == GGML_TYPE_Q5_K) { ggml_tensor_extra_cl_q5_K * extra = (ggml_tensor_extra_cl_q5_K *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_k_trans4_ns; int ne00 = tensor->ne[0]; @@ -12159,10 +12232,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q6_k_trans4_ns; @@ -12249,10 +12320,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, } #endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask = 0xFF; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); @@ -12380,6 +12449,21 @@ static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_b cl_int err; cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + // On Adreno X1-85 the device pool intermittently fails at hundreds of MB + // once the heap fragments (e.g. graph-allocator compute-buffer reserve + // after model load). Four-step retry: + // 1. normal alloc (fast path) + // 2. clFinish + retry (drains in-flight allocs) + // 3. cl_qcom_large_buffer (X2-class driver only, OpenCL 3.0 only) + // 4. ALLOC_HOST_PTR (host-pinned pool) — last-resort fallback. This + // buffer backs compute scratch read/written by every kernel in the + // graph, so kernel accesses fall to host memory and runtime perf + // degrades meaningfully. Better than failing to load, but the user + // should see the warning and consider -ngl reduction. + if (err != CL_SUCCESS) { + clFinish(backend_ctx->queue); + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + } #if GGML_OPENCL_TARGET_VERSION >= 300 // clCreateBufferWithProperties and cl_mem_properties are OpenCL 3.0. Drivers older than // that do not export the symbol, so a build targeting them fails to link. The large @@ -12390,9 +12474,20 @@ static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_b mem = clCreateBufferWithProperties(backend_ctx->context, props, CL_MEM_READ_WRITE, size, NULL, &err); } #endif + if (err != CL_SUCCESS) { + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, size, NULL, &err); + if (err == CL_SUCCESS) { + GGML_LOG_WARN("%s: %.2f MiB allocated via CL_MEM_ALLOC_HOST_PTR fallback — " + "device pool exhausted; runtime perf will be degraded. " + "Consider lowering -ngl or context size.\n", + __func__, size / 1024.0 / 1024.0); + } + } if (err != CL_SUCCESS) { - GGML_LOG_INFO("%s: failed to allocate %.2f MiB\n", __func__, size / 1024.0 / 1024.0); + GGML_LOG_ERROR("%s: failed to allocate %.2f MiB (err=%d). " + "Consider reducing -ngl, lowering -c / -ub, or using quantized KV cache.\n", + __func__, size / 1024.0 / 1024.0, err); return nullptr; } @@ -13147,6 +13242,7 @@ static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, c (size_t)ne03}; size_t local_work_size[] = {(size_t)nth0, (size_t)rows_per_workgroup, 1}; + // ne01 == 0 makes global_work_size[0] zero here; enqueue_ndrange_kernel drops the empty range. backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } From c069aa7f5f2beeead1a3a8e9f71510f1b64d0725 Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 12 Sep 2026 07:38:50 +0200 Subject: [PATCH 52/65] server: frame the router child state command as a whole line (#28747) The child writes its state commands on stdout while the logger writes on stderr, and both share a single pipe. The logger emits the trailing color reset after the newline of a debug, warn or error entry, so that escape sequence has no newline of its own and the router reads it glued in front of the next command. The line prefix check then fails and the command is forwarded as a log line instead of being handled, which leaves a finished download stuck in the downloading state. Writing the command with a leading newline closes the pending line so it always starts at a line boundary. --- tools/server/server-models.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index f1783c083036..4984f1be68cc 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1786,7 +1786,10 @@ void server_child::notify_to_router(const std::string & state, const json & payl std::lock_guard lk(mtx_stdout); common_log_pause(common_log_main()); fflush(stdout); - fprintf(stdout, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str()); + // the router matches the command on a line prefix, so the leading newline + // closes whatever the logger left open on the shared pipe, down to the + // trailing color reset that carries no newline of its own + fprintf(stdout, "\n%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str()); fflush(stdout); common_log_resume(common_log_main()); } From f3a33dff26f5d5ba8fbf47a26d4857c6edfe69a8 Mon Sep 17 00:00:00 2001 From: Ed Addario <29247825+EAddario@users.noreply.github.com> Date: Sat, 12 Sep 2026 07:22:57 +0100 Subject: [PATCH 53/65] rpc : fix linking when compiling with BUILD_SHARED_LIBS=OFF (#28492) --- ggml/src/ggml-rpc/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-rpc/CMakeLists.txt b/ggml/src/ggml-rpc/CMakeLists.txt index af3bd0290f7c..e3d0c9b4c200 100644 --- a/ggml/src/ggml-rpc/CMakeLists.txt +++ b/ggml/src/ggml-rpc/CMakeLists.txt @@ -36,8 +36,10 @@ if (GGML_RPC_RDMA) target_compile_definitions(ggml-rpc PRIVATE GGML_RPC_RDMA) if (APPLE) # librdma.dylib only exists on macOS 26.2 and later. Link it weakly so a build made - # where it exists still loads where it does not; checked at runtime before use. - target_link_options(ggml-rpc PRIVATE "LINKER:-weak_library,${RDMA_LIB}") + # where it exists still loads where it does not; checked at runtime before use + # but with BUILD_SHARED_LIBS=OFF ggml-rpc is a static archive and never links + # so the librdma symbols used by transport-apple.cpp stay undefined. + target_link_options(ggml-rpc PUBLIC "LINKER:-weak_library,${RDMA_LIB}") target_compile_definitions(ggml-rpc PRIVATE GGML_RPC_RDMA_APPLE) target_sources(ggml-rpc PRIVATE transport-apple.cpp) else() From 2a3005c23f60cb38dab70b8ea2ddbd969bcf3e87 Mon Sep 17 00:00:00 2001 From: Michael Taylor <162068037+mctylr-gh@users.noreply.github.com> Date: Sat, 12 Sep 2026 04:05:38 -0300 Subject: [PATCH 54/65] syscl : Handle (fail gracefully) unsupported tq1_0 quants (#28681) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 4091f73a4674..f225682f288e 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6326,7 +6326,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons return false; } - if (src0_type == GGML_TYPE_TQ2_0) { + if (src0_type == GGML_TYPE_TQ2_0 || src0_type == GGML_TYPE_TQ1_0) { return false; } @@ -6380,7 +6380,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_SET_ROWS: { - if (op->type == GGML_TYPE_TQ2_0) { + if (op->type == GGML_TYPE_TQ2_0 || op->type == GGML_TYPE_TQ1_0) { return false; } auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || @@ -6502,12 +6502,14 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons src1_type == GGML_TYPE_IQ3_S || src1_type == GGML_TYPE_IQ1_S || src1_type == GGML_TYPE_IQ1_M || - src1_type == GGML_TYPE_TQ2_0) { + src1_type == GGML_TYPE_TQ2_0 || + src1_type == GGML_TYPE_TQ1_0) { return false; } } - if (src0_type == GGML_TYPE_TQ2_0 || src1_type == GGML_TYPE_TQ2_0) { + if (src0_type == GGML_TYPE_TQ2_0 || src1_type == GGML_TYPE_TQ2_0 || + src0_type == GGML_TYPE_TQ1_0 || src1_type == GGML_TYPE_TQ1_0) { return false; } From 718f7b4175bf8b6af6f5eac09fee10754b3ecddd Mon Sep 17 00:00:00 2001 From: "Alessandro de Oliveira Faria (A.K.A.CABELO)" Date: Sat, 12 Sep 2026 04:15:08 -0300 Subject: [PATCH 55/65] vendor : update cpp-httplib to 0.56.0 (#28787) --- scripts/sync_vendor.py | 2 +- vendor/cpp-httplib/httplib.cpp | 296 ++++++++++++++++++++++++--------- vendor/cpp-httplib/httplib.h | 92 ++++++++-- 3 files changed, 298 insertions(+), 92 deletions(-) diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 0170b5b168f4..a73ae1193e8c 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -5,7 +5,7 @@ import sys import subprocess -HTTPLIB_VERSION = "refs/tags/v0.54.1" +HTTPLIB_VERSION = "refs/tags/v0.56.0" # used by examples/gguf-hash, these repos have no release tag, so we pin a commit XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68" diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp index 7fd10b3939bc..c82ff1e71de8 100644 --- a/vendor/cpp-httplib/httplib.cpp +++ b/vendor/cpp-httplib/httplib.cpp @@ -912,17 +912,42 @@ bool write_websocket_frame(Stream &strm, ws::Opcode opcode, namespace ws { namespace impl { -bool read_websocket_frame(Stream &strm, Opcode &opcode, - std::string &payload, bool &fin, - bool expect_masked, size_t max_len) { - // Read first 2 bytes +// Read exactly `size` bytes. Stream::read may return less than asked for -- it +// hands back whatever its buffer already holds -- so every multi-byte field has +// to loop. Reading a 2-byte header with a single read() fails whenever the +// header straddles the read buffer's boundary. +// +// Timeout is reported only when nothing at all was consumed. Once a byte has +// been taken the stream sits mid-field and cannot be resumed, so a timeout +// there is a failure like any other. (When read() fails it always records why, +// so the error belongs to this call and not to an earlier one.) +FrameRead read_exact(Stream &strm, void *buf, size_t size) { + auto p = static_cast(buf); + size_t total = 0; + while (total < size) { + auto n = strm.read(p + total, size - total); + if (n <= 0) { + auto timed_out = total == 0 && strm.get_error() == Error::Timeout; + return timed_out ? FrameRead::Timeout : FrameRead::Fail; + } + total += static_cast(n); + } + return FrameRead::Ok; +} + +FrameRead read_websocket_frame(Stream &strm, Opcode &opcode, + std::string &payload, bool &fin, + bool expect_masked, size_t max_len) { + // Read first 2 bytes. This is the only read that may report a timeout: it + // sits on a frame boundary, where nothing has been consumed yet. uint8_t header[2]; - if (strm.read(reinterpret_cast(header), 2) != 2) { return false; } + FrameRead first = read_exact(strm, header, 2); + if (first != FrameRead::Ok) { return first; } fin = (header[0] & 0x80) != 0; // RSV1, RSV2, RSV3 must be 0 when no extension is negotiated - if (header[0] & 0x70) { return false; } + if (header[0] & 0x70) { return FrameRead::Fail; } opcode = static_cast(header[0] & 0x0F); bool masked = (header[1] & 0x80) != 0; @@ -932,46 +957,44 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode, // MUST have a payload length of 125 bytes or less bool is_control = (static_cast(opcode) & 0x08) != 0; if (is_control) { - if (!fin) { return false; } - if (payload_len > 125) { return false; } + if (!fin) { return FrameRead::Fail; } + if (payload_len > 125) { return FrameRead::Fail; } } - if (masked != expect_masked) { return false; } + if (masked != expect_masked) { return FrameRead::Fail; } // Extended payload length if (payload_len == 126) { uint8_t ext[2]; - if (strm.read(reinterpret_cast(ext), 2) != 2) { return false; } + if (read_exact(strm, ext, 2) != FrameRead::Ok) { return FrameRead::Fail; } payload_len = (static_cast(ext[0]) << 8) | ext[1]; } else if (payload_len == 127) { uint8_t ext[8]; - if (strm.read(reinterpret_cast(ext), 8) != 8) { return false; } + if (read_exact(strm, ext, 8) != FrameRead::Ok) { return FrameRead::Fail; } // RFC 6455 Section 5.2: the most significant bit MUST be 0 - if (ext[0] & 0x80) { return false; } + if (ext[0] & 0x80) { return FrameRead::Fail; } payload_len = 0; for (int i = 0; i < 8; i++) { payload_len = (payload_len << 8) | ext[i]; } } - if (payload_len > max_len) { return false; } + if (payload_len > max_len) { return FrameRead::Fail; } // Read mask key if present uint8_t mask_key[4] = {0}; if (masked) { - if (strm.read(reinterpret_cast(mask_key), 4) != 4) { return false; } + if (read_exact(strm, mask_key, 4) != FrameRead::Ok) { + return FrameRead::Fail; + } } // Read payload payload.resize(static_cast(payload_len)); - if (payload_len > 0) { - size_t total_read = 0; - while (total_read < payload_len) { - auto n = strm.read(&payload[total_read], - static_cast(payload_len - total_read)); - if (n <= 0) { return false; } - total_read += static_cast(n); - } + if (payload_len > 0 && + read_exact(strm, &payload[0], static_cast(payload_len)) != + FrameRead::Ok) { + return FrameRead::Fail; } // Unmask if needed @@ -981,7 +1004,7 @@ bool read_websocket_frame(Stream &strm, Opcode &opcode, } } - return true; + return FrameRead::Ok; } } // namespace impl @@ -1728,7 +1751,9 @@ ssize_t select_impl(socket_t sock, short events, time_t sec, pfd.events = events; pfd.revents = 0; - auto timeout = static_cast(sec * 1000 + usec / 1000); + // A negative timeout waits forever, poll's own convention. 0 keeps meaning + // "return immediately", which callers here rely on to probe a socket. + auto timeout = sec < 0 ? -1 : static_cast(sec * 1000 + usec / 1000); return handle_EINTR([&]() { return poll_wrapper(&pfd, 1, timeout); }); } @@ -1810,8 +1835,11 @@ class SocketStream final : public Stream { bool ensure_readable(); socket_t sock_; - time_t read_timeout_sec_; - time_t read_timeout_usec_; + // Atomic because ws::WebSocket::set_read_timeout() reaches this from another + // thread while a read is in flight -- that is the point of it, for a caller + // holding one connection and wanting control back to send on it. + std::atomic read_timeout_sec_; + std::atomic read_timeout_usec_; time_t write_timeout_sec_; time_t write_timeout_usec_; time_t max_timeout_msec_; @@ -2204,12 +2232,10 @@ int getaddrinfo_with_timeout(const char *node, const char *service, // actually finish before letting the stack frame go. The trade-off is that // a wedged DNS server can hold this thread for the system resolver timeout // (~30s by default) past the caller's connection timeout. - struct gaicb request {}; + struct gaicb request{}; struct gaicb *requests[1] = {&request}; - struct sigevent sevp {}; - struct timespec timeout { - timeout_sec, 0 - }; + struct sigevent sevp{}; + struct timespec timeout{timeout_sec, 0}; request.ar_name = node; request.ar_service = service; @@ -2948,8 +2974,21 @@ EncodingType encoding_type(const Request &req, return best; } +// `content_type` is taken separately because a file-backed response has not +// been given one yet when its coding has to be decided. +EncodingType encoding_type(const Request &req, const Response &res, + const std::string &content_type) { + // The response already names a content coding of its own: a handler serving + // a body it encoded itself (pre-compressed static assets, say), or a mount + // point whose headers name the coding its files are stored in. Applying one + // on top of that would double-encode the body and append a second + // `Content-Encoding` field line. + if (res.has_header("Content-Encoding")) { return EncodingType::None; } + return encoding_type(req, content_type); +} + EncodingType encoding_type(const Request &req, const Response &res) { - return encoding_type(req, res.get_header_value("Content-Type")); + return encoding_type(req, res, res.get_header_value("Content-Type")); } std::unique_ptr make_compressor(EncodingType type) { @@ -3677,6 +3716,17 @@ bool is_chunked_transfer_encoding(const Headers &headers) { return case_ignore::equal(last_coding, "chunked"); } +bool has_conflicting_content_length(const Headers &headers) { + // RFC 9112 §6.3: a message carrying both Transfer-Encoding and a non-zero + // Content-Length is framed ambiguously. The body readers here delimit it by + // the transfer coding and drop Content-Length, while an intermediary may do + // the reverse, so the two disagree on where the body ends and a reused + // connection is desynchronised (request/response smuggling). Content-Length: + // 0 is tolerated for compatibility with existing peers. + return has_header(headers, "Transfer-Encoding") && + get_header_value_u64(headers, "Content-Length", 0, 0) > 0; +} + template bool prepare_content_receiver(T &x, int &status, ContentReceiverWithProgress receiver, @@ -4035,7 +4085,7 @@ void set_file_content_provider(Response &res, return true; }); - res.file_content_encoding_ = encoding; + res.content_coding_ = encoding; } template @@ -4361,13 +4411,20 @@ bool parse_range_header(const std::string &s, Ranges &ranges) try { ssize_t first = -1; if (!lhs.empty()) { - ssize_t v; - auto res = detail::from_chars(lhs.data(), lhs.data() + lhs.size(), v); - if (res.ec == std::errc{}) { first = v; } + // Reject an overflowing first-byte-pos; treating it as absent (-1) + // would turn the range into a suffix range. + auto res = + detail::from_chars(lhs.data(), lhs.data() + lhs.size(), first); + if (res.ec != std::errc{}) { + all_valid_ranges = false; + return; + } } ssize_t last = -1; if (!rhs.empty()) { + // An overflowing last-byte-pos is past any content length, so keeping + // -1 ("remainder", RFC 9110 14.1.2) is correct here. ssize_t v; auto res = detail::from_chars(rhs.data(), rhs.data() + rhs.size(), v); if (res.ec == std::errc{}) { last = v; } @@ -6902,7 +6959,7 @@ void Response::set_content(const char *s, size_t n, auto rng = headers.equal_range("Content-Type"); headers.erase(rng.first, rng.second); set_header("Content-Type", content_type); - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content(const std::string &s, @@ -6917,7 +6974,7 @@ void Response::set_content(std::string &&s, auto rng = headers.equal_range("Content-Type"); headers.erase(rng.first, rng.second); set_header("Content-Type", content_type); - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content_provider( @@ -6928,7 +6985,7 @@ void Response::set_content_provider( if (in_length > 0) { content_provider_ = std::move(provider); } content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_content_provider( @@ -6939,7 +6996,7 @@ void Response::set_content_provider( content_provider_ = detail::ContentProviderAdapter(std::move(provider)); content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = false; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_chunked_content_provider( @@ -6950,7 +7007,7 @@ void Response::set_chunked_content_provider( content_provider_ = detail::ContentProviderAdapter(std::move(provider)); content_provider_resource_releaser_ = std::move(resource_releaser); is_chunked_content_provider_ = true; - file_content_encoding_ = detail::EncodingType::None; + content_coding_ = detail::EncodingType::None; } void Response::set_file_content(const std::string &path, @@ -7991,12 +8048,19 @@ ssize_t WebSocketSSLStream::read(char *ptr, size_t size) { needs_readable || (err.code == tls::ErrorCode::SyscallError && WSAGetLastError() == WSAETIMEDOUT); #endif - if (!needs_readable && err.code != tls::ErrorCode::WantWrite) { return -1; } + if (!needs_readable && err.code != tls::ErrorCode::WantWrite) { + error_ = Error::Read; + return -1; + } if (!(needs_readable ? wait_readable() : wait_writable())) { error_ = Error::Timeout; return -1; } } + // Out of retries. Recording a reason matters: a caller that reads get_error() + // to tell a timeout from a close would otherwise see whatever the previous + // failure left behind (error_ is never cleared on success). + error_ = Error::Read; return -1; } @@ -8653,9 +8717,10 @@ Server::write_content_with_provider(Stream &strm, const Request &req, } } else { if (res.is_chunked_content_provider_) { - auto type = detail::encoding_type(req, res); - - auto compressor = detail::make_compressor(type); + // Use the coding `apply_ranges()` chose when it wrote the headers; + // re-negotiating here would disagree with them, e.g. once a handler's + // own Content-Encoding header suppresses the negotiation. + auto compressor = detail::make_compressor(res.content_coding_); if (!compressor) { compressor = detail::make_unique(); } @@ -8881,7 +8946,8 @@ bool Server::handle_file_request(Request &req, Response &res) { auto encoding = detail::EncodingType::None; if (static_file_compression_) { content_type = content_type_of(); - encoding = static_file_encoding(req, content_type, stat.size()); + encoding = + static_file_encoding(req, res, content_type, stat.size()); } // The ETag names the representation actually sent, so a client that @@ -9296,8 +9362,10 @@ bool Server::dispatch_request(Request &req, Response &res, // the ETag, which has to name the representation actually sent, and // `apply_static_file_compression()` go through this, so the two cannot drift // apart. -detail::EncodingType Server::static_file_encoding( - const Request &req, const std::string &content_type, size_t length) const { +detail::EncodingType +Server::static_file_encoding(const Request &req, const Response &res, + const std::string &content_type, + size_t length) const { if (!static_file_compression_) { return detail::EncodingType::None; } // Nothing to compress, and an empty file already answers with @@ -9322,14 +9390,14 @@ detail::EncodingType Server::static_file_encoding( return detail::EncodingType::None; } - return detail::encoding_type(req, content_type); + return detail::encoding_type(req, res, content_type); } // Compresses a file-backed content provider into `res.body` and takes over the // framing headers. Returns false when the response is left untouched. bool Server::apply_static_file_compression(const Request &req, Response &res) const { - auto type = res.file_content_encoding_; + auto type = res.content_coding_; if (type == detail::EncodingType::None || !res.content_provider_) { return false; } @@ -9353,7 +9421,7 @@ bool Server::apply_static_file_compression(const Request &req, res.content_provider_success_ = true; res.content_provider_ = nullptr; res.content_length_ = 0; - res.file_content_encoding_ = detail::EncodingType::None; + res.content_coding_ = detail::EncodingType::None; res.set_header("Content-Encoding", detail::encoding_name(type)); res.set_header("Vary", "Accept-Encoding"); @@ -9412,6 +9480,7 @@ void Server::apply_ranges(const Request &req, Response &res, if (res.content_provider_) { if (res.is_chunked_content_provider_) { res.set_header("Transfer-Encoding", "chunked"); + res.content_coding_ = type; if (type != detail::EncodingType::None) { res.set_header("Content-Encoding", detail::encoding_name(type)); res.set_header("Vary", "Accept-Encoding"); @@ -9568,8 +9637,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr, // coding is not chunked, which leaves the body length undeterminable. The // latter must not fall through to the "no body" path, or the body bytes are // parsed as the next request on a persistent connection. - if (req.has_header("Transfer-Encoding") && - (req.get_header_value_u64("Content-Length") > 0 || + if (detail::has_conflicting_content_length(req.headers) || + (req.has_header("Transfer-Encoding") && !detail::is_chunked_transfer_encoding(req.headers))) { connection_closed = true; res.status = StatusCode::BadRequest_400; @@ -9734,7 +9803,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr, auto ws_strm = std::unique_ptr(new detail::WebSocketSSLStream( strm.socket(), const_cast(req.ssl), - CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0, + CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND, 0, write_timeout_sec_, write_timeout_usec_)); ws::WebSocket ws(std::move(ws_strm), req, true, websocket_ping_interval_sec_, @@ -9744,7 +9813,8 @@ Server::process_request(Stream &strm, const std::string &remote_addr, } #endif // Use WebSocket-specific read timeout instead of HTTP timeout - strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND, 0); + strm.set_read_timeout(CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND, + 0); ws::WebSocket ws(strm, req, true, websocket_ping_interval_sec_, websocket_max_missed_pongs_); entry.handler(req, ws); @@ -9808,7 +9878,7 @@ Server::process_request(Stream &strm, const std::string &remote_addr, detail::set_file_content_provider( res, mm, content_type, - static_file_encoding(req, content_type, mm->size())); + static_file_encoding(req, res, content_type, mm->size())); } } @@ -10228,8 +10298,12 @@ Result ClientImpl::send_(Request &&req) { void ClientImpl::prepare_default_headers(Request &r, bool for_stream, const std::string &ct) { (void)for_stream; - for (const auto &header : default_headers_) { - if (!r.has_header(header.first)) { r.headers.insert(header); } + // Default headers are meant for the origin and may carry its credentials, so + // keep them off the CONNECT request the proxy reads. + if (r.method != "CONNECT") { + for (const auto &header : default_headers_) { + if (!r.has_header(header.first)) { r.headers.insert(header); } + } } // RFC 9110 5.3 recommends sending control data such as Host first, so @@ -10379,6 +10453,17 @@ ClientImpl::open_stream(const std::string &method, const std::string &path, return handle; } + // Same framing check as ClientImpl::process_request(). A HEAD or bodyless + // (204/304) response legitimately carries framing headers with no body. + if (method != "HEAD" && + handle.response->status != StatusCode::NoContent_204 && + handle.response->status != StatusCode::NotModified_304 && + detail::has_conflicting_content_length(handle.response->headers)) { + handle.error = Error::Read; + handle.response.reset(); + return handle; + } + handle.body_reader_.stream = handle.stream_; handle.body_reader_.payload_max_length = payload_max_length_; @@ -10910,24 +10995,24 @@ bool ClientImpl::write_request(Stream &strm, Request &req, } } - if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { - if (!req.has_header("Authorization")) { + // A CONNECT request is read by the proxy; everything sent through the tunnel + // it opens is read by the origin. Each credential goes only to its own hop. + auto is_connect = req.method == "CONNECT"; + + if (!is_connect && !req.has_header("Authorization")) { + if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) { req.headers.insert(make_basic_authentication_header( basic_auth_username_, basic_auth_password_, false)); - } - } - - if (!bearer_token_auth_token_.empty()) { - if (!req.has_header("Authorization")) { + } else if (!bearer_token_auth_token_.empty()) { req.headers.insert(make_bearer_token_authentication_header( bearer_token_auth_token_, false)); } } - // Proxy-Authorization is only sent when the proxy is actually used for - // this target — otherwise NO_PROXY-matched requests would leak proxy - // credentials directly to the destination server. - if (is_proxy_enabled_for_host(host_)) { + // Proxy-Authorization is only sent when the proxy reads this message — + // otherwise NO_PROXY-matched requests, and requests inside a TLS tunnel, + // would leak proxy credentials to the destination server. + if (is_proxy_enabled_for_host(host_) && (!is_ssl() || is_connect)) { if (!proxy_basic_auth_username_.empty() && !proxy_basic_auth_password_.empty() && !req.has_header("Proxy-Authorization")) { @@ -11323,6 +11408,17 @@ bool ClientImpl::process_request(Stream &strm, Request &req, // Body if ((res.status != StatusCode::NoContent_204) && req.method != "HEAD" && req.method != "CONNECT") { + // Reject ambiguous framing (RFC 9112 §6.3). Unlike a request, a response + // whose final transfer coding is not chunked is not ambiguous: its body + // runs until the server closes the connection, so it is not rejected. + // HEAD/204 are excluded above and a 304 carries no body. + if (res.status != StatusCode::NotModified_304 && + detail::has_conflicting_content_length(res.headers)) { + error = Error::Read; + output_error_log(error, &req); + return false; + } + auto redirect = 300 < res.status && res.status < 400 && res.status != StatusCode::NotModified_304 && follow_location_; @@ -17562,8 +17658,16 @@ ReadResult WebSocket::read(std::string &msg) { std::string payload; bool fin; - if (!impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_, - CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + impl::FrameRead r = + impl::read_websocket_frame(strm_, opcode, payload, fin, is_server_, + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH); + // A timeout landed on a frame boundary: the connection is untouched and + // still usable, so hand control back without closing it. That is only + // useful to a caller who asked for the timeout; the compile-time default + // is a backstop against a peer gone quiet, and elapsing it closes the + // connection so a plain `while (ws.read(msg))` loop ends. + if (r == impl::FrameRead::Timeout && read_timeout_set_) { return Timeout; } + if (r != impl::FrameRead::Ok) { closed_ = true; return Fail; } @@ -17600,9 +17704,14 @@ ReadResult WebSocket::read(std::string &msg) { Opcode cont_opcode; std::string cont_payload; bool cont_fin; - if (!impl::read_websocket_frame( + // A timeout is not reportable here: half of a fragmented message is + // already in `msg` and read() has no way to resume it, so it is a + // failure like any other. Timeouts are only ever seen on a message + // boundary. + if (impl::read_websocket_frame( strm_, cont_opcode, cont_payload, cont_fin, is_server_, - CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH)) { + CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH) != + impl::FrameRead::Ok) { closed_ = true; return Fail; } @@ -17696,7 +17805,8 @@ void WebSocket::close(CloseStatus status, const std::string &reason) { Opcode op; std::string resp; bool fin; - while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125)) { + while (impl::read_websocket_frame(strm_, op, resp, fin, is_server_, 125) == + impl::FrameRead::Ok) { if (op == Opcode::Close) { break; } } } @@ -17741,6 +17851,15 @@ const Request &WebSocket::request() const { return req_; } bool WebSocket::is_open() const { return !closed_; } +void WebSocket::set_read_timeout(time_t sec, time_t usec) { + // 0 waits forever here, as it does for SO_RCVTIMEO. The stream waits with + // poll(), where 0 would instead mean "return immediately", so hand it the + // negative poll uses for an unbounded wait. + if (sec == 0 && usec == 0) { sec = -1; } + strm_.set_read_timeout(sec, usec); + read_timeout_set_ = true; +} + // WebSocketClient implementation WebSocketClient::WebSocketClient( const std::string &scheme_host_port_path, const Headers &headers) @@ -17843,6 +17962,16 @@ void WebSocketClient::shutdown_and_close() { bool WebSocketClient::create_stream(std::unique_ptr &strm, Error &error, int &ssl_error, uint64_t &ssl_backend_error) { + // A read timeout of 0 means "wait forever", the way SO_RCVTIMEO reads it. + // The streams wait with poll(), where 0 instead means "return immediately", + // so they are given the negative poll uses for an unbounded wait. + auto unbounded = read_timeout_sec_ == 0 && read_timeout_usec_ == 0; + time_t strm_read_sec = unbounded ? -1 : read_timeout_sec_; + time_t strm_read_usec = unbounded ? 0 : read_timeout_usec_; + // The handshake belongs to establishing the connection, so an unset read + // timeout leaves it bounded by the connection timeout instead of forever. + time_t hs_sec = unbounded ? connection_timeout_sec_ : read_timeout_sec_; + time_t hs_usec = unbounded ? connection_timeout_usec_ : read_timeout_usec_; #ifdef CPPHTTPLIB_SSL_ENABLED if (is_ssl_) { // A plain flag rather than SSLClient::load_certs()'s call_once: connect() @@ -17862,8 +17991,8 @@ bool WebSocketClient::create_stream(std::unique_ptr &strm, detail::ClientTlsSessionError tls_error; if (!detail::setup_client_tls_session(host_, tls_ctx_, tls_session_, sock_, server_certificate_verification_, - read_timeout_sec_, read_timeout_usec_, - &tls_error, options)) { + hs_sec, hs_usec, &tls_error, + options)) { error = tls_error.error; ssl_error = tls_error.ssl_error; ssl_backend_error = tls_error.backend_error; @@ -17871,17 +18000,19 @@ bool WebSocketClient::create_stream(std::unique_ptr &strm, } strm = std::unique_ptr(new detail::WebSocketSSLStream( - sock_, tls_session_, read_timeout_sec_, read_timeout_usec_, - write_timeout_sec_, write_timeout_usec_)); + sock_, tls_session_, strm_read_sec, strm_read_usec, write_timeout_sec_, + write_timeout_usec_)); return true; } #else (void)error; (void)ssl_error; (void)ssl_backend_error; + (void)hs_sec; + (void)hs_usec; #endif strm = std::unique_ptr( - new detail::SocketStream(sock_, read_timeout_sec_, read_timeout_usec_, + new detail::SocketStream(sock_, strm_read_sec, strm_read_usec, write_timeout_sec_, write_timeout_usec_)); return true; } @@ -17951,6 +18082,9 @@ Result WebSocketClient::connect() { ws_ = std::unique_ptr(new WebSocket(std::move(strm), req, false, websocket_ping_interval_sec_, websocket_max_missed_pongs_)); + // The stream was created with the timeout already; tell the WebSocket + // whether it came from the caller, so read() knows to report it as Timeout. + ws_->read_timeout_set_ = read_timeout_set_; return Result{Error::Success, upgrade.status, std::move(upgrade.headers)}; } @@ -17983,6 +18117,10 @@ const std::string &WebSocketClient::subprotocol() const { void WebSocketClient::set_read_timeout(time_t sec, time_t usec) { read_timeout_sec_ = sec; read_timeout_usec_ = usec; + read_timeout_set_ = true; + // The members above only seed the next connect(); read() consults the + // stream, so an already-open connection has to be told directly. + if (ws_) { ws_->set_read_timeout(sec, usec); } } void WebSocketClient::set_write_timeout(time_t sec, time_t usec) { diff --git a/vendor/cpp-httplib/httplib.h b/vendor/cpp-httplib/httplib.h index ca7c96a41a38..a3a2ff45afc4 100644 --- a/vendor/cpp-httplib/httplib.h +++ b/vendor/cpp-httplib/httplib.h @@ -8,8 +8,8 @@ #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.54.1" -#define CPPHTTPLIB_VERSION_NUM "0x003601" +#define CPPHTTPLIB_VERSION "0.56.0" +#define CPPHTTPLIB_VERSION_NUM "0x003800" #ifdef _WIN32 #if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00 @@ -215,8 +215,36 @@ #define CPPHTTPLIB_WEBSOCKET_MAX_PAYLOAD_LENGTH 16777216 #endif -#ifndef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND -#define CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND 300 +// One macro used to set the read timeout for both sides. They want different +// defaults: a client's read timeout is the caller's own tool (it waits forever +// until asked not to), while a server keeps a ceiling that reclaims a worker +// from a peer that has gone quiet. The old name still works and sets both. +#ifdef CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#pragma message( \ + "CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND is deprecated; define " \ + "CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND and/or " \ + "CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND instead") +#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND \ + CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#endif +#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND \ + CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND +#endif +#endif + +// 0 waits forever. A read timeout is how a caller gets control back to send on +// the same connection; it is not a liveness check (that is ping/pong). Only a +// timeout set at runtime through set_read_timeout() is reported as +// ws::Timeout; when one of these compile-time defaults elapses, read() returns +// ws::Fail and closes the connection. +#ifndef CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND 0 +#endif + +#ifndef CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND 300 #endif #ifndef CPPHTTPLIB_WEBSOCKET_CLOSE_TIMEOUT_SECOND @@ -1817,10 +1845,12 @@ struct Response { std::string file_content_path_; std::string file_content_content_type_; - // Content coding chosen for a file-backed content provider, decided once - // where the file is opened so that the ETag and the body cannot disagree. - // `EncodingType::None` for every other kind of response. - detail::EncodingType file_content_encoding_ = detail::EncodingType::None; + // Content coding chosen for the response body, decided once so that the + // headers and the body cannot disagree: where the file is opened for a + // file-backed content provider (keeping the ETag honest), and in + // `apply_ranges()` for a chunked content provider. `EncodingType::None` + // for every other kind of response. + detail::EncodingType content_coding_ = detail::EncodingType::None; }; enum class Error { @@ -2359,6 +2389,7 @@ class Server { bool parse_request_line(const char *s, Request &req) const; detail::EncodingType static_file_encoding(const Request &req, + const Response &res, const std::string &content_type, size_t length) const; bool apply_static_file_compression(const Request &req, Response &res) const; @@ -3663,6 +3694,9 @@ ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags); EncodingType encoding_type(const Request &req, const std::string &content_type); +EncodingType encoding_type(const Request &req, const Response &res, + const std::string &content_type); + EncodingType encoding_type(const Request &req, const Response &res); class BufferStream final : public Stream { @@ -4345,7 +4379,11 @@ enum class CloseStatus : uint16_t { InternalError = 1011, }; -enum ReadResult : int { Fail = 0, Text = 1, Binary = 2 }; +// Timeout is returned only when a read timeout was set and it elapsed before +// any byte of a frame arrived: nothing was consumed and the connection is +// still open, so the caller can send on it and read again. `msg` is left +// untouched, so a `while (ws.read(msg))` loop must not treat it as a message. +enum ReadResult : int { Fail = 0, Text = 1, Binary = 2, Timeout = 3 }; // Result of WebSocketClient::connect(). Truthy only when the WebSocket // upgrade handshake fully succeeded. On failure error() identifies the @@ -4405,6 +4443,18 @@ class WebSocket { const Request &request() const; bool is_open() const; + // Bound how long read() waits before returning Timeout. 0 waits forever. + // A server handler owns its connection's timeout this way; a client sets it + // through WebSocketClient. Safe to call while another thread is in read(). + // + // Only a timeout set here is reported as Timeout. The compile-time default + // (CPPHTTPLIB_WEBSOCKET_SERVER_READ_TIMEOUT_SECOND) is a backstop rather + // than a request for control, so when it elapses read() returns Fail and + // closes the connection, and `while (ws.read(msg))` ends as it always has. + void set_read_timeout(time_t sec, time_t usec = 0); + template + void set_read_timeout(const std::chrono::duration &duration); + private: friend class httplib::Server; friend class WebSocketClient; @@ -4440,6 +4490,10 @@ class WebSocket { int max_missed_pongs_; int unacked_pings_ = 0; std::atomic closed_{false}; + // Set once the caller has bounded read() through set_read_timeout(). Until + // then the timeout in effect is the compile-time default, and elapsing it + // is a failure that closes the connection, not a Timeout. + std::atomic read_timeout_set_{false}; std::mutex write_mutex_; // Owned by whichever thread is parsing frames off strm_. Only one thread // may do so: read_websocket_frame() reads a payload until it has the whole @@ -4527,8 +4581,9 @@ class WebSocketClient { bool is_valid_ = false; socket_t sock_ = INVALID_SOCKET; std::unique_ptr ws_; - time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_READ_TIMEOUT_SECOND; + time_t read_timeout_sec_ = CPPHTTPLIB_WEBSOCKET_CLIENT_READ_TIMEOUT_SECOND; time_t read_timeout_usec_ = 0; + bool read_timeout_set_ = false; // see WebSocket::read_timeout_set_ time_t write_timeout_sec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND; time_t write_timeout_usec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND; time_t websocket_ping_interval_sec_ = @@ -4560,6 +4615,13 @@ class WebSocketClient { #endif }; +template +inline void WebSocket::set_read_timeout( + const std::chrono::duration &duration) { + detail::duration_to_sec_and_usec( + duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); }); +} + template inline void WebSocketClient::set_read_timeout( const std::chrono::duration &duration) { @@ -4586,8 +4648,14 @@ namespace impl { bool is_valid_utf8(const std::string &s); -bool read_websocket_frame(Stream &strm, Opcode &opcode, std::string &payload, - bool &fin, bool expect_masked, size_t max_len); +// Three states, because a failure that consumed bytes and one that consumed +// none are not the same thing: the first has left the stream in the middle of +// a frame and the connection cannot be reused, the second can just be retried. +enum class FrameRead { Ok, Fail, Timeout }; + +FrameRead read_websocket_frame(Stream &strm, Opcode &opcode, + std::string &payload, bool &fin, + bool expect_masked, size_t max_len); } // namespace impl From e192abb406a35e9fbd2859892286144a6bda6ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Gallou=C3=ABt?= Date: Sat, 12 Sep 2026 11:23:54 +0200 Subject: [PATCH 56/65] server : add missing headers (#28795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Adrien Gallouët --- tools/server/server-common.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index eade7db21256..483391333538 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include #ifdef _WIN32 // windows.h defines min and max as macros, which breaks std::min and std::max From c8edceb0615d859b2c0d9fa08c3ed07020ebf9b8 Mon Sep 17 00:00:00 2001 From: thelittlefireman <5165783+thelittlefireman@users.noreply.github.com> Date: Sat, 12 Sep 2026 11:26:53 +0200 Subject: [PATCH 57/65] ggml-cuda: hip add specific config table for AMD GCN (#27841) --- ggml/src/ggml-cuda/mmq-config-gcn.cuh | 281 ++++++++++++++++++++++++++ ggml/src/ggml-cuda/mmq.cuh | 8 +- 2 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 ggml/src/ggml-cuda/mmq-config-gcn.cuh diff --git a/ggml/src/ggml-cuda/mmq-config-gcn.cuh b/ggml/src/ggml-cuda/mmq-config-gcn.cuh new file mode 100644 index 000000000000..24af2ef2b05b --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-gcn.cuh @@ -0,0 +1,281 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_gcn(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 24afedd1432b..6923f3510c03 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -218,6 +218,7 @@ struct ggml_cuda_mmq_config { #include "mmq-config-ampere.cuh" #include "mmq-config-blackwell.cuh" +#include "mmq-config-gcn.cuh" #include "mmq-config-cdna.cuh" #include "mmq-config-rdna2.cuh" #include "mmq-config-rdna3.cuh" @@ -228,6 +229,9 @@ struct ggml_cuda_mmq_config { static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc) { if (GGML_CUDA_CC_IS_AMD(cc)) { + if (GGML_CUDA_CC_IS_GCN(cc)) { + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); + } if (GGML_CUDA_CC_IS_CDNA(cc)) { return ggml_cuda_mmq_get_config_cdna(type, J, fallback); } @@ -256,7 +260,9 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback) { #ifdef GGML_USE_HIP -#ifdef CDNA +#ifdef GCN + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); +#elif defined(CDNA) return ggml_cuda_mmq_get_config_cdna(type, J, fallback); #elif defined(RDNA4) return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); From 56381e407c0ccfb3a6f71e668a27a901001d22ce Mon Sep 17 00:00:00 2001 From: MiaoMing Chen Date: Sat, 12 Sep 2026 17:50:35 +0800 Subject: [PATCH 58/65] server : allow model downloads at model limit fix issue #26809 (#28530) --- tools/server/server-models.cpp | 3 ++- tools/server/tests/unit/test_router.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 4984f1be68cc..3d134acf3621 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1129,7 +1129,8 @@ void server_models::load(const std::string & name, const load_options & opts) { // exceeding models_max. Without this, the window between unload_lru() // releasing its lock and this lock_guard acquiring allows multiple // threads to each observe capacity and all proceed to load. - if (base_params.models_max > 0) { + // Download workers do not use models_max slots. + if (opts.mode == SERVER_CHILD_MODE_NORMAL && base_params.models_max > 0) { size_t count_active = 0; for (const auto & m : mapping) { if (m.second.meta.is_running()) { diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index e4b7f9fe4826..bae156517749 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -540,13 +540,17 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i def test_router_download_model(): - """Case 1: download a model, verify SSE events and GET /models.""" + """Case 1: download a model at the model limit, verify SSE events and GET /models.""" global server + server.models_max = 1 server.start() # Ensure the model is not present before we start server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + # A download worker must not consume or evict a model slot + _load_model_and_wait(MODEL_B, timeout=120) + sse_events: list = [] stop = threading.Event() sse_ready = threading.Event() @@ -580,6 +584,7 @@ def test_router_download_model(): # Model should now appear in GET /models ids = _get_model_ids(is_reload=False) assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download" + assert _get_model_status(MODEL_B) == "loaded" def test_router_delete_model(): From 3057bb66c86c46d5781e50e85462a760ba7d1feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Gallou=C3=ABt?= Date: Sat, 12 Sep 2026 16:09:46 +0200 Subject: [PATCH 59/65] ui : add cache (#28802) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Adrien Gallouët --- scripts/ui-assets.cmake | 157 ++++++++++++++++++++++++++-------------- 1 file changed, 104 insertions(+), 53 deletions(-) diff --git a/scripts/ui-assets.cmake b/scripts/ui-assets.cmake index 402f95bd4f34..dec34ebda96c 100644 --- a/scripts/ui-assets.cmake +++ b/scripts/ui-assets.cmake @@ -21,6 +21,7 @@ set(DIST_DIR "${UI_BINARY_DIR}/dist") set(SRC_DIST_DIR "${UI_SOURCE_DIR}/dist") set(WORK_DIR "${UI_BINARY_DIR}/ui-src") set(STAMP_FILE "${UI_BINARY_DIR}/.ui-stamp") +set(EMBED_STAMP "${UI_BINARY_DIR}/.ui-embed.sha256") set(UI_CPP "${UI_BINARY_DIR}/ui.cpp") set(UI_H "${UI_BINARY_DIR}/ui.h") @@ -141,9 +142,59 @@ function(ui_validate_assets files in_dir) endfunction() # Generate ui.cpp/ui.h embedding every file of ${dist_dir} (empty table when -# it has no index.html). When LLAMA_UI_GZIP is enabled, assets are compressed -# first and served pre-gzipped (llama_ui_use_gzip()). +# it has no index.html), gzip-compressed when LLAMA_UI_GZIP is enabled. function(emit_files dist_dir) + set(UI_TEMPLATE_DIR "${LLAMA_SOURCE_DIR}/tools/ui") + + # Collect the asset list once and reuse it for the fingerprint, + # validation, compression and embedding. + set(assets "") + if(EXISTS "${dist_dir}/index.html") + file(GLOB_RECURSE assets + LIST_DIRECTORIES false + RELATIVE "${dist_dir}" + "${dist_dir}/*") + list(FILTER assets EXCLUDE REGEX "^_gzip/") + list(SORT assets) + endif() + + if(LLAMA_UI_GZIP AND NOT DEFINED ENV{SOURCE_DATE_EPOCH}) + # Zero the gzip header timestamp so identical inputs give identical + # bytes (and therefore stable ETags) on every machine. + set(ENV{SOURCE_DATE_EPOCH} 0) + endif() + + # Fingerprint of every input that determines ui.cpp/ui.h: compression + # settings, the asset tree (names + SHA-256) and this script + templates. + set(fp "${LLAMA_UI_GZIP}|$ENV{SOURCE_DATE_EPOCH}|${CMAKE_VERSION}\n") + foreach(f ${assets}) + file(SHA256 "${dist_dir}/${f}" h) + string(APPEND fp "${f} ${h}\n") + endforeach() + foreach(g + "${CMAKE_CURRENT_FUNCTION_LIST_FILE}" + "${UI_TEMPLATE_DIR}/ui.h.in" + "${UI_TEMPLATE_DIR}/ui.cpp.in") + file(SHA256 "${g}" h) + string(APPEND fp "gen ${h}\n") + endforeach() + string(SHA256 fingerprint "${fp}") + + if(EXISTS "${EMBED_STAMP}" AND EXISTS "${UI_CPP}" AND EXISTS "${UI_H}") + file(READ "${EMBED_STAMP}" fp_saved) + string(STRIP "${fp_saved}" fp_saved) + if(fp_saved STREQUAL "${fingerprint}") + message(STATUS "UI: assets unchanged, skipping embedding") + return() + endif() + endif() + + # Drop the old stamp up front so a crash mid-generation cannot leave + # outputs and stamp out of sync. + file(REMOVE "${EMBED_STAMP}") + + ui_validate_assets("${assets}" "${dist_dir}") + set(embed_dir "${dist_dir}") set(use_gzip FALSE) @@ -156,21 +207,11 @@ function(emit_files dist_dir) endif() if(LLAMA_UI_GZIP) # Compress every asset into a parallel _gzip/ tree under the build - # directory (never write into the source or dist tree); the - # structure stays the same: /abc/def --> /_gzip/abc/def. - # FORMAT raw produces a bare gzip stream (no archive container) - # that can be served with Content-Encoding: gzip. SOURCE_DATE_EPOCH - # zeroes the header timestamp so identical inputs give identical - # bytes (and therefore stable ETags) on every machine. - if(NOT DEFINED ENV{SOURCE_DATE_EPOCH}) - set(ENV{SOURCE_DATE_EPOCH} 0) - endif() + # directory, served with Content-Encoding: gzip. set(gzip_root "${UI_BINARY_DIR}/ui-gzip") set(gzip_dir "${gzip_root}/_gzip") file(REMOVE_RECURSE "${gzip_root}") - file(GLOB_RECURSE all_files RELATIVE "${dist_dir}" "${dist_dir}/*") - list(FILTER all_files EXCLUDE REGEX "^_gzip/") - foreach(f ${all_files}) + foreach(f IN LISTS assets) get_filename_component(asset_path "${dist_dir}/${f}" REALPATH) get_filename_component(dst_dir "${gzip_dir}/${f}" DIRECTORY) file(MAKE_DIRECTORY "${dst_dir}") @@ -187,21 +228,10 @@ function(emit_files dist_dir) endif() endif() - set(assets "") - if(EXISTS "${embed_dir}/index.html") - file(GLOB_RECURSE assets RELATIVE "${embed_dir}" "${embed_dir}/*") - list(FILTER assets EXCLUDE REGEX "^_gzip/") - list(SORT assets) - ui_validate_assets("${assets}" "${embed_dir}") - endif() - list(LENGTH assets n_assets) - # Only the per-asset data arrays and table rows are built here; all - # static C++ lives in the ui.h.in / ui.cpp.in templates. configure_file - # rewrites an output only when its contents change, so the library is - # not recompiled needlessly. @ONLY keeps ${...} in the content literal; - # mime types come from a fixed list. + # Per-asset arrays and table rows go into the ui.h.in / ui.cpp.in templates; + # configure_file only rewrites on content change, avoiding needless recompiles. set(ASSET_ARRAYS "") set(ASSET_TABLE "") set(idx 0) @@ -235,9 +265,11 @@ function(emit_files dist_dir) set(USE_GZIP true) endif() - set(UI_TEMPLATE_DIR "${LLAMA_SOURCE_DIR}/tools/ui") configure_file("${UI_TEMPLATE_DIR}/ui.h.in" "${UI_H}" @ONLY) configure_file("${UI_TEMPLATE_DIR}/ui.cpp.in" "${UI_CPP}" @ONLY) + + # Write the embed stamp last, after both generated files succeeded. + file(WRITE "${EMBED_STAMP}" "${fingerprint}") message(STATUS "UI: embedded ${n_assets} assets") endfunction() @@ -419,16 +451,8 @@ function(hf_download version out_var out_resolved) message(STATUS "UI: downloading from ${resolved}: ${base}/dist.tar.gz") - file(DOWNLOAD "${base}/dist.tar.gz?download=true" "${archive}" - STATUS status TIMEOUT 300 ${auth_headers} - ) - list(GET status 0 rc) - if(NOT rc EQUAL 0) - list(GET status 1 errmsg) - message(STATUS "UI: download dist.tar.gz from ${resolved} failed: ${errmsg}") - continue() - endif() - + # Fetch the checksum first: when the archive we already have matches + # it, the expensive download is skipped and only extraction repeats. file(DOWNLOAD "${base}/dist.tar.gz.sha256?download=true" "${archive}.sha256" STATUS status TIMEOUT 30 ${auth_headers} ) @@ -439,17 +463,44 @@ function(hf_download version out_var out_resolved) continue() endif() - # Validate sha256 checkums + # Validate the sha256 checksum: reject anything that is not a full + # 64-hex-digit digest before touching the archive. file(READ "${archive}.sha256" expected) string(REGEX MATCH "^[0-9a-fA-F]+" expected "${expected}") string(TOLOWER "${expected}" expected) - file(SHA256 "${archive}" actual) - if("${expected}" STREQUAL "" OR NOT "${actual}" STREQUAL "${expected}") - message(STATUS "UI: checksum mismatch for dist.tar.gz from ${resolved}") + string(LENGTH "${expected}" expected_len) + if(NOT expected_len EQUAL 64) + message(STATUS "UI: invalid checksum from ${resolved}") continue() endif() - # Clear DIST_DIR to remove stale files first + set(actual "") + if(EXISTS "${archive}") + file(SHA256 "${archive}" actual) + endif() + + if("${actual}" STREQUAL "${expected}") + message(STATUS "UI: local dist.tar.gz matches checksum from ${resolved}, skipping download") + else() + file(DOWNLOAD "${base}/dist.tar.gz?download=true" "${archive}" + STATUS status TIMEOUT 300 ${auth_headers} + ) + list(GET status 0 rc) + if(NOT rc EQUAL 0) + list(GET status 1 errmsg) + message(STATUS "UI: download dist.tar.gz from ${resolved} failed: ${errmsg}") + continue() + endif() + + file(SHA256 "${archive}" actual) + if(NOT "${actual}" STREQUAL "${expected}") + message(STATUS "UI: checksum mismatch for dist.tar.gz from ${resolved}") + continue() + endif() + endif() + + # Remove the stamp with the dist tree it describes, together. + file(REMOVE "${STAMP_FILE}") file(REMOVE_RECURSE "${DIST_DIR}") file(ARCHIVE_EXTRACT INPUT "${archive}" DESTINATION "${DIST_DIR}") @@ -495,27 +546,27 @@ endif() if(NOT provisioned AND HF_ENABLED) resolve_version(VERSION) + # Stamp a successful HF download: records bucket + requested version and + # lets later steps distinguish downloaded assets from locally built ones. + set(stamp_key "${HF_BUCKET}|${VERSION}") + set(stamp_ok FALSE) - if(EXISTS "${STAMP_FILE}" AND NOT "${VERSION}" STREQUAL "") + if(EXISTS "${STAMP_FILE}" AND EXISTS "${DIST_DIR}/index.html" AND NOT "${VERSION}" STREQUAL "") file(READ "${STAMP_FILE}" stamped) string(STRIP "${stamped}" stamped) - if("${stamped}" STREQUAL "${VERSION}") + if(stamped STREQUAL "${stamp_key}") set(stamp_ok TRUE) endif() endif() - set(have_assets FALSE) - if(EXISTS "${DIST_DIR}/index.html") - set(have_assets TRUE) - endif() - if(stamp_ok AND have_assets) - message(STATUS "UI: HF stamp '${stamped}' matches version, skipping HF fetch") + if(stamp_ok) + message(STATUS "UI: HF stamp matches '${stamp_key}', skipping HF fetch") set(provisioned TRUE) else() hf_download("${VERSION}" HF_OK HF_RESOLVED) if(HF_OK) - file(WRITE "${STAMP_FILE}" "${HF_RESOLVED}") - message(STATUS "UI: HF download succeeded, stamp updated (${HF_RESOLVED})") + file(WRITE "${STAMP_FILE}" "${stamp_key}") + message(STATUS "UI: HF download succeeded, stamp updated (${stamp_key}, resolved: ${HF_RESOLVED})") set(provisioned TRUE) else() message(STATUS "UI: HF download failed") From 737e0980fef1c2d573afedc8b00f7caf30617652 Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 12 Sep 2026 22:47:14 +0200 Subject: [PATCH 60/65] cmake: leave the timestamp out of precompiled headers on clang (#28816) Clang stores the modification time of the precompiled header sources inside the header and refuses the header when they differ. A cached header restored from another checkout carries the timestamps of that checkout, so the build fails. The option covers the compilers ccache treats as MSVC while they are clang underneath, clang-cl and the Intel LLVM drivers. --- CMakeLists.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 86b09dfd4640..4052fa3d6154 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -197,6 +197,16 @@ llama_option_depr(WARNING LLAMA_CURL) include("cmake/license.cmake") license_add_file("llama.cpp" "LICENSE") +# +# compile options +# + +# clang stores the modification time of the precompiled header sources inside the +# header and rejects it when they differ, so the timestamp is left out of it +add_compile_options( + "$<$:SHELL:-Xclang -fno-pch-timestamp>" + "$<$:SHELL:-Xclang -fno-pch-timestamp>") + # # 3rd-party # From ae9afff8d2c012ca760eb9c2adf41961cf6f6232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 12 Sep 2026 22:49:53 +0200 Subject: [PATCH 61/65] jinja : support dot property integer literals (#28817) --- common/jinja/runtime.cpp | 6 ++++++ tests/test-jinja.cpp | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index 49354c7c9cd2..252ab55de20e 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -842,6 +842,12 @@ value member_expression::execute_impl(context & ctx) { } else { property = this->property->execute(ctx); } + } else if (is_stmt(this->property)) { + // syntax: obj.index + property = mk_val(cast_stmt(this->property)->val); + if (property->as_int() < 0) { + throw std::runtime_error("Static member property cannot be negative"); + } } else { // syntax: obj.prop if (!is_stmt(this->property)) { diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index ab551d7b38fd..00de91ddf93e 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -398,6 +398,18 @@ static void test_expressions(testing & t) { "Bob" ); + test_template(t, "dot notation (integer property)", + "{{ {10: 'Bob'}.10 }}", + json::object(), + "Bob" + ); + + test_template(t, "dot notation (array index)", + "{{ user.10 }}", + {{"user", json::array({"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"})}}, + "k" + ); + test_template(t, "negative float (not dot notation)", "{{ -1.0 }}", json::object(), From acecd56032ddc34bada14a2d978f110d9c987095 Mon Sep 17 00:00:00 2001 From: Aldehir Rojas Date: Sat, 12 Sep 2026 16:14:50 -0500 Subject: [PATCH 62/65] common : implement common_schema internal representation for JSON schemas (#28736) * common : implement common_schema types * common : implement a json schema optimizer * common : reduce optimizations * common : refactor json-schema-to-grammar to use common_schema * common : use common_trie * common/schema : implement type/kind resolution * cont : cleanup * cont : remove common_chat_tool_parameters * cont : simplify schema resolution * cont : pass common_schema through the json-schema-to-grammar builder * cont : cleanup * cont : move enums under common_schema and add type enum * cont : reduce test cases * cont : clean up * cont : clean up * refactor : rename common_schema_parse to common_schema_from_json * tests : fix gcc dangling-reference warning in test-json-schema * tests : take the schema label as const char * to satisfy gcc dangling-reference * refactor : rename common_schema_builder parse_* methods to build_* * cont : fix may_be_string * cont : properly handle empty tool parameters * cont : add tests for empty $ref * cont : remove dead code * cont : update docs * cont : make "{}" mean any object for json_object as well * cont : restore (min|max)Length to imply string type * cont : rename common_schema to common_chat_schema --- .github/workflows/build-cpu.yml | 1 - common/CMakeLists.txt | 2 + common/arg.cpp | 4 +- common/chat-auto-parser-generator.cpp | 52 +- common/chat-peg-parser.cpp | 10 +- common/chat.cpp | 10 + common/chat.h | 3 + common/json-schema-to-grammar.cpp | 597 ++++++------------ common/json-schema-to-grammar.h | 30 +- common/json-schema.cpp | 514 ++++++++++++++++ common/json-schema.h | 198 ++++++ common/parsers/cohere2moe.cpp | 9 - common/parsers/deepseek.cpp | 36 +- common/parsers/functionary-v3-2.cpp | 7 +- common/parsers/gemma4.cpp | 9 - common/parsers/gigachat-v3.cpp | 7 +- common/parsers/gpt-oss.cpp | 11 +- common/parsers/kimi-k2.cpp | 7 +- common/parsers/kimi-k3.cpp | 9 +- common/parsers/lfm2.cpp | 9 - common/parsers/minicpm5.cpp | 52 +- common/parsers/minimax-m3.cpp | 82 +-- common/parsers/ministral3.cpp | 11 +- common/parsers/muse-glimmer.cpp | 46 +- common/parsers/parsers.cpp | 23 +- common/parsers/parsers.h | 4 +- common/parsers/qwen3-coder.cpp | 29 +- common/peg-parser.cpp | 42 +- common/peg-parser.h | 10 +- docs/development/parsing.md | 9 +- examples/json_schema_to_grammar.py | 842 -------------------------- examples/regex_to_grammar.py | 20 - examples/ts-type-to-grammar.sh | 28 - grammars/README.md | 8 +- tests/CMakeLists.txt | 7 +- tests/test-chat-peg-parser.cpp | 15 - tests/test-chat.cpp | 13 + tests/test-grammar-integration.cpp | 12 +- tests/test-json-schema-to-grammar.cpp | 347 +++++------ tests/test-json-schema.cpp | 513 ++++++++++++++++ tools/cli/README.md | 4 +- tools/completion/README.md | 6 +- tools/server/README.md | 4 +- tools/server/server-common.cpp | 5 + tools/server/server-schema.cpp | 4 + 45 files changed, 1733 insertions(+), 1928 deletions(-) create mode 100644 common/json-schema.cpp create mode 100644 common/json-schema.h delete mode 100755 examples/json_schema_to_grammar.py delete mode 100644 examples/regex_to_grammar.py delete mode 100755 examples/ts-type-to-grammar.sh create mode 100644 tests/test-json-schema.cpp diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml index 9e92314bc2f2..ddda55f1c6b0 100644 --- a/.github/workflows/build-cpu.yml +++ b/.github/workflows/build-cpu.yml @@ -221,7 +221,6 @@ jobs: # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar # $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe) # cd build - # $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1 # & $sde -future -- ctest -L main -C Release --verbose --timeout 900 - name: ccache-clear diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 9a43911d3547..38dab96ab814 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -84,6 +84,8 @@ add_library(${TARGET} imatrix-loader.cpp imatrix-loader.h json-schema-to-grammar.cpp + json-schema.cpp + json-schema.h json.cpp json.h llguidance.cpp diff --git a/common/arg.cpp b/common/arg.cpp index 43052d58d1d1..b1c0f23526ef 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2277,14 +2277,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_sampling()); add_opt(common_arg( {"-j", "--json-schema"}, "SCHEMA", - "JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead", + "JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object", [](common_params & params, const std::string & value) { params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, json_schema_to_grammar(json::parse(value))}; } ).set_sampling()); add_opt(common_arg( {"-jf", "--json-schema-file"}, "FILE", - "File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead", + "File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object", [](common_params & params, const std::string & value) { std::ifstream file(value); if (!file) { diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index d7e117e4d98b..b78789d8c0df 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -5,6 +5,7 @@ #include "common.h" #include "json-schema-to-grammar.h" #include "log.h" +#include "parsers/parsers.h" #include "peg-parser.h" #include @@ -12,16 +13,6 @@ using json = common_json; -// Helper to iterate over tools/functions -static void foreach_function(const json & tools, const std::function & fn) { - for (const auto & tool : tools) { - if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) { - continue; - } - fn(tool); - } -} - namespace autoparser { parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) : @@ -87,15 +78,6 @@ common_chat_params peg_generator::generate_parser(const common_chat_template & if (include_grammar) { data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } parser.build_grammar(builder, data.grammar_lazy); }); @@ -312,7 +294,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context foreach_function(inputs.tools, [&](const json & tool) { const auto & func = tool.at("function"); std::string name = func.at("name"); - const auto & schema = func.contains("parameters") ? func.at("parameters") : json::object(); + const auto schema = common_chat_tool_parameters(func); // Build call_id parser based on position (if supported) bool have_call_id = false; @@ -383,43 +365,31 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte common_peg_parser tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { - const auto & func = tool.at("function"); - std::string name = func.at("name"); - auto params = func.contains("parameters") ? func.at("parameters") : json::object(); - const auto & properties = params.contains("properties") ? params.at("properties") : json::object(); - - std::set required; - if (params.contains("required")) { - required = params.at("required").get>(); - } - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); + const auto & func = tool.at("function"); + std::string name = func.at("name"); // Build parser for each argument, separating required and optional std::vector required_parsers; std::vector optional_parsers; - for (const auto & [param_name, param_schema] : properties.items()) { - bool is_required = required.find(param_name) != required.end(); - + foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { auto arg = - p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) + + p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) + arguments.name_suffix) + arguments.value_prefix + - (schema_info.resolves_to_string(param_schema) ? + (param.schema->may_be_string() ? p.ac(p.tool_arg_string_value(until_suffix) + p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) : (p.tool_arg_json_value(p.schema( - p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) + + p.json(), "tool-" + name + "-arg-" + param.name + "-schema", doc, *param.schema)) + p.tool_arg_close(p.literal(arguments.value_suffix))))); - auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); - if (is_required) { + auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg); + if (param.required) { required_parsers.push_back(named_arg); } else { optional_parsers.push_back(named_arg); } - } + }); // Build required arg sequence in definition order common_peg_parser args_seq = p.eps(); diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 79b97a80f1b2..ffa43a318888 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -488,7 +488,7 @@ common_peg_parser common_chat_peg_builder::standard_constructed_tools( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); // Build argument parsers auto args = eps(); @@ -565,7 +565,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); auto args = eps(); if (params.contains("properties") && !params["properties"].empty()) { @@ -640,7 +640,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); // Build inner object fields std::vector inner_fields; @@ -726,7 +726,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() + atomic(literal("\"") + tool_name(literal(name)) + literal("\"")); @@ -795,7 +795,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys( } const auto & function = tool_def.at("function"); std::string name = function.at("name"); - ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object(); + ordered_json params = common_chat_tool_parameters(function); auto tool_name_ = name_key_parser + space() + literal(":") + space() + atomic(literal("\"") + tool_name(literal(name)) + literal("\"")); diff --git a/common/chat.cpp b/common/chat.cpp index faf27f78672d..3a204e12d758 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -574,6 +574,16 @@ json common_chat_tools_to_json_oaicompat(const std::vector & t return result; } +json common_chat_tool_parameters(const json & function) { + if (function.contains("parameters")) { + const auto & params = function.at("parameters"); + if (!params.is_null() && !(params.is_object() && params.empty())) { + return params; + } + } + return json{{"type", "object"}, {"properties", json::object()}}; +} + std::vector common_chat_tools_parse_oaicompat(const json & tools) { std::vector result; diff --git a/common/chat.h b/common/chat.h index cb39e3458f44..0e1423a5a3b1 100644 --- a/common/chat.h +++ b/common/chat.h @@ -360,6 +360,9 @@ common_json common_chat_msgs_to_json_oaicompat(const std::vector & tools); +// The parameters schema of a function tool. A tool without parameters, or with an empty {}, takes zero arguments. +common_json common_chat_tool_parameters(const common_json & function); + // get template caps, useful for reporting to server /props endpoint std::map common_chat_templates_get_caps(const common_chat_templates * chat_templates); diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index a7a18857d713..e0426098c08a 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -1,5 +1,7 @@ #include "json-schema-to-grammar.h" #include "common.h" +#include "trie.h" +#include "unicode.h" #include #include @@ -336,18 +338,20 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) { return 2 + n_hex; } -class common_schema_converter { +class common_chat_schema_converter { private: - friend class common_schema_info; friend std::string build_grammar(const std::function & cb, const common_grammar_options & options); - std::function _fetch_json; bool _dotall; std::map _rules; - std::unordered_map _refs; std::unordered_set _refs_being_resolved; std::vector _errors; std::vector _warnings; + template + static const T & as(const common_chat_schema & node) { + return static_cast(node); + } + std::string _add_rule(const std::string & name, const std::string & rule) { std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-"); if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) { @@ -363,11 +367,11 @@ class common_schema_converter { return key; } - std::string _generate_union_rule(const std::string & name, const std::vector & alt_schemas) { + std::string _generate_union_rule(const std::string & name, const std::vector & alt_schemas) { std::vector rules; rules.reserve(alt_schemas.size()); for (size_t i = 0; i < alt_schemas.size(); i++) { - rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i))); + rules.push_back(visit(*alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i))); } return string_join(rules, " | "); } @@ -634,85 +638,68 @@ class common_schema_converter { -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] */ std::string _not_strings(const std::vector & strings) { - - struct TrieNode { - std::map children; - bool is_end_of_string; - - TrieNode() : is_end_of_string(false) {} - - void insert(const std::string & string) { - auto *node = this; - for (char c : string) { - node = &node->children[c]; - } - node->is_end_of_string = true; - } - }; - - TrieNode trie; - for (const auto & s : strings) { - trie.insert(s); - } + common_trie trie(strings); std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); std::ostringstream out; out << "[\"] ( "; - std::function visit = [&](const TrieNode & node) { - std::ostringstream rejects; + std::function visit = [&](size_t idx) { + const auto & node = trie.nodes[idx]; + std::string rejects; auto first = true; - for (const auto & kv : node.children) { - rejects << kv.first; + for (const auto & [cpt, child] : node.children) { + std::string c = common_unicode_cpt_to_utf8(cpt); + rejects += c; if (first) { first = false; } else { out << " | "; } - out << "[" << kv.first << "]"; - if (!kv.second.children.empty()) { + out << "[" << c << "]"; + if (!trie.nodes[child].children.empty()) { out << " ("; - visit(kv.second); + visit(child); out << ")"; - } else if (kv.second.is_end_of_string) { + } else { out << " " << char_rule << "+"; } } if (!node.children.empty()) { - if (!first) { - out << " | "; - } - out << "[^\"" << rejects.str() << "] " << char_rule << "*"; + out << " | [^\"" << rejects << "] " << char_rule << "*"; } }; - visit(trie); + visit(0); out << " )"; - if (!trie.is_end_of_string) { + if (trie.nodes[0].pattern < 0) { out << "?"; } out << " [\"]"; return out.str(); } - std::string _resolve_ref(const std::string & ref) { - auto it = ref.find('#'); - std::string ref_fragment = it != std::string::npos ? ref.substr(it + 1) : ref; + std::string _resolve_ref(const common_chat_schema_ref & schema) { + auto it = schema.ref.find('#'); + std::string ref_fragment = it != std::string::npos ? schema.ref.substr(it + 1) : schema.ref; static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)"); std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-"); - if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) { - _refs_being_resolved.insert(ref); - json resolved = _refs[ref]; - ref_name = visit(resolved, ref_name); - _refs_being_resolved.erase(ref); + if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(schema.ref) == _refs_being_resolved.end()) { + if (!schema.target) { + _errors.push_back("Unresolved $ref " + schema.ref); + return ""; + } + _refs_being_resolved.insert(schema.ref); + ref_name = visit(*schema.target, ref_name); + _refs_being_resolved.erase(schema.ref); } return ref_name; } std::string _build_object_rule( - const std::vector> & properties, + const std::vector> & properties, const std::unordered_set & required, const std::string & name, - const json & additional_properties) + const common_chat_schema * additional_properties) { std::vector required_props; std::vector optional_props; @@ -722,7 +709,7 @@ class common_schema_converter { const auto &prop_name = kv.first; const auto &prop_schema = kv.second; - std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name); + std::string prop_rule_name = visit(*prop_schema, name + (name.empty() ? "" : "-") + prop_name); prop_kv_rule_names[prop_name] = _add_rule( name + (name.empty() ? "" : "-") + prop_name + "-kv", format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name @@ -734,10 +721,10 @@ class common_schema_converter { } prop_names.push_back(prop_name); } - if ((additional_properties.is_boolean() && additional_properties.get()) || additional_properties.is_object()) { + if (additional_properties) { std::string sub_name = name + (name.empty() ? "" : "-") + "additional"; std::string value_rule = - additional_properties.is_object() ? visit(additional_properties, sub_name + "-value") + additional_properties->kind() != common_chat_schema::KIND_ANY ? visit(*additional_properties, sub_name + "-value") : _add_primitive("value", PRIMITIVE_RULES.at("value")); auto key_rule = @@ -825,267 +812,163 @@ class common_schema_converter { } public: - common_schema_converter( - const std::function & fetch_json, - bool dotall) - : _fetch_json(fetch_json), _dotall(dotall) - { + explicit common_chat_schema_converter(bool dotall) : _dotall(dotall) { _rules["space"] = SPACE_RULE; } - void resolve_refs(json & schema, const std::string & url) { - /* - * Resolves all $ref fields in the given schema, fetching any remote schemas, - * replacing each $ref with absolute reference URL and populates _refs with the - * respective referenced (sub)schema dictionaries. - */ - std::function visit_refs = [&](json & n) { - if (n.is_array()) { - for (auto & x : n) { - visit_refs(x); - } - } else if (n.is_object()) { - if (n.contains("$ref")) { - std::string ref = n["$ref"]; - if (_refs.find(ref) == _refs.end()) { - json target; - if (ref.find("https://") == 0) { - std::string base_url = ref.substr(0, ref.find('#')); - auto it = _refs.find(base_url); - if (it != _refs.end()) { - target = it->second; - } else { - // Fetch the referenced schema and resolve its refs - auto referenced = _fetch_json(ref); - resolve_refs(referenced, base_url); - _refs[base_url] = referenced; - } - if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) { - return; - } - } else if (ref.find("#/") == 0) { - target = schema; - n["$ref"] = url + ref; - ref = url + ref; - } else { - _errors.push_back("Unsupported ref: " + ref); - return; - } - std::string pointer = ref.substr(ref.find('#') + 1); - std::vector tokens = string_split(pointer, "/"); - for (size_t i = 1; i < tokens.size(); ++i) { - const std::string& sel = tokens[i]; - if (target.is_object() && target.contains(sel)) { - target = target[sel]; - } else if (target.is_array()) { - size_t sel_index; - try { - sel_index = std::stoull(sel); - } catch (const std::invalid_argument & e) { - sel_index = target.size(); - } - if (sel_index >= target.size()) { - _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump()); - return; - } - target = target[sel_index]; - } else { - _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump()); - return; - } - } - _refs[ref] = target; - } - } else { - for (const auto & kv : n.items()) { - visit_refs(kv.value()); - } - } - } - }; - - visit_refs(schema); + std::string add_schema(const std::string & name, const common_chat_schema & schema) { + return visit(schema, name); } static std::string _generate_constant_rule(const json & value) { return format_literal(value.dump()); } - std::string visit(const json & schema, const std::string & name) { - json schema_type = schema.contains("type") ? schema["type"] : json(); - std::string schema_format = schema.contains("format") ? schema["format"].get() : ""; - std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name; + std::string _visit_primitive(const std::string & rule_name, const std::string & type) { + return _add_primitive(rule_name == "root" ? "root" : type, PRIMITIVE_RULES.at(type)); + } - if (schema.contains("$ref")) { - return _add_rule(rule_name, _resolve_ref(schema["$ref"])); - } - if (schema.contains("oneOf") || schema.contains("anyOf")) { - const json & alts = schema.contains("oneOf") ? schema.at("oneOf") : schema.at("anyOf"); - std::vector alt_schemas; - for (const auto & alt : alts) { - alt_schemas.push_back(alt); - } - return _add_rule(rule_name, _generate_union_rule(name, alt_schemas)); - } - if (schema_type.is_array()) { - std::vector schema_types; - for (const auto & t : schema_type) { - json schema_copy(schema); - schema_copy["type"] = t; - schema_types.push_back(schema_copy); - } - return _add_rule(rule_name, _generate_union_rule(name, schema_types)); - } - if (schema.contains("const")) { - return _add_rule(rule_name, _generate_constant_rule(schema["const"])); - } - if (schema.contains("enum")) { - std::vector enum_values; - for (const auto & v : schema["enum"]) { - enum_values.push_back(_generate_constant_rule(v)); - } - return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")"); - } - if ((schema_type.is_null() || schema_type == "object") - && (schema.contains("properties") || - (schema.contains("additionalProperties") && schema["additionalProperties"] != true))) { - std::unordered_set required; - if (schema.contains("required") && schema["required"].is_array()) { - for (const auto & item : schema["required"]) { - if (item.is_string()) { - required.insert(item.get()); + std::string _visit_all_of(const common_chat_schema_all_of & schema, const std::string & name, const std::string & rule_name) { + std::unordered_set required; + std::vector> properties; + std::map enum_values; + std::function add_component = [&](const common_chat_schema & comp, bool is_required) { + if (comp.kind() == common_chat_schema::KIND_REF) { + if (const auto * target = as(comp).target) { + add_component(*target, is_required); + } + } else if (comp.kind() == common_chat_schema::KIND_OBJECT) { + for (const auto & prop : as(comp).properties) { + properties.emplace_back(prop.name, prop.schema.get()); + if (is_required) { + required.insert(prop.name); } } + } else if (comp.kind() == common_chat_schema::KIND_ENUM) { + for (const auto & v : as(comp).values) { + enum_values[_generate_constant_rule(v)] += 1; + } } - std::vector> properties; - if (schema.contains("properties")) { - for (const auto & prop : schema["properties"].items()) { - properties.emplace_back(prop.key(), prop.value()); + }; + for (const auto & child : schema.children) { + if (child->kind() == common_chat_schema::KIND_ANY_OF) { + for (const auto & alt : as(*child).children) { + add_component(*alt, false); } + } else { + add_component(*child, true); } - return _add_rule(rule_name, - _build_object_rule( - properties, required, name, - schema.contains("additionalProperties") ? schema["additionalProperties"] : json())); } - if ((schema_type.is_null() || schema_type == "object" || schema_type == "string") && schema.contains("allOf")) { - std::unordered_set required; - std::vector> properties; - std::map enum_values; - const std::string& hybrid_name = name; - std::function add_component = [&](const json & comp_schema, bool is_required) { - if (comp_schema.contains("$ref")) { - add_component(_refs[comp_schema["$ref"]], is_required); - } else if (comp_schema.contains("properties")) { - for (const auto & prop : comp_schema["properties"].items()) { - properties.emplace_back(prop.key(), prop.value()); - if (is_required) { - required.insert(prop.key()); - } - } - } else if (comp_schema.contains("enum")) { - for (const auto & v : comp_schema["enum"]) { - const auto rule = _generate_constant_rule(v); - if (enum_values.find(rule) == enum_values.end()) { - enum_values[rule] = 0; - } - enum_values[rule] += 1; - } - } else { - // todo warning + if (!enum_values.empty()) { + std::vector enum_intersection; + for (const auto & p : enum_values) { + if (p.second == schema.children.size()) { + enum_intersection.push_back(p.first); } - }; - for (const auto & t : schema["allOf"]) { - if (t.contains("anyOf")) { - for (const auto & tt : t["anyOf"]) { - add_component(tt, false); - } - } else { - add_component(t, true); + } + if (!enum_intersection.empty()) { + return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")"); + } + } + return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr)); + } + + std::string visit(const common_chat_schema & schema, const std::string & name) { + std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name; + std::string sub_name = name + (name.empty() ? "" : "-"); + + switch (schema.kind()) { + case common_chat_schema::KIND_REF: + return _add_rule(rule_name, _resolve_ref(as(schema))); + case common_chat_schema::KIND_ANY_OF: + return _add_rule(rule_name, _generate_union_rule(name, as(schema).children)); + case common_chat_schema::KIND_ALL_OF: + return _visit_all_of(as(schema), name, rule_name); + case common_chat_schema::KIND_CONST: + return _add_rule(rule_name, _generate_constant_rule(as(schema).value)); + case common_chat_schema::KIND_ENUM: { + std::vector enum_values; + for (const auto & v : as(schema).values) { + enum_values.push_back(_generate_constant_rule(v)); } + return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")"); } - if (!enum_values.empty()) { - std::vector enum_intersection; - for (const auto & p : enum_values) { - if (p.second == schema["allOf"].size()) { - enum_intersection.push_back(p.first); - } + case common_chat_schema::KIND_OBJECT: { + const auto & obj = as(schema); + if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == common_chat_schema::KIND_ANY) { + return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object"))); } - if (!enum_intersection.empty()) { - return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")"); + std::vector> properties; + std::unordered_set required; + for (const auto & prop : obj.properties) { + properties.emplace_back(prop.name, prop.schema.get()); + if (prop.required) { + required.insert(prop.name); + } } + return _add_rule(rule_name, _build_object_rule(properties, required, name, obj.additional_properties.get())); } - return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json())); - } - if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) { - json items = schema.contains("items") ? schema["items"] : schema["prefixItems"]; - if (items.is_array()) { + case common_chat_schema::KIND_TUPLE: { + const auto & items = as(schema).items; std::string rule = "\"[\" space "; for (size_t i = 0; i < items.size(); i++) { if (i > 0) { rule += " \",\" space "; } - rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i)); + rule += visit(*items[i], sub_name + "tuple-" + std::to_string(i)); } rule += " space \"]\""; return _add_rule(rule_name, rule); } - std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item"); - int min_items = schema.contains("minItems") ? schema["minItems"].get() : 0; - json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json(); - int max_items = max_items_json.is_number_integer() ? max_items_json.get() : std::numeric_limits::max(); - - return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " space \"]\""); - } - if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) { - return _visit_pattern(schema["pattern"], rule_name); - } - if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) { - return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid")); - } - if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) { - auto prim_name = schema_format + "-string"; - return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name))); - } - if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) { - std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); - int min_len = schema.contains("minLength") ? schema["minLength"].get() : 0; - int max_len = schema.contains("maxLength") ? schema["maxLength"].get() : std::numeric_limits::max(); - return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\""); - } - if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) { - int64_t min_value = std::numeric_limits::min(); - int64_t max_value = std::numeric_limits::max(); - if (schema.contains("minimum")) { - min_value = schema["minimum"].get(); - } else if (schema.contains("exclusiveMinimum")) { - min_value = schema["exclusiveMinimum"].get() + 1; + case common_chat_schema::KIND_ARRAY: { + const auto & arr = as(schema); + if (arr.items->kind() == common_chat_schema::KIND_ANY && arr.min_items == 0 && arr.max_items < 0) { + return _visit_primitive(rule_name, "array"); + } + std::string item_rule_name = visit(*arr.items, sub_name + "item"); + int max_items = arr.max_items < 0 ? std::numeric_limits::max() : arr.max_items; + return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, arr.min_items, max_items, "\",\" space") + " space \"]\""); } - if (schema.contains("maximum")) { - max_value = schema["maximum"].get(); - } else if (schema.contains("exclusiveMaximum")) { - max_value = schema["exclusiveMaximum"].get() - 1; + case common_chat_schema::KIND_STRING: { + const auto & str = as(schema); + if (!str.pattern.empty()) { + return _visit_pattern(str.pattern, rule_name); + } + if (str.format == common_chat_schema::FORMAT_UUID) { + return _visit_primitive(rule_name, "uuid"); + } + if (str.format != common_chat_schema::FORMAT_NONE) { + std::string prim_name = std::string(str.format == common_chat_schema::FORMAT_DATE ? "date" : str.format == common_chat_schema::FORMAT_TIME ? "time" : "date-time") + "-string"; + return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name))); + } + if (str.min_length > 0 || str.max_length >= 0) { + std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); + int max_len = str.max_length < 0 ? std::numeric_limits::max() : str.max_length; + return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, str.min_length, max_len) + " \"\\\"\""); + } + return _visit_primitive(rule_name, "string"); } - std::stringstream out; - out << "("; - build_min_max_int(min_value, max_value, out); - out << ")"; - return _add_rule(rule_name, out.str()); - } - if (schema.empty() || schema_type == "object") { - return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object"))); - } - if (schema_type.is_null() && schema.is_object()) { - // No type constraint and no recognized structural keywords (e.g. {"description": "..."}). - // Per JSON Schema semantics this is equivalent to {} and accepts any value. - return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value"))); - } - if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get()) == PRIMITIVE_RULES.end()) { - _errors.push_back("Unrecognized schema: " + schema.dump()); - return ""; + case common_chat_schema::KIND_INTEGER: { + const auto & i = as(schema); + if (i.minimum == std::numeric_limits::min() && i.maximum == std::numeric_limits::max()) { + return _visit_primitive(rule_name, "integer"); + } + std::stringstream out; + out << "("; + build_min_max_int(i.minimum, i.maximum, out); + out << ")"; + return _add_rule(rule_name, out.str()); + } + case common_chat_schema::KIND_NUMBER: + return _visit_primitive(rule_name, "number"); + case common_chat_schema::KIND_BOOLEAN: + return _visit_primitive(rule_name, "boolean"); + case common_chat_schema::KIND_NULL: + return _visit_primitive(rule_name, "null"); + case common_chat_schema::KIND_ANY: + return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value"))); } - // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero - return _add_primitive(rule_name == "root" ? "root" : schema_type.get(), PRIMITIVE_RULES.at(schema_type.get())); + return ""; } void check_errors() { @@ -1106,134 +989,6 @@ class common_schema_converter { } }; -// common_schema_info implementation (pimpl) - -common_schema_info::common_schema_info() - : impl_(std::make_unique( - [](const std::string &) { return json(); }, - false)) {} - -common_schema_info::~common_schema_info() = default; - -common_schema_info::common_schema_info(common_schema_info &&) noexcept = default; -common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default; - -void common_schema_info::resolve_refs(common_json & schema) { - impl_->resolve_refs(schema, ""); -} - -// Determines if a JSON schema can resolve to a string type through any path. -// Some models emit raw string values rather than JSON-encoded strings for string parameters. -// If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns -// true, allowing callers to handle the value as a raw string for simplicity. -bool common_schema_info::resolves_to_string(const common_json & schema) { - std::unordered_set visited_refs; - - std::function check = [&](const json & s) -> bool { - if (!s.is_object()) { - return false; - } - - // Handle $ref - if (s.contains("$ref")) { - const std::string & ref = s["$ref"]; - if (visited_refs.find(ref) != visited_refs.end()) { - // Circular reference, assume not a string to be safe - return false; - } - visited_refs.insert(ref); - auto it = impl_->_refs.find(ref); - if (it != impl_->_refs.end()) { - return check(it->second); - } - return false; - } - - // Check type field - if (s.contains("type")) { - const json & schema_type = s["type"]; - if (schema_type.is_string()) { - if (schema_type == "string") { - return true; - } - } else if (schema_type.is_array()) { - // Type can be an array like ["string", "null"] - for (const auto & t : schema_type) { - if (t == "string") { - return true; - } - } - } - } - - // Check oneOf/anyOf - if any alternative can be a string - if (s.contains("oneOf")) { - for (const auto & alt : s["oneOf"]) { - if (check(alt)) { - return true; - } - } - } - if (s.contains("anyOf")) { - for (const auto & alt : s["anyOf"]) { - if (check(alt)) { - return true; - } - } - } - - // Check allOf - all components must be compatible with string type - if (s.contains("allOf")) { - bool all_string = true; - for (const auto & component : s["allOf"]) { - if (!check(component)) { - all_string = false; - break; - } - } - if (all_string) { - return true; - } - } - - // Check const - if the constant value is a string - if (s.contains("const")) { - if (s["const"].is_string()) { - return true; - } - } - - // Check enum - if any enum value is a string - if (s.contains("enum")) { - for (const auto & val : s["enum"]) { - if (val.is_string()) { - return true; - } - } - } - - // String-specific keywords imply string type - if (s.contains("pattern") || s.contains("minLength") || s.contains("maxLength")) { - return true; - } - - // Check format - many formats imply string - if (s.contains("format")) { - const std::string & fmt = s["format"]; - if (fmt == "date" || fmt == "time" || fmt == "date-time" || - fmt == "uri" || fmt == "email" || fmt == "hostname" || - fmt == "ipv4" || fmt == "ipv6" || fmt == "uuid" || - fmt.find("uuid") == 0) { - return true; - } - } - - return false; - }; - - return check(schema); -} - std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) { #ifdef LLAMA_USE_LLGUIDANCE if (!force_gbnf) { @@ -1242,25 +997,29 @@ std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) #else (void)force_gbnf; #endif // LLAMA_USE_LLGUIDANCE - return build_grammar([&](const common_grammar_builder & callbacks) { - auto copy = schema; - callbacks.resolve_refs(copy); - callbacks.add_schema("", copy); - }); + try { + return json_schema_to_grammar(common_chat_schema_from_json(schema)); + } catch (const std::runtime_error & e) { + throw std::invalid_argument(std::string("JSON schema conversion failed:\n") + e.what()); + } +} + +std::string json_schema_to_grammar(const common_chat_schema_document & schema) { + common_chat_schema_converter converter(false); + converter.visit(*schema.root, ""); + converter.check_errors(); + return converter.format_grammar(); } std::string build_grammar(const std::function & cb, const common_grammar_options & options) { - common_schema_converter converter([&](const std::string &) { return json(); }, options.dotall); + common_chat_schema_converter converter(options.dotall); common_grammar_builder builder { /* .add_rule = */ [&](const std::string & name, const std::string & rule) { return converter._add_rule(name, rule); }, - /* .add_schema = */ [&](const std::string & name, const common_json & schema) { - return converter.visit(schema, name == "root" ? "" : name); + /* .add_schema = */ [&](const std::string & name, const common_chat_schema & schema) { + return converter.add_schema(name == "root" ? "" : name, schema); }, - /* .resolve_refs = */ [&](common_json & schema) { - converter.resolve_refs(schema, ""); - } }; cb(builder); converter.check_errors(); diff --git a/common/json-schema-to-grammar.h b/common/json-schema-to-grammar.h index 84ed71c76a13..b928c250bbef 100644 --- a/common/json-schema-to-grammar.h +++ b/common/json-schema-to-grammar.h @@ -1,37 +1,17 @@ #pragma once +#include "json-schema.h" #include "json.h" #include -#include #include -std::string json_schema_to_grammar(const common_json & schema, - bool force_gbnf = false); - -class common_schema_converter; - -// Probes a JSON schema to extract information about its structure and type constraints. -class common_schema_info { - std::unique_ptr impl_; - - public: - common_schema_info(); - ~common_schema_info(); - - common_schema_info(const common_schema_info &) = delete; - common_schema_info & operator=(const common_schema_info &) = delete; - common_schema_info(common_schema_info &&) noexcept; - common_schema_info & operator=(common_schema_info &&) noexcept; - - void resolve_refs(common_json & schema); - bool resolves_to_string(const common_json & schema); -}; +std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf = false); +std::string json_schema_to_grammar(const common_chat_schema_document & schema); struct common_grammar_builder { - std::function add_rule; - std::function add_schema; - std::function resolve_refs; + std::function add_rule; + std::function add_schema; }; struct common_grammar_options { diff --git a/common/json-schema.cpp b/common/json-schema.cpp new file mode 100644 index 000000000000..6898840e7d1f --- /dev/null +++ b/common/json-schema.cpp @@ -0,0 +1,514 @@ +#include "json-schema.h" +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include + +class common_chat_schema_builder { + const common_json & root_; + common_chat_schema_document & doc_; + + // the targets built here, moved into doc_ once the whole schema is built + std::map refs_; + + // ref nodes get their target once every $ref is built, a cycle would otherwise need it too early + std::vector pending_; + + [[noreturn]] static void fail(const std::string & path, const std::string & msg) { + throw std::runtime_error("JSON schema error at " + path + ": " + msg); + } + + static int get_count(const common_json & schema, const std::string & key, const std::string & path, int def) { + if (!schema.contains(key)) { + return def; + } + const common_json & value = schema.at(key); + if (!value.is_number_integer() || value.get() < 0) { + fail(path, key + " must be a non-negative integer"); + } + return value.get(); + } + + // a fractional bound is rounded inwards, towards the integers it still admits + static int64_t get_bound(const common_json & schema, const std::string & key, const std::string & path, bool round_up) { + const common_json & value = schema.at(key); + if (value.is_number_integer()) { + return value.get(); + } + if (!value.is_number()) { + fail(path, key + " must be a number"); + } + double d = value.get(); + return (int64_t) (round_up ? std::ceil(d) : std::floor(d)); + } + + static common_chat_schema::string_format get_format(const common_json & schema, const std::string & path) { + if (!schema.contains("format")) { + return common_chat_schema::FORMAT_NONE; + } + const common_json & value = schema.at("format"); + if (!value.is_string()) { + fail(path, "format must be a string"); + } + std::string format = value.get(); + if (format == "date") { + return common_chat_schema::FORMAT_DATE; + } + if (format == "time") { + return common_chat_schema::FORMAT_TIME; + } + if (format == "date-time") { + return common_chat_schema::FORMAT_DATE_TIME; + } + if (format == "uuid" || (format.size() == 5 && format.compare(0, 4, "uuid") == 0 && format[4] >= '1' && format[4] <= '5')) { + return common_chat_schema::FORMAT_UUID; + } + return common_chat_schema::FORMAT_NONE; + } + + const common_json & resolve_ref(const std::string & ref, const std::string & path) { + const common_json * target = &root_; + auto tokens = string_split(ref.substr(1), "/"); + for (size_t i = 1; i < tokens.size(); i++) { + const std::string & sel = tokens[i]; + if (target->is_object() && target->contains(sel)) { + target = &target->at(sel); + } else if (target->is_array()) { + size_t idx; + try { + idx = std::stoull(sel); + } catch (const std::logic_error &) { + idx = target->size(); + } + if (idx >= target->size()) { + fail(path, "cannot resolve $ref " + ref + ", " + sel + " is out of range"); + } + target = &target->at(idx); + } else { + fail(path, "cannot resolve $ref " + ref + ", " + sel + " not found"); + } + } + return *target; + } + + common_chat_schema_ptr build_ref(const common_json & value, const std::string & path) { + if (!value.is_string()) { + fail(path, "$ref must be a string"); + } + std::string ref = value.get(); + if (ref.compare(0, 2, "#/") != 0) { + fail(path, "unsupported $ref " + ref + ", only references into the same document are supported"); + } + if (refs_.find(ref) == refs_.end()) { + // reserve the key first, so that a cycle back to this $ref stops here + refs_[ref] = nullptr; + refs_[ref] = build_node(resolve_ref(ref, path), ref); + } + auto node = std::make_unique(ref); + pending_.push_back(node.get()); + return node; + } + + template + common_chat_schema_ptr build_alternatives(const common_json & alts, const std::string & path) { + if (!alts.is_array()) { + fail(path, "must be an array of schemas"); + } + if (alts.empty()) { + fail(path, "must not be empty"); + } + auto node = std::make_unique(); + size_t i = 0; + for (const auto & alt : alts) { + node->children.push_back(build_node(alt, path + "/" + std::to_string(i++))); + } + return node; + } + + common_chat_schema_ptr build_object(const common_json & schema, const std::string & path) { + auto node = std::make_unique(); + + std::unordered_set required; + if (schema.contains("required") && schema.at("required").is_array()) { + for (const auto & name : schema.at("required")) { + if (name.is_string()) { + required.insert(name.get()); + } + } + } + + if (schema.contains("properties")) { + const common_json & properties = schema.at("properties"); + if (!properties.is_object()) { + fail(path, "properties must be an object"); + } + for (const auto & [name, prop] : properties.items()) { + node->properties.push_back({name, build_node(prop, path + "/properties/" + name), required.count(name) > 0}); + } + } + + if (schema.contains("additionalProperties")) { + const common_json & additional = schema.at("additionalProperties"); + if (additional.is_boolean()) { + if (additional.get()) { + node->additional_properties = std::make_unique(); + } + } else if (additional.is_object()) { + node->additional_properties = build_node(additional, path + "/additionalProperties"); + } else { + fail(path, "additionalProperties must be a boolean or a schema"); + } + } else if (!schema.contains("properties")) { + // {"type": "object"} on its own accepts any object + node->additional_properties = std::make_unique(); + } + + return node; + } + + common_chat_schema_ptr build_array(const common_json & schema, const std::string & path) { + auto node = std::make_unique(); + if (schema.contains("items") || schema.contains("prefixItems")) { + // "items" wins when both are present; as in the converter, a schema instead of an array is the item schema + const std::string key = schema.contains("items") ? "items" : "prefixItems"; + const common_json & items = schema.at(key); + if (items.is_array()) { + auto tuple = std::make_unique(); + size_t i = 0; + for (const auto & item : items) { + tuple->items.push_back(build_node(item, path + "/" + key + "/" + std::to_string(i++))); + } + return tuple; + } + node->items = build_node(items, path + "/" + key); + } else { + node->items = std::make_unique(); + } + node->min_items = get_count(schema, "minItems", path, 0); + node->max_items = get_count(schema, "maxItems", path, -1); + return node; + } + + common_chat_schema_ptr build_string(const common_json & schema, const std::string & path) { + auto node = std::make_unique(); + if (schema.contains("pattern")) { + const common_json & pattern = schema.at("pattern"); + if (!pattern.is_string()) { + fail(path, "pattern must be a string"); + } + node->pattern = pattern.get(); + } + node->format = get_format(schema, path); + node->min_length = get_count(schema, "minLength", path, 0); + node->max_length = get_count(schema, "maxLength", path, -1); + return node; + } + + common_chat_schema_ptr build_integer(const common_json & schema, const std::string & path) { + auto node = std::make_unique(); + if (schema.contains("minimum")) { + node->minimum = get_bound(schema, "minimum", path, /* round_up */ true); + } else if (schema.contains("exclusiveMinimum")) { + node->minimum = get_bound(schema, "exclusiveMinimum", path, /* round_up */ false) + 1; + } + if (schema.contains("maximum")) { + node->maximum = get_bound(schema, "maximum", path, /* round_up */ false); + } else if (schema.contains("exclusiveMaximum")) { + node->maximum = get_bound(schema, "exclusiveMaximum", path, /* round_up */ true) - 1; + } + return node; + } + + common_chat_schema_ptr build_node(const common_json & schema, const std::string & path) { + if (!schema.is_object()) { + fail(path, "schema must be an object"); + } + if (schema.contains("$ref")) { + return build_ref(schema.at("$ref"), path); + } + if (schema.contains("oneOf") || schema.contains("anyOf")) { + const std::string key = schema.contains("oneOf") ? "oneOf" : "anyOf"; + return build_alternatives(schema.at(key), path + "/" + key); + } + + common_json type; + if (schema.contains("type")) { + type = schema.at("type"); + } + if (type.is_array()) { + // {"type": ["a", "b"], ...} is {"anyOf": [{"type": "a", ...}, {"type": "b", ...}]} + if (type.empty()) { + fail(path, "type must not be empty"); + } + auto node = std::make_unique(); + size_t i = 0; + for (const auto & t : type) { + common_json alt = schema; + alt["type"] = t; + node->children.push_back(build_node(alt, path + "/type/" + std::to_string(i++))); + } + return node; + } + if (schema.contains("const")) { + return std::make_unique(schema.at("const")); + } + if (schema.contains("enum")) { + const common_json & values = schema.at("enum"); + if (!values.is_array() || values.empty()) { + fail(path, "enum must be a non-empty array"); + } + auto node = std::make_unique(); + for (const auto & value : values) { + node->values.push_back(value); + } + return node; + } + if (!type.is_null() && !type.is_string()) { + fail(path, "type must be a string or an array of strings"); + } + + const std::string type_name = type.is_string() ? type.get() : ""; + const bool has_properties = schema.contains("properties") || + (schema.contains("additionalProperties") && schema.at("additionalProperties") != true); + + if (type_name.empty()) { + // without a type the structural keywords decide, in the same order as the converter + if (has_properties) { + return build_object(schema, path); + } + if (schema.contains("allOf")) { + return build_alternatives(schema.at("allOf"), path + "/allOf"); + } + if (schema.contains("items") || schema.contains("prefixItems")) { + return build_array(schema, path); + } + if (schema.contains("pattern") || schema.contains("minLength") || schema.contains("maxLength") || get_format(schema, path) != common_chat_schema::FORMAT_NONE) { + return build_string(schema, path); + } + return std::make_unique(); + } + if (type_name == "object") { + if (!has_properties && schema.contains("allOf")) { + return build_alternatives(schema.at("allOf"), path + "/allOf"); + } + return build_object(schema, path); + } + if (type_name == "string") { + if (schema.contains("allOf")) { + return build_alternatives(schema.at("allOf"), path + "/allOf"); + } + return build_string(schema, path); + } + if (type_name == "array") { + return build_array(schema, path); + } + if (type_name == "integer") { + return build_integer(schema, path); + } + if (type_name == "number") { + return std::make_unique(); + } + if (type_name == "boolean") { + return std::make_unique(); + } + if (type_name == "null") { + return std::make_unique(); + } + fail(path, "unrecognized type " + type_name); + } + + public: + common_chat_schema_builder(const common_json & root, common_chat_schema_document & doc) : root_(root), doc_(doc) {} + + common_chat_schema_ptr build() { + auto node = build_node(root_, "#"); + for (auto & entry : refs_) { + doc_.refs[entry.first] = std::move(entry.second); + } + for (auto * ref : pending_) { + ref->target = doc_.refs.at(ref->ref).get(); + } + return node; + } +}; + +common_chat_schema_document common_chat_schema_from_json(const common_json & schema) { + common_chat_schema_document doc; + doc.root = common_chat_schema_builder(schema, doc).build(); + return doc; +} + +static common_chat_schema::value_type json_type(const common_json & value) { + if (value.is_null()) { + return common_chat_schema::TYPE_NULL; + } + if (value.is_boolean()) { + return common_chat_schema::TYPE_BOOLEAN; + } + if (value.is_number_integer()) { + return common_chat_schema::TYPE_INTEGER; + } + if (value.is_number()) { + return common_chat_schema::TYPE_NUMBER; + } + if (value.is_string()) { + return common_chat_schema::TYPE_STRING; + } + if (value.is_array()) { + return common_chat_schema::TYPE_ARRAY; + } + return common_chat_schema::TYPE_OBJECT; +} + +static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set & visited) { + switch (s.kind()) { + case common_chat_schema::KIND_ANY: + return common_chat_schema::type_set::all(); + case common_chat_schema::KIND_NULL: + return { common_chat_schema::TYPE_NULL }; + case common_chat_schema::KIND_BOOLEAN: + return { common_chat_schema::TYPE_BOOLEAN }; + case common_chat_schema::KIND_NUMBER: + return { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER }; + case common_chat_schema::KIND_INTEGER: + return { common_chat_schema::TYPE_INTEGER }; + case common_chat_schema::KIND_STRING: + return { common_chat_schema::TYPE_STRING }; + case common_chat_schema::KIND_ARRAY: + case common_chat_schema::KIND_TUPLE: + return { common_chat_schema::TYPE_ARRAY }; + case common_chat_schema::KIND_OBJECT: + return { common_chat_schema::TYPE_OBJECT }; + case common_chat_schema::KIND_CONST: + return { json_type(static_cast(s).value) }; + case common_chat_schema::KIND_ENUM: { + common_chat_schema::type_set types; + for (const auto & value : static_cast(s).values) { + types.add(json_type(value)); + } + return types; + } + case common_chat_schema::KIND_REF: { + const auto * target = static_cast(s).target; + if (!target || !visited.insert(target).second) { + // a cycle contributes no type, to be safe + return {}; + } + auto types = value_types_impl(*target, visited); + visited.erase(target); + return types; + } + case common_chat_schema::KIND_ANY_OF: { + common_chat_schema::type_set types; + for (const auto & child : static_cast(s).children) { + types |= value_types_impl(*child, visited); + } + return types; + } + case common_chat_schema::KIND_ALL_OF: { + auto types = common_chat_schema::type_set::all(); + for (const auto & child : static_cast(s).children) { + types &= value_types_impl(*child, visited); + } + return types; + } + } + return {}; +} + +common_chat_schema::type_set common_chat_schema::value_types() const { + std::unordered_set visited; + return value_types_impl(*this, visited); +} + +static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set & visited) { + switch (s.kind()) { + case common_chat_schema::KIND_STRING: + return true; + case common_chat_schema::KIND_CONST: + return static_cast(s).value.is_string(); + case common_chat_schema::KIND_ENUM: + for (const auto & v : static_cast(s).values) { + if (v.is_string()) { + return true; + } + } + return false; + case common_chat_schema::KIND_REF: { + // a cycle is taken as not a string, to be safe + const auto * target = static_cast(s).target; + if (!target || !visited.insert(target).second) { + return false; + } + bool result = may_be_string_impl(*target, visited); + visited.erase(target); + return result; + } + case common_chat_schema::KIND_ANY_OF: + for (const auto & child : static_cast(s).children) { + if (may_be_string_impl(*child, visited)) { + return true; + } + } + return false; + case common_chat_schema::KIND_ALL_OF: { + // every child must allow a string, an any child constrains nothing + bool any_string = false; + for (const auto & child : static_cast(s).children) { + if (child->kind() == common_chat_schema::KIND_ANY) { + continue; + } + if (!may_be_string_impl(*child, visited)) { + return false; + } + any_string = true; + } + return any_string; + } + default: + return false; + } +} + +bool common_chat_schema::may_be_string() const { + std::unordered_set visited; + return may_be_string_impl(*this, visited); +} + +const char * common_chat_schema::kind_name(node_kind kind) { + switch (kind) { + case KIND_ANY: return "any"; + case KIND_REF: return "ref"; + case KIND_ANY_OF: return "anyOf"; + case KIND_ALL_OF: return "allOf"; + case KIND_CONST: return "const"; + case KIND_ENUM: return "enum"; + case KIND_NULL: return "null"; + case KIND_BOOLEAN: return "boolean"; + case KIND_NUMBER: return "number"; + case KIND_INTEGER: return "integer"; + case KIND_STRING: return "string"; + case KIND_ARRAY: return "array"; + case KIND_TUPLE: return "tuple"; + case KIND_OBJECT: return "object"; + } + return "?"; +} + +const char * common_chat_schema::type_name(value_type type) { + switch (type) { + case TYPE_NULL: return "null"; + case TYPE_BOOLEAN: return "boolean"; + case TYPE_NUMBER: return "number"; + case TYPE_INTEGER: return "integer"; + case TYPE_STRING: return "string"; + case TYPE_ARRAY: return "array"; + case TYPE_OBJECT: return "object"; + } + return "?"; +} diff --git a/common/json-schema.h b/common/json-schema.h new file mode 100644 index 000000000000..084208c96201 --- /dev/null +++ b/common/json-schema.h @@ -0,0 +1,198 @@ +#pragma once + +#include "json.h" + +#include +#include +#include +#include +#include +#include + +// JSON schema, covering the subset that json_schema_to_grammar() can convert. + +struct common_chat_schema { + enum node_kind { + KIND_ANY, + KIND_REF, + KIND_ANY_OF, + KIND_ALL_OF, + KIND_CONST, + KIND_ENUM, + KIND_NULL, + KIND_BOOLEAN, + KIND_NUMBER, + KIND_INTEGER, + KIND_STRING, + KIND_ARRAY, + KIND_TUPLE, + KIND_OBJECT, + }; + + enum value_type { + TYPE_NULL, + TYPE_BOOLEAN, + TYPE_NUMBER, + TYPE_INTEGER, + TYPE_STRING, + TYPE_ARRAY, + TYPE_OBJECT, + }; + + enum string_format { + FORMAT_NONE, + FORMAT_UUID, // uuid, uuid1 .. uuid5 + FORMAT_DATE, + FORMAT_TIME, + FORMAT_DATE_TIME, + }; + + class type_set { + uint32_t mask_ = 0; + + public: + type_set() = default; + type_set(std::initializer_list types) { + for (auto type : types) { + add(type); + } + } + + static type_set all() { + return { TYPE_NULL, TYPE_BOOLEAN, TYPE_NUMBER, TYPE_INTEGER, TYPE_STRING, TYPE_ARRAY, TYPE_OBJECT }; + } + + void add(value_type type) { mask_ |= 1u << type; } + + bool has(value_type type) const { return (mask_ & (1u << type)) != 0; } + bool is_only(value_type type) const { return mask_ == (1u << type); } + bool empty() const { return mask_ == 0; } + + type_set & operator|=(const type_set & other) { mask_ |= other.mask_; return *this; } + type_set & operator&=(const type_set & other) { mask_ &= other.mask_; return *this; } + + bool operator==(const type_set & other) const { return mask_ == other.mask_; } + bool operator!=(const type_set & other) const { return mask_ != other.mask_; } + }; + + virtual ~common_chat_schema() = default; + virtual node_kind kind() const = 0; + + type_set value_types() const; + + // Whether a value matching the schema may be a string, through any branch of it. + bool may_be_string() const; + + static const char * kind_name(node_kind kind); + static const char * type_name(value_type type); +}; + +using common_chat_schema_ptr = std::unique_ptr; + +struct common_chat_schema_any : common_chat_schema { + node_kind kind() const override { return KIND_ANY; } +}; + +// {"$ref": "#/..."}, only references into the same document are supported +struct common_chat_schema_ref : common_chat_schema { + std::string ref; + const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs + + explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {} + + node_kind kind() const override { return KIND_REF; } +}; + +// oneOf / anyOf, or a "type" array expanded to one alternative per type +struct common_chat_schema_any_of : common_chat_schema { + std::vector children; + + node_kind kind() const override { return KIND_ANY_OF; } +}; + +struct common_chat_schema_all_of : common_chat_schema { + std::vector children; + + node_kind kind() const override { return KIND_ALL_OF; } +}; + +struct common_chat_schema_const : common_chat_schema { + common_json value; + + explicit common_chat_schema_const(common_json value) : value(std::move(value)) {} + + node_kind kind() const override { return KIND_CONST; } +}; + +struct common_chat_schema_enum : common_chat_schema { + std::vector values; + + node_kind kind() const override { return KIND_ENUM; } +}; + +struct common_chat_schema_null : common_chat_schema { + node_kind kind() const override { return KIND_NULL; } +}; + +struct common_chat_schema_boolean : common_chat_schema { + node_kind kind() const override { return KIND_BOOLEAN; } +}; + +struct common_chat_schema_number : common_chat_schema { + node_kind kind() const override { return KIND_NUMBER; } +}; + +// bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in +struct common_chat_schema_integer : common_chat_schema { + int64_t minimum = INT64_MIN; // INT64_MIN for unbounded + int64_t maximum = INT64_MAX; // INT64_MAX for unbounded + + node_kind kind() const override { return KIND_INTEGER; } +}; + +struct common_chat_schema_string : common_chat_schema { + std::string pattern; // empty when absent + string_format format = FORMAT_NONE; + int min_length = 0; + int max_length = -1; // -1 for unbounded + + node_kind kind() const override { return KIND_STRING; } +}; + +struct common_chat_schema_array : common_chat_schema { + common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent + int min_items = 0; + int max_items = -1; // -1 for unbounded + + node_kind kind() const override { return KIND_ARRAY; } +}; + +struct common_chat_schema_tuple : common_chat_schema { + std::vector items; + + node_kind kind() const override { return KIND_TUPLE; } +}; + +struct common_chat_schema_property { + std::string name; + common_chat_schema_ptr schema; + bool required = false; +}; + +struct common_chat_schema_object : common_chat_schema { + std::vector properties; // in schema order + common_chat_schema_ptr additional_properties; // null when not allowed + + node_kind kind() const override { return KIND_OBJECT; } +}; + +struct common_chat_schema_document { + common_chat_schema_ptr root; + std::map refs; +}; + +// A document shared by the PEG parsers built from its nodes, which it keeps alive +using common_chat_schema_document_ptr = std::shared_ptr; + +// Throws std::runtime_error when the schema falls outside the supported subset. +common_chat_schema_document common_chat_schema_from_json(const common_json & schema); diff --git a/common/parsers/cohere2moe.cpp b/common/parsers/cohere2moe.cpp index 46a2a01baae1..59595368dc3e 100644 --- a/common/parsers/cohere2moe.cpp +++ b/common/parsers/cohere2moe.cpp @@ -129,15 +129,6 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_template if (include_grammar) { data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp index 5e2581727204..640fa9e1560d 100644 --- a/common/parsers/deepseek.cpp +++ b/common/parsers/deepseek.cpp @@ -149,39 +149,28 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - const auto & props = params.contains("properties") ? params.at("properties") : json::object(); - - std::set required; - if (params.contains("required")) { - required = params.at("required").get>(); - } - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); std::vector required_parsers; std::vector optional_parsers; - for (const auto & [param_name, param_schema] : props.items()) { - bool is_required = required.find(param_name) != required.end(); - bool is_string = schema_info.resolves_to_string(param_schema); + foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { + bool is_string = param.schema->may_be_string(); auto arg = p.tool_arg( - p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) + + p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param.name)) + p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + (is_string ? p.tool_arg_string_value(p.until(PARAM_END)) : - p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema", - param_schema, false))) + + p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param.name + "-schema", + doc, *param.schema))) + p.tool_arg_close(p.literal(PARAM_END))); - auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); - if (is_required) { + auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg); + if (param.required) { required_parsers.push_back(named_arg); } else { optional_parsers.push_back(named_arg); } - } + }); common_peg_parser args_seq = p.eps(); for (size_t i = 0; i < required_parsers.size(); i++) { @@ -266,15 +255,6 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ if (include_grammar) { data.grammar_lazy = has_tools && !require_tools; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/functionary-v3-2.cpp b/common/parsers/functionary-v3-2.cpp index 349b8065ac1a..9d47f0a3328f 100644 --- a/common/parsers/functionary-v3-2.cpp +++ b/common/parsers/functionary-v3-2.cpp @@ -45,7 +45,7 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); - const auto & schema = function.at("parameters"); + const auto schema = common_chat_tool_parameters(function); // Tool format: >>>function_name\n{json_args} auto tool_parser = p.tool( @@ -82,11 +82,6 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cpp index 041523acb8d2..ad48226e611b 100644 --- a/common/parsers/gemma4.cpp +++ b/common/parsers/gemma4.cpp @@ -291,15 +291,6 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template & if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/gigachat-v3.cpp b/common/parsers/gigachat-v3.cpp index 41da5554acbc..48abcb3f33c1 100644 --- a/common/parsers/gigachat-v3.cpp +++ b/common/parsers/gigachat-v3.cpp @@ -33,7 +33,7 @@ common_chat_params common_chat_params_init_gigachat_v3( for (const auto & tool : inputs.tools) { const auto & function = tool.at("function"); std::string name = function.at("name"); - const auto & schema = function.at("parameters"); + const auto schema = common_chat_tool_parameters(function); auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\""); auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))); @@ -65,11 +65,6 @@ common_chat_params common_chat_params_init_gigachat_v3( data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cpp index d7dbfbfb57b0..00beb41a47ee 100644 --- a/common/parsers/gpt-oss.cpp +++ b/common/parsers/gpt-oss.cpp @@ -109,7 +109,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); - const auto & params = function.at("parameters"); + const auto params = common_chat_tool_parameters(function); auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name)); auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); @@ -143,15 +143,6 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/kimi-k2.cpp b/common/parsers/kimi-k2.cpp index 57f6bfdcb60d..5ee9121abab2 100644 --- a/common/parsers/kimi-k2.cpp +++ b/common/parsers/kimi-k2.cpp @@ -82,7 +82,7 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); - const auto & schema = function.at("parameters"); + const auto schema = common_chat_tool_parameters(function); // Match: functions.: // Capture the full call id (functions.:) using tool_id tag @@ -116,11 +116,6 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & if (include_grammar) { data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/kimi-k3.cpp b/common/parsers/kimi-k3.cpp index 56a49903f701..989e39f956f0 100644 --- a/common/parsers/kimi-k3.cpp +++ b/common/parsers/kimi-k3.cpp @@ -98,7 +98,7 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); - const json schema = function.contains("parameters") ? function.at("parameters") : json::object(); + const json schema = common_chat_tool_parameters(function); // arguments come one tag per key, with the JSON type in a type="..." // attribute. the type is taken from the tool schema instead, as it tells @@ -155,13 +155,6 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & if (include_grammar) { data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - if (function.contains("parameters")) { - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - } - }); parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/lfm2.cpp b/common/parsers/lfm2.cpp index 4514f908b956..280788509eca 100644 --- a/common/parsers/lfm2.cpp +++ b/common/parsers/lfm2.cpp @@ -98,15 +98,6 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template & if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.at("parameters"); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/minicpm5.cpp b/common/parsers/minicpm5.cpp index e6e0abf066c1..4d18d3d9600e 100644 --- a/common/parsers/minicpm5.cpp +++ b/common/parsers/minicpm5.cpp @@ -71,32 +71,27 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template & foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); const std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - auto args = p.eps(); - if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - auto arg_choice = p.choice(); - for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { - auto value_parser = p.eps(); - if (schema_info.resolves_to_string(prop_schema)) { - value_parser = string_value; - } else { - value_parser = p.tool_arg_json_value( - p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false) - ) + p.tool_arg_close(p.literal("")); - } - - auto arg_rule = p.tool_arg( - p.tool_arg_open(p.literal("")) + - value_parser - ); - - arg_choice |= arg_rule; + std::vector arg_rules; + foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) { + auto value_parser = p.eps(); + if (prop.schema->may_be_string()) { + value_parser = string_value; + } else { + value_parser = p.tool_arg_json_value( + p.schema(p.json(), "tool-" + name + "-arg-" + prop.name + "-schema", doc, *prop.schema) + ) + p.tool_arg_close(p.literal("")); } - args = p.zero_or_more(arg_choice + p.space()); + + arg_rules.push_back(p.tool_arg( + p.tool_arg_open(p.literal("")) + + value_parser + )); + }); + + auto args = p.eps(); + if (!arg_rules.empty()) { + args = p.zero_or_more(p.choice(arg_rules) + p.space()); } auto tool_parser = p.tool( @@ -123,15 +118,6 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template & if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/parsers/minimax-m3.cpp b/common/parsers/minimax-m3.cpp index ff23ea153c40..7ea9bfe5a083 100644 --- a/common/parsers/minimax-m3.cpp +++ b/common/parsers/minimax-m3.cpp @@ -84,29 +84,18 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template return generation_prompt + reasoning + p.content(p.rest()) + end; } - auto alternatives_of = [](const json & schema) -> std::optional { - for (const auto * keyword : { "oneOf", "anyOf" }) { - if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) { - return schema.at(keyword); - } - } - return std::nullopt; - }; - auto tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); + auto params = common_chat_tool_parameters(function); + auto doc = std::make_shared(common_chat_schema_from_json(params)); // The template expands argument values recursively in XML (see the to_xml() macro) - std::function value_of; - std::function members_of; + std::function value_of; + std::function members_of; - auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) { + auto element_of = [&](const std::string & tag, const common_chat_schema & schema, const std::string & rule_name) { const std::string close = NS + ""; return p.rule(rule_name, p.tool_arg( @@ -117,69 +106,57 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template value_of(schema, rule_name, close))); }; - value_of = [&](const json & schema, + value_of = [&](const common_chat_schema & schema, const std::string & rule_name, const std::string & close) -> common_peg_parser { auto close_tag = p.tool_arg_close(p.literal(close)); // A string accepts anything, so a union with a string alternative is a string - if (schema_info.resolves_to_string(schema)) { + if (schema.may_be_string()) { return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close); } - if (auto alternatives = alternatives_of(schema)) { + if (schema.kind() == common_chat_schema::KIND_ANY_OF) { std::vector choices; size_t index = 0; - for (const auto & alternative : *alternatives) { + for (const auto & alternative : static_cast(schema).children) { const std::string alt_name = rule_name + "-" + std::to_string(index++); // There is a risk that this breaks streaming deltas, but that's a risk we // assume to provide tool arg streaming. - choices.push_back(value_of(alternative, alt_name, close)); + choices.push_back(value_of(*alternative, alt_name, close)); } return p.choice(choices); } - const std::string type = schema.contains("type") && schema.at("type").is_string() - ? schema.at("type").get() - : ""; - - if (type == "object" && schema.contains("properties")) { - return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag; + if (schema.kind() == common_chat_schema::KIND_OBJECT) { + const auto & object = static_cast(schema); + if (!object.properties.empty()) { + return p.tag(mm3::TOOL_ARG_OBJECT, members_of(object, rule_name)) + p.space() + close_tag; + } } - if (type == "array" && schema.contains("items")) { + if (schema.kind() == common_chat_schema::KIND_ARRAY) { const std::string item_close = NS + ""; auto item = p.rule(rule_name + "-item", p.tag(mm3::TOOL_ARG_ITEM, p.literal(NS + "") + - value_of(schema.at("items"), rule_name + "-item", item_close))); + value_of(*static_cast(schema).items, rule_name + "-item", item_close))); return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag; } - return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag; + return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, schema)) + close_tag; }; // Required properties in schema order, then any number of optional ones in any order. - members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser { - const auto & props = schema.at("properties"); - - std::set required; - if (schema.contains("required")) { - required = schema.at("required").get>(); - } - + members_of = [&](const common_chat_schema_object & object, const std::string & rule_prefix) -> common_peg_parser { std::vector required_elements; std::vector optional_elements; - for (const auto & [key, key_schema] : props.items()) { - auto element = element_of(key, key_schema, rule_prefix + "-" + key); - if (required.find(key) != required.end()) { - required_elements.push_back(element); - } else { - optional_elements.push_back(element); - } + for (const auto & prop : object.properties) { + auto element = element_of(prop.name, *prop.schema, rule_prefix + "-" + prop.name); + (prop.required ? required_elements : optional_elements).push_back(element); } common_peg_parser members = p.eps(); @@ -201,8 +178,10 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template return members; }; - common_peg_parser invoke_body = - params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps(); + common_peg_parser invoke_body = p.eps(); + if (doc->root->kind() == common_chat_schema::KIND_OBJECT) { + invoke_body = members_of(static_cast(*doc->root), "tool-" + name + "-arg"); + } auto func_parser = p.tool( p.tool_open(p.literal(NS + "")); - } - - auto arg_rule = p.tool_arg( - p.tool_arg_open(p.literal("")) + - value_parser); - - arg_choice |= arg_rule; + std::vector arg_rules; + foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) { + auto value_parser = p.eps(); + if (prop.schema->may_be_string()) { + value_parser = string_value; + } else { + value_parser = p.tool_arg_json_value( + p.schema(p.json(), "tool-" + name + "-arg-" + prop.name + "-schema", doc, *prop.schema)) + + p.tool_arg_close(p.literal("")); } - args = p.zero_or_more(arg_choice + p.space()); + + arg_rules.push_back(p.tool_arg( + p.tool_arg_open(p.literal("")) + + value_parser)); + }); + + auto args = p.eps(); + if (!arg_rules.empty()) { + args = p.zero_or_more(p.choice(arg_rules) + p.space()); } auto tool_parser = p.tool( @@ -131,11 +126,6 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa if (include_grammar) { data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/parsers.cpp b/common/parsers/parsers.cpp index 0a4d5cfbb522..643186c34cf2 100644 --- a/common/parsers/parsers.cpp +++ b/common/parsers/parsers.cpp @@ -2,8 +2,6 @@ #include "log.h" -#include - void foreach_function(const json & tools, const std::function & fn) { for (const auto & tool : tools) { if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) { @@ -14,21 +12,14 @@ void foreach_function(const json & tools, const std::function & fn) { - if (!function.contains("parameters") || !function.at("parameters").is_object()) { - return; - } - const auto & params = function.at("parameters"); - if (!params.contains("properties") || !params.at("properties").is_object()) { +void foreach_parameter(const json & function, const std::function & fn) { + auto params = common_chat_tool_parameters(function); + auto doc = std::make_shared(common_chat_schema_from_json(params)); + const auto * object = dynamic_cast(doc->root.get()); + if (!object) { return; } - const auto & props = params.at("properties"); - std::set required; - if (params.contains("required") && params.at("required").is_array()) { - required = params.at("required").get>(); - } - for (const auto & [name, prop] : props.items()) { - bool is_required = (required.find(name) != required.end()); - fn(name, prop, is_required); + for (const auto & prop : object->properties) { + fn(prop, doc); } } diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h index 7898f0007107..73fc719fddde 100644 --- a/common/parsers/parsers.h +++ b/common/parsers/parsers.h @@ -20,8 +20,8 @@ using json = common_json; // iterate over the function tools of an OpenAI-style tools array void foreach_function(const json & tools, const std::function & fn); -// iterate over the parameters of a function tool, flagging the ones listed as required -void foreach_parameter(const json & function, const std::function & fn); +// iterate over the parameters of a function tool, with the document that owns them +void foreach_parameter(const json & function, const std::function & fn); // render a template; the override arguments let a parser feed in messages, tools or context it has rewritten std::string common_chat_template_direct_apply_impl( diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cpp index 8a1e5213700f..dfc74408472f 100644 --- a/common/parsers/qwen3-coder.cpp +++ b/common/parsers/qwen3-coder.cpp @@ -93,28 +93,24 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat auto tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto parameters = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(parameters); + const auto & function = tool.at("function"); + std::string name = function.at("name"); std::vector required_args; std::vector optional_args; - foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { - auto rule_name = "tool-" + name + "-arg-" + param_name; + foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { + auto rule_name = "tool-" + name + "-arg-" + param.name; - auto arg_open = p.tool_arg_open("\n"); + auto arg_open = p.tool_arg_open("\n"); - auto arg_value = schema_info.resolves_to_string(param_schema) ? + auto arg_value = param.schema->may_be_string() ? arg_string : - p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; + p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close; auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); - (is_required ? required_args : optional_args).push_back(arg_rule); + (param.required ? required_args : optional_args).push_back(arg_rule); }); // Accept required arguments in any order, as Qwen does not always adhere to the @@ -158,15 +154,6 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } parser.build_grammar(builder, data.grammar_lazy); }); diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 46fc29bf2f8b..10735389ea19 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -953,7 +953,7 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id } else if constexpr (std::is_same_v) { return "Until(" + string_join(p.delimiters, " | ") + ")"; } else if constexpr (std::is_same_v) { - return "Schema(" + dump_impl(p.child, visited) + ", " + (p.schema ? p.schema->dump() : "null") + ")"; + return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")"; } else if constexpr (std::is_same_v) { return "Rule(" + p.name + ", " + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { @@ -1119,8 +1119,13 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes, return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max})); } +common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw) { + return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::move(doc), &node, raw})); +} + common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) { - return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared(schema), raw})); + auto doc = std::make_shared(common_chat_schema_from_json(schema)); + return this->schema(p, name, doc, *doc->root, raw); } common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) { @@ -1573,30 +1578,9 @@ static std::set collect_reachable_rules( // GBNF generation implementation void common_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const { + // A raw string value is parsed by the child rather than constrained by the schema auto schema_delegates = [](const common_peg_schema_parser & s) -> bool { - if (!s.schema) { - return true; - } - if (s.raw && s.schema->contains("type")) { - const auto & type_val = s.schema->at("type"); - if (type_val.is_string() && type_val == "string") { - return true; - } - // Handle nullable types like ["string", "null"] - delegate when the - // non-null type is string, since the tagged format uses raw text - if (type_val.is_array()) { - for (const auto & t : type_val) { - if (t.is_string() && t.get() != "null") { - return t.get() == "string"; - } - } - } - } - // Delegate for enum schemas in raw mode - enum values are literal strings - if (s.raw && !s.schema->contains("type") && s.schema->contains("enum")) { - return true; - } - return false; + return !s.node || (s.raw && s.node->may_be_string()); }; // Unwrap the parser so we can properly check if it's a sequence or choice @@ -1731,7 +1715,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo if (schema_delegates(p)) { return to_gbnf(p.child); } - return builder.add_schema(p.name, *p.schema); + return builder.add_schema(p.name, *p.node); } else if constexpr (std::is_same_v) { return p.name; } else if constexpr (std::is_same_v) { @@ -1859,7 +1843,6 @@ static common_json serialize_parser_variant(const common_peg_parser_variant & va {"type", "schema"}, {"child", p.child}, {"name", p.name}, - {"schema", p.schema ? *p.schema : json(nullptr)}, {"raw", p.raw} }; } else if constexpr (std::is_same_v) { @@ -1999,15 +1982,12 @@ static common_peg_parser_variant deserialize_parser_variant(const common_json & return common_peg_until_parser{j["delimiters"].get>()}; } if (type == "schema") { - if (!j.contains("child") || !j.contains("name") || !j.contains("schema") || !j.contains("raw")) { + if (!j.contains("child") || !j.contains("name") || !j.contains("raw")) { throw std::runtime_error("schema parser missing required fields"); } common_peg_schema_parser parser; parser.child = j["child"].get(); parser.name = j["name"]; - if (!j["schema"].is_null()) { - parser.schema = std::make_shared(j["schema"]); - } parser.raw = j["raw"].get(); return parser; } diff --git a/common/peg-parser.h b/common/peg-parser.h index ab095cc7d671..fb5d82b30fdc 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -1,5 +1,6 @@ #pragma once +#include "json-schema.h" #include "json.h" #include @@ -245,7 +246,8 @@ struct common_peg_until_parser { struct common_peg_schema_parser { common_peg_parser_id child; std::string name; - std::shared_ptr schema; + common_chat_schema_document_ptr doc; // owns node + const common_chat_schema * node = nullptr; // Indicates if the GBNF should accept a raw string that matches the schema. bool raw; @@ -488,8 +490,10 @@ class common_peg_parser_builder { // A marker, i.e. text delimited by a pair of <> or [] common_peg_parser marker(); - // Wraps a parser with JSON schema metadata for grammar generation. - // Used internally to convert JSON schemas to GBNF grammar rules. + // Wraps a parser with the schema its GBNF is generated from, a node of the document that owns it + common_peg_parser schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw = false); + + // Parses the JSON schema into a document of its own common_peg_parser schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw = false); // Creates a named rule, stores it in the grammar, and returns a ref. diff --git a/docs/development/parsing.md b/docs/development/parsing.md index a41057db2b8a..0cb372eca56a 100644 --- a/docs/development/parsing.md +++ b/docs/development/parsing.md @@ -28,7 +28,7 @@ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { for (const auto & tool : tools) { const auto & function = tool.at("function"); std::string name = function.at("name"); - const auto & schema = function.at("parameters"); + const auto schema = common_chat_tool_parameters(function); auto tool_name = p.json_member("name", "\"" + p.literal(name) + "\""); auto tool_args = p.json_member("arguments", p.schema(p.json(), "tool-" + name + "-schema", schema)); @@ -108,6 +108,7 @@ For a more complete example, see `test_example_native()` in - **`rule(name, p, trigger)`** - Creates a named rule and returns a reference - **`trigger_rule(name, p)`** - Creates a trigger rule (entry point for lazy grammar generation) - **`schema(p, name, schema, raw)`** - Wraps parser with JSON schema metadata for grammar generation +- **`schema(p, name, doc, node, raw)`** - Same, for a node of a `common_chat_schema_document` built earlier, e.g. one tool parameter ### AST Control @@ -121,9 +122,6 @@ some exceptions. ```cpp data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(params.tools, [&](const json & fn) { - builder.resolve_refs(fn.at("parameters")); - }); parser.build_grammar(builder, data.grammar_lazy); }); ``` @@ -151,7 +149,8 @@ implementation to generate the grammar instead of the underlying parser. The `raw` option emits a grammar suitable for a raw string instead of a JSON string. In other words, it won't be wrapped in quotes or require escaping -quotes. It should only be used when `type == "string"`. +quotes. It only takes effect when the schema may be a string, as reported by +`common_chat_schema::may_be_string()`, otherwise the JSON grammar is used. The downside is that it can potentially lead to ambiguous grammars. For example, if a user provides the pattern `^.*$`, the following grammar may be diff --git a/examples/json_schema_to_grammar.py b/examples/json_schema_to_grammar.py deleted file mode 100755 index 02b7ef15ae98..000000000000 --- a/examples/json_schema_to_grammar.py +++ /dev/null @@ -1,842 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import itertools -import json -import re -import sys -from typing import Any, List, Optional, Set, Tuple, Union - -def _build_repetition(item_rule, min_items, max_items, separator_rule=None): - - if max_items == 0: - return "" - - if min_items == 0 and max_items == 1: - return f'{item_rule}?' - - if not separator_rule: - if min_items == 1 and max_items is None: - return f'{item_rule}+' - elif min_items == 0 and max_items is None: - return f'{item_rule}*' - else: - return f'{item_rule}{{{min_items},{max_items if max_items is not None else ""}}}' - - result = item_rule + ' ' + _build_repetition(f'({separator_rule} {item_rule})', min_items - 1 if min_items > 0 else 0, max_items - 1 if max_items is not None else None) - return f'({result})?' if min_items == 0 else result - -def _generate_min_max_int(min_value: Optional[int], max_value: Optional[int], out: list, decimals_left: int = 16, top_level: bool = True): - def digit_range(from_char: str, to_char: str): - out.append("[") - if from_char == to_char: - out.append(from_char) - else: - out.append(from_char) - out.append("-") - out.append(to_char) - out.append("]") - - def more_digits(min_digits: int, max_digits: int): - out.append("[0-9]") - if min_digits == max_digits and min_digits == 1: - return - out.append("{") - out.append(str(min_digits)) - if max_digits != min_digits: - out.append(",") - if max_digits != sys.maxsize: - out.append(str(max_digits)) - out.append("}") - - def uniform_range(from_str: str, to_str: str): - i = 0 - while i < len(from_str) and from_str[i] == to_str[i]: - i += 1 - if i > 0: - out.append("\"") - out.append(from_str[:i]) - out.append("\"") - if i < len(from_str): - if i > 0: - out.append(" ") - sub_len = len(from_str) - i - 1 - if sub_len > 0: - from_sub = from_str[i+1:] - to_sub = to_str[i+1:] - sub_zeros = "0" * sub_len - sub_nines = "9" * sub_len - - to_reached = False - out.append("(") - if from_sub == sub_zeros: - digit_range(from_str[i], chr(ord(to_str[i]) - 1)) - out.append(" ") - more_digits(sub_len, sub_len) - else: - out.append("[") - out.append(from_str[i]) - out.append("] ") - out.append("(") - uniform_range(from_sub, sub_nines) - out.append(")") - if ord(from_str[i]) < ord(to_str[i]) - 1: - out.append(" | ") - if to_sub == sub_nines: - digit_range(chr(ord(from_str[i]) + 1), to_str[i]) - to_reached = True - else: - digit_range(chr(ord(from_str[i]) + 1), chr(ord(to_str[i]) - 1)) - out.append(" ") - more_digits(sub_len, sub_len) - if not to_reached: - out.append(" | ") - digit_range(to_str[i], to_str[i]) - out.append(" ") - uniform_range(sub_zeros, to_sub) - out.append(")") - else: - out.append("[") - out.append(from_str[i]) - out.append("-") - out.append(to_str[i]) - out.append("]") - - if min_value is not None and max_value is not None: - if min_value < 0 and max_value < 0: - out.append("\"-\" (") - _generate_min_max_int(-max_value, -min_value, out, decimals_left, top_level=True) - out.append(")") - return - - if min_value < 0: - out.append("\"-\" (") - _generate_min_max_int(0, -min_value, out, decimals_left, top_level=True) - out.append(") | ") - min_value = 0 - - min_s = str(min_value) - max_s = str(max_value) - min_digits = len(min_s) - max_digits = len(max_s) - - for digits in range(min_digits, max_digits): - uniform_range(min_s, "9" * digits) - min_s = "1" + "0" * digits - out.append(" | ") - uniform_range(min_s, max_s) - return - - less_decimals = max(decimals_left - 1, 1) - - if min_value is not None: - if min_value < 0: - out.append("\"-\" (") - _generate_min_max_int(None, -min_value, out, decimals_left, top_level=False) - out.append(") | [0] | [1-9] ") - more_digits(0, decimals_left - 1) - elif min_value == 0: - if top_level: - out.append("[0] | [1-9] ") - more_digits(0, less_decimals) - else: - more_digits(1, decimals_left) - elif min_value <= 9: - c = str(min_value) - range_start = '1' if top_level else '0' - if c > range_start: - digit_range(range_start, chr(ord(c) - 1)) - out.append(" ") - more_digits(1, less_decimals) - out.append(" | ") - digit_range(c, "9") - out.append(" ") - more_digits(0, less_decimals) - else: - min_s = str(min_value) - length = len(min_s) - c = min_s[0] - - if c > "1": - digit_range("1" if top_level else "0", chr(ord(c) - 1)) - out.append(" ") - more_digits(length, less_decimals) - out.append(" | ") - digit_range(c, c) - out.append(" (") - _generate_min_max_int(int(min_s[1:]), None, out, less_decimals, top_level=False) - out.append(")") - if c < "9": - out.append(" | ") - digit_range(chr(ord(c) + 1), "9") - out.append(" ") - more_digits(length - 1, less_decimals) - return - - if max_value is not None: - if max_value >= 0: - if top_level: - out.append("\"-\" [1-9] ") - more_digits(0, less_decimals) - out.append(" | ") - _generate_min_max_int(0, max_value, out, decimals_left, top_level=True) - else: - out.append("\"-\" (") - _generate_min_max_int(-max_value, None, out, decimals_left, top_level=False) - out.append(")") - return - - raise RuntimeError("At least one of min_value or max_value must be set") - -class BuiltinRule: - def __init__(self, content: str, deps: list | None = None): - self.content = content - self.deps = deps or [] - -# Constraining spaces to prevent model "running away". -SPACE_RULE = '| " " | "\\n"{1,2} [ \\t]{0,20}' - -PRIMITIVE_RULES = { - 'boolean' : BuiltinRule('("true" | "false")', []), - 'decimal-part' : BuiltinRule('[0-9]{1,16}', []), - 'integral-part': BuiltinRule('[0] | [1-9] [0-9]{0,15}', []), - 'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)?', ['integral-part', 'decimal-part']), - 'integer' : BuiltinRule('("-"? integral-part)', ['integral-part']), - 'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']), - 'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? space "}"', ['string', 'value']), - 'array' : BuiltinRule('"[" space ( value ("," space value)* )? space "]"', ['value']), - 'uuid' : BuiltinRule(r'"\"" [0-9a-fA-F]{8} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{12} "\""', []), - 'char' : BuiltinRule(r'[^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})', []), - 'string' : BuiltinRule(r'"\"" char* "\""', ['char']), - 'null' : BuiltinRule('"null"', []), -} - -# TODO: support "uri", "email" string formats -STRING_FORMAT_RULES = { - 'date' : BuiltinRule('[0-9]{4} "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []), - 'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9]{3} )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []), - 'date-time' : BuiltinRule('date "T" time', ['date', 'time']), - 'date-string' : BuiltinRule('"\\"" date "\\""', ['date']), - 'time-string' : BuiltinRule('"\\"" time "\\""', ['time']), - 'date-time-string': BuiltinRule('"\\"" date-time "\\""', ['date-time']), -} - -DOTALL = '[\\U00000000-\\U0010FFFF]' -DOT = '[^\\x0A\\x0D]' - -RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()]) - -INVALID_RULE_CHARS_RE = re.compile(r'[^a-zA-Z0-9-]+') -GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\\]') -GRAMMAR_RANGE_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\]\-\\]') -GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]', '\\': '\\\\'} - -NON_LITERAL_SET = set('|.()[]{}*+?') -ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('^$.[]()|{}*+?') - - -class SchemaConverter: - def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern): - self._prop_order = prop_order - self._allow_fetch = allow_fetch - self._dotall = dotall - self._raw_pattern = raw_pattern - self._rules = { - 'space': SPACE_RULE, - } - self._refs = {} - self._refs_being_resolved = set() - - def _format_literal(self, literal): - escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub( - lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)) or m.group(0), literal - ) - return f'"{escaped}"' - - def not_literal(self, literal: str, dotall: bool = True, maybe_escaped_underscores = False) -> str: - ''' - not_literal('a') -> '[^a]' - not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?' - ''' - assert len(literal) > 0, 'Empty literal not supported' - def recurse(i: int): - c = literal[i] - if maybe_escaped_underscores and c == '_': - yield f'[^{c}\\\\]' - yield ' | ' - yield f'"\\\\"? "{c}"' - else: - yield f'[^{c}]' - if i < len(literal) - 1: - yield ' | ' - yield self._format_literal(c) - yield ' (' - yield from recurse(i + 1) - yield ')?' - - return ''.join(('(', *recurse(0), ')')) - - def _not_strings(self, strings): - class TrieNode: - def __init__(self): - self.children = {} - self.is_end_of_string = False - - def insert(self, string): - node = self - for c in string: - node = node.children.setdefault(c, TrieNode()) - node.is_end_of_string = True - - trie = TrieNode() - for s in strings: - trie.insert(s) - - char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) - out = ['["] ( '] - - def visit(node): - rejects = [] - first = True - for c in sorted(node.children.keys()): - child = node.children[c] - rejects.append(c) - if first: - first = False - else: - out.append(' | ') - out.append(f'[{c}]') - if child.children: - out.append(f' (') - visit(child) - out.append(')') - elif child.is_end_of_string: - out.append(f' {char_rule}+') - if node.children: - if not first: - out.append(' | ') - out.append(f'[^"{"".join(rejects)}] {char_rule}*') - visit(trie) - - out.append(f' ){"" if trie.is_end_of_string else "?"} ["]') - return ''.join(out) - - def _add_rule(self, name, rule): - esc_name = INVALID_RULE_CHARS_RE.sub('-', name) - if esc_name not in self._rules or self._rules[esc_name] == rule: - key = esc_name - else: - i = 0 - while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule: - i += 1 - key = f'{esc_name}{i}' - self._rules[key] = rule - return key - - def resolve_refs(self, schema: dict, url: str): - ''' - Resolves all $ref fields in the given schema, fetching any remote schemas, - replacing $ref with absolute reference URL and populating self._refs with the - respective referenced (sub)schema dictionaries. - ''' - def visit(n: dict): - if isinstance(n, list): - return [visit(x) for x in n] - elif isinstance(n, dict): - ref = n.get('$ref') - if ref is not None and ref not in self._refs: - if ref.startswith('https://'): - assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)' - import requests - - frag_split = ref.split('#') - base_url = frag_split[0] - - target = self._refs.get(base_url) - if target is None: - target = self.resolve_refs(requests.get(ref).json(), base_url) - self._refs[base_url] = target - - if len(frag_split) == 1 or frag_split[-1] == '': - return target - elif ref.startswith('#/'): - target = schema - ref = f'{url}{ref}' - n['$ref'] = ref - else: - raise ValueError(f'Unsupported ref {ref}') - - for sel in ref.split('#')[-1].split('/')[1:]: - assert target is not None, f'Error resolving ref {ref}: {sel} not in {target}' - if isinstance(target, list): - try: - sel_index = int(sel) - except ValueError: - raise ValueError(f'Error resolving ref {ref}: {sel} not in {target}') - assert 0 <= sel_index < len(target), f'Error resolving ref {ref}: {sel} not in {target}' - target = target[sel_index] - else: - assert sel in target, f'Error resolving ref {ref}: {sel} not in {target}' - target = target[sel] - - self._refs[ref] = target - else: - for v in n.values(): - visit(v) - - return n - return visit(schema) - - def _generate_union_rule(self, name, alt_schemas): - return ' | '.join(( - self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') - for i, alt_schema in enumerate(alt_schemas) - )) - - def _visit_pattern(self, pattern, name): - ''' - Transforms a regular expression pattern into a GBNF rule. - - Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions - Output: https://github.com/ggml-org/llama.cpp/blob/master/grammars/README.md - - Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. - - Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which - we define sub-rules to keep the output lean. - ''' - - assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"' - pattern = pattern[1:-1] - sub_rule_ids = {} - - i = 0 - length = len(pattern) - - def to_rule(s: tuple[str, bool]) -> str: - (txt, is_literal) = s - return "\"" + txt + "\"" if is_literal else txt - - def transform() -> tuple[str, bool]: - ''' - Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. - ''' - nonlocal i - nonlocal pattern - nonlocal sub_rule_ids - - start = i - # For each component of this sequence, store its string representation and whether it's a literal. - # We only need a flat structure here to apply repetition operators to the last item, and - # to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially - # (GBNF's syntax is luckily very close to regular expressions!) - seq: list[tuple[str, bool]] = [] - - def get_dot(): - if self._dotall: - rule = DOTALL - else: - # Accept any character... except \n and \r line break chars (\x0A and \xOD) - rule = DOT - return self._add_rule(f'dot', rule) - - def join_seq(): - nonlocal seq - ret = [] - for is_literal, g in itertools.groupby(seq, lambda x: x[1]): - if is_literal: - ret.append((''.join(x[0] for x in g), True)) - else: - ret.extend(g) - if len(ret) == 1: - return ret[0] - return (' '.join(to_rule(x) for x in seq), False) - - while i < length: - c = pattern[i] - if c == '.': - seq.append((get_dot(), False)) - i += 1 - elif c == '(': - i += 1 - if i < length: - assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' - seq.append((f'({to_rule(transform())})', False)) - elif c == ')': - i += 1 - assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}' - return join_seq() - elif c == '[': - square_brackets = c - i += 1 - while i < length and pattern[i] != ']': - if pattern[i] == '\\': - square_brackets += pattern[i:i+2] - i += 2 - else: - square_brackets += pattern[i] - i += 1 - assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}' - square_brackets += ']' - i += 1 - seq.append((square_brackets, False)) - elif c == '|': - seq.append(('|', False)) - i += 1 - elif c in ('*', '+', '?'): - seq[-1] = (to_rule(seq[-1]) + c, False) - i += 1 - elif c == '{': - curly_brackets = c - i += 1 - while i < length and pattern[i] != '}': - curly_brackets += pattern[i] - i += 1 - assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}' - curly_brackets += '}' - i += 1 - nums = [s.strip() for s in curly_brackets[1:-1].split(',')] - min_times = 0 - max_times = None - try: - if len(nums) == 1: - min_times = int(nums[0]) - max_times = min_times - else: - assert len(nums) == 2 - min_times = int(nums[0]) if nums[0] else 0 - max_times = int(nums[1]) if nums[1] else None - except ValueError: - raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/') - - (sub, sub_is_literal) = seq[-1] - - if not sub_is_literal: - id = sub_rule_ids.get(sub) - if id is None: - id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub) - sub_rule_ids[sub] = id - sub = id - - seq[-1] = (_build_repetition(f'"{sub}"' if sub_is_literal else sub, min_times, max_times), False) - else: - literal = '' - while i < length: - if pattern[i] == '\\' and i < length - 1: - next = pattern[i + 1] - if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS: - i += 1 - literal += pattern[i] - i += 1 - else: - literal += pattern[i:i+2] - i += 2 - elif pattern[i] == '"' and not self._raw_pattern: - literal += '\\"' - i += 1 - elif pattern[i] not in NON_LITERAL_SET and \ - (i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET): - literal += pattern[i] - i += 1 - else: - break - if literal: - seq.append((literal, True)) - - return join_seq() - - return self._add_rule( - name, - to_rule(transform()) if self._raw_pattern \ - else "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\"") - - - def _resolve_ref(self, ref): - ref_fragment = ref.split('#')[-1] - ref_name = 'ref' + re.sub(r'[^a-zA-Z0-9-]+', '-', ref_fragment) - if ref_name not in self._rules and ref not in self._refs_being_resolved: - self._refs_being_resolved.add(ref) - resolved = self._refs[ref] - ref_name = self.visit(resolved, ref_name) - self._refs_being_resolved.remove(ref) - return ref_name - - def _generate_constant_rule(self, value): - return self._format_literal(json.dumps(value)) - - def visit(self, schema, name): - schema_type = schema.get('type') - schema_format = schema.get('format') - rule_name = name + '-' if name in RESERVED_NAMES else name or 'root' - - if (ref := schema.get('$ref')) is not None: - return self._add_rule(rule_name, self._resolve_ref(ref)) - - elif 'oneOf' in schema or 'anyOf' in schema: - return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf'])) - - elif isinstance(schema_type, list): - return self._add_rule(rule_name, self._generate_union_rule(name, [{**schema, 'type': t} for t in schema_type])) - - elif 'const' in schema: - return self._add_rule(rule_name, self._generate_constant_rule(schema['const'])) - - elif 'enum' in schema: - rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + ')' - return self._add_rule(rule_name, rule) - - elif schema_type in (None, 'object') and \ - ('properties' in schema or \ - ('additionalProperties' in schema and schema['additionalProperties'] is not True)): - required = set(schema.get('required', [])) - properties = list(schema.get('properties', {}).items()) - return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties'))) - - elif schema_type in (None, 'object', 'string') and 'allOf' in schema: - required = set() - properties = [] - enum_sets = [] - hybrid_name = name - def add_component(comp_schema, is_required): - if (ref := comp_schema.get('$ref')) is not None: - comp_schema = self._refs[ref] - - if 'properties' in comp_schema: - for prop_name, prop_schema in comp_schema['properties'].items(): - properties.append((prop_name, prop_schema)) - if is_required: - required.add(prop_name) - - if 'enum' in comp_schema: - enum_sets.append(set(comp_schema['enum'])) - - for t in schema['allOf']: - if 'anyOf' in t: - for tt in t['anyOf']: - add_component(tt, is_required=False) - else: - add_component(t, is_required=True) - - if enum_sets: - enum_intersection = enum_sets[0] - for s in enum_sets[1:]: - enum_intersection &= s - - if enum_intersection: - rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in sorted(enum_intersection))) + ')' - return self._add_rule(rule_name, rule) - - return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=None)) - - elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema): - items = schema.get('items', schema.get('prefixItems')) - if isinstance(items, list): - return self._add_rule( - rule_name, - '"[" space ' + - ' "," space '.join( - self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') - for i, item in enumerate(items)) + - ' space "]"') - else: - item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') - min_items = schema.get("minItems", 0) - max_items = schema.get("maxItems") - return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' space "]"') - - elif schema_type in (None, 'string') and 'pattern' in schema: - return self._visit_pattern(schema['pattern'], rule_name) - - elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''): - return self._add_primitive( - 'root' if rule_name == 'root' else schema_format, - PRIMITIVE_RULES['uuid'] - ) - - elif schema_type in (None, 'string') and f'{schema_format}-string' in STRING_FORMAT_RULES: - prim_name = f'{schema_format}-string' - return self._add_rule(rule_name, self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name])) - - elif schema_type == 'string' and ('minLength' in schema or 'maxLength' in schema): - char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) - min_len = schema.get('minLength', 0) - max_len = schema.get('maxLength') - - return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\""') - - elif schema_type in (None, 'integer') and \ - ('minimum' in schema or 'exclusiveMinimum' in schema or 'maximum' in schema or 'exclusiveMaximum' in schema): - min_value = None - max_value = None - if 'minimum' in schema: - min_value = schema['minimum'] - elif 'exclusiveMinimum' in schema: - min_value = schema['exclusiveMinimum'] + 1 - if 'maximum' in schema: - max_value = schema['maximum'] - elif 'exclusiveMaximum' in schema: - max_value = schema['exclusiveMaximum'] - 1 - - out = ["("] - _generate_min_max_int(min_value, max_value, out) - out.append(")") - return self._add_rule(rule_name, ''.join(out)) - - elif (schema_type == 'object') or (len(schema) == 0): - return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object'])) - - elif schema_type is None and isinstance(schema, dict): - # No type constraint and no recognized structural keywords (e.g. {"description": "..."}). - # Per JSON Schema semantics this is equivalent to {} and accepts any value. - return self._add_rule(rule_name, self._add_primitive('value', PRIMITIVE_RULES['value'])) - - else: - assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}' - # TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero - return self._add_primitive('root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type]) - - def _add_primitive(self, name: str, rule: BuiltinRule): - n = self._add_rule(name, rule.content) - - for dep in rule.deps: - dep_rule = PRIMITIVE_RULES.get(dep) or STRING_FORMAT_RULES.get(dep) - assert dep_rule, f'Rule {dep} not known' - if dep not in self._rules: - self._add_primitive(dep, dep_rule) - return n - - def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Optional[Union[bool, Any]]): - prop_order = self._prop_order - # sort by position in prop_order (if specified) then by original order - sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))] - - prop_kv_rule_names = {} - for prop_name, prop_schema in properties: - prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') - prop_kv_rule_names[prop_name] = self._add_rule( - f'{name}{"-" if name else ""}{prop_name}-kv', - fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}' - ) - required_props = [k for k in sorted_props if k in required] - optional_props = [k for k in sorted_props if k not in required] - - if additional_properties is not None and additional_properties != False: - sub_name = f'{name}{"-" if name else ""}additional' - value_rule = self.visit(additional_properties, f'{sub_name}-value') if isinstance(additional_properties, dict) else \ - self._add_primitive('value', PRIMITIVE_RULES['value']) - key_rule = self._add_primitive('string', PRIMITIVE_RULES['string']) if not sorted_props \ - else self._add_rule(f'{sub_name}-k', self._not_strings(sorted_props)) - - prop_kv_rule_names["*"] = self._add_rule( - f'{sub_name}-kv', - f'{key_rule} ":" space {value_rule}' - ) - optional_props.append("*") - - if not required_props and not optional_props: - return '"{" space "}"' - - rule = '"{" space ' - rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props) - - if optional_props: - rule += ' (' - if required_props: - rule += ' "," space ( ' - - def get_recursive_refs(ks, first_is_optional): - [k, *rest] = ks - kv_rule_name = prop_kv_rule_names[k] - comma_ref = f'( "," space {kv_rule_name} )' - if first_is_optional: - res = comma_ref + ('*' if k == '*' else '?') - else: - res = kv_rule_name + (' ' + comma_ref + "*" if k == '*' else '') - if len(rest) > 0: - res += ' ' + self._add_rule( - f'{name}{"-" if name else ""}{k}-rest', - get_recursive_refs(rest, first_is_optional=True) - ) - return res - - rule += ' | '.join( - get_recursive_refs(optional_props[i:], first_is_optional=False) - for i in range(len(optional_props)) - ) - if required_props: - rule += ' )' - rule += ' )?' - - rule += ' space "}"' - - return rule - - def format_grammar(self): - return '\n'.join( - f'{name} ::= {rule}' - for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0]) - ) - - -def main(args_in = None): - parser = argparse.ArgumentParser( - description=''' - Generates a grammar (suitable for use in ./llama-cli) that produces JSON conforming to a - given JSON schema. Only a subset of JSON schema features are supported; more may be - added in the future. - ''', - ) - parser.add_argument( - '--prop-order', - default=[], - type=lambda s: s.split(','), - help=''' - comma-separated property names defining the order of precedence for object properties; - properties not specified here are given lower precedence than those that are, and - are kept in their original order from the schema. Required properties are always - given precedence over optional properties. - ''' - ) - parser.add_argument( - '--allow-fetch', - action='store_true', - default=False, - help='Whether to allow fetching referenced schemas over HTTPS') - parser.add_argument( - '--dotall', - action='store_true', - default=False, - help='Whether to treat dot (".") as matching all chars including line breaks in regular expression patterns') - parser.add_argument( - '--raw-pattern', - action='store_true', - default=False, - help='Treats string patterns as raw patterns w/o quotes (or quote escapes)') - - parser.add_argument('schema', help='file containing JSON schema ("-" for stdin)') - args = parser.parse_args(args_in) - - if args.schema.startswith('https://'): - url = args.schema - import requests - schema = requests.get(url).json() - elif args.schema == '-': - url = 'stdin' - schema = json.load(sys.stdin) - else: - url = f'file://{args.schema}' - with open(args.schema) as f: - schema = json.load(f) - converter = SchemaConverter( - prop_order={name: idx for idx, name in enumerate(args.prop_order)}, - allow_fetch=args.allow_fetch, - dotall=args.dotall, - raw_pattern=args.raw_pattern) - schema = converter.resolve_refs(schema, url) - converter.visit(schema, '') - print(converter.format_grammar()) - - -if __name__ == '__main__': - main() diff --git a/examples/regex_to_grammar.py b/examples/regex_to_grammar.py deleted file mode 100644 index 5cd9210a4dfc..000000000000 --- a/examples/regex_to_grammar.py +++ /dev/null @@ -1,20 +0,0 @@ -import json, subprocess, sys, os - -assert len(sys.argv) >= 2 -[_, pattern, *rest] = sys.argv - -print(subprocess.check_output( - [ - "python", - os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "json_schema_to_grammar.py"), - *rest, - "-", - "--raw-pattern", - ], - text=True, - input=json.dumps({ - "type": "string", - "pattern": pattern, - }, indent=2))) diff --git a/examples/ts-type-to-grammar.sh b/examples/ts-type-to-grammar.sh deleted file mode 100755 index 966050407888..000000000000 --- a/examples/ts-type-to-grammar.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# ./examples/ts-type-to-grammar.sh "{a:string,b:string,c?:string}" -# python examples/json_schema_to_grammar.py https://json.schemastore.org/tsconfig.json -# -set -euo pipefail - -readonly type="$1" - -# Create a temporary directory -TMPDIR="" -trap 'rm -fR "$TMPDIR"' EXIT -TMPDIR=$(mktemp -d) - -DTS_FILE="$TMPDIR/type.d.ts" -SCHEMA_FILE="$TMPDIR/schema.json" - -echo "export type MyType = $type" > "$DTS_FILE" - -# This is a fork of typescript-json-schema, actively maintained as of March 2024: -# https://github.com/vega/ts-json-schema-generator -npx ts-json-schema-generator --unstable --no-top-ref --path "$DTS_FILE" --type MyType -e none > "$SCHEMA_FILE" - -# Alternative, not actively maintained as of March 2024: -# https://github.com/YousefED/typescript-json-schema -# npx typescript-json-schema --defaultProps --required "$DTS_FILE" MyType | tee "$SCHEMA_FILE" >&2 - -./examples/json_schema_to_grammar.py "$SCHEMA_FILE" diff --git a/grammars/README.md b/grammars/README.md index 9478b3e1b5aa..f005fc2522b8 100644 --- a/grammars/README.md +++ b/grammars/README.md @@ -146,8 +146,6 @@ You can use GBNF grammars: - For any completion endpoints, passed as the `json_schema` body field - For the `/chat/completions` endpoint, passed inside the `response_format` body field (e.g. `{"type", "json_object", "schema": {"items": {}}}` or `{ type: "json_schema", json_schema: {"schema": ...} }`) - In [llama-cli](../tools/cli) and [llama-completion](../tools/completion), passed as the `--json` / `-j` flag -- To convert to a grammar ahead of time: - - in CLI, with [examples/json_schema_to_grammar.py](../examples/json_schema_to_grammar.py) > [!NOTE] > The JSON schema is only used to constrain the model output and is not injected into the prompt. The model has no visibility into the schema, so if you want it to understand the expected structure, describe it explicitly in your prompt. This does not apply to tool calling, where schemas are injected into the prompt. @@ -187,11 +185,7 @@ llama-cli \ Show grammar -You can convert any schema in command-line with: - -```bash -examples/json_schema_to_grammar.py name-age-schema.json -``` +The schema above converts to: ``` char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b3559a173f1e..2c05c0c9306a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -163,11 +163,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) llama_build_and_test(test-chat.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) target_include_directories(test-chat PRIVATE ${PROJECT_SOURCE_DIR}/tools/server) target_link_libraries(test-chat PRIVATE server-context) - # TODO: disabled on loongarch64 because the ggml-ci node lacks Python 3.8 - if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64") - llama_build_and_test(test-json-schema-to-grammar.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) - target_include_directories(test-json-schema-to-grammar PRIVATE ${PROJECT_SOURCE_DIR}/tools/server) - endif() + llama_build_and_test(test-json-schema-to-grammar.cpp) if (NOT GGML_BACKEND_DL) llama_build(test-quantize-stats.cpp) @@ -262,6 +258,7 @@ endif() llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple-tokenize.cpp) llama_build_and_test(test-jinja.cpp) llama_test(test-jinja NAME test-jinja-py ARGS -py LABEL python) +llama_build_and_test(test-json-schema.cpp) llama_build_and_test(test-chat-auto-parser.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) llama_build_and_test(test-chat-template.cpp) # debug tool for chat template differential analysis (not registered as a test, run it manually) diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 793891394ce6..9d15796f7aef 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -358,11 +358,6 @@ static void test_example_native(testing & t) { auto parser = build_parser(tc); auto lazy = !tc.tools.empty() && tc.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; auto grammar = build_grammar([&](const common_grammar_builder & builder) { - for (const auto & def : tc.tools) { - auto function = def.at("function"); - auto parameters = function.at("parameters"); - builder.resolve_refs(parameters); - }; parser.build_grammar(builder, lazy); }); @@ -440,11 +435,6 @@ static void test_example_qwen3_coder(testing & t) { }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { - for (const auto & def : tools) { - auto function = def.at("function"); - auto parameters = function.at("parameters"); - builder.resolve_refs(parameters); - }; parser.build_grammar(builder); }); @@ -513,11 +503,6 @@ static void test_example_qwen3_non_coder(testing & t) { }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { - for (const auto & def : tools) { - auto function = def.at("function"); - auto parameters = function.at("parameters"); - builder.resolve_refs(parameters); - }; parser.build_grammar(builder); }); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index f27c91e4d4cc..1aef83f430d7 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -472,6 +472,12 @@ static common_chat_tool empty_args_tool_no_properties{ })", }; +static common_chat_tool empty_args_tool_no_schema{ + /* .name = */ "empty_args_no_schema", + /* .description = */ "A tool that takes no arguments and has no parameters schema", + /* .parameters = */ "{}", +}; + static common_chat_tool python_tool{ /* .name = */ "python", /* .description = */ "an ipython interpreter", @@ -5071,6 +5077,13 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect(simple_assist_msg("", "", "empty_args", "{}")) .run(); + // Tool call with no parameters schema, {} means no arguments + tst.test("\n{\"name\": \"empty_args_no_schema\", \"arguments\": {}}") + .enable_thinking(false) + .tools({ empty_args_tool_no_schema }) + .expect(simple_assist_msg("", "", "empty_args_no_schema", "{}")) + .run(); + // fake tool call marker in reasoning tst.test( "Let me think about \n{\"name\": \"special_function\", \"arguments\": {\"arg1\": 2}} hmm\n\n\n" diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index eb4b7c78f50f..2af96f8b4797 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -918,7 +918,7 @@ static void test_json_schema() { // Otherwise, this test structure is the same. test_schema( - "empty schema (object)", + "empty schema (any value)", // Schema R"""( {} @@ -927,14 +927,16 @@ static void test_json_schema() { { R"""({})""", R"""({"foo": "bar"})""", - }, - // Failing strings - { - "", "[]", "null", R"""("")""", "true", + }, + // Failing strings + { + "", + R"""({"foo"})""", + "foo", } ); diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index 2a1b6348c951..4c4206c6e690 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -9,8 +9,6 @@ #include "json.h" #include -#include -#include #include static std::string trim(const std::string & source) { @@ -64,21 +62,8 @@ struct TestCase { } }; -static void write(const std::string & file, const std::string & content) { - std::ofstream f; - f.open(file.c_str()); - f << content.c_str(); - f.close(); -} - -static std::string read(const std::string & file) { - std::ostringstream actuals; - actuals << std::ifstream(file.c_str()).rdbuf(); - return actuals.str(); -} - -static void test_all(const std::string & lang, std::function runner) { - fprintf(stderr, "#\n# Testing JSON schema conversion (%s)\n#\n", lang.c_str()); +static void test_all(const std::string & title, std::function runner) { + fprintf(stderr, "#\n# %s\n#\n", title.c_str()); auto test = [&](const TestCase & tc) { fprintf(stderr, "- %s%s\n", tc.name.c_str(), tc.expected_status == FAILURE ? " (failure expected)" : ""); runner(tc); @@ -330,7 +315,7 @@ static void test_all(const std::string & lang, std::function test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE); - tc.verify(read("test-grammar-output.tmp")); - }); - } else { - fprintf(stderr, "\033[33mWARNING: Python not found (min version required is 3.8), skipping Python JSON schema -> grammar tests.\n\033[0m"); - } + }; + auto doc = common_chat_schema_from_json(parameters); + tc.verify(build_grammar([&](const common_grammar_builder & builder) { + const auto & item = static_cast(*doc.root).properties.at(0); + builder.add_schema("root", *item.schema); + })); } - test_all("Check Expectations Validity", [](const TestCase & tc) { + test_all("Check the expectations parse", [](const TestCase & tc) { if (tc.expected_status == SUCCESS) { tc.verify_expectation_parseable(); } diff --git a/tests/test-json-schema.cpp b/tests/test-json-schema.cpp new file mode 100644 index 000000000000..fb8cee18b9fb --- /dev/null +++ b/tests/test-json-schema.cpp @@ -0,0 +1,513 @@ +#include "json-schema.h" +#include "json.h" +#include "testing.h" + +#include +#include +#include +#include +#include + +static common_chat_schema_document parse(const std::string & schema) { + return common_chat_schema_from_json(common_json::parse(schema)); +} + +// the node as T, aborting the current test when it is some other kind +template +static const T & as(testing & t, const common_chat_schema * node, const char * what) { + const T * typed = dynamic_cast(node); + if (!t.assert_true(std::string(what) + " has the expected kind", typed != nullptr)) { + throw std::runtime_error(std::string(what) + " has the wrong kind"); + } + return *typed; +} + +template +static const T & root(testing & t, const common_chat_schema_document & doc) { + return as(t, doc.root.get(), "root"); +} + +static void assert_error(testing & t, const std::string & schema, const std::string & needle) { + try { + parse(schema); + t.assert_true(schema + " is rejected", false); + } catch (const std::runtime_error & e) { + std::string what = e.what(); + t.assert_true(schema + " -> " + what, what.find(needle) != std::string::npos); + } +} + +static void test_any(testing & t) { + t.test("empty schema", [](testing & t) { + auto doc = parse("{}"); + root(t, doc); + t.assert_true("no refs", doc.refs.empty()); + }); + + t.test("keywords that do not imply a type", [](testing & t) { + auto doc = parse(R"({"description": "x", "format": "email", "additionalProperties": true})"); + root(t, doc); + }); +} + +static void test_primitives(testing & t) { + t.test("null, boolean, number", [](testing & t) { + auto doc_null = parse(R"({"type": "null"})"); + root(t, doc_null); + auto doc_bool = parse(R"({"type": "boolean"})"); + root(t, doc_bool); + auto doc_num = parse(R"({"type": "number", "minimum": 1, "maximum": 2})"); + root(t, doc_num); + }); +} + +static void test_integer(testing & t) { + t.test("unbounded", [](testing & t) { + auto doc = parse(R"({"type": "integer"})"); + const auto & i = root(t, doc); + t.assert_equal("minimum", INT64_MIN, i.minimum); + t.assert_equal("maximum", INT64_MAX, i.maximum); + }); + + t.test("inclusive bounds", [](testing & t) { + auto doc = parse(R"({"type": "integer", "minimum": -5, "maximum": 10})"); + const auto & i = root(t, doc); + t.assert_equal("minimum", -5, i.minimum); + t.assert_equal("maximum", 10, i.maximum); + }); + + t.test("exclusive bounds are folded", [](testing & t) { + auto doc = parse(R"({"type": "integer", "exclusiveMinimum": 0, "exclusiveMaximum": 10})"); + const auto & i = root(t, doc); + t.assert_equal("minimum", 1, i.minimum); + t.assert_equal("maximum", 9, i.maximum); + }); + + t.test("fractional bounds round inwards", [](testing & t) { + auto doc = parse(R"({"type": "integer", "minimum": 1.5, "exclusiveMaximum": 9.5})"); + const auto & i = root(t, doc); + t.assert_equal("minimum", 2, i.minimum); + t.assert_equal("maximum", 9, i.maximum); + }); +} + +static void test_string(testing & t) { + t.test("defaults", [](testing & t) { + auto doc = parse(R"({"type": "string"})"); + const auto & s = root(t, doc); + t.assert_equal("pattern", "", s.pattern); + t.assert_equal("format", common_chat_schema::FORMAT_NONE, s.format); + t.assert_equal("min_length", 0, s.min_length); + t.assert_equal("max_length", -1, s.max_length); + }); + + t.test("all keywords are kept", [](testing & t) { + auto doc = parse(R"({"type": "string", "pattern": "^[a-z]+$", "format": "date", "minLength": 2, "maxLength": 8})"); + const auto & s = root(t, doc); + t.assert_equal("pattern", "^[a-z]+$", s.pattern); + t.assert_equal("format", common_chat_schema::FORMAT_DATE, s.format); + t.assert_equal("min_length", 2, s.min_length); + t.assert_equal("max_length", 8, s.max_length); + }); + + t.test("formats", [](testing & t) { + auto expect = [&](const char * format, common_chat_schema::string_format expected) { + auto doc = parse(std::string(R"({"type": "string", "format": ")") + format + "\"}"); + t.assert_equal(format, expected, root(t, doc).format); + }; + expect("time", common_chat_schema::FORMAT_TIME); + expect("date-time", common_chat_schema::FORMAT_DATE_TIME); + expect("uuid", common_chat_schema::FORMAT_UUID); + expect("uuid5", common_chat_schema::FORMAT_UUID); + expect("email", common_chat_schema::FORMAT_NONE); + }); + + t.test("pattern, length and known format imply a string", [](testing & t) { + auto doc_pattern = parse(R"({"pattern": "^a$"})"); + t.assert_equal("pattern", "^a$", root(t, doc_pattern).pattern); + auto doc_length = parse(R"({"minLength": 1, "maxLength": 3})"); + t.assert_equal("min_length", 1, root(t, doc_length).min_length); + t.assert_equal("max_length", 3, root(t, doc_length).max_length); + auto doc_format = parse(R"({"format": "uuid"})"); + t.assert_equal("format", common_chat_schema::FORMAT_UUID, root(t, doc_format).format); + }); +} + +static void test_array(testing & t) { + t.test("items with bounds", [](testing & t) { + auto doc = parse(R"({"type": "array", "items": {"type": "integer"}, "minItems": 1, "maxItems": 3})"); + const auto & a = root(t, doc); + as(t, a.items.get(), "items"); + t.assert_equal("min_items", 1, a.min_items); + t.assert_equal("max_items", 3, a.max_items); + }); + + t.test("no items", [](testing & t) { + auto doc = parse(R"({"type": "array"})"); + const auto & a = root(t, doc); + as(t, a.items.get(), "items"); + t.assert_equal("min_items", 0, a.min_items); + t.assert_equal("max_items", -1, a.max_items); + }); + + t.test("items imply an array", [](testing & t) { + auto doc = parse(R"({"items": {"type": "string"}})"); + const auto & a = root(t, doc); + as(t, a.items.get(), "items"); + }); +} + +static void test_tuple(testing & t) { + t.test("prefixItems", [](testing & t) { + auto doc = parse(R"({"prefixItems": [{"type": "string"}, {"type": "number"}]})"); + const auto & tup = root(t, doc); + t.assert_equal("size", (size_t) 2, tup.items.size()); + as(t, tup.items[0].get(), "items[0]"); + as(t, tup.items[1].get(), "items[1]"); + }); + + t.test("items as an array", [](testing & t) { + auto doc = parse(R"({"type": "array", "items": [{"type": "boolean"}]})"); + const auto & tup = root(t, doc); + t.assert_equal("size", (size_t) 1, tup.items.size()); + as(t, tup.items[0].get(), "items[0]"); + }); +} + +static void test_object(testing & t) { + t.test("type alone accepts any object", [](testing & t) { + auto doc = parse(R"({"type": "object"})"); + const auto & o = root(t, doc); + t.assert_true("no properties", o.properties.empty()); + as(t, o.additional_properties.get(), "additional_properties"); + }); + + t.test("properties", [](testing & t) { + auto doc = parse(R"({ + "type": "object", + "properties": { + "b": {"type": "string"}, + "a": {"type": "integer"}, + "c": {"type": "boolean"} + }, + "required": ["a", "c"] + })"); + const auto & o = root(t, doc); + t.assert_equal("size", (size_t) 3, o.properties.size()); + t.assert_equal("order", "b", o.properties[0].name); + t.assert_equal("order", "a", o.properties[1].name); + t.assert_equal("order", "c", o.properties[2].name); + t.assert_true("b optional", !o.properties[0].required); + t.assert_true("a required", o.properties[1].required); + t.assert_true("c required", o.properties[2].required); + as(t, o.properties[0].schema.get(), "b"); + as(t, o.properties[1].schema.get(), "a"); + as(t, o.properties[2].schema.get(), "c"); + t.assert_true("closed", o.additional_properties == nullptr); + }); + + t.test("unknown required entries are ignored", [](testing & t) { + auto doc = parse(R"({"properties": {"a": {}}, "required": ["a", "zzz", 1]})"); + const auto & o = root(t, doc); + t.assert_equal("size", (size_t) 1, o.properties.size()); + t.assert_true("a required", o.properties[0].required); + }); + + t.test("additionalProperties false implies an object", [](testing & t) { + auto doc = parse(R"({"additionalProperties": false})"); + const auto & o = root(t, doc); + t.assert_true("no properties", o.properties.empty()); + t.assert_true("closed", o.additional_properties == nullptr); + }); + + t.test("additionalProperties schema", [](testing & t) { + auto doc = parse(R"({"properties": {"a": {}}, "additionalProperties": {"type": "integer", "minimum": 0}})"); + const auto & o = root(t, doc); + t.assert_equal("size", (size_t) 1, o.properties.size()); + const auto & v = as(t, o.additional_properties.get(), "additional_properties"); + t.assert_equal("minimum", 0, v.minimum); + }); + + t.test("nested", [](testing & t) { + auto doc = parse(R"({"properties": {"inner": {"properties": {"leaf": {"type": "null"}}, "required": ["leaf"]}}})"); + const auto & o = root(t, doc); + const auto & inner = as(t, o.properties[0].schema.get(), "inner"); + t.assert_equal("leaf name", "leaf", inner.properties[0].name); + t.assert_true("leaf required", inner.properties[0].required); + as(t, inner.properties[0].schema.get(), "leaf"); + }); +} + +static void test_const_enum(testing & t) { + t.test("const", [](testing & t) { + auto doc = parse(R"({"const": {"a": [1, null]}})"); + t.assert_equal("value", R"({"a":[1,null]})", root(t, doc).value.dump()); + }); + + t.test("enum", [](testing & t) { + auto doc = parse(R"({"enum": ["a", 1, null, true]})"); + const auto & e = root(t, doc); + t.assert_equal("size", (size_t) 4, e.values.size()); + t.assert_equal("values[0]", "\"a\"", e.values[0].dump()); + t.assert_equal("values[1]", "1", e.values[1].dump()); + t.assert_equal("values[2]", "null", e.values[2].dump()); + t.assert_equal("values[3]", "true", e.values[3].dump()); + }); + + t.test("const wins over enum, enum wins over type", [](testing & t) { + auto doc_enum = parse(R"({"type": "integer", "enum": [1, 2]})"); + root(t, doc_enum); + auto doc_const = parse(R"({"type": "string", "const": "x", "enum": ["y"]})"); + t.assert_equal("value", "\"x\"", root(t, doc_const).value.dump()); + }); +} + +static void test_any_of(testing & t) { + t.test("anyOf and oneOf", [](testing & t) { + auto doc_any = parse(R"({"anyOf": [{"type": "string"}, {"type": "number"}]})"); + const auto & u = root(t, doc_any); + t.assert_equal("size", (size_t) 2, u.children.size()); + as(t, u.children[0].get(), "children[0]"); + as(t, u.children[1].get(), "children[1]"); + + auto doc_one = parse(R"({"oneOf": [{"type": "null"}]})"); + const auto & o = root(t, doc_one); + t.assert_equal("size", (size_t) 1, o.children.size()); + as(t, o.children[0].get(), "children[0]"); + }); + + t.test("oneOf wins over anyOf and type", [](testing & t) { + auto doc = parse(R"({"type": "string", "oneOf": [{"type": "null"}], "anyOf": [{"type": "number"}, {"type": "boolean"}]})"); + const auto & u = root(t, doc); + t.assert_equal("size", (size_t) 1, u.children.size()); + as(t, u.children[0].get(), "children[0]"); + }); + + t.test("type array expands with sibling keywords", [](testing & t) { + auto doc = parse(R"({"type": ["string", "null", "integer"], "minLength": 2, "minimum": 5})"); + const auto & u = root(t, doc); + t.assert_equal("size", (size_t) 3, u.children.size()); + t.assert_equal("min_length", 2, as(t, u.children[0].get(), "children[0]").min_length); + as(t, u.children[1].get(), "children[1]"); + t.assert_equal("minimum", 5, as(t, u.children[2].get(), "children[2]").minimum); + }); +} + +static void test_all_of(testing & t) { + t.test("components", [](testing & t) { + auto doc = parse(R"({"allOf": [{"properties": {"a": {}}}, {"anyOf": [{"properties": {"b": {}}}, {"type": "null"}]}]})"); + const auto & all = root(t, doc); + t.assert_equal("size", (size_t) 2, all.children.size()); + as(t, all.children[0].get(), "children[0]"); + as(t, all.children[1].get(), "children[1]"); + + auto doc_typed = parse(R"({"type": "object", "allOf": [{"properties": {"a": {}}}]})"); + root(t, doc_typed); + }); + + t.test("properties win over allOf", [](testing & t) { + auto doc = parse(R"({"type": "object", "properties": {"a": {}}, "allOf": [{"properties": {"b": {}}}]})"); + t.assert_equal("size", (size_t) 1, root(t, doc).properties.size()); + }); + + t.test("other types ignore allOf", [](testing & t) { + auto doc = parse(R"({"type": "integer", "allOf": [{"minimum": 1}]})"); + root(t, doc); + }); +} + +static void test_ref(testing & t) { + t.test("target is owned by the document", [](testing & t) { + auto doc = parse(R"({"$ref": "#/$defs/t", "type": "string", "$defs": {"t": {"type": "boolean"}}})"); + const auto & r = root(t, doc); + t.assert_equal("ref", "#/$defs/t", r.ref); + t.assert_equal("refs", (size_t) 1, doc.refs.size()); + t.assert_true("target", r.target != nullptr && r.target == doc.refs.at("#/$defs/t").get()); + as(t, r.target, "target"); + }); + + t.test("definitions", [](testing & t) { + auto doc = parse(R"({"properties": {"a": {"$ref": "#/definitions/t"}}, "definitions": {"t": {"type": "number"}}})"); + const auto & o = root(t, doc); + const auto & r = as(t, o.properties[0].schema.get(), "a"); + as(t, r.target, "target"); + }); + + t.test("recursive", [](testing & t) { + auto doc = parse(R"({ + "$ref": "#/$defs/node", + "$defs": { + "node": { + "type": "object", + "properties": { + "value": {"type": "number"}, + "next": {"$ref": "#/$defs/node"} + }, + "required": ["value"] + } + } + })"); + const auto & r = root(t, doc); + const auto & node = as(t, r.target, "node"); + t.assert_equal("properties", (size_t) 2, node.properties.size()); + const auto & next = as(t, node.properties[1].schema.get(), "next"); + t.assert_true("cycle", next.target == r.target); + t.assert_equal("refs", (size_t) 1, doc.refs.size()); + }); + + t.test("pointer through an array", [](testing & t) { + auto doc = parse(R"({"oneOf": [{"type": "null"}, {"$ref": "#/oneOf/0"}]})"); + const auto & u = root(t, doc); + const auto & r = as(t, u.children[1].get(), "children[1]"); + as(t, r.target, "target"); + }); + + t.test("targets survive moving the document", [](testing & t) { + auto parsed = parse(R"({"items": {"$ref": "#/$defs/t"}, "$defs": {"t": {"type": "null"}}})"); + common_chat_schema_document doc = std::move(parsed); + const auto & a = root(t, doc); + const auto & r = as(t, a.items.get(), "items"); + t.assert_true("target", r.target == doc.refs.at("#/$defs/t").get()); + as(t, r.target, "target"); + }); +} + +static void test_may_be_string(testing & t) { + auto check = [](testing & t, const std::string & schema, bool expected) { + t.assert_equal(schema, expected, parse(schema).root->may_be_string()); + }; + + t.test("leaves", [&](testing & t) { + check(t, R"({"type": "string"})", true); + check(t, R"({"type": "integer"})", false); + check(t, R"({"minLength": 1})", true); + check(t, R"({"pattern": "^[a-z]+$"})", true); + check(t, R"({"const": "hello"})", true); + check(t, R"({"const": 123})", false); + check(t, R"({"enum": [1, "a", null]})", true); + check(t, R"({"enum": [1, 2, 3]})", false); + }); + + t.test("composites", [&](testing & t) { + check(t, R"({"type": ["integer", "string"]})", true); + check(t, R"({"anyOf": [{"type": "integer"}, {"type": "boolean"}]})", false); + check(t, R"({"allOf": [{"type": "string"}, {"minLength": 1}]})", true); + check(t, R"({"allOf": [{"type": "string"}, {"type": "integer"}]})", false); + check(t, R"({"allOf": [{"minLength": 1}, {"maxLength": 2}]})", true); + }); + + t.test("ref", [&](testing & t) { + check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"anyOf": [{"$ref": "#/$defs/n"}, {"type": "string"}]}}})", true); + check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"$ref": "#/$defs/n"}}})", false); + check(t, R"({"anyOf": [{"$ref": "#/$defs/a"}, {"$ref": "#/$defs/b"}], "$defs": {"a": {"allOf": [{"$ref": "#/$defs/b"}, {"type": "integer"}]}, "b": {"type": "string"}}})", true); + }); +} + +// e.g. {number, integer}, in type order +static std::string dump(const common_chat_schema::type_set & types) { + static const common_chat_schema::value_type order[] = { common_chat_schema::TYPE_NULL, common_chat_schema::TYPE_BOOLEAN, common_chat_schema::TYPE_NUMBER, + common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_ARRAY, + common_chat_schema::TYPE_OBJECT }; + std::string out; + for (auto type : order) { + if (types.has(type)) { + out += (out.empty() ? "" : ", ") + std::string(common_chat_schema::type_name(type)); + } + } + return "{" + out + "}"; +} + +static void test_value_types(testing & t) { + auto check = [](testing & t, const std::string & schema, const common_chat_schema::type_set & expected) { + t.assert_equal(schema, dump(expected), dump(parse(schema).root->value_types())); + }; + + t.test("leaves", [&](testing & t) { + check(t, R"({"type": "string"})", { common_chat_schema::TYPE_STRING }); + check(t, R"({"type": "number"})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER }); + check(t, R"({"description": "x"})", common_chat_schema::type_set::all()); + check(t, R"({"properties": {"a": {"type": "string"}}})", { common_chat_schema::TYPE_OBJECT }); + check(t, R"({"items": {"type": "string"}})", { common_chat_schema::TYPE_ARRAY }); + check(t, R"({"const": 1.5})", { common_chat_schema::TYPE_NUMBER }); + check(t, R"({"enum": [1, "a", null]})", { common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL }); + }); + + t.test("any_of is the union, all_of is the intersection", [&](testing & t) { + check(t, R"({"type": ["string", "null"]})", { common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL }); + check(t, R"({"allOf": [{"type": ["string", "number"]}, {"type": ["number", "object"]}]})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER }); + check(t, R"({"allOf": [{"type": "string"}, {"type": "integer"}]})", {}); + }); + + t.test("ref", [&](testing & t) { + check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"anyOf": [{"$ref": "#/$defs/n"}, {"type": "string"}]}}})", + { common_chat_schema::TYPE_STRING }); + }); +} + +static void test_errors(testing & t) { + t.test("not a schema", [](testing & t) { + assert_error(t, R"([])", "#: schema must be an object"); + }); + + t.test("type", [](testing & t) { + assert_error(t, R"({"type": 5})", "#: type must be a string or an array of strings"); + assert_error(t, R"({"type": []})", "#: type must not be empty"); + assert_error(t, R"({"type": ["string", "bad"]})", "#/type/1: unrecognized type bad"); + }); + + t.test("ref", [](testing & t) { + assert_error(t, R"({"$ref": 5})", "#: $ref must be a string"); + assert_error(t, R"({"$ref": "https://example.com/x.json"})", "#: unsupported $ref https://example.com/x.json"); + assert_error(t, R"({"$ref": ""})", "#: unsupported $ref ,"); + assert_error(t, R"({"$ref": "#"})", "#: unsupported $ref #,"); + assert_error(t, R"({"$defs": {}, "$ref": "#/$defs/missing"})", "#: cannot resolve $ref #/$defs/missing, missing not found"); + assert_error(t, R"({"oneOf": [{}], "$ref": "#/oneOf/1"})", "#: cannot resolve $ref #/oneOf/1, 1 is out of range"); + assert_error(t, R"({"$defs": {"a": {"$ref": "#/$defs/a/nope"}}, "$ref": "#/$defs/a"})", "#/$defs/a: cannot resolve $ref #/$defs/a/nope, nope not found"); + }); + + t.test("alternatives", [](testing & t) { + assert_error(t, R"({"oneOf": []})", "#/oneOf: must not be empty"); + assert_error(t, R"({"anyOf": {}})", "#/anyOf: must be an array of schemas"); + assert_error(t, R"({"anyOf": [{"type": "string"}, {"items": {"type": "x"}}]})", "#/anyOf/1/items: unrecognized type x"); + }); + + t.test("keywords", [](testing & t) { + assert_error(t, R"({"enum": []})", "#: enum must be a non-empty array"); + assert_error(t, R"({"type": "string", "pattern": 5})", "#: pattern must be a string"); + assert_error(t, R"({"type": "string", "minLength": -1})", "#: minLength must be a non-negative integer"); + assert_error(t, R"({"type": "integer", "minimum": "1"})", "#: minimum must be a number"); + assert_error(t, R"({"type": "array", "maxItems": 1.5})", "#: maxItems must be a non-negative integer"); + assert_error(t, R"({"properties": []})", "#: properties must be an object"); + assert_error(t, R"({"properties": {"a": {"type": "nope"}}})", "#/properties/a: unrecognized type nope"); + assert_error(t, R"({"additionalProperties": null})", "#: additionalProperties must be a boolean or a schema"); + }); +} + +int main(int argc, char * argv[]) { + testing t(std::cout); + if (argc >= 2) { + t.set_filter(argv[1]); + } + + const char * verbose = getenv("LLAMA_TEST_VERBOSE"); + if (verbose) { + t.verbose = std::string(verbose) == "1"; + } + + t.test("any", test_any); + t.test("primitives", test_primitives); + t.test("integer", test_integer); + t.test("string", test_string); + t.test("array", test_array); + t.test("tuple", test_tuple); + t.test("object", test_object); + t.test("const and enum", test_const_enum); + t.test("any_of", test_any_of); + t.test("all_of", test_all_of); + t.test("ref", test_ref); + t.test("may_be_string", test_may_be_string); + t.test("value_types", test_value_types); + t.test("errors", test_errors); + + return t.summary(); +} diff --git a/tools/cli/README.md b/tools/cli/README.md index 77a5e6fe3259..ea1f7aaf8bd2 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -133,8 +133,8 @@ | `-l, --logit-bias TOKEN_ID(+/-)BIAS` | modifies the likelihood of token appearing in the completion,
i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',
or `--logit-bias 15043-1` to decrease likelihood of token ' Hello' | | `--grammar GRAMMAR` | BNF-like grammar to constrain generations (see samples in grammars/ dir) | | `--grammar-file FNAME` | file to read grammar from | -| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object
For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | -| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object
For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | +| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | +| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | | `-bs, --backend-sampling` | enable backend sampling (experimental) (default: disabled)
(env: LLAMA_ARG_BACKEND_SAMPLING) | diff --git a/tools/completion/README.md b/tools/completion/README.md index 08485a95f593..c9a4cccfc271 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -216,8 +216,8 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-l, --logit-bias TOKEN_ID(+/-)BIAS` | modifies the likelihood of token appearing in the completion,
i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',
or `--logit-bias 15043-1` to decrease likelihood of token ' Hello' | | `--grammar GRAMMAR` | BNF-like grammar to constrain generations (see samples in grammars/ dir) | | `--grammar-file FNAME` | file to read grammar from | -| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object
For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | -| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object
For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | +| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | +| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | | `-bs, --backend-sampling` | enable backend sampling (experimental) (default: disabled)
(env: LLAMA_ARG_BACKEND_SAMPLING) | @@ -556,7 +556,7 @@ These options help improve the performance and memory usage of the LLaMA models. - `--grammar GRAMMAR`, `--grammar-file FILE`: Specify a grammar (defined inline or in a file) to constrain model output to a specific format. For example, you could force the model to output JSON or to speak only in emojis. See the [GBNF guide](../../grammars/README.md) for details on the syntax. -- `--json-schema SCHEMA`: Specify a [JSON schema](https://json-schema.org/) to constrain model output to (e.g. `{}` for any JSON object, or `{"items": {"type": "string", "minLength": 10, "maxLength": 100}, "minItems": 10}` for a JSON array of strings with size constraints). If a schema uses external `$ref`s, you should use `--grammar "$( python examples/json_schema_to_grammar.py myschema.json )"` instead. +- `--json-schema SCHEMA`: Specify a [JSON schema](https://json-schema.org/) to constrain model output to (e.g. `{"type": "object"}` for any JSON object, or `{"items": {"type": "string", "minLength": 10, "maxLength": 100}, "minItems": 10}` for a JSON array of strings with size constraints). ### Quantization diff --git a/tools/server/README.md b/tools/server/README.md index 19090763281a..ef9033404824 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -150,8 +150,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `-l, --logit-bias TOKEN_ID(+/-)BIAS` | modifies the likelihood of token appearing in the completion,
i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',
or `--logit-bias 15043-1` to decrease likelihood of token ' Hello' | | `--grammar GRAMMAR` | BNF-like grammar to constrain generations (see samples in grammars/ dir) | | `--grammar-file FNAME` | file to read grammar from | -| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object
For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | -| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object
For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead | +| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | +| `-jf, --json-schema-file FILE` | File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{"type": "object"}` for any JSON object | | `-bs, --backend-sampling` | enable backend sampling (experimental) (default: disabled)
(env: LLAMA_ARG_BACKEND_SAMPLING) | diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 483391333538..7bf1138c8a8e 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1198,6 +1198,11 @@ json oaicompat_chat_params_parse( } } + // an absent or empty schema means any object + if (json_schema.is_object() && json_schema.empty()) { + json_schema["type"] = "object"; + } + // get input files if (!body.contains("messages")) { throw std::invalid_argument("'messages' is required"); diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 64b9251295ce..27ecafb7a595 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -257,6 +257,10 @@ std::vector> make_llama_cmpl_schema(const common_params & if (data.contains("json_schema") && !data.contains("grammar")) { try { auto schema = json_value(data, "json_schema", json::object()); + if (schema.is_object() && schema.empty()) { + // an empty schema means any object + schema["type"] = "object"; + } SRV_DBG("JSON schema: %s\n", schema.dump(2).c_str()); std::string grammar_str = json_schema_to_grammar(schema); SRV_DBG("Converted grammar: %s\n", grammar_str.c_str()); From 8e330954adb6e86c329c9d7e338f01f93ffe4b88 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sun, 13 Sep 2026 01:36:34 +0200 Subject: [PATCH 63/65] common: add LOG_JSON macro to log structured data (#28586) * add LOG_JSON macro * fit: add demo LOG_JSON --- common/arg.cpp | 2 +- common/fit.cpp | 41 +++++++++++++++++++++++++++ common/log.cpp | 76 +++++++++++++++++++++++++++++++++++++++++--------- common/log.h | 19 ++++++++++++- 4 files changed, 123 insertions(+), 15 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index b1c0f23526ef..c4c4e143c987 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -3875,7 +3875,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--no-log-jsonl"}, "Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)", [](common_params &, bool value) { - common_log_set_jsonl(common_log_main(), value); + common_log_set_jsonl(value); } ).set_env("LLAMA_ARG_LOG_JSONL")); add_opt(common_arg( diff --git a/common/fit.cpp b/common/fit.cpp index c601fe405ea5..7a0300829508 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -1,5 +1,6 @@ #include "fit.h" +#include "json.h" #include "log.h" #include "../src/llama-ext.h" @@ -915,6 +916,9 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { std::vector> table_data; table_data.reserve(devices.size()); + + // same data as the table below, for --log-jsonl consumers + common_json rows = common_json::array(); const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n"; const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n"; const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n"; @@ -989,6 +993,19 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { std::to_string(mb.context / MiB), std::to_string(mb.compute / MiB), std::to_string(unaccounted / static_cast(MiB))}); + + rows.push_back({ + {"kind", "device"}, + {"name", name}, + {"description", desc}, + {"total", total / MiB}, + {"free", free / MiB}, + {"self", self / MiB}, + {"model", mb.model / MiB}, + {"context", mb.context / MiB}, + {"compute", mb.compute / MiB}, + {"unaccounted", unaccounted / static_cast(MiB)}, + }); } // print memory breakdown for host: @@ -1004,6 +1021,15 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { std::to_string(mb_host.context / MiB), std::to_string(mb_host.compute / MiB), ""}); // unaccounted + + rows.push_back({ + {"kind", "host"}, + {"name", "Host"}, + {"self", self / MiB}, + {"model", mb_host.model / MiB}, + {"context", mb_host.context / MiB}, + {"compute", mb_host.compute / MiB}, + }); } // print memory breakdown for all remaining buffer types: @@ -1025,6 +1051,16 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { std::to_string(mb.context / MiB), std::to_string(mb.compute / MiB), ""}); // unaccounted + + rows.push_back({ + {"kind", "buffer_type"}, + {"name", name}, + {"self", self / MiB}, + {"model", mb.model / MiB}, + {"context", mb.context / MiB}, + {"compute", mb.compute / MiB}, + }); + seen_buffer_types.insert(buft); } @@ -1042,6 +1078,11 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { __func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(), td[6].c_str(), td[7].c_str(), td[8].c_str()); } + + LOG_JSON("fit_memory_breakdown", common_json({ + {"unit", "MiB"}, + {"rows", rows}, + })); } void common_fit_print( diff --git a/common/log.cpp b/common/log.cpp index 42951190c082..0a9a4eb9ea49 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -37,6 +37,16 @@ void common_log_set_verbosity_thold(int verbosity) { common_log_verbosity_thold = verbosity; } +static bool common_log_jsonl = false; + +bool common_log_get_jsonl(void) { + return common_log_jsonl; +} + +void common_log_set_jsonl(bool jsonl) { + common_log_jsonl = jsonl; +} + static int64_t t_us() { return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); } @@ -87,6 +97,7 @@ struct common_log_entry { bool is_end { false }; // signals the worker thread to stop bool prefix { false }; bool jsonl { false }; + bool is_json { false }; // msg already holds a serialized JSON object common_log_entry(size_t size = 256) : msg(size) { } @@ -107,6 +118,12 @@ struct common_log_entry { } if (jsonl) { + if (is_json) { + fprintf(fcur, "%s\n", msg.data()); + fflush(fcur); + return; + } + common_json obj = { {"type", "log"}, {"time", timestamp}, @@ -156,7 +173,6 @@ struct common_log { file = nullptr; prefix = false; timestamps = false; - jsonl = false; running = false; t_start = t_us(); @@ -184,7 +200,6 @@ struct common_log { bool prefix; bool timestamps; - bool jsonl; bool running; int64_t t_start; @@ -273,7 +288,8 @@ struct common_log { entry.is_end = false; entry.level = level; entry.prefix = prefix; - entry.jsonl = jsonl; + entry.jsonl = common_log_jsonl; + entry.is_json = false; entry.timestamp = 0; if (timestamps) { entry.timestamp = t_us() - t_start; @@ -283,6 +299,42 @@ struct common_log { cv_new.notify_one(); } + void add_json(const char * type, const common_json & obj) { + const common_json full = { + {"type", type}, + {"data", obj}, + }; + + const std::string text = full.dump_safe(); + + std::unique_lock lock(mtx); + + // block if the queue is full + cv_full.wait(lock, [this]() { return !running || !is_full(); }); + + if (!running) { + // discard messages while the worker thread is paused + return; + } + + auto & entry = queue[tail]; + + if (entry.msg.size() < text.size() + 1) { + entry.msg.resize(text.size() + 1); + } + memcpy(entry.msg.data(), text.c_str(), text.size() + 1); + + entry.is_end = false; + entry.level = GGML_LOG_LEVEL_NONE; + entry.prefix = false; + entry.jsonl = true; + entry.is_json = true; + entry.timestamp = 0; + + tail = (tail + 1) % queue.size(); + cv_new.notify_one(); + } + void resume() { std::lock_guard lock(mtx); @@ -388,12 +440,6 @@ struct common_log { this->timestamps = timestamps; } - - void set_jsonl(bool jsonl) { - std::lock_guard lock(mtx); - - this->jsonl = jsonl; - } }; // @@ -440,6 +486,14 @@ void common_log_add(struct common_log * log, enum ggml_log_level level, const ch va_end(args); } +void common_log_add_json(struct common_log * log, const char * type, const common_json & obj) { + if (!common_log_jsonl) { + return; + } + + log->add_json(type, obj); +} + void common_log_set_file(struct common_log * log, const char * file) { log->set_file(file); } @@ -467,10 +521,6 @@ void common_log_set_timestamps(struct common_log * log, bool timestamps) { log->set_timestamps(timestamps); } -void common_log_set_jsonl(struct common_log * log, bool jsonl) { - log->set_jsonl(jsonl); -} - void common_log_flush(struct common_log * log) { log->pause(); log->resume(); diff --git a/common/log.h b/common/log.h index 37f4de92b212..e36b09463e60 100644 --- a/common/log.h +++ b/common/log.h @@ -43,6 +43,10 @@ int common_log_get_verbosity_thold(void); void common_log_set_verbosity_thold(int verbosity); // not thread-safe +bool common_log_get_jsonl(void); + +void common_log_set_jsonl(bool jsonl); // not thread-safe + int common_log_get_verbosity(enum ggml_log_level level); void common_log_default_callback(enum ggml_log_level level, const char * text, void * user_data); @@ -91,7 +95,6 @@ void common_log_set_file (struct common_log * log, const char * file); // n void common_log_set_colors (struct common_log * log, log_colors colors); // not thread-safe void common_log_set_prefix (struct common_log * log, bool prefix); // whether to output prefix to each log void common_log_set_timestamps(struct common_log * log, bool timestamps); // whether to output timestamps in the prefix -void common_log_set_jsonl (struct common_log * log, bool jsonl); // print each log as a JSON object on one line, not thread-safe void common_log_flush (struct common_log * log); // flush all pending log messages // helper macros for logging @@ -127,3 +130,17 @@ void common_log_flush (struct common_log * log); // f #define LOG_WRNV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_WARN, verbosity, __VA_ARGS__) #define LOG_ERRV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, verbosity, __VA_ARGS__) #define LOG_CNTV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_CONT, verbosity, __VA_ARGS__) + +class common_json; // defined in common/json.h + +// helper allows different types of json output +// no-op if --log-jsonl is not set +void common_log_add_json(struct common_log * log, const char * type, const common_json & data); + +// will only print if --log-jsonl is set +#define LOG_JSON(type, data) \ + do { \ + if (common_log_get_jsonl()) { \ + common_log_add_json(common_log_main(), type, data); \ + } \ + } while (0) From 790cf51aabd61763486050dec7451d9147cb7c61 Mon Sep 17 00:00:00 2001 From: Aldehir Rojas Date: Sat, 12 Sep 2026 19:08:52 -0500 Subject: [PATCH 64/65] chat : improve parsing of complex types in qwen3-coder (#28742) * chat : improve schema support in qwen3 parser * cont : clean up grammar a bit --- common/parsers/qwen3-coder.cpp | 31 ++++++++++++++++-- tests/test-chat.cpp | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cpp index dfc74408472f..7938a2027932 100644 --- a/common/parsers/qwen3-coder.cpp +++ b/common/parsers/qwen3-coder.cpp @@ -104,9 +104,34 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat auto arg_open = p.tool_arg_open("\n"); - auto arg_value = param.schema->may_be_string() ? - arg_string : - p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close; + auto types = param.schema->value_types(); + + auto arg_value = p.eps(); + if (!types.has(common_chat_schema::TYPE_STRING)) { + arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close; + } else if (types.is_only(common_chat_schema::TYPE_STRING)) { + arg_value = arg_string; + } else { + // The string alternative accepts any text, so the grammar only keeps the raw string + // rule. The parser still tries the JSON alternatives first to type the value. + auto json_value = p.choice(); + if (types.has(common_chat_schema::TYPE_OBJECT)) { + json_value |= p.json_object(); + } + if (types.has(common_chat_schema::TYPE_ARRAY)) { + json_value |= p.json_array(); + } + if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) { + json_value |= p.json_number(); + } + if (types.has(common_chat_schema::TYPE_BOOLEAN)) { + json_value |= p.json_bool(); + } + if (types.has(common_chat_schema::TYPE_NULL)) { + json_value |= p.json_null(); + } + arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "xml-arg-string"); + } auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 1aef83f430d7..30a7237e314b 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -846,6 +846,25 @@ static common_chat_tool nullable_int_tool{ })", }; +static common_chat_tool string_union_tool{ + /* .name = */ "set_union", + /* .description = */ "Set values whose types are unions with string", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "value": { + "type": ["string", "object"], + "description": "A string or object value" + }, + "amount": { + "type": ["string", "integer"], + "description": "A string or integer value" + } + }, + "required": ["value", "amount"] + })", +}; + static common_chat_tool enum_no_type_tool{ /* .name = */ "set_unit", /* .description = */ "Set a temperature unit", @@ -3805,6 +3824,46 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); + // nullable string given null - parses as JSON null, not the string "null" + tst.test( + "\n" + "\n" + "\nnull\n\n" + "\n" + "") + .tools({ nullable_string_tool }) + .expect_tool_calls({ + { "set_nullable_str", R"({"name": null})", {} }, + }) + .run(); + + // unions with string - JSON values of the other types are typed, everything else is a string + tst.test( + "\n" + "\n" + "\n{\"a\": 1}\n\n" + "\n2 dollars\n\n" + "\n" + "") + .tools({ string_union_tool }) + .expect_tool_calls({ + { "set_union", R"({"value": {"a": 1}, "amount": "2 dollars"})", {} }, + }) + .run(); + + tst.test( + "\n" + "\n" + "\n{not valid json\n\n" + "\n42\n\n" + "\n" + "") + .tools({ string_union_tool }) + .expect_tool_calls({ + { "set_union", R"({"value": "{not valid json", "amount": 42})", {} }, + }) + .run(); + // enum without explicit type key - should infer string from enum values tst.test( "\n" From 2de4d0f97f2286c3471846a6394ebf43a4eb77b3 Mon Sep 17 00:00:00 2001 From: linsen <251731047+linsen458-spec@users.noreply.github.com> Date: Sun, 13 Sep 2026 09:09:12 +0800 Subject: [PATCH 65/65] server : forward --api-key to router-spawned child instances unset_reserved_args() dropped LLAMA_API_KEY from the base preset before it was merged into the per-model presets, so children spawned by the router only re-validated against --api-key-file keys: a client that authenticated with the router's --api-key key could list models but got 401 from every chat completion (#28820). Keep LLAMA_API_KEY in the preset so children accept the same key union the router accepts (they bind to router-assigned loopback ports); the /api/models preset rendering masks it separately. Fixes #28820 --- tools/server/server-models.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 3d134acf3621..10b5aef803c5 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -468,7 +468,10 @@ static std::filesystem::path get_server_exec_path() { static void unset_reserved_args(common_preset & preset, bool unset_model_args) { preset.unset_option("LLAMA_ARG_SSL_KEY_FILE"); preset.unset_option("LLAMA_ARG_SSL_CERT_FILE"); - preset.unset_option("LLAMA_API_KEY"); + // note: LLAMA_API_KEY is deliberately kept here so that child instances + // spawned by the router re-validate against the same key union the router + // accepts (--api-key and --api-key-file); the /api/models preset rendering + // masks it separately. preset.unset_option("LLAMA_ARG_MODELS_DIR"); preset.unset_option("LLAMA_ARG_MODELS_MAX"); preset.unset_option("LLAMA_ARG_MODELS_PRESET"); @@ -2028,6 +2031,8 @@ void server_models_routes::init_routes() { if (!meta.preset.name.empty()) { common_preset preset_copy = meta.preset; unset_reserved_args(preset_copy, false); + // don't render the API key into the preset shown to clients + preset_copy.unset_option("LLAMA_API_KEY"); preset_copy.unset_option("LLAMA_ARG_HOST"); preset_copy.unset_option("LLAMA_ARG_PORT"); preset_copy.unset_option("LLAMA_ARG_ALIAS");