Npu decode funcall gating

ggml/src/ggml-openvino/ggml-decoder.cpp

@@ -459,6 +459,9 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
     for (int i = 0; i < cgraph->n_nodes; i++) {
         auto * node = cgraph->nodes[i];
         std::string name = std::string(node->name);
+        if (name.find("per_layer") != std::string::npos) {
+            model_params.has_per_layer_embd = true;
+        }
         const int attention_pattern_case = get_attention_pattern_case(node);
         if (attention_pattern_case != -1) {
             ggml_tensor * cache_k_permute = nullptr;

ggml/src/ggml-openvino/ggml-decoder.h

@@ -23,6 +23,10 @@ struct ModelParams {
     int state_size = -1;  // for SSM molels, eg qwen35
     int32_t rope_params[15];
     bool mixed_rope_params = false;
+    // True for gemma3/gemma4-style archs that carry a per-layer input embedding
+    // (inp_per_layer / per_layer_proj tensors). Such layers make each NPUW folded
+    // funcall expensive; used to disable NPUW_FUNCALL_FOR_ALL for the decode model.
+    bool has_per_layer_embd = false;
     std::vector<int> swa_layers;
 
     std::vector<std::string> kv_names;

ggml/src/ggml-openvino/ggml-openvino-extra.cpp

@@ -45,6 +45,10 @@ void ggml_openvino_device_config::init() {
         "GGML_OPENVINO_DISABLE_CACHE",
         "GGML_OPENVINO_DISABLE_KV_SLICE",
         "GGML_OPENVINO_MANUAL_GQA_ATTN",
+        "GGML_OPENVINO_NPUW_FOLD",
+        "GGML_OPENVINO_NPUW_FUNCALL_FOR_ALL",
+        "GGML_OPENVINO_DECODE_NPUW_FOLD",
+        "GGML_OPENVINO_DECODE_NPUW_FUNCALL_FOR_ALL",
     };
 
     for (const char * const & env_var : env_var_names) {
@@ -75,15 +79,48 @@ void ggml_openvino_device_config::init() {
             {"NPUW_DQ",                           "YES"   },
             {"NPUW_DQ_FULL",                      "NO"    },
         };
+        // Optional env overrides for the prefill/base config.
+        auto apply_npuw_env = [](ov::AnyMap & cfg) {
+            if (const char * v = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_FOLD")) {
+                cfg["NPUW_FOLD"] = (std::atoi(v) != 0) ? "YES" : "NO";
+            }
+            if (const char * v = ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_FUNCALL_FOR_ALL")) {
+                cfg["NPUW_FUNCALL_FOR_ALL"] = (std::atoi(v) != 0) ? "YES" : "NO";
+            }
+        };
+        apply_npuw_env(compile_config);
+
+        // Decode (1-token) model starts from the same config as prefill. For gemma3/4-style
+        // archs (per-layer input embedding) the NPUW funcall per-call overhead dominates
+        // decode, so NPUW_FUNCALL_FOR_ALL is disabled there -- but that decision needs the
+        // model, so it is applied at compile time in ov_graph_compute_static, not here.
+        // This just carries the base config + any explicit env overrides.
+        compile_config_decode = compile_config;
+        apply_npuw_env(compile_config_decode);
+        // Decode-only overrides (take precedence over the shared ones above).
+        if (const char * v = ggml_openvino_getenv_str("GGML_OPENVINO_DECODE_NPUW_FOLD")) {
+            compile_config_decode["NPUW_FOLD"] = (std::atoi(v) != 0) ? "YES" : "NO";
+        }
+        if (const char * v = ggml_openvino_getenv_str("GGML_OPENVINO_DECODE_NPUW_FUNCALL_FOR_ALL")) {
+            compile_config_decode["NPUW_FUNCALL_FOR_ALL"] = (std::atoi(v) != 0) ? "YES" : "NO";
+        }
+
         if (cache_dir && strlen(cache_dir) > 0) {
             compile_config["NPUW_CACHE_DIR"] = cache_dir;
             compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
+            compile_config_decode["NPUW_CACHE_DIR"] = cache_dir;
+            compile_config_decode.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
         }
     } else if (cache_dir && strlen(cache_dir) > 0) {
         compile_config.insert(ov::cache_dir(cache_dir));
         compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
     }
 
+    // On CPU/GPU the decode model uses the same config as prefill.
+    if (device_name != "NPU") {
+        compile_config_decode = compile_config;
+    }
+
     // Initialize remote context with queue sharing for GPU
     if (device_name == "GPU") {
         // Create OpenCL context and queue
@@ -183,6 +220,10 @@ const ov::AnyMap & ggml_openvino_get_compile_config() {
     return ggml_openvino_get_device_config().compile_config;
 }
 
+const ov::AnyMap & ggml_openvino_get_compile_config_decode() {
+    return ggml_openvino_get_device_config().compile_config_decode;
+}
+
 // Get the OpenCL command queue for GPU operations
 cl_command_queue ggml_openvino_get_cl_queue() {
     return ggml_openvino_get_device_config().cl_queue;

ggml/src/ggml-openvino/ggml-openvino-extra.h

@@ -25,6 +25,10 @@ std::optional<ov::RemoteContext> ggml_openvino_get_remote_context();
 // Get the compile config for the current device
 const ov::AnyMap & ggml_openvino_get_compile_config();
 
+// Get the compile config for the NPU decode (1-token) model. On CPU/GPU this is
+// identical to ggml_openvino_get_compile_config().
+const ov::AnyMap & ggml_openvino_get_compile_config_decode();
+
 // Get the OpenCL command queue for GPU operations (returns nullptr for CPU/NPU)
 cl_command_queue ggml_openvino_get_cl_queue();
 
@@ -64,6 +68,12 @@ struct ggml_openvino_device_config {
     bool initialized = false;
     std::optional<ov::RemoteContext> remote_context;
     ov::AnyMap compile_config;
+    // NPU-only: compile config for the decode (1-token) model. Starts as a copy of
+    // compile_config plus optional decode-specific env overrides; NPUW_FUNCALL_FOR_ALL
+    // is additionally disabled for gemma3/4-style archs at compile time (see
+    // ov_graph_compute_static), where per-call funcall overhead dominates decode.
+    // Identical to compile_config on CPU/GPU.
+    ov::AnyMap compile_config_decode;
     std::unordered_map<std::string, std::string> environment_variables;
     cl_command_queue cl_queue = nullptr;
 

ggml/src/ggml-openvino/utils.cpp

@@ -530,13 +530,22 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
 
         ov::CompiledModel compiled_model_prefill;
         ov::CompiledModel compiled_model_decode;
+        ov::AnyMap config_decode = ggml_openvino_get_compile_config_decode();
+        // Arch gating: gemma3/4-style per-layer-embedding models pay heavy NPUW funcall
+        // per-call overhead in the 1-token decode graph (~8x slower). Disable funcall for
+        // decode on those archs only, unless the user explicitly pinned the decode knob.
+        if (ggml_openvino_is_npu() && m_params.has_per_layer_embd &&
+            ggml_openvino_getenv_str("GGML_OPENVINO_DECODE_NPUW_FUNCALL_FOR_ALL") == nullptr &&
+            ggml_openvino_getenv_str("GGML_OPENVINO_NPUW_FUNCALL_FOR_ALL") == nullptr) {
+            config_decode["NPUW_FUNCALL_FOR_ALL"] = "NO";
+        }
         auto remote_context = ggml_openvino_get_remote_context();
         if (remote_context.has_value()) {
             compiled_model_prefill = core.compile_model(model_prefill, remote_context.value(), config);
-            compiled_model_decode = core.compile_model(model_decode, remote_context.value(), config);
+            compiled_model_decode = core.compile_model(model_decode, remote_context.value(), config_decode);
         } else {
             compiled_model_prefill = core.compile_model(model_prefill, device, config);
-            compiled_model_decode = core.compile_model(model_decode, device, config);
+            compiled_model_decode = core.compile_model(model_decode, device, config_decode);
         }
 
         auto infer_request_prefill = std::make_shared<ov::InferRequest>(compiled_model_prefill.create_infer_request());