diff --git a/src/llama-context.cpp b/src/llama-context.cpp index cb6cf1f2854a..8683ac26e77b 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -182,6 +182,12 @@ llama_context::llama_context( throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ)); } + { + const char * rows_plain = getenv("GGML_GDN_ROWS_PLAIN"); + const char * rows_plain_max = getenv("GGML_GDN_ROWS_PLAIN_MAX_TOKENS"); + cparams.gdn_rows_plain = rows_plain && atoi(rows_plain) == 1; + cparams.gdn_rows_plain_max_tokens = rows_plain_max ? std::max(0, atoi(rows_plain_max)) : 0; + } cparams.n_rs_seq = params.n_rs_seq; if (cparams.n_rs_seq > 0 && !llm_arch_supports_rs_rollback(model.arch)) { LLAMA_LOG_DEBUG("%s: n_rs_seq=%u requested but model does not support recurrent partial rollback; clamping to 0\n", diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 1f013e2f0027..65b60b6a12df 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -51,6 +51,8 @@ struct llama_cparams { bool offload_kqv; bool flash_attn; bool auto_fa; + bool gdn_rows_plain = false; // GGML_GDN_ROWS_PLAIN=1: in-place recurrent state rows for plain decode + int gdn_rows_plain_max_tokens = 0; // GGML_GDN_ROWS_PLAIN_MAX_TOKENS: only up to this many tokens per sequence (0: any) bool fused_gdn_ar; // use fused gated delta net (autoregressive) bool fused_gdn_ch; // use fused gated delta net (chunked) bool auto_fgdn; diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index ae67400c6131..e9413f3d0b54 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -551,9 +551,8 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn( const int64_t n_seqs = v->ne[3]; const int64_t n_seq_tokens = q->ne[2]; - const bool keep = cparams.n_rs_seq > 0; - - GGML_ASSERT(state_rows == nullptr || keep); // rows mode is a ring-path optimization + // rows mode also serves plain decode (GGML_GDN_ROWS_PLAIN): one snapshot slot, K = 1 + const bool keep = cparams.n_rs_seq > 0 || state_rows != nullptr; if (!keep) { auto attn_out = build_delta_net(q, k, v, g, b, s, il); diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 2f143d367a4d..4769f8455900 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -479,7 +479,12 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( // GPU device in the model is Metal. static const bool gdn_state_rows_env = getenv("GGML_GDN_STATE_GATHER") == nullptr; - const bool gdn_state_rows = gdn_state_rows_env && gdn_state_rows_dev_ok && cparams.n_rs_seq > 0; + // GGML_GDN_ROWS_PLAIN=1: plain decode also updates the state rows in place (bitwise identical), but only when no extra cells are relocated, as that relocation runs before the in-place read + // GGML_GDN_ROWS_PLAIN_MAX_TOKENS caps it by tokens per sequence (A19: the in-place recurrence is ~18% slower at 512-token prefill) + const bool gdn_rows_plain_ok = cparams.gdn_rows_plain && mctx_cur->get_n_rs() == (uint32_t) n_seqs && + (cparams.gdn_rows_plain_max_tokens <= 0 || n_seq_tokens <= cparams.gdn_rows_plain_max_tokens); + + const bool gdn_state_rows = gdn_state_rows_env && gdn_state_rows_dev_ok && (cparams.n_rs_seq > 0 || gdn_rows_plain_ok); ggml_tensor * state; if (gdn_state_rows) {