From 2787e12fc1d0abb3c40c61ee2d14786c28fc6788 Mon Sep 17 00:00:00 2001 From: Jason Titus <870238+jasontitus@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:41:02 -0700 Subject: [PATCH 1/2] qwen35 : in-place recurrent state rows for plain decode (GGML_GDN_ROWS_PLAIN=1) The rows path of the gated delta net (the fused op reads and writes the per-sequence state rows of the cache) is used today only when n_rs_seq > 0 (speculative contexts with snapshots). Plain decode instead gathers each layer's state, runs the recurrence, and copies it back: 10-14% of a decode token on the Bonsai models (M5 Max per-op profile). GGML_GDN_ROWS_PLAIN=1 takes the rows path for plain decode too, with one snapshot slot (K = 1); the output is bitwise identical. - only when no extra cells are relocated (n_rs == n_seqs): the relocation in build_rs_cache_view runs before the GDN read and, after a cell reorder, could overwrite a row another sequence reads (the gathered path reads first); graph reuse compares the s_copy_extra size, so such a batch rebuilds and takes the gathered path - GGML_GDN_ROWS_PLAIN_MAX_TOKENS caps it by tokens per sequence (on an A19 the in-place recurrence is ~18% slower at 512-token prefill); contexts with n_rs_seq > 0 keep rows mode at every width - off by default; read per context Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01YHqq1nTe46u7euASncLsee --- src/llama-context.cpp | 6 ++++++ src/llama-cparams.h | 2 ++ src/models/delta-net-base.cpp | 5 ++--- src/models/qwen35.cpp | 12 +++++++++++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index cb6cf1f2854a..8683ac26e77b 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -182,6 +182,12 @@ llama_context::llama_context( throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ)); } + { + const char * rows_plain = getenv("GGML_GDN_ROWS_PLAIN"); + const char * rows_plain_max = getenv("GGML_GDN_ROWS_PLAIN_MAX_TOKENS"); + cparams.gdn_rows_plain = rows_plain && atoi(rows_plain) == 1; + cparams.gdn_rows_plain_max_tokens = rows_plain_max ? std::max(0, atoi(rows_plain_max)) : 0; + } cparams.n_rs_seq = params.n_rs_seq; if (cparams.n_rs_seq > 0 && !llm_arch_supports_rs_rollback(model.arch)) { LLAMA_LOG_DEBUG("%s: n_rs_seq=%u requested but model does not support recurrent partial rollback; clamping to 0\n", diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 1f013e2f0027..65b60b6a12df 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -51,6 +51,8 @@ struct llama_cparams { bool offload_kqv; bool flash_attn; bool auto_fa; + bool gdn_rows_plain = false; // GGML_GDN_ROWS_PLAIN=1: in-place recurrent state rows for plain decode + int gdn_rows_plain_max_tokens = 0; // GGML_GDN_ROWS_PLAIN_MAX_TOKENS: only up to this many tokens per sequence (0: any) bool fused_gdn_ar; // use fused gated delta net (autoregressive) bool fused_gdn_ch; // use fused gated delta net (chunked) bool auto_fgdn; diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index ae67400c6131..e9413f3d0b54 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -551,9 +551,8 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn( const int64_t n_seqs = v->ne[3]; const int64_t n_seq_tokens = q->ne[2]; - const bool keep = cparams.n_rs_seq > 0; - - GGML_ASSERT(state_rows == nullptr || keep); // rows mode is a ring-path optimization + // rows mode also serves plain decode (GGML_GDN_ROWS_PLAIN): one snapshot slot, K = 1 + const bool keep = cparams.n_rs_seq > 0 || state_rows != nullptr; if (!keep) { auto attn_out = build_delta_net(q, k, v, g, b, s, il); diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 2f143d367a4d..49b8ea62ade0 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -479,7 +479,17 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( // GPU device in the model is Metal. static const bool gdn_state_rows_env = getenv("GGML_GDN_STATE_GATHER") == nullptr; - const bool gdn_state_rows = gdn_state_rows_env && gdn_state_rows_dev_ok && cparams.n_rs_seq > 0; + // GGML_GDN_ROWS_PLAIN=1 also takes the rows path for plain decode (no snapshots), which removes the per-layer + // state gather and copy-back; the result is bitwise identical. Only when no extra cells are relocated + // (n_rs == n_seqs): the relocation in build_rs_cache_view runs before the GDN read and, after a cell reorder, + // could overwrite a row another sequence reads (the gathered path reads first). Graph reuse compares the + // s_copy_extra size, so a batch with extra cells rebuilds and takes the gathered path. + // GGML_GDN_ROWS_PLAIN_MAX_TOKENS caps it by tokens per sequence: on an A19 the in-place recurrence is ~18% + // slower at 512-token prefill while decode gains. Contexts with n_rs_seq > 0 keep rows mode at every width. + const bool gdn_rows_plain_ok = cparams.gdn_rows_plain && mctx_cur->get_n_rs() == (uint32_t) n_seqs && + (cparams.gdn_rows_plain_max_tokens <= 0 || n_seq_tokens <= cparams.gdn_rows_plain_max_tokens); + + const bool gdn_state_rows = gdn_state_rows_env && gdn_state_rows_dev_ok && (cparams.n_rs_seq > 0 || gdn_rows_plain_ok); ggml_tensor * state; if (gdn_state_rows) { From 1ad814a9e2c65a1953f5be225b201625466c7fb2 Mon Sep 17 00:00:00 2001 From: Jason Titus <870238+jasontitus@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:34:49 -0700 Subject: [PATCH 2/2] qwen35 : shorten comments Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01YHqq1nTe46u7euASncLsee --- src/models/qwen35.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 49b8ea62ade0..4769f8455900 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -479,13 +479,8 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( // GPU device in the model is Metal. static const bool gdn_state_rows_env = getenv("GGML_GDN_STATE_GATHER") == nullptr; - // GGML_GDN_ROWS_PLAIN=1 also takes the rows path for plain decode (no snapshots), which removes the per-layer - // state gather and copy-back; the result is bitwise identical. Only when no extra cells are relocated - // (n_rs == n_seqs): the relocation in build_rs_cache_view runs before the GDN read and, after a cell reorder, - // could overwrite a row another sequence reads (the gathered path reads first). Graph reuse compares the - // s_copy_extra size, so a batch with extra cells rebuilds and takes the gathered path. - // GGML_GDN_ROWS_PLAIN_MAX_TOKENS caps it by tokens per sequence: on an A19 the in-place recurrence is ~18% - // slower at 512-token prefill while decode gains. Contexts with n_rs_seq > 0 keep rows mode at every width. + // GGML_GDN_ROWS_PLAIN=1: plain decode also updates the state rows in place (bitwise identical), but only when no extra cells are relocated, as that relocation runs before the in-place read + // GGML_GDN_ROWS_PLAIN_MAX_TOKENS caps it by tokens per sequence (A19: the in-place recurrence is ~18% slower at 512-token prefill) const bool gdn_rows_plain_ok = cparams.gdn_rows_plain && mctx_cur->get_n_rs() == (uint32_t) n_seqs && (cparams.gdn_rows_plain_max_tokens <= 0 || n_seq_tokens <= cparams.gdn_rows_plain_max_tokens);