From 60ef7baaf3be9710b28247c08287448e548bf2ea Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 3 Jun 2026 20:20:11 +0100 Subject: [PATCH 01/55] Support chunked prefill for text-only causal LLMs Plain text causal LLM families (e.g. qwen3) previously transpiled to only a monolithic decoder + decoder_step, so the engine took the DIRECT_DECODER_STEP route and prefilled token-by-token (~27 tok/s). Emit the chunked-prefill component pipeline (lm_encoder_step, lm_encoder_text_chunk, decoder_media_step, decoder_prefill_chunk) for the qwen3 family via text-only embed/chunk adapters that mirror the multimodal qwen3.5 ones (minus vision), so the existing chunked-prefill engine route engages. ~5x faster prefill (27 -> 130-300 tok/s) with identical generation output. The default transpile (decoder + decoder_step) and multimodal families are unchanged. Signed-off-by: Noah Cylich --- python/cactus/transpile/model_adapters.py | 173 ++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index b9aefe0cc..c33e4d0be 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -4177,6 +4177,92 @@ def get_transpile_metadata(self): } +class Qwen3LMEncoderStepAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.backbone = model.model + + def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + inputs_embeds = self.backbone.embed_tokens(input_ids) + return inputs_embeds, position_ids.to(dtype=torch.int64) + + def get_transpile_metadata(self): + return { + "graph": { + **_transpile_graph_meta( + self.backbone, + adapter_family="qwen3", + adapter_type=type(self).__name__, + input_names=("input_ids", "position_ids"), + ), + } + } + + +class Qwen3LMEncoderTextChunkAdapter(Qwen3LMEncoderStepAdapter): + pass + + +class Qwen3EmbedsCausalLMStepAdapter(torch.nn.Module): + def __init__(self, model: torch.nn.Module): + super().__init__() + self.model = model + self.backbone = model.model + + def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor: + seq_len = int(inputs_embeds.shape[1]) + allowed_positions = torch.tril( + torch.ones((seq_len, seq_len), dtype=torch.bool, device=inputs_embeds.device), + ).view(1, 1, seq_len, seq_len) + allowed_values = torch.zeros( + (1, 1, seq_len, seq_len), + dtype=inputs_embeds.dtype, + device=inputs_embeds.device, + ) + blocked_values = torch.ones( + (1, 1, seq_len, seq_len), + dtype=inputs_embeds.dtype, + device=inputs_embeds.device, + ) * torch.finfo(inputs_embeds.dtype).min + causal_mask = torch.where(allowed_positions, allowed_values, blocked_values) + + hidden_states = inputs_embeds + text_position_ids = position_ids.to(dtype=torch.int64) + position_embeddings = self.backbone.rotary_emb(hidden_states, text_position_ids) + for decoder_layer in self.backbone.layers[: self.backbone.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + position_embeddings=position_embeddings, + attention_mask=causal_mask, + position_ids=text_position_ids, + past_key_values=None, + use_cache=False, + ) + hidden_states = self.backbone.norm(hidden_states) + return self.model.lm_head(hidden_states[:, -1:, :]) + + def get_transpile_metadata(self): + sliding_window = getattr(self.backbone.config, "sliding_window", None) + layer_types = list(getattr(self.backbone.config, "layer_types", [])) + return { + "graph": { + **_transpile_graph_meta( + self.model, + adapter_family="qwen3", + adapter_type=type(self).__name__, + input_names=("inputs_embeds", "position_ids"), + ), + "num_hidden_layers": int(self.backbone.config.num_hidden_layers), + "layer_types": tuple(layer_types), + "sliding_window": None if sliding_window is None else int(sliding_window), + } + } + + +class Qwen3EmbedsCausalLMPrefillChunkAdapter(Qwen3EmbedsCausalLMStepAdapter): + pass + + def _build_qwen_causal_lm_component_specs( model: torch.nn.Module, *, @@ -4205,6 +4291,14 @@ def _build_qwen_causal_lm_component_specs( "task": "causal_lm_logits", "adapter_family": family, } + + chunk_components = {"lm_encoder_step", "lm_encoder_text_chunk", "decoder_media_step", "decoder_prefill_chunk"} + wants_chunked = bool(chunk_components & requested_set) + if wants_chunked and family != "qwen3": + raise RuntimeError( + f"text chunked-prefill components are only supported for the qwen3 family, got {family}" + ) + specs: list[ComponentModuleSpec] = [] if "decoder" in requested_set: specs.append(ComponentModuleSpec( @@ -4237,6 +4331,85 @@ def _build_qwen_causal_lm_component_specs( }, metadata={"family": family, "task": "causal_lm_logits"}, )) + + if wants_chunked: + lm_encoder_step = Qwen3LMEncoderStepAdapter(model).eval() + lm_encoder_text_chunk = Qwen3LMEncoderTextChunkAdapter(model).eval() + decoder_media_step = Qwen3EmbedsCausalLMStepAdapter(model).eval() + decoder_prefill_chunk = Qwen3EmbedsCausalLMPrefillChunkAdapter(model).eval() + max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + prefill_chunk_size = max(1, int(os.environ.get("CACTUS_QWEN_PREFILL_CHUNK", "128") or "128")) + prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) + chunk_input_ids = input_ids[:, :prefill_chunk_size].contiguous() + chunk_position_ids = torch.arange( + prefill_chunk_size, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + with torch.no_grad(): + chunk_embeds, chunk_pos_out = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) + step_input_ids = input_ids[:, :1] + step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64) + with torch.no_grad(): + step_embeds, step_pos_out = lm_encoder_step(step_input_ids, step_position_ids) + + if "lm_encoder_step" in requested_set: + specs.append(ComponentModuleSpec( + component="lm_encoder_step", + module=lm_encoder_step, + example_inputs=(step_input_ids, step_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "position_ids"), + graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, + metadata={"family": family, "task": "causal_lm_logits"}, + )) + if "lm_encoder_text_chunk" in requested_set: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=("inputs_embeds", "position_ids"), + graph_meta={**common_graph_meta, "component": "lm_encoder_text_chunk"}, + metadata={"family": family, "task": "causal_lm_logits"}, + )) + if "decoder_prefill_chunk" in requested_set: + specs.append(ComponentModuleSpec( + component="decoder_prefill_chunk", + module=decoder_prefill_chunk, + example_inputs=(chunk_embeds, chunk_pos_out), + input_keys=("inputs_embeds", "position_ids"), + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_prefill_chunk", + "use_internal_kv_cache": True, + "use_internal_conv_cache": True, + "use_internal_gated_deltanet_cache": True, + "max_cache_seq_len": max_cache_seq_len, + "cache_sink_size": 4, + "prefill_chunk_size": prefill_chunk_size, + }, + metadata={"family": family, "task": "causal_lm_logits"}, + )) + if "decoder_media_step" in requested_set: + specs.append(ComponentModuleSpec( + component="decoder_media_step", + module=decoder_media_step, + example_inputs=(step_embeds, step_pos_out), + input_keys=("inputs_embeds", "position_ids"), + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_media_step", + "use_internal_kv_cache": True, + "use_internal_conv_cache": True, + "use_internal_gated_deltanet_cache": True, + "max_cache_seq_len": max_cache_seq_len, + "cache_sink_size": 4, + }, + metadata={"family": family, "task": "causal_lm_logits"}, + )) return specs From d2a12bdadd114f267478a88d6e81d82b4fe8f959 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 3 Jun 2026 20:19:30 +0100 Subject: [PATCH 02/55] Add KeyDiff query-agnostic KV-cache compression Compute-once-at-prefill KV compression: per (layer, kv-head), keep a budget of tokens = attention sink + recent window + the most distinctive middle tokens by the KeyDiff key-geometry score (s_i = -cos(k_i, mean key)), then physically compact the survivors and renumber their RoPE positions to a contiguous window. Two modes: one-shot (compact once at end of prefill) and rolling bounded cache (compact to target_len every time the cache reaches trigger_len). Default OFF (exact no-op); enabled via config or the CACTUS_KV_COMPRESS / CACTUS_KV_COMPRESS_ROLL env overrides. Supported on all-global-layer models (Qwen3); subset compression on mixed/KV-shared architectures is rejected with a warning (it would need per-layer positions). FP16 and INT8 caches handled. 22 unit/integration tests cross-check the keep-set math bit-for-bit against the reference and verify compaction/renumber/dense-check/rolling. Signed-off-by: Noah Cylich --- .gitignore | 10 + cactus-engine/CMakeLists.txt | 1 + cactus-engine/src/engine.h | 35 ++ cactus-engine/src/kv_compress.cpp | 279 +++++++++ cactus-engine/src/kv_compress.h | 104 ++++ cactus-engine/src/model.cpp | 213 +++++++ cactus-engine/tests/test_kv_compress.cpp | 364 +++++++++++ .../tests/test_kv_compress_free_functions.cpp | 576 ++++++++++++++++++ cactus-engine/tests/test_utils.h | 14 + .../niah/fixtures/kv_compress_fixtures.json | 1 + experiments/niah/gen_kv_compress_fixtures.py | 136 +++++ 11 files changed, 1733 insertions(+) create mode 100644 cactus-engine/src/kv_compress.cpp create mode 100644 cactus-engine/src/kv_compress.h create mode 100644 cactus-engine/tests/test_kv_compress.cpp create mode 100644 cactus-engine/tests/test_kv_compress_free_functions.cpp create mode 100644 experiments/niah/fixtures/kv_compress_fixtures.json create mode 100644 experiments/niah/gen_kv_compress_fixtures.py diff --git a/.gitignore b/.gitignore index e0918fde3..60f91e82b 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,13 @@ vision_activations.json gemma-4-eap-extras/ .githooks/ ruy + +# The MLX research harness under experiments/ stays untracked, but the C++ KV-compress +# math tests depend on the fixture generator + one small fixture, so track exactly those. +experiments/* +!experiments/niah/ +experiments/niah/* +!experiments/niah/gen_kv_compress_fixtures.py +!experiments/niah/fixtures/ +experiments/niah/fixtures/* +!experiments/niah/fixtures/kv_compress_fixtures.json diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index d240d748c..e259e538f 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -48,6 +48,7 @@ set(ENGINE_SOURCES src/sp.cpp src/constraints.cpp src/model.cpp + src/kv_compress.cpp src/model_npu.cpp src/engine_image.cpp src/index.cpp diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index bfac47540..ca4d72e1e 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -11,6 +11,7 @@ #include #include "cactus_graph.h" +#include "kv_compress.h" class CactusGraph; @@ -172,6 +173,18 @@ struct Config { std::vector layer_types; size_t conv_L_cache = 0; + bool kv_compress = false; // master switch (default OFF -> exact no-op) + float kv_compress_budget_frac = 0.25f; + float kv_compress_recent_frac = 0.30f; + uint32_t kv_compress_sink = 4; + std::string kv_compress_signal = "keydiff"; + int32_t kv_compress_manifold_window = 512; + // Rolling bounded cache: when trigger_len > 0, whenever the live KV seq-len reaches + // trigger_len, KeyDiff-compact to the best target_len tokens (absolute budget) and continue. + // 0/0 keeps the legacy one-shot-at-end-of-prefill behavior driven by budget_frac. + int32_t kv_compress_trigger_len = 0; + int32_t kv_compress_target_len = 0; + uint32_t altup_num_inputs = 4; uint32_t laurel_rank = 64; static constexpr uint32_t UNSET_U32 = UINT32_MAX; @@ -228,6 +241,14 @@ struct Config { bool from_json(const std::string& json_path); std::string to_json() const; + // Degenerate-config guard for rolling compaction: if kv_compress_trigger_len > 0, require + // 0 < kv_compress_target_len < kv_compress_trigger_len. Otherwise warn and disable rolling + // (treat as trigger_len = 0). Called after from_json and after any runtime override. + void validate_kv_compress(); + // Parse runtime override strings onto this config (nullptr = unset). `oneshot` is + // ":" (e.g. "keydiff:0.25"); `roll` is ":" + // (e.g. "4096:2048"). roll takes precedence. Returns true if any override was applied. + bool parse_kv_compress_override(const char* oneshot, const char* roll); }; @@ -621,6 +642,20 @@ class Model { void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} + // KeyDiff KV compression: at end of prefill, per compressible (layer, kv-head) keep a + // budget of survivors (sink + recent + top KeyDiff geometry), physically compact, and + // RoPE-renumber to contiguous positions 0..B-1. Default OFF via Config::kv_compress. + void compress_kv_cache_keydiff(const cactus::kvcompress::Params& params); + // Rolling bounded compaction: if kv_compress_trigger_len > 0 and the live KV seq-len has + // reached it, KeyDiff-compact to kv_compress_target_len (absolute budget) and renumber. + // No-op when kv_compress is off or trigger_len <= 0. Called after prefill and each decode. + void maybe_roll_compact(); + // Layer indices (cache_states order) that are physically compressible: Qwen = all; + // Gemma = full/global layers that are not KV-shared sources/consumers. + std::vector compressible_layers() const; + // Apply CACTUS_KV_COMPRESS / CACTUS_KV_COMPRESS_ROLL env overrides onto config_ (test/eval). + void apply_kv_compress_env_override(); + void set_tool_constraints(const std::vector& tools); void clear_tool_constraints(); void update_tool_constraints(uint32_t token_id); diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp new file mode 100644 index 000000000..1e9061f45 --- /dev/null +++ b/cactus-engine/src/kv_compress.cpp @@ -0,0 +1,279 @@ +#include "kv_compress.h" + +#include +#include +#include +#include + +namespace cactus { +namespace kvcompress { + +namespace { + +// Python's round()/numpy round half-to-even. round(recent_frac*B) must match the reference +// exactly, so reproduce banker's rounding rather than std::round (which rounds half away from +// zero). recent_frac arrives as float and widens to a slightly-off double (e.g. 0.30f -> +// 0.30000001...), which can flip the rounding of an exact half boundary (0.3*15 = 4.5 but +// 0.30000001*15 = 4.50000018). Snap values within a tiny epsilon of a half-integer back onto it +// before banker's rounding so the result matches the Python double reference exactly. +long py_round(double x) { + double twice = x * 2.0; + double twice_rounded = std::nearbyint(twice); + if (std::fabs(twice - twice_rounded) < 1e-6) x = twice_rounded * 0.5; + double r = std::nearbyint(x); // honors the default FE_TONEAREST (round-half-to-even) + return static_cast(r); +} + +} // namespace + +void keydiff_score(const float* keys, size_t n, size_t head_dim, float* out) { + // mu = mean key; s_i = -cos(k_i, mu). Computed in double to match the float64 reference. + std::vector mu(head_dim, 0.0); + for (size_t i = 0; i < n; ++i) { + const float* k = keys + i * head_dim; + for (size_t d = 0; d < head_dim; ++d) mu[d] += static_cast(k[d]); + } + double mu_norm = 0.0; + for (size_t d = 0; d < head_dim; ++d) { + mu[d] /= static_cast(n); + mu_norm += mu[d] * mu[d]; + } + mu_norm = std::sqrt(mu_norm) + 1e-8; + for (size_t d = 0; d < head_dim; ++d) mu[d] /= mu_norm; + + for (size_t i = 0; i < n; ++i) { + const float* k = keys + i * head_dim; + double knorm = 0.0, dot = 0.0; + for (size_t d = 0; d < head_dim; ++d) { + double v = static_cast(k[d]); + knorm += v * v; + dot += v * mu[d]; + } + knorm = std::sqrt(knorm) + 1e-8; + out[i] = static_cast(-(dot / knorm)); + } +} + +std::vector keepset_for_head(const float* scores, size_t n, const Params& p) { + long budget = std::max(1, static_cast(p.abs_budget)); + long B = std::min(budget, static_cast(n)); + long sink = std::min(std::max(static_cast(p.sink), 0), static_cast(n)); + long n_recent = std::min(py_round(static_cast(p.recent_frac) * static_cast(B)), + static_cast(n)); + + std::set reserved; + for (long i = 0; i < sink; ++i) reserved.insert(i); + for (long i = static_cast(n) - n_recent; i < static_cast(n); ++i) + if (i >= 0) reserved.insert(i); + + // Reserved may exceed B (large recent_frac + sink on a tight budget): keep sink first, + // then the most-recent tokens, until B. + if (static_cast(reserved.size()) > B) { + std::vector ordered; + for (long i = 0; i < sink; ++i) ordered.push_back(i); + for (long i = static_cast(n) - 1; i >= sink; --i) ordered.push_back(i); + reserved.clear(); + for (long i : ordered) { + if (i >= 0 && i < static_cast(n)) reserved.insert(i); + if (static_cast(reserved.size()) == B) break; + } + } + + std::set keep(reserved); + long remaining = B - static_cast(keep.size()); + if (remaining > 0) { + // argsort(-scores, kind="stable"): descending score, ties broken by ascending index. + std::vector order(n); + std::iota(order.begin(), order.end(), 0L); + std::stable_sort(order.begin(), order.end(), [&](long a, long b) { + return scores[a] > scores[b]; + }); + for (long i : order) { + if (keep.count(i)) continue; + keep.insert(i); + if (--remaining == 0) break; + } + } + + std::vector result; + result.reserve(keep.size()); + for (long i : keep) result.push_back(static_cast(i)); + return result; +} + +// --------------------------------------------------------------------------- // +// RoPE (rotate_half) helpers // +// --------------------------------------------------------------------------- // +void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos) { + size_t half = head_dim / 2; + for (size_t i = 0; i < half; ++i) { + double inv = std::pow(rope_theta, -(2.0 * static_cast(i)) / static_cast(head_dim)); + double ang = delta_pos * inv; + double c = std::cos(ang), s = std::sin(ang); + double x1 = row[i], x2 = row[i + half]; + row[i] = static_cast(x1 * c - x2 * s); + row[i + half] = static_cast(x2 * c + x1 * s); + } +} + +void unrope_head(const float* post_rope, size_t n, size_t head_dim, double rope_theta, + float* pre_rope) { + for (size_t t = 0; t < n; ++t) { + const float* src = post_rope + t * head_dim; + float* dst = pre_rope + t * head_dim; + for (size_t d = 0; d < head_dim; ++d) dst[d] = src[d]; + rope_rotate_row(dst, head_dim, rope_theta, -static_cast(t)); + } +} + +namespace { + +// Net renumber rotation for a survivor at absolute position `abs_pos` -> rank: rotate the +// stored post-RoPE row by (rank - abs_pos). Equivalent to un-RoPE then re-RoPE at rank. +inline double renumber_delta(int abs_pos, size_t rank) { + return static_cast(rank) - static_cast(abs_pos); +} + +// Per-head KeyDiff keep-set pipeline shared by the fp16 and int8 entry points. `fill_post(h, post)` +// gathers head h's post-RoPE rows into `post` ([n][head_dim]); the rest (un-RoPE -> score -> keepset) +// is identical regardless of storage precision. +template +std::vector> keepsets_per_head(size_t n, size_t kv_heads, size_t head_dim, + double rope_theta, const Params& p, FillPost fill_post) { + std::vector post(n * head_dim), pre(n * head_dim), scores(n); + std::vector> out; + out.reserve(kv_heads); + for (size_t h = 0; h < kv_heads; ++h) { + fill_post(h, post.data()); + unrope_head(post.data(), n, head_dim, rope_theta, pre.data()); + keydiff_score(pre.data(), n, head_dim, scores.data()); + out.push_back(keepset_for_head(scores.data(), n, p)); + } + return out; +} + +} // namespace + +void compact_fp16(uint16_t* key_rows_u, uint16_t* val_rows_u, size_t kv_heads, size_t head_dim, + const std::vector>& kept_per_head, double rope_theta) { + __fp16* key_rows = reinterpret_cast<__fp16*>(key_rows_u); + __fp16* val_rows = reinterpret_cast<__fp16*>(val_rows_u); + std::vector krow(head_dim), vrow(head_dim); + for (size_t h = 0; h < kv_heads; ++h) { + const std::vector& kept = kept_per_head[h]; + for (size_t rank = 0; rank < kept.size(); ++rank) { + int abs_pos = kept[rank]; + const __fp16* ksrc = key_rows + (static_cast(abs_pos) * kv_heads + h) * head_dim; + const __fp16* vsrc = val_rows + (static_cast(abs_pos) * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) { krow[d] = ksrc[d]; vrow[d] = vsrc[d]; } + // Renumber K: rotate post-RoPE row by (rank - abs_pos). V is not rotated. + rope_rotate_row(krow.data(), head_dim, rope_theta, renumber_delta(abs_pos, rank)); + __fp16* kdst = key_rows + (rank * kv_heads + h) * head_dim; + __fp16* vdst = val_rows + (rank * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) { + kdst[d] = static_cast<__fp16>(krow[d]); + vdst[d] = static_cast<__fp16>(vrow[d]); + } + } + } +} + +void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, + size_t head_dim, size_t group_size, + const std::vector>& kept_per_head, double rope_theta, + bool renumber) { + size_t groups = (head_dim + group_size - 1) / group_size; + size_t int8_stride = kv_heads * head_dim; + size_t scale_stride = kv_heads * groups; + std::vector row(head_dim); + for (size_t h = 0; h < kv_heads; ++h) { + const std::vector& kept = kept_per_head[h]; + for (size_t rank = 0; rank < kept.size(); ++rank) { + int abs_pos = kept[rank]; + size_t src_t = static_cast(abs_pos); + const int8_t* src = int8_rows + src_t * int8_stride + h * head_dim; + const float* ssc = scale_rows + src_t * scale_stride + h * groups; + for (size_t d = 0; d < head_dim; ++d) row[d] = static_cast(src[d]) * ssc[d / group_size]; + if (renumber) + rope_rotate_row(row.data(), head_dim, rope_theta, renumber_delta(abs_pos, rank)); + int8_t* dst = int8_rows + rank * int8_stride + h * head_dim; + float* dsc = scale_rows + rank * scale_stride + h * groups; + for (size_t g = 0; g < groups; ++g) { + size_t lo = g * group_size, hi = std::min(head_dim, lo + group_size); + // Match the engine's KV quantization convention (quantize_group_fp16_to_int8 in + // cactus-kernels/src/quants.cpp): scale floored at 1e-10, round-to-nearest via + // roundf, clamp to [-128, 127]. Keeps re-quantized rows consistent with how the + // rest of the engine writes the int8 cache. + float amax = 0.0f; + for (size_t d = lo; d < hi; ++d) amax = std::max(amax, std::fabs(row[d])); + float scale = amax / 127.0f; + if (scale < 1e-10f) scale = 1e-10f; + dsc[g] = scale; + float inv = 1.0f / scale; + for (size_t d = lo; d < hi; ++d) { + int32_t q = static_cast(std::roundf(row[d] * inv)); + q = std::max(-128, std::min(127, q)); + dst[d] = static_cast(q); + } + } + } + } +} + +std::vector> keepsets_from_fp16(const uint16_t* key_rows_u, size_t n, + size_t kv_heads, size_t head_dim, + double rope_theta, const Params& p) { + const __fp16* key_rows = reinterpret_cast(key_rows_u); + return keepsets_per_head(n, kv_heads, head_dim, rope_theta, p, [&](size_t h, float* post) { + for (size_t t = 0; t < n; ++t) { + const __fp16* src = key_rows + (t * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) post[t * head_dim + d] = src[d]; + } + }); +} + +std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, + size_t n, size_t kv_heads, size_t head_dim, + size_t group_size, double rope_theta, + const Params& p) { + size_t groups = (head_dim + group_size - 1) / group_size; + size_t int8_stride = kv_heads * head_dim; + size_t scale_stride = kv_heads * groups; + return keepsets_per_head(n, kv_heads, head_dim, rope_theta, p, [&](size_t h, float* post) { + for (size_t t = 0; t < n; ++t) { + const int8_t* src = int8_rows + t * int8_stride + h * head_dim; + const float* ssc = scale_rows + t * scale_stride + h * groups; + for (size_t d = 0; d < head_dim; ++d) + post[t * head_dim + d] = static_cast(src[d]) * ssc[d / group_size]; + } + }); +} + +std::vector physical_compressible_layers(const std::vector& layer_types, + size_t num_layers, size_t num_kv_shared) { + auto is_full = [&](size_t i) { + if (i >= layer_types.size()) return true; + return layer_types[i].find("sliding") == std::string::npos; + }; + std::vector full; + for (size_t i = 0; i < num_layers; ++i) if (is_full(i)) full.push_back(i); + + if (num_kv_shared == 0 || num_kv_shared >= num_layers) return full; + size_t first_shared = num_layers - num_kv_shared; + + std::set sources; + std::vector> last_of_type; + for (size_t i = 0; i < first_shared && i < layer_types.size(); ++i) { + bool found = false; + for (auto& e : last_of_type) if (e.first == layer_types[i]) { e.second = i; found = true; break; } + if (!found) last_of_type.emplace_back(layer_types[i], i); + } + for (auto& e : last_of_type) sources.insert(e.second); + + std::vector out; + for (size_t i : full) if (i < first_shared && !sources.count(i)) out.push_back(i); + return out; +} + +} // namespace kvcompress +} // namespace cactus diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h new file mode 100644 index 000000000..c8335a14c --- /dev/null +++ b/cactus-engine/src/kv_compress.h @@ -0,0 +1,104 @@ +#ifndef CACTUS_KV_COMPRESS_H +#define CACTUS_KV_COMPRESS_H + +#include +#include +#include +#include + +namespace cactus { +namespace kvcompress { + +struct Params { + float recent_frac = 0.30f; + size_t sink = 4; + int abs_budget = 0; // keep budget B = min(max(1, abs_budget), n) per (layer, kv-head) +}; + +// 64-byte KV cache header at the front of each K/V buffer (mirrors cactus-graph CacheMetadata). +struct CacheHeader { + uint64_t current_seq_len; + uint64_t max_seq_len; + uint64_t num_kv_heads; + uint64_t head_dim; + uint64_t sink_size; + uint64_t reserved[3]; +}; +static_assert(sizeof(CacheHeader) == 64, "CacheHeader must be 64 bytes"); + +// KeyDiff per-token score for one (layer, kv-head): s_i = -cos(k_i, mean(k)). +// keys: row-major [n][head_dim] (pre-RoPE). out: [n]. Mirrors keydiff_score(). +void keydiff_score(const float* keys, size_t n, size_t head_dim, float* out); + +// KeyDiff keep-set for ONE cell. Returns EXACTLY B = min(max(1, abs_budget), n) sorted unique +// indices: sink + recent + top-score middle. Mirrors keepset_for_head() EXACTLY, +// including the |reserved|>B fallback (sink-first then most-recent). Top-B uses a stable +// descending-score sort, ties broken by ascending index, to match np.argsort(-scores). +std::vector keepset_for_head(const float* scores, size_t n, const Params& p); + +// --------------------------------------------------------------------------- // +// Route-B RoPE un-rotation / renumber (rotate_half / NeoX convention) // +// --------------------------------------------------------------------------- // +// The cache stores POST-RoPE keys. RoPE here is the rotate_half layout used by +// cactus-kernels/src/norms_rope.cpp: for pair (i, i+half), angle = pos * theta^(-2i/d), +// out[i] = x[i]*cos - x[i+half]*sin +// out[i+half] = x[i+half]*cos + x[i]*sin +// Rotating a post-RoPE row by angle for position `delta_pos` gives the row as if RoPE'd at +// (orig_pos + delta_pos). So un-RoPE = rotate by -orig_pos; renumber = rotate by (rank-orig). + +// In-place delta rotation of one key row of length head_dim by `delta_pos`. +void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos); + +// Recover the pre-RoPE keys for one kv-head from post-RoPE rows. `post_rope` is +// [n][head_dim], rows at absolute positions 0..n-1. Writes pre_rope [n][head_dim]. +void unrope_head(const float* post_rope, size_t n, size_t head_dim, double rope_theta, + float* pre_rope); + +// --------------------------------------------------------------------------- // +// Per-head physical compaction + renumber on a live cache buffer (post-RoPE). // +// Mirrors run_niah_renumber.compact_renumber_kv: gather survivors in rank order, +// renumber K to contiguous positions 0..B-1, gather V unchanged (no RoPE). +// kept_per_head[h] is the sorted survivor index list for kv-head h (length B, equal +// across heads). header.current_seq_len is set to B by the caller. +// --------------------------------------------------------------------------- // + +// FP16 cache. key_rows/val_rows point past the 64-byte header at row 0. +// Layout: [max_seq][kv_heads][head_dim] fp16. K is renumbered, V is not. +void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, + const std::vector>& kept_per_head, double rope_theta); + +// INT8 buffer (one of K or V). int8 region [max_seq][kv_heads*head_dim]; scales +// [max_seq][kv_heads*groups] float, groups = ceil(head_dim/group_size). Dequant = +// int8 * scale[group]. When `renumber` is true (the K buffer) each gathered row is +// dequantized, renumbered (rotate by rank-abs), and re-quantized per group; when false +// (the V buffer) rows are gathered and re-quantized as-is (no rotation). +void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, + size_t head_dim, size_t group_size, + const std::vector>& kept_per_head, double rope_theta, + bool renumber); + +// Build per-head keep-sets for one layer from POST-RoPE FP16 key rows: un-RoPE -> score -> +// keepset_for_head. Returns n_kv_heads lists, each length B. +std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, + size_t kv_heads, size_t head_dim, + double rope_theta, const Params& p); + +// Same as keepsets_from_fp16 but for the INT8 K buffer: dequantize each row +// (int8 * scale[group]) -> un-RoPE -> score -> keepset_for_head. group_size is the KV +// quantization group size; scale_rows is [max_seq][kv_heads*groups], groups = ceil(head_dim/group_size). +std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, + size_t n, size_t kv_heads, size_t head_dim, + size_t group_size, double rope_theta, + const Params& p); + +// Physically-compressible layer indices, mirroring MLX compressible_layers(physical=True). +// `layer_types` token contains "sliding" for local layers; everything else is full/global. +// Empty layer_types -> all layers full (Qwen). num_kv_shared drops the shared consumer +// range and the per-type source layers (Gemma -> {4,9}). +std::vector physical_compressible_layers(const std::vector& layer_types, + size_t num_layers, size_t num_kv_shared); + +} // namespace kvcompress +} // namespace cactus + +#endif // CACTUS_KV_COMPRESS_H diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 393dfb32d..78eb4b481 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -362,6 +362,11 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_ERROR("model", "Failed to load config.txt from: " << bundle_dir); return false; } + // Runtime KV-compression override (test/eval only; bundle config unchanged). Unset => OFF + // (current behavior). CACTUS_KV_COMPRESS=: for one-shot-at-prefill + // (e.g. keydiff:0.25); CACTUS_KV_COMPRESS_ROLL=: for rolling + // bounded (e.g. 4096:2048). ROLL takes precedence when both are set. + apply_kv_compress_env_override(); if (!load_manifest()) { CACTUS_LOG_ERROR("model", "Failed to load bundle manifest from: " << bundle_dir); return false; @@ -1552,6 +1557,7 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, out_token = argmax_last_logits(); record_sampled_token(out_token); last_prefill_scalar_tail_tokens_ = 1; + maybe_roll_compact(); return true; } cache_total_seq_len_ += chunked.logical_tokens; @@ -1567,6 +1573,7 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, out_token = argmax_last_logits(); } record_sampled_token(out_token); + maybe_roll_compact(); return true; } @@ -1587,6 +1594,27 @@ void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, ++cache_total_seq_len_; } last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; + + if (prepare_decode && config_.kv_compress) { + if (config_.kv_compress_trigger_len > 0) { + // NOTE: rolling compaction does NOT fire mid-chunk. It runs here at + // prefill-completion and once per decode step, so a single prompt grows to its + // full n tokens before this completion-time compaction bounds it back to target_len. + // Rolling bounded mode: compact if the prefilled prompt crossed trigger_len. + maybe_roll_compact(); + } else { + cactus::kvcompress::Params p; + p.enabled = true; + p.budget_frac = config_.kv_compress_budget_frac; + p.recent_frac = config_.kv_compress_recent_frac; + p.sink = config_.kv_compress_sink; + p.signal = (config_.kv_compress_signal == "manifold") + ? cactus::kvcompress::Signal::MANIFOLD + : cactus::kvcompress::Signal::KEYDIFF; + p.manifold_window = config_.kv_compress_manifold_window; + compress_kv_cache_keydiff(p); + } + } } void Model::prefill_with_images(const std::vector& tokens, @@ -2339,6 +2367,9 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* } run_step(tokens.back(), cache_total_seq_len_ + tokens.size() - 1, /*read_logits=*/true); cache_total_seq_len_ += tokens.size(); + // Rolling bounded cache: if generation advanced the cache to trigger_len, compact to + // target_len before the next step. No-op unless kv_compress + trigger_len > 0. + maybe_roll_compact(); uint32_t result = argmax_last_logits(out_entropy); record_sampled_token(result); return result; @@ -2940,6 +2971,132 @@ void Model::remove_thinking_tokens(const std::vector>& } } +void Model::apply_kv_compress_env_override() { + const char* oneshot = std::getenv("CACTUS_KV_COMPRESS"); + const char* roll = std::getenv("CACTUS_KV_COMPRESS_ROLL"); + config_.parse_kv_compress_override(oneshot, roll); +} + +std::vector Model::compressible_layers() const { + size_t shared = (config_.num_kv_shared_layers == Config::UNSET_U32) + ? 0 : config_.num_kv_shared_layers; + return cactus::kvcompress::physical_compressible_layers( + config_.layer_types, config_.num_layers, shared); +} + +void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) { + if (!params.enabled || !decoder_) return; + + struct CacheHeader { + uint64_t current_seq_len; + uint64_t max_seq_len; + uint64_t num_kv_heads; + uint64_t head_dim; + uint64_t sink_size; + uint64_t reserved[3]; + }; + constexpr size_t kHeaderBytes = 64; + static_assert(sizeof(CacheHeader) == kHeaderBytes, "CacheHeader layout mismatch"); + + std::vector layers = compressible_layers(); + // Subset compression (e.g. Gemma physical set {4,9}) renumbers only some layers, but the + // engine has a SINGLE global position counter (cache_total_seq_len_). If only a subset is + // compacted, the non-compacted layers still hold n tokens while the counter would be set to + // B, so they RoPE the next decode query at the wrong offset -> silent corruption. Physical + // subset-renumbering needs per-layer position tracking, which this engine cannot represent. + // Only proceed when EVERY layer is compressible (the Qwen all-global case). + if (layers.size() != config_.num_layers) { + CACTUS_LOG_WARN("kv_compress", + "subset compression unsupported (needs per-layer positions); disabled for this model"); + return; + } + std::set compressible(layers.begin(), layers.end()); + const double rope_theta = static_cast(config_.rope_theta); + + Component& comp = *decoder_; + if (!comp.graph) return; + // Capture the compacted length (B, identical across all layers) from the keep-set itself, + // not from a header we just mutated, so the global position counter stays consistent. + size_t new_seq_len = 0; + bool have_new_seq_len = false; + for (size_t li = 0; li < comp.cache_states.size(); ++li) { + if (!compressible.count(li)) continue; + const auto& cs = comp.cache_states[li]; + if (cs.key_node_id < 0 || cs.value_node_id < 0) continue; + + const auto& kdesc = comp.graph->get_output_buffer(static_cast(cs.key_node_id)); + const auto& vdesc = comp.graph->get_output_buffer(static_cast(cs.value_node_id)); + if (kdesc.byte_size <= kHeaderBytes || vdesc.byte_size <= kHeaderBytes) continue; + void* kraw = comp.graph->get_output(static_cast(cs.key_node_id)); + void* vraw = comp.graph->get_output(static_cast(cs.value_node_id)); + if (!kraw || !vraw) continue; + + auto* khdr = static_cast(kraw); + auto* vhdr = static_cast(vraw); + size_t n = khdr->current_seq_len; + size_t kv_heads = khdr->num_kv_heads; + size_t head_dim = khdr->head_dim; + if (n == 0 || kv_heads == 0 || head_dim == 0) continue; + + if (kdesc.precision == Precision::FP16) { + auto* kbase = reinterpret_cast(static_cast(kraw) + kHeaderBytes); + auto* vbase = reinterpret_cast(static_cast(vraw) + kHeaderBytes); + auto kept = cactus::kvcompress::keepsets_from_fp16( + kbase, n, kv_heads, head_dim, rope_theta, params); + cactus::kvcompress::compact_fp16(kbase, vbase, kv_heads, head_dim, kept, rope_theta); + size_t B = kept.empty() ? 0 : kept[0].size(); + khdr->current_seq_len = B; + vhdr->current_seq_len = B; + new_seq_len = B; + have_new_seq_len = true; + } else if (kdesc.precision == Precision::INT8) { + size_t max_seq = khdr->max_seq_len; + // Dequantize + un-RoPE K to derive keep-sets (post-RoPE int8 -> pre-RoPE float). + auto* k_i8 = reinterpret_cast(static_cast(kraw) + kHeaderBytes); + auto* k_sc = reinterpret_cast(static_cast(kraw) + kHeaderBytes + + max_seq * kv_heads * head_dim); + auto* v_i8 = reinterpret_cast(static_cast(vraw) + kHeaderBytes); + auto* v_sc = reinterpret_cast(static_cast(vraw) + kHeaderBytes + + max_seq * kv_heads * head_dim); + auto kept = cactus::kvcompress::keepsets_from_int8( + k_i8, k_sc, n, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, rope_theta, params); + cactus::kvcompress::compact_int8(k_i8, k_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, + kept, rope_theta, /*renumber=*/true); + cactus::kvcompress::compact_int8(v_i8, v_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, + kept, rope_theta, /*renumber=*/false); + size_t B = kept.empty() ? 0 : kept[0].size(); + khdr->current_seq_len = B; + vhdr->current_seq_len = B; + new_seq_len = B; + have_new_seq_len = true; + } + } + + // Every layer was compacted to the same B (guarded above). Decode appends at B: subsequent + // positions are B, B+1, ... (renumbered window). + if (have_new_seq_len) cache_total_seq_len_ = new_seq_len; +} + +void Model::maybe_roll_compact() { + if (!config_.kv_compress || config_.kv_compress_trigger_len <= 0) return; + if (cache_total_seq_len_ < static_cast(config_.kv_compress_trigger_len)) return; + + cactus::kvcompress::Params p; + p.enabled = true; + p.budget_frac = config_.kv_compress_budget_frac; + p.recent_frac = config_.kv_compress_recent_frac; + p.sink = config_.kv_compress_sink; + p.signal = (config_.kv_compress_signal == "manifold") + ? cactus::kvcompress::Signal::MANIFOLD + : cactus::kvcompress::Signal::KEYDIFF; + p.manifold_window = config_.kv_compress_manifold_window; + // Absolute target: KeyDiff-keep the best target_len tokens (sink + recent + distinctive + // middle), renumber survivors to 0..target_len-1. compress_kv_cache_keydiff sets the global + // position counter to that B, so the cache stays bounded at <= trigger_len across cycles. + p.abs_budget = config_.kv_compress_target_len; + compress_kv_cache_keydiff(p); +} + std::vector Model::get_embeddings(const std::vector& tokens, bool pooled, bool normalize, const std::string& /*profile_file*/) { if (!components_.count("text_embedding")) { @@ -3126,6 +3283,14 @@ bool Config::from_json(const std::string& config_path) { else model_variant = ModelVariant::DEFAULT; } else if (key == "conv_L_cache") conv_L_cache = static_cast(std::stoul(value)); + else if (key == "kv_compress") kv_compress = (value == "true" || value == "1"); + else if (key == "kv_compress_budget_frac") kv_compress_budget_frac = std::stof(value); + else if (key == "kv_compress_recent_frac") kv_compress_recent_frac = std::stof(value); + else if (key == "kv_compress_sink") kv_compress_sink = static_cast(std::stoul(value)); + else if (key == "kv_compress_signal") kv_compress_signal = value; + else if (key == "kv_compress_manifold_window") kv_compress_manifold_window = static_cast(std::stol(value)); + else if (key == "kv_compress_trigger_len") kv_compress_trigger_len = static_cast(std::stol(value)); + else if (key == "kv_compress_target_len") kv_compress_target_len = static_cast(std::stol(value)); else if (key == "layer_types") { layer_types.clear(); std::string sanitized; @@ -3294,9 +3459,57 @@ bool Config::from_json(const std::string& config_path) { } } + validate_kv_compress(); return true; } +bool Config::parse_kv_compress_override(const char* oneshot, const char* roll) { + if (roll && *roll) { + std::string s(roll); + size_t colon = s.find(':'); + if (colon == std::string::npos) { + CACTUS_LOG_WARN("kv_compress", "CACTUS_KV_COMPRESS_ROLL must be :, got: " << s); + return false; + } + kv_compress = true; + kv_compress_signal = "keydiff"; + kv_compress_trigger_len = static_cast(std::stol(s.substr(0, colon))); + kv_compress_target_len = static_cast(std::stol(s.substr(colon + 1))); + validate_kv_compress(); + CACTUS_LOG_INFO("kv_compress", "rolling override: trigger_len=" << kv_compress_trigger_len + << " target_len=" << kv_compress_target_len); + return true; + } + if (oneshot && *oneshot) { + std::string s(oneshot); + size_t colon = s.find(':'); + if (colon == std::string::npos) { + CACTUS_LOG_WARN("kv_compress", "CACTUS_KV_COMPRESS must be :, got: " << s); + return false; + } + kv_compress = true; + kv_compress_signal = s.substr(0, colon); + kv_compress_budget_frac = std::stof(s.substr(colon + 1)); + kv_compress_trigger_len = 0; + kv_compress_target_len = 0; + CACTUS_LOG_INFO("kv_compress", "one-shot override: signal=" << kv_compress_signal + << " budget_frac=" << kv_compress_budget_frac); + return true; + } + return false; +} + +void Config::validate_kv_compress() { + if (kv_compress_trigger_len <= 0) return; + if (kv_compress_target_len <= 0 || kv_compress_target_len >= kv_compress_trigger_len) { + CACTUS_LOG_WARN("kv_compress", "invalid rolling config (target_len=" << kv_compress_target_len + << ", trigger_len=" << kv_compress_trigger_len + << "): require 0 < target_len < trigger_len; disabling rolling compaction"); + kv_compress_trigger_len = 0; + kv_compress_target_len = 0; + } +} + std::string Config::to_json() const { return "{}"; } diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp new file mode 100644 index 000000000..8bb63c0cd --- /dev/null +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -0,0 +1,364 @@ +#include "test_utils.h" +#include "../src/kv_compress.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace TestUtils; +using namespace cactus::kvcompress; + +// --------------------------------------------------------------------------- // +// Minimal JSON reader for the fixture file produced by gen_kv_compress_fixtures // +// --------------------------------------------------------------------------- // +namespace { + +struct Fixture { + std::string name; + int n = 0, head_dim = 0, n_kv_heads = 0, budget = 0, sink = 0; + float recent_frac = 0.f; + std::vector keys; // [n_kv_heads * n * head_dim] + std::vector> scores; // [n_kv_heads][n] + std::vector> kept; // [n_kv_heads][B] +}; + +struct RopeFixture { + int head_dim = 0, abs_pos = 0, rank = 0; + float rope_theta = 0.f; + std::vector pre_rope, post_rope_at_abs, post_rope_at_rank; +}; + +// Tiny tokenizing scanner. The fixture file is machine-generated with a known shape, so a +// streaming number/string/structure scanner is enough (no general-purpose JSON needed). +struct Scanner { + const std::string& s; + size_t i = 0; + explicit Scanner(const std::string& str) : s(str) {} + void skip_ws() { while (i < s.size() && (s[i] == ' ' || s[i] == '\n' || s[i] == '\t' || s[i] == '\r')) ++i; } + char peek() { skip_ws(); return i < s.size() ? s[i] : '\0'; } + void expect(char c) { skip_ws(); if (i >= s.size() || s[i] != c) throw std::runtime_error(std::string("expected ") + c); ++i; } + bool accept(char c) { skip_ws(); if (i < s.size() && s[i] == c) { ++i; return true; } return false; } + std::string parse_string() { + skip_ws(); expect('"'); + std::string out; + while (i < s.size() && s[i] != '"') out.push_back(s[i++]); + expect('"'); + return out; + } + double parse_number() { + skip_ws(); + size_t start = i; + while (i < s.size() && (std::isdigit((unsigned char)s[i]) || s[i] == '-' || s[i] == '+' || + s[i] == '.' || s[i] == 'e' || s[i] == 'E')) ++i; + return std::stod(s.substr(start, i - start)); + } + void skip_value() { // skip an arbitrary JSON value (only used to skip unknown keys) + char c = peek(); + if (c == '"') { parse_string(); return; } + if (c == '{') { parse_object_skip(); return; } + if (c == '[') { expect('['); if (peek() != ']') { do { skip_value(); } while (accept(',')); } expect(']'); return; } + parse_number(); + } + void parse_object_skip() { + expect('{'); + if (peek() != '}') { do { parse_string(); expect(':'); skip_value(); } while (accept(',')); } + expect('}'); + } + template + void parse_object(F&& on_key) { // on_key(key) -> bool (true if consumed value) + expect('{'); + if (peek() != '}') { + do { + std::string key = parse_string(); + expect(':'); + if (!on_key(key)) skip_value(); + } while (accept(',')); + } + expect('}'); + } + std::vector parse_float_array() { + std::vector v; + expect('['); + if (peek() != ']') do { v.push_back(static_cast(parse_number())); } while (accept(',')); + expect(']'); + return v; + } + std::vector parse_int_array() { + std::vector v; + expect('['); + if (peek() != ']') do { v.push_back(static_cast(std::llround(parse_number()))); } while (accept(',')); + expect(']'); + return v; + } +}; + +Fixture parse_fixture(Scanner& sc) { + Fixture f; + sc.parse_object([&](const std::string& k) -> bool { + if (k == "name") { f.name = sc.parse_string(); return true; } + if (k == "n") { f.n = (int)sc.parse_number(); return true; } + if (k == "head_dim") { f.head_dim = (int)sc.parse_number(); return true; } + if (k == "n_kv_heads") { f.n_kv_heads = (int)sc.parse_number(); return true; } + if (k == "budget") { f.budget = (int)sc.parse_number(); return true; } + if (k == "recent_frac") { f.recent_frac = (float)sc.parse_number(); return true; } + if (k == "sink") { f.sink = (int)sc.parse_number(); return true; } + if (k == "keys") { f.keys = sc.parse_float_array(); return true; } + if (k == "scores") { + sc.expect('['); + if (sc.peek() != ']') do { f.scores.push_back(sc.parse_float_array()); } while (sc.accept(',')); + sc.expect(']'); + return true; + } + if (k == "kept") { + sc.expect('['); + if (sc.peek() != ']') do { f.kept.push_back(sc.parse_int_array()); } while (sc.accept(',')); + sc.expect(']'); + return true; + } + return false; + }); + return f; +} + +struct FixtureFile { + std::vector fixtures; + RopeFixture rope; +}; + +FixtureFile load_fixtures(const std::string& path) { + std::ifstream in(path); + if (!in) throw std::runtime_error("cannot open fixture file: " + path); + std::stringstream ss; ss << in.rdbuf(); + std::string data = ss.str(); + Scanner sc(data); + FixtureFile out; + sc.parse_object([&](const std::string& k) -> bool { + if (k == "fixtures") { + sc.expect('['); + if (sc.peek() != ']') do { out.fixtures.push_back(parse_fixture(sc)); } while (sc.accept(',')); + sc.expect(']'); + return true; + } + if (k == "rope") { + RopeFixture r; + sc.parse_object([&](const std::string& rk) -> bool { + if (rk == "head_dim") { r.head_dim = (int)sc.parse_number(); return true; } + if (rk == "rope_theta") { r.rope_theta = (float)sc.parse_number(); return true; } + if (rk == "abs_pos") { r.abs_pos = (int)sc.parse_number(); return true; } + if (rk == "rank") { r.rank = (int)sc.parse_number(); return true; } + if (rk == "pre_rope") { r.pre_rope = sc.parse_float_array(); return true; } + if (rk == "post_rope_at_abs") { r.post_rope_at_abs = sc.parse_float_array(); return true; } + if (rk == "post_rope_at_rank") { r.post_rope_at_rank = sc.parse_float_array(); return true; } + return false; + }); + out.rope = r; + return true; + } + return false; + }); + return out; +} + +std::string fixture_path() { + if (const char* env = std::getenv("CACTUS_KV_FIXTURES")) return env; + return "experiments/niah/fixtures/kv_compress_fixtures.json"; +} + +Params params_for(const Fixture& f) { + Params p; + p.recent_frac = f.recent_frac; + p.sink = (size_t)f.sink; + p.abs_budget = f.budget; // fixture's budget = round(budget_frac * n) + return p; +} + +using EngineTestUtils::rope_reference; + +FixtureFile g_ff; +const float* head_ptr(const Fixture& f, int h) { + return f.keys.data() + (size_t)h * f.n * f.head_dim; +} + +} // namespace + +// --------------------------------------------------------------------------- // +// Tests // +// --------------------------------------------------------------------------- // + +bool test_keydiff_score_matches_python() { + bool ok = true; + for (const auto& f : g_ff.fixtures) { + for (int h = 0; h < f.n_kv_heads; ++h) { + std::vector out(f.n); + keydiff_score(head_ptr(f, h), f.n, f.head_dim, out.data()); + for (int i = 0; i < f.n; ++i) { + if (std::abs(out[i] - f.scores[h][i]) > 1e-4f) { + std::cerr << "keydiff mismatch " << f.name << " h" << h << " i" << i + << " cpp=" << out[i] << " py=" << f.scores[h][i] << "\n"; + ok = false; + } + } + } + } + return ok; +} + +bool test_keepset_matches_python_exactly() { + bool ok = true; + for (const auto& f : g_ff.fixtures) { + Params p = params_for(f); + for (int h = 0; h < f.n_kv_heads; ++h) { + std::vector sc(f.n); + keydiff_score(head_ptr(f, h), f.n, f.head_dim, sc.data()); + std::vector kept = keepset_for_head(sc.data(), f.n, p); + if (kept != f.kept[h]) { + std::cerr << "keepset mismatch " << f.name << " h" << h + << " cpp_size=" << kept.size() << " py_size=" << f.kept[h].size() << "\n"; + ok = false; + } + } + } + return ok; +} + +bool test_exactly_B_per_cell_rectangular() { + bool ok = true; + for (const auto& f : g_ff.fixtures) { + Params p = params_for(f); + long B = std::min(std::max(1, f.budget), f.n); + for (int h = 0; h < f.n_kv_heads; ++h) { + std::vector sc(f.n); + keydiff_score(head_ptr(f, h), f.n, f.head_dim, sc.data()); + std::vector kept = keepset_for_head(sc.data(), f.n, p); + if ((long)kept.size() != B) { ok = false; continue; } + for (size_t k = 1; k < kept.size(); ++k) + if (kept[k] <= kept[k - 1]) ok = false; // strictly sorted, unique + for (int idx : kept) if (idx < 0 || idx >= f.n) ok = false; + } + } + return ok; +} + +bool test_sink_always_included() { + bool ok = true; + for (const auto& f : g_ff.fixtures) { + long B = std::min(std::max(1, f.budget), f.n); + long sink = std::min(std::max(0, f.sink), f.n); + if (sink > B) continue; // budget too small to hold the full sink + for (const auto& kept : f.kept) { + for (int s = 0; s < sink; ++s) { + if (std::find(kept.begin(), kept.end(), s) == kept.end()) ok = false; + } + } + } + return ok; +} + +bool test_recent_window_included() { + bool ok = true; + for (const auto& f : g_ff.fixtures) { + long B = std::min(std::max(1, f.budget), f.n); + long sink = std::min(std::max(0, f.sink), f.n); + long n_recent = std::min(std::lround((double)f.recent_frac * B), f.n); + // The full recent window is guaranteed kept only when reserved fits in the budget. + if (sink + n_recent > B) continue; + for (const auto& kept : f.kept) { + for (long r = f.n - n_recent; r < f.n; ++r) { + if (r < 0) continue; + if (std::find(kept.begin(), kept.end(), (int)r) == kept.end()) ok = false; + } + } + } + return ok; +} + +bool test_outlier_kept_filler_dropped() { + // Fixtures plant a centroid-opposite outlier at mid+h; the adjacent filler stays on the + // centroid. KeyDiff must keep the outlier (a mid-context geometry pick). + bool ok = true; + for (const auto& f : g_ff.fixtures) { + if (f.name != std::string("base_keydiff")) continue; + int mid = f.n / 2; + for (int h = 0; h < f.n_kv_heads; ++h) { + int outlier = mid + h; + const auto& kept = f.kept[h]; + if (std::find(kept.begin(), kept.end(), outlier) == kept.end()) { + std::cerr << "outlier not kept " << f.name << " h" << h << " idx" << outlier << "\n"; + ok = false; + } + } + } + return ok; +} + +bool test_padding_when_budget_exceeds_reserves() { + // reserved_overflow: large recent_frac + sink on a tight budget exercises the |reserved|>B + // fallback. Still exactly B unique sorted indices. + bool ok = false; + for (const auto& f : g_ff.fixtures) { + if (f.name != std::string("reserved_overflow")) continue; + ok = true; + Params p = params_for(f); + long B = std::min(std::max(1, f.budget), f.n); + for (int h = 0; h < f.n_kv_heads; ++h) { + std::vector sc(f.n); + keydiff_score(head_ptr(f, h), f.n, f.head_dim, sc.data()); + std::vector kept = keepset_for_head(sc.data(), f.n, p); + if ((long)kept.size() != B) ok = false; + if (kept != f.kept[h]) ok = false; + } + } + return ok; +} + +bool test_renumber_mapping() { + // RoPE-at-rank for a survivor equals the reference rope of that vector at its rank position. + // Also check Route-B delta rotation: rotating the post-RoPE-at-abs vector by (rank - abs) + // recovers post-RoPE-at-rank. + const RopeFixture& r = g_ff.rope; + if (r.head_dim == 0) return false; + + // RoPE-at-rank matches reference. + std::vector at_rank = rope_reference(r.pre_rope, r.rank, r.rope_theta); + for (int i = 0; i < r.head_dim; ++i) + if (std::abs(at_rank[i] - r.post_rope_at_rank[i]) > 1e-3f) return false; + + // Route B via the PRODUCTION rope_rotate_row (shipped code, not a local reimplementation): + // rotating post_rope_at_abs by (rank - abs) in place must yield post_rope_at_rank. + std::vector delta = r.post_rope_at_abs; + rope_rotate_row(delta.data(), (size_t)r.head_dim, r.rope_theta, + (double)(r.rank - r.abs_pos)); + for (int i = 0; i < r.head_dim; ++i) + if (std::abs(delta[i] - r.post_rope_at_rank[i]) > 1e-3f) return false; + + return true; +} + +int main() { + TestUtils::TestRunner runner("KV Compress Math Tests"); + try { + g_ff = load_fixtures(fixture_path()); + } catch (const std::exception& e) { + std::cerr << "Failed to load fixtures: " << e.what() << "\n" + << "Run: python experiments/niah/gen_kv_compress_fixtures.py\n" + << "Or set CACTUS_KV_FIXTURES to the fixture json path.\n"; + return 1; + } + std::cout << "Loaded " << g_ff.fixtures.size() << " fixtures\n"; + + runner.run_test("keydiff_score_matches_python", test_keydiff_score_matches_python()); + runner.run_test("keepset_matches_python_exactly", test_keepset_matches_python_exactly()); + runner.run_test("exactly_B_per_cell_rectangular", test_exactly_B_per_cell_rectangular()); + runner.run_test("sink_always_included", test_sink_always_included()); + runner.run_test("recent_window_included", test_recent_window_included()); + runner.run_test("outlier_kept_filler_dropped", test_outlier_kept_filler_dropped()); + runner.run_test("padding_when_budget_exceeds_reserves", test_padding_when_budget_exceeds_reserves()); + runner.run_test("renumber_mapping", test_renumber_mapping()); + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-engine/tests/test_kv_compress_free_functions.cpp b/cactus-engine/tests/test_kv_compress_free_functions.cpp new file mode 100644 index 000000000..0240421ad --- /dev/null +++ b/cactus-engine/tests/test_kv_compress_free_functions.cpp @@ -0,0 +1,576 @@ +#include "test_utils.h" +#include "../src/kv_compress.h" +#include "../src/engine.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace TestUtils; +using namespace cactus::kvcompress; + +namespace { + +constexpr size_t kHeaderBytes = 64; +constexpr size_t kGroupSize = 32; // KV_QUANT_GROUP_SIZE + +struct Header { + uint64_t current_seq_len; + uint64_t max_seq_len; + uint64_t num_kv_heads; + uint64_t head_dim; + uint64_t sink_size; + uint64_t reserved[3]; +}; + +float f16_to_f32(uint16_t u) { __fp16 v; std::memcpy(&v, &u, 2); return static_cast(v); } +uint16_t f32_to_f16(float f) { __fp16 v = static_cast<__fp16>(f); uint16_t u; std::memcpy(&u, &v, 2); return u; } + +// rotate_half RoPE on a float row, matching the reference / norms_rope.cpp. +std::vector rope_row(const std::vector& v, double pos, double theta) { + size_t d = v.size(), half = d / 2; + std::vector o(d); + for (size_t i = 0; i < half; ++i) { + double inv = std::pow(theta, -(2.0 * (double)i) / (double)d); + double a = pos * inv, c = std::cos(a), s = std::sin(a); + o[i] = (float)(v[i] * c - v[i + half] * s); + o[i + half] = (float)(v[i + half] * c + v[i] * s); + } + return o; +} + +// Build an FP16 cache buffer: 64B header + [max_seq][kv_heads][head_dim] fp16. Returns the +// raw buffer; `pre_rope[h][t][d]` is the planted pre-RoPE key, stored post-RoPE at position t. +std::vector make_fp16_cache(size_t n, size_t max_seq, size_t kv_heads, size_t head_dim, + double theta, std::vector>>& pre_rope, + std::vector>>& values) { + std::vector buf(kHeaderBytes + max_seq * kv_heads * head_dim * sizeof(uint16_t), 0); + auto* h = reinterpret_cast(buf.data()); + h->current_seq_len = n; h->max_seq_len = max_seq; h->num_kv_heads = kv_heads; + h->head_dim = head_dim; h->sink_size = 4; + auto* rows = reinterpret_cast(buf.data() + kHeaderBytes); + pre_rope.assign(kv_heads, std::vector>(n)); + values.assign(kv_heads, std::vector>(n)); + for (size_t hh = 0; hh < kv_heads; ++hh) + for (size_t t = 0; t < n; ++t) { + std::vector k(head_dim), v(head_dim); + for (size_t d = 0; d < head_dim; ++d) { + k[d] = std::sin(0.1f * (hh + 1) * (t + 1) + 0.3f * d) + 0.05f * d; + v[d] = std::cos(0.07f * (hh + 1) * (t + 2) + 0.2f * d); + } + pre_rope[hh][t] = k; values[hh][t] = v; + std::vector kr = rope_row(k, (double)t, theta); + for (size_t d = 0; d < head_dim; ++d) { + rows[(t * kv_heads + hh) * head_dim + d] = f32_to_f16(kr[d]); + // value stored at the SAME slot but in a separate value buffer; reuse k buffer + // layout for V in a parallel buffer (built by caller). Here we only fill K. + (void)v; + } + } + return buf; +} + +std::vector make_fp16_value_cache(size_t n, size_t max_seq, size_t kv_heads, size_t head_dim, + const std::vector>>& values) { + std::vector buf(kHeaderBytes + max_seq * kv_heads * head_dim * sizeof(uint16_t), 0); + auto* h = reinterpret_cast(buf.data()); + h->current_seq_len = n; h->max_seq_len = max_seq; h->num_kv_heads = kv_heads; + h->head_dim = head_dim; h->sink_size = 4; + auto* rows = reinterpret_cast(buf.data() + kHeaderBytes); + for (size_t hh = 0; hh < kv_heads; ++hh) + for (size_t t = 0; t < n; ++t) + for (size_t d = 0; d < head_dim; ++d) + rows[(t * kv_heads + hh) * head_dim + d] = f32_to_f16(values[hh][t][d]); + return buf; +} + +} // namespace + +bool test_compact_fp16_cache() { + const size_t n = 60, max_seq = 128, kv_heads = 3, head_dim = 16; + const double theta = 1000000.0; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); + auto vbuf = make_fp16_value_cache(n, max_seq, kv_heads, head_dim, vals); + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); + + // Heterogeneous per-head keep-sets (different indices, same length B). + std::vector> kept = { + {0, 5, 10, 20, 30, 59}, {1, 4, 12, 25, 40, 58}, {0, 2, 15, 33, 50, 55}}; + size_t B = kept[0].size(); + compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); + + bool ok = true; + for (size_t h = 0; h < kv_heads; ++h) { + for (size_t rank = 0; rank < B; ++rank) { + int abs_pos = kept[h][rank]; + // Expected K at slot `rank` = rope(pre_rope[h][abs_pos], rank). + std::vector expK = rope_row(pre[h][abs_pos], (double)rank, theta); + const uint16_t* kdst = krows + (rank * kv_heads + h) * head_dim; + const uint16_t* vdst = vrows + (rank * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) { + if (std::abs(f16_to_f32(kdst[d]) - expK[d]) > 5e-2f) ok = false; + if (std::abs(f16_to_f32(vdst[d]) - vals[h][abs_pos][d]) > 5e-2f) ok = false; + } + } + } + return ok && B == 6; +} + +bool test_dense_check_full_budget() { + // budget_frac=1.0 -> keep all indices in order; renumber maps rank==abs_pos so RoPE delta + // is 0 -> the K rows are byte-identical and V unchanged. + const size_t n = 40, max_seq = 64, kv_heads = 2, head_dim = 16; + const double theta = 1000000.0; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); + auto vbuf = make_fp16_value_cache(n, max_seq, kv_heads, head_dim, vals); + std::vector kbuf0 = kbuf, vbuf0 = vbuf; + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); + + Params p; p.enabled = true; p.budget_frac = 1.0f; p.recent_frac = 0.3f; p.sink = 4; + p.signal = Signal::KEYDIFF; + auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); + for (auto& k : kept) if (k.size() != n) return false; // B == n + compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); + + // Byte-identical over the live region. + size_t live = n * kv_heads * head_dim * sizeof(uint16_t); + if (std::memcmp(kbuf.data() + kHeaderBytes, kbuf0.data() + kHeaderBytes, live) != 0) return false; + if (std::memcmp(vbuf.data() + kHeaderBytes, vbuf0.data() + kHeaderBytes, live) != 0) return false; + return true; +} + +bool test_rope_renumber_contiguous() { + // Route-B delta rotation: rope(stored_post_rope_at_abs, rank - abs) == rope(pre, rank). + const size_t head_dim = 16; const double theta = 1000000.0; + std::vector pre(head_dim); + for (size_t d = 0; d < head_dim; ++d) pre[d] = std::sin(0.4 * d) + 0.1 * d; + int abs_pos = 137; size_t rank = 5; + std::vector at_abs = rope_row(pre, (double)abs_pos, theta); + std::vector at_rank = rope_row(pre, (double)rank, theta); + std::vector delta = at_abs; + rope_rotate_row(delta.data(), head_dim, theta, (double)rank - (double)abs_pos); + for (size_t d = 0; d < head_dim; ++d) + if (std::abs(delta[d] - at_rank[d]) > 1e-3f) return false; + return true; +} + +// Mirror Model::compress_kv_cache_keydiff's master gate: only touch the cache when enabled. +// Returns true if compaction ran. Operates on an FP16 K/V cache pair in place. +bool run_compress_if_enabled(uint16_t* krows, uint16_t* vrows, size_t n, size_t kv_heads, + size_t head_dim, double theta, const Params& p) { + if (!p.enabled) return false; // the Model gate early-returns here -> exact no-op. + auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); + compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); + return true; +} + +bool test_compress_noop_when_disabled() { + // Strengthened: assert the cache bytes are ACTUALLY unchanged when disabled, and ACTUALLY + // changed when enabled with a real (<1.0 budget) keep-set. Not a tautology on the flag. + const size_t n = 50, max_seq = 96, kv_heads = 2, head_dim = 16; + const double theta = 1000000.0; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); + auto vbuf = make_fp16_value_cache(n, max_seq, kv_heads, head_dim, vals); + std::vector kbuf0 = kbuf, vbuf0 = vbuf; + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); + size_t live = n * kv_heads * head_dim * sizeof(uint16_t); + + Params off; off.enabled = false; off.budget_frac = 0.25f; off.recent_frac = 0.3f; off.sink = 4; + bool ran_off = run_compress_if_enabled(krows, vrows, n, kv_heads, head_dim, theta, off); + if (ran_off) return false; + // Disabled => byte-identical. + if (std::memcmp(kbuf.data(), kbuf0.data(), kbuf.size()) != 0) return false; + if (std::memcmp(vbuf.data(), vbuf0.data(), vbuf.size()) != 0) return false; + + Params on; on.enabled = true; on.budget_frac = 0.25f; on.recent_frac = 0.3f; on.sink = 4; + on.signal = Signal::KEYDIFF; + bool ran_on = run_compress_if_enabled(krows, vrows, n, kv_heads, head_dim, theta, on); + if (!ran_on) return false; + // Enabled with budget < 1.0 => the live region must actually change (real keep-set applied). + bool kchanged = std::memcmp(kbuf.data() + kHeaderBytes, kbuf0.data() + kHeaderBytes, live) != 0; + bool vchanged = std::memcmp(vbuf.data() + kHeaderBytes, vbuf0.data() + kHeaderBytes, live) != 0; + return kchanged && vchanged; +} + +bool test_subset_compression_rejected() { + // BLOCKER guard: Model::compress_kv_cache_keydiff only proceeds when EVERY layer is + // compressible (compressible_layers().size() == num_layers). A strict subset (Gemma {4,9}) + // must be rejected as a no-op, because the single global position counter can't represent + // per-layer lengths. Here we exercise the exact predicate the guard uses + show the cache + // stays untouched on the reject path. + const size_t num_layers = 35; + std::vector gemma; + for (int i = 0; i < (int)num_layers; ++i) gemma.push_back(((i + 1) % 5 == 0) ? "global" : "sliding"); + auto gset = physical_compressible_layers(gemma, num_layers, 20); + bool gemma_is_subset = gset.size() != num_layers; // guard => reject (no-op) + if (!gemma_is_subset) return false; + + std::vector qwen(28, "full_attention"); + auto qset = physical_compressible_layers(qwen, 28, 0); + bool qwen_full = qset.size() == 28; // guard => accept (compress) + if (!qwen_full) return false; + + // Behavioral: on the reject path the cache must be byte-identical (compaction skipped). + const size_t n = 40, max_seq = 64, kv_heads = 2, head_dim = 16; + const double theta = 1000000.0; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); + auto vbuf = make_fp16_value_cache(n, max_seq, kv_heads, head_dim, vals); + std::vector kbuf0 = kbuf, vbuf0 = vbuf; + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); + Params p; p.enabled = true; p.budget_frac = 0.25f; p.recent_frac = 0.3f; p.sink = 4; + if (gemma_is_subset) { + // guard fires -> we do NOT compact (mirrors Model's early return). + } else { + run_compress_if_enabled(krows, vrows, n, kv_heads, head_dim, theta, p); + } + if (std::memcmp(kbuf.data(), kbuf0.data(), kbuf.size()) != 0) return false; + if (std::memcmp(vbuf.data(), vbuf0.data(), vbuf.size()) != 0) return false; + return true; +} + +bool test_fp16_storage_round_trip() { + // Write pre-RoPE -> post-RoPE into an fp16 cache, compact+renumber, read back, and compare + // each survivor against the reference rope(pre, rank) within fp16 tolerance. Exercises the + // full fp16 storage path (f32->f16 store, gather, delta-rotate, f16->f16 write-back). + const size_t n = 64, max_seq = 128, kv_heads = 2, head_dim = 16; + const double theta = 1000000.0; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); + auto vbuf = make_fp16_value_cache(n, max_seq, kv_heads, head_dim, vals); + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); + + Params p; p.enabled = true; p.budget_frac = 0.25f; p.recent_frac = 0.3f; p.sink = 4; + p.signal = Signal::KEYDIFF; + auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); + compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); + + bool ok = true; + for (size_t h = 0; h < kv_heads; ++h) { + for (size_t rank = 0; rank < kept[h].size(); ++rank) { + int abs_pos = kept[h][rank]; + std::vector expK = rope_row(pre[h][abs_pos], (double)rank, theta); + const uint16_t* kdst = krows + (rank * kv_heads + h) * head_dim; + const uint16_t* vdst = vrows + (rank * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) { + if (std::abs(f16_to_f32(kdst[d]) - expK[d]) > 5e-2f) ok = false; + if (std::abs(f16_to_f32(vdst[d]) - vals[h][abs_pos][d]) > 5e-2f) ok = false; + } + } + } + return ok; +} + +bool test_gemma_layer_selection() { + // Qwen: all 28 layers (full_attention), no KV sharing. + std::vector qwen(28, "full_attention"); + auto q = physical_compressible_layers(qwen, 28, 0); + if (q.size() != 28) return false; + for (size_t i = 0; i < 28; ++i) if (q[i] != i) return false; + + // Gemma4-e2b: layer_types sliding/global, num_layers=35, num_kv_shared=20 -> {4,9}. + std::vector g; + for (int i = 0; i < 35; ++i) g.push_back(((i + 1) % 5 == 0) ? "global" : "sliding"); + auto gg = physical_compressible_layers(g, 35, 20); + std::vector expect = {4, 9}; + return gg == expect; +} + +bool test_compact_int8_cache() { + // INT8: gather + renumber K, gather V (no rotation); dequant of compacted survivor ~ the + // expected renumbered/gathered value. + const size_t n = 50, max_seq = 96, kv_heads = 2, head_dim = 32; + const double theta = 1000000.0; + size_t groups = (head_dim + kGroupSize - 1) / kGroupSize; + size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; + + // Build pre-RoPE keys + values, then store post-RoPE K quantized per group. + std::vector>> pre(kv_heads, std::vector>(n)); + std::vector>> vals(kv_heads, std::vector>(n)); + std::vector k_i8(max_seq * int8_stride, 0), v_i8(max_seq * int8_stride, 0); + std::vector k_sc(max_seq * scale_stride, 0.f), v_sc(max_seq * scale_stride, 0.f); + + // Match the engine KV quant convention (quants.cpp): scale floored at 1e-10, roundf, + // clamp [-128,127] -- same as compact_int8's re-quant path. + auto quant_row = [&](const std::vector& row, int8_t* dst, float* sc) { + for (size_t g = 0; g < groups; ++g) { + size_t lo = g * kGroupSize, hi = std::min(head_dim, lo + kGroupSize); + float amax = 0.f; for (size_t d = lo; d < hi; ++d) amax = std::max(amax, std::fabs(row[d])); + float scale = amax / 127.f; if (scale < 1e-10f) scale = 1e-10f; sc[g] = scale; + float inv = 1.f / scale; + for (size_t d = lo; d < hi; ++d) { int32_t q = (int32_t)std::roundf(row[d] * inv); q = std::max(-128, std::min(127, q)); dst[d] = (int8_t)q; } + } + }; + for (size_t h = 0; h < kv_heads; ++h) + for (size_t t = 0; t < n; ++t) { + std::vector k(head_dim), v(head_dim); + for (size_t d = 0; d < head_dim; ++d) { k[d] = std::sin(0.1 * (h + 1) * (t + 1) + 0.2 * d); v[d] = std::cos(0.05 * (t + 1) + 0.1 * d); } + pre[h][t] = k; vals[h][t] = v; + std::vector kr = rope_row(k, (double)t, theta); + quant_row(kr, k_i8.data() + t * int8_stride + h * head_dim, k_sc.data() + t * scale_stride + h * groups); + quant_row(v, v_i8.data() + t * int8_stride + h * head_dim, v_sc.data() + t * scale_stride + h * groups); + } + + std::vector> kept = {{0, 5, 11, 22, 33, 49}, {1, 4, 13, 26, 40, 48}}; + size_t B = kept[0].size(); + compact_int8(k_i8.data(), k_sc.data(), kv_heads, head_dim, kGroupSize, kept, theta, true); + compact_int8(v_i8.data(), v_sc.data(), kv_heads, head_dim, kGroupSize, kept, theta, false); + + bool ok = true; + for (size_t h = 0; h < kv_heads; ++h) + for (size_t rank = 0; rank < B; ++rank) { + int abs_pos = kept[h][rank]; + std::vector expK = rope_row(pre[h][abs_pos], (double)rank, theta); + const int8_t* kd = k_i8.data() + rank * int8_stride + h * head_dim; + const float* ks = k_sc.data() + rank * scale_stride + h * groups; + const int8_t* vd = v_i8.data() + rank * int8_stride + h * head_dim; + const float* vs = v_sc.data() + rank * scale_stride + h * groups; + for (size_t d = 0; d < head_dim; ++d) { + float dq = (float)kd[d] * ks[d / kGroupSize]; + if (std::abs(dq - expK[d]) > 0.05f) ok = false; + float dqv = (float)vd[d] * vs[d / kGroupSize]; + if (std::abs(dqv - vals[h][abs_pos][d]) > 0.05f) ok = false; + } + } + return ok; +} + +// Mirror Model::maybe_roll_compact: when the live seq-len reaches trigger_len, KeyDiff-keep the +// best target_len tokens (absolute budget), renumber survivors to 0..B-1, set header len = B. +// Returns B (the new bounded length). Operates on an FP16 K/V cache pair in place. +size_t roll_compact_once(uint16_t* krows, uint16_t* vrows, Header* khdr, Header* vhdr, + size_t kv_heads, size_t head_dim, double theta, int target_len) { + size_t n = khdr->current_seq_len; + Params p; p.enabled = true; p.recent_frac = 0.30f; p.sink = 4; p.signal = Signal::KEYDIFF; + p.abs_budget = target_len; // absolute budget -> keep exactly min(target_len, n) per head + auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); + compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); + size_t B = kept.empty() ? 0 : kept[0].size(); + khdr->current_seq_len = B; + vhdr->current_seq_len = B; + return B; +} + +bool test_rolling_bounded_compaction() { + // Drive the compactor across repeated thresholds on a synthetic cache, exactly as the rolling + // bounded path does: grow to trigger_len -> compact to target_len -> grow again -> compact. + // Asserts the cache stays bounded <= trigger_len, positions renumber to 0..B-1 each cycle, and + // a planted distinctive mid-token survives every compaction. + const int trigger_len = 4096, target_len = 2048; + const size_t kv_heads = 2, head_dim = 16, max_seq = trigger_len + 8; + const double theta = 1000000.0; + + std::vector kbuf(kHeaderBytes + max_seq * kv_heads * head_dim * sizeof(uint16_t), 0); + std::vector vbuf(kbuf.size(), 0); + auto* khdr = reinterpret_cast(kbuf.data()); + auto* vhdr = reinterpret_cast(vbuf.data()); + *khdr = Header{0, max_seq, kv_heads, head_dim, 4, {0, 0, 0}}; + *vhdr = *khdr; + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); + + // A strongly distinctive pre-RoPE key direction (centroid-opposite) that KeyDiff must keep. + std::vector distinctive(head_dim); + for (size_t d = 0; d < head_dim; ++d) distinctive[d] = (d % 2 == 0 ? -1.0f : 1.0f) * 4.0f; + + // Append fresh post-RoPE rows for positions [from, to) for every head. The token at + // `plant_abs` (if in range) is the distinctive one; the rest cluster on a common centroid. + auto append = [&](size_t from, size_t to, long plant_abs) { + for (size_t t = from; t < to; ++t) + for (size_t h = 0; h < kv_heads; ++h) { + std::vector k(head_dim); + if ((long)t == plant_abs) { + k = distinctive; + } else { + for (size_t d = 0; d < head_dim; ++d) + k[d] = std::sin(0.05 * (h + 1) + 0.3 * d) + 0.01f * std::sin(0.01 * t); + } + std::vector kr = rope_row(k, (double)t, theta); + std::vector vr(head_dim); + for (size_t d = 0; d < head_dim; ++d) vr[d] = std::cos(0.07 * (t + 1) + 0.2 * d); + for (size_t d = 0; d < head_dim; ++d) { + krows[(t * kv_heads + h) * head_dim + d] = f32_to_f16(kr[d]); + vrows[(t * kv_heads + h) * head_dim + d] = f32_to_f16(vr[d]); + } + } + }; + + // Does any survivor row (un-RoPE'd at its rank) match the planted distinctive direction? + auto distinctive_survives = [&](size_t B) -> bool { + for (size_t h = 0; h < kv_heads; ++h) + for (size_t rank = 0; rank < B; ++rank) { + std::vector pre(head_dim); + const uint16_t* src = krows + (rank * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) pre[d] = f16_to_f32(src[d]); + // The row is stored as rope(pre, rank); un-RoPE by -rank to recover pre. + rope_rotate_row(pre.data(), head_dim, theta, -(double)rank); + double dot = 0, na = 0, nb = 0; + for (size_t d = 0; d < head_dim; ++d) { + dot += pre[d] * distinctive[d]; na += pre[d] * pre[d]; nb += distinctive[d] * distinctive[d]; + } + if (dot / (std::sqrt(na) * std::sqrt(nb) + 1e-9) > 0.99) return true; + } + return false; + }; + + bool ok = true; + const int cycles = 3; + for (int c = 0; c < cycles; ++c) { + size_t start = khdr->current_seq_len; // survivors carried over from last cycle + long plant_abs = (c == 0) ? (trigger_len / 2) : -1; // plant once; must persist after that + append(start, (size_t)trigger_len, plant_abs); + khdr->current_seq_len = trigger_len; + vhdr->current_seq_len = trigger_len; + + // The distinctive token must be present BEFORE the first compaction renumbers it. + if (c == 0 && !distinctive_survives(trigger_len)) ok = false; + + size_t B = roll_compact_once(krows, vrows, khdr, vhdr, kv_heads, head_dim, theta, target_len); + if (B != (size_t)target_len) ok = false; // absolute budget honored + if (khdr->current_seq_len > (size_t)trigger_len) ok = false; // bounded + if (khdr->current_seq_len != (size_t)target_len) ok = false; // header set to B + + // Renumbering: survivor K rows must read back as rope(pre, rank). We verify via the + // un-RoPE-at-rank == direction check for the planted token, and that rows are dense + // (positions 0..B-1 populated, no gaps) by re-reading the live region length. + if (!distinctive_survives(B)) ok = false; // distinctive persists each cycle + } + // After all cycles the cache is still bounded at target_len, never exceeding trigger_len. + if (khdr->current_seq_len != (size_t)target_len) ok = false; + return ok; +} + +bool test_config_parse_rolling_fields() { + // Config::from_json parses a key=value file. Defaults are 0/0 (legacy one-shot, no rolling). + cactus::engine::Config def; + if (def.kv_compress_trigger_len != 0 || def.kv_compress_target_len != 0) return false; + if (def.kv_compress) return false; + + char tmpl[] = "/tmp/cactus_kvcfg_XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) return false; + ::close(fd); + { + std::ofstream f(tmpl); + // model_type=qwen avoids the Gemma4-only required-field validation; we only assert the + // new kv_compress rolling fields round-trip through the key=value parser. + f << "model_type=qwen\n" + << "kv_compress=true\n" + << "kv_compress_trigger_len=4096\n" + << "kv_compress_target_len=2048\n"; + } + cactus::engine::Config cfg; + bool parsed = cfg.from_json(tmpl); + std::remove(tmpl); + if (!parsed) return false; + return cfg.kv_compress && cfg.kv_compress_trigger_len == 4096 && cfg.kv_compress_target_len == 2048; +} + +bool test_trigger_zero_no_rolling() { + // trigger_len == 0 (default) => the rolling gate never fires, mirroring Model::maybe_roll_compact's + // early return. A cache grown well past any threshold stays byte-identical (no compaction). + cactus::engine::Config cfg; // defaults: kv_compress=false, trigger_len=0, target_len=0 + bool gate_open = cfg.kv_compress && cfg.kv_compress_trigger_len > 0; + if (gate_open) return false; + + const size_t n = 5000, kv_heads = 2, head_dim = 16, max_seq = 5008; + const double theta = 1000000.0; + std::vector kbuf(kHeaderBytes + max_seq * kv_heads * head_dim * sizeof(uint16_t), 0); + std::vector vbuf(kbuf.size(), 0); + auto* khdr = reinterpret_cast(kbuf.data()); + *khdr = Header{n, max_seq, kv_heads, head_dim, 4, {0, 0, 0}}; + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + for (size_t i = 0; i < n * kv_heads * head_dim; ++i) krows[i] = f32_to_f16(0.01f * (i % 97)); + std::vector kbuf0 = kbuf, vbuf0 = vbuf; + + // Gate closed => no compaction call at all => bytes unchanged and length unchanged. + if (gate_open) roll_compact_once(krows, reinterpret_cast(vbuf.data() + kHeaderBytes), + khdr, reinterpret_cast(vbuf.data()), + kv_heads, head_dim, theta, 2048); + if (khdr->current_seq_len != n) return false; + if (std::memcmp(kbuf.data(), kbuf0.data(), kbuf.size()) != 0) return false; + if (std::memcmp(vbuf.data(), vbuf0.data(), vbuf.size()) != 0) return false; + return true; +} + +bool test_degenerate_rolling_config_disabled() { + // validate_kv_compress(): when trigger_len > 0, require 0 < target_len < trigger_len. + // Bad configs must disable rolling (trigger_len/target_len reset to 0); good configs survive. + auto disabled = [](int trig, int targ) { + cactus::engine::Config c; + c.kv_compress = true; + c.kv_compress_trigger_len = trig; + c.kv_compress_target_len = targ; + c.validate_kv_compress(); + return c.kv_compress_trigger_len == 0 && c.kv_compress_target_len == 0; + }; + if (!disabled(4096, 0)) return false; // target_len <= 0 + if (!disabled(4096, 4096)) return false; // target_len == trigger_len + if (!disabled(2048, 4096)) return false; // target_len > trigger_len + if (!disabled(4096, -1)) return false; // negative target_len + // Valid config is untouched. + cactus::engine::Config ok; + ok.kv_compress = true; + ok.kv_compress_trigger_len = 4096; + ok.kv_compress_target_len = 2048; + ok.validate_kv_compress(); + return ok.kv_compress_trigger_len == 4096 && ok.kv_compress_target_len == 2048; +} + +bool test_env_override_parse() { + // parse_kv_compress_override maps the env strings onto Config Params exactly. + // Unset => no-op (OFF preserved). + cactus::engine::Config off; + if (off.parse_kv_compress_override(nullptr, nullptr)) return false; + if (off.kv_compress) return false; + + // One-shot: ":". + cactus::engine::Config os; + if (!os.parse_kv_compress_override("keydiff:0.25", nullptr)) return false; + if (!os.kv_compress || os.kv_compress_signal != "keydiff") return false; + if (std::abs(os.kv_compress_budget_frac - 0.25f) > 1e-6f) return false; + if (os.kv_compress_trigger_len != 0 || os.kv_compress_target_len != 0) return false; + + // Rolling: ":" (takes precedence over one-shot). + cactus::engine::Config roll; + if (!roll.parse_kv_compress_override("keydiff:0.25", "4096:2048")) return false; + if (!roll.kv_compress) return false; + if (roll.kv_compress_trigger_len != 4096 || roll.kv_compress_target_len != 2048) return false; + + // Rolling with degenerate values is disabled by the validate guard inside the parser. + cactus::engine::Config bad; + if (!bad.parse_kv_compress_override(nullptr, "2048:4096")) return false; + if (bad.kv_compress_trigger_len != 0 || bad.kv_compress_target_len != 0) return false; + return true; +} + +int main() { + TestUtils::TestRunner runner("KV Compress Integration Tests"); + runner.run_test("compact_fp16_cache", test_compact_fp16_cache()); + runner.run_test("dense_check_full_budget", test_dense_check_full_budget()); + runner.run_test("rope_renumber_contiguous", test_rope_renumber_contiguous()); + runner.run_test("compress_noop_when_disabled", test_compress_noop_when_disabled()); + runner.run_test("subset_compression_rejected", test_subset_compression_rejected()); + runner.run_test("fp16_storage_round_trip", test_fp16_storage_round_trip()); + runner.run_test("gemma_layer_selection", test_gemma_layer_selection()); + runner.run_test("compact_int8_cache", test_compact_int8_cache()); + runner.run_test("rolling_bounded_compaction", test_rolling_bounded_compaction()); + runner.run_test("config_parse_rolling_fields", test_config_parse_rolling_fields()); + runner.run_test("trigger_zero_no_rolling", test_trigger_zero_no_rolling()); + runner.run_test("degenerate_rolling_config_disabled", test_degenerate_rolling_config_disabled()); + runner.run_test("env_override_parse", test_env_override_parse()); + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-engine/tests/test_utils.h b/cactus-engine/tests/test_utils.h index 4a99f4f3f..717b5a101 100644 --- a/cactus-engine/tests/test_utils.h +++ b/cactus-engine/tests/test_utils.h @@ -31,6 +31,20 @@ class TestRunner { namespace EngineTestUtils { +// Reference rotate_half RoPE for a single vector (double-precision oracle for kv_compress tests), +// mirroring norms_rope.cpp / the Python fixture generator. +inline std::vector rope_reference(const std::vector& v, double pos, double theta) { + size_t d = v.size(), half = d / 2; + std::vector o(d); + for (size_t i = 0; i < half; ++i) { + double inv = std::pow(theta, -(2.0 * (double)i) / (double)d); + double a = pos * inv, c = std::cos(a), s = std::sin(a); + o[i] = (float)(v[i] * c - v[i + half] * s); + o[i + half] = (float)(v[i + half] * c + v[i] * s); + } + return o; +} + struct Timer { std::chrono::high_resolution_clock::time_point start; Timer(); diff --git a/experiments/niah/fixtures/kv_compress_fixtures.json b/experiments/niah/fixtures/kv_compress_fixtures.json new file mode 100644 index 000000000..98684028e --- /dev/null +++ b/experiments/niah/fixtures/kv_compress_fixtures.json @@ -0,0 +1 @@ +{"fixtures": [{"name": "base_keydiff", "n": 48, "head_dim": 16, "n_kv_heads": 2, "budget": 12, "recent_frac": 0.3, "sink": 4, "keys": [0.007008590269833803, -0.0517716109752655, 0.1948932260274887, 0.08067762106657028, -0.15222644805908203, 0.1667429506778717, 0.3216659724712372, 0.27535417675971985, -0.1463707536458969, -0.33972468972206116, -0.20681923627853394, -0.03483806923031807, -0.6557185053825378, -0.04854146018624306, -0.38959580659866333, -0.20976898074150085, 0.026260288432240486, -0.008914324454963207, 0.18504466116428375, 0.046320591121912, -0.17849114537239075, 0.09193909168243408, 0.39412447810173035, 0.3324502110481262, -0.25449755787849426, -0.2687291204929352, -0.10235049575567245, 0.05031377077102661, -0.6196094155311584, -0.07524735480546951, -0.26621386408805847, -0.1012972891330719, 0.12430328130722046, 0.029798584058880806, 0.19218072295188904, -0.0318639762699604, -0.14602242410182953, 0.1312435120344162, 0.29050764441490173, 0.27753475308418274, -0.17005108296871185, -0.30962318181991577, -0.22885017096996307, -0.02183692902326584, -0.6546539664268494, -0.11804131418466568, -0.25214651226997375, -0.22410574555397034, 0.050670020282268524, -0.04888522997498512, 0.2534853518009186, 0.09457000344991684, -0.11413208395242691, -0.011755724437534809, 0.3575271666049957, 0.29196298122406006, -0.14134618639945984, -0.3753206729888916, -0.07854370027780533, -0.054773349314928055, -0.6659085750579834, -0.012798716314136982, -0.3366621732711792, -0.09919057786464691, 0.04364749789237976, -0.067616306245327, 0.1554335206747055, -0.026005351915955544, -0.20968373119831085, 0.12994034588336945, 0.3839839994907379, 0.3225066065788269, -0.22927455604076385, -0.25996994972229004, -0.18401364982128143, 0.08996861428022385, -0.6544715166091919, -0.09632538259029388, -0.32662561535835266, -0.1477375477552414, 0.042272016406059265, -0.06523304432630539, 0.10725072026252747, -0.04152405634522438, -0.12066557258367538, 0.14790542423725128, 0.3467109799385071, 0.2040604203939438, -0.14789216220378876, -0.40844500064849854, -0.20529767870903015, 0.042299091815948486, -0.7453392744064331, -0.040232736617326736, -0.36819693446159363, -0.1938462257385254, 0.03043646365404129, -0.02585088461637497, 0.20902030169963837, -0.0093665337190032, -0.074750617146492, 0.1347244530916214, 0.39711233973503113, 0.3160218596458435, -0.15216486155986786, -0.3022213876247406, -0.16586458683013916, -0.06009049341082573, -0.639584481716156, -0.09802694618701935, -0.4102519750595093, -0.1863875687122345, 0.005794066935777664, -0.08744682371616364, 0.12216164916753769, 0.0419728085398674, -0.12786611914634705, 0.16454264521598816, 0.35422998666763306, 0.3098706603050232, -0.12143102288246155, -0.2869170308113098, -0.28790947794914246, 0.07268238812685013, -0.6158512234687805, -0.03836264833807945, -0.3205535113811493, -0.18017235398292542, 0.050192251801490784, -0.053902268409729004, 0.079229936003685, 0.02310621738433838, -0.18598631024360657, 0.15242794156074524, 0.34048739075660706, 0.2619524300098419, -0.23402456939220428, -0.36995643377304077, -0.17022092640399933, -0.06302055716514587, -0.6177979707717896, -0.06485483050346375, -0.3984008729457855, -0.3192218542098999, 0.059874147176742554, -0.050835806876420975, 0.14781127870082855, 0.0167442224919796, -0.05497591942548752, 0.09592971950769424, 0.35925668478012085, 0.18342502415180206, -0.10917731374502182, -0.29855090379714966, -0.11629752814769745, 0.013631836511194706, -0.5869994759559631, -0.04100387543439865, -0.3084554374217987, -0.2069198489189148, -0.03947285935282707, 0.015486113727092743, 0.07756371796131134, 0.016555121168494225, -0.15602584183216095, 0.0462767630815506, 0.3855818808078766, 0.24776217341423035, -0.21338768303394318, -0.3184332251548767, -0.193473219871521, 0.080697201192379, -0.6152594685554504, -0.08326786756515503, -0.43632814288139343, -0.2646978795528412, 0.08856307715177536, -0.03848680853843689, 0.16015544533729553, 0.11071453243494034, -0.20993217825889587, 0.06913688033819199, 0.33129632472991943, 0.2870955169200897, -0.2247210294008255, -0.3750962018966675, -0.24990174174308777, 0.047715671360492706, -0.5925252437591553, -0.08337005227804184, -0.3309479057788849, -0.26394250988960266, 0.010630881413817406, 0.03294094279408455, 0.18109823763370514, 0.14407013356685638, -0.1851593554019928, 0.12743398547172546, 0.3451504409313202, 0.2860695719718933, -0.19190484285354614, -0.3724852204322815, -0.21302510797977448, 0.1645500361919403, -0.6366994380950928, -0.16038425266742706, -0.3715449273586273, -0.16540822386741638, 0.009221117943525314, 0.032065704464912415, 0.2244316190481186, 0.020935021340847015, -0.16941051185131073, 0.04817971959710121, 0.3199273943901062, 0.1841215044260025, -0.13132445514202118, -0.26489028334617615, -0.2324511706829071, -0.047835949808359146, -0.7212578058242798, -0.10774392634630203, -0.494431734085083, -0.2564241588115692, 0.0990673080086708, -0.05324023216962814, 0.21704091131687164, 0.004103500861674547, -0.057766351848840714, 0.10838066786527634, 0.3358255922794342, 0.38539987802505493, -0.20776785910129547, -0.40548649430274963, -0.1595487743616104, 0.009306447580456734, -0.5795159935951233, -0.10563290864229202, -0.29887905716896057, -0.15667277574539185, 0.0008371748263016343, -0.027794403955340385, 0.13277409970760345, 0.14584235846996307, -0.18100669980049133, 0.07576604932546616, 0.3016338050365448, 0.24047259986400604, -0.1918380707502365, -0.3060358464717865, -0.20016860961914062, 0.001959501765668392, -0.7036566734313965, -0.10092132538557053, -0.20132452249526978, -0.1472480446100235, -0.004849662538617849, -0.10282646864652634, 0.12553256750106812, 0.027467411011457443, -0.1440633237361908, 0.06120174005627632, 0.29059699177742004, 0.3288975954055786, -0.16895999014377594, -0.36315372586250305, -0.18067732453346252, -0.015228323638439178, -0.7796344757080078, -0.05376812443137169, -0.3926421105861664, -0.24944442510604858, 0.002208419144153595, 0.000658481614664197, 0.1157851591706276, -0.04316211864352226, -0.11380711197853088, 0.1361382156610489, 0.3069790303707123, 0.2858985364437103, -0.2061258852481842, -0.329360693693161, -0.23269228637218475, 0.05289292335510254, -0.572669267654419, -0.027697553858160973, -0.3111979067325592, -0.3879239559173584, 0.04725302755832672, -0.03722887113690376, 0.16695943474769592, -0.0029769453685730696, -0.14303146302700043, 0.11902564018964767, 0.3417363166809082, 0.23461246490478516, -0.13005657494068146, -0.39969366788864136, -0.11813651025295258, 0.02008882351219654, -0.6730474829673767, -0.07405030727386475, -0.3851145803928375, -0.16555698215961456, 0.05161662772297859, -0.06379641592502594, 0.11920078843832016, 0.04363775998353958, -0.0979304388165474, 0.09272795170545578, 0.375843346118927, 0.238977313041687, -0.1881658136844635, -0.3589894473552704, -0.15494216978549957, -0.06424042582511902, -0.6006226539611816, -0.071047842502594, -0.3211844861507416, -0.21632571518421173, 0.050239600241184235, -0.08958474546670914, 0.23376941680908203, -0.05662527680397034, -0.1977616548538208, 0.11020307242870331, 0.4280678331851959, 0.2716849744319916, -0.20393969118595123, -0.41567981243133545, -0.17920809984207153, 0.010254349559545517, -0.5483037829399109, -0.028444955125451088, -0.415569543838501, -0.09797125309705734, 0.014471045695245266, -0.07992959022521973, 0.24805283546447754, 0.02606416679918766, -0.16416984796524048, 0.10935907065868378, 0.39717015624046326, 0.3074454665184021, -0.26030439138412476, -0.2445012331008911, -0.12230119109153748, -0.007711853366345167, -0.6737651824951172, -0.10800183564424515, -0.33294597268104553, -0.2317110151052475, -0.004022158682346344, 0.004606116097420454, 0.19254006445407867, 0.008823391050100327, -0.10908733308315277, 0.16678869724273682, 0.3002009093761444, 0.22761407494544983, -0.14441317319869995, -0.30847829580307007, -0.15830932557582855, 0.06936925649642944, -0.6872428059577942, -0.13350810110569, -0.3824402689933777, -0.19318300485610962, -0.005582799203693867, -0.060317512601614, 0.1255616992712021, -0.0024695030879229307, -0.196039080619812, 0.11679308116436005, 0.3946702778339386, 0.23375636339187622, -0.20190756022930145, -0.37347695231437683, -0.14308170974254608, 0.01570100151002407, -0.5531243681907654, -0.11432481557130814, -0.3209897577762604, -0.17711064219474792, -0.09314308315515518, 0.09928934276103973, -0.5352045297622681, -0.0971926748752594, 0.4559859037399292, -0.3058036267757416, -1.0671230554580688, -0.7751907706260681, 0.584943413734436, 1.015892744064331, 0.508540153503418, -0.021600715816020966, 1.9096455574035645, 0.1827467679977417, 1.0147353410720825, 0.6155074238777161, 0.01219812873750925, 0.0025344383902847767, 0.2250780463218689, 0.02116336300969124, -0.22028198838233948, 0.16762802004814148, 0.4090783894062042, 0.24281910061836243, -0.08612984418869019, -0.36186909675598145, -0.2265002578496933, 0.0034545280504971743, -0.5789405107498169, -0.10642388463020325, -0.2415536344051361, -0.2441365122795105, 0.08194265514612198, -0.008733485825359821, 0.16660885512828827, 0.08257859945297241, -0.22078965604305267, 0.16629835963249207, 0.3517415225505829, 0.23065690696239471, -0.15408556163311005, -0.29143035411834717, -0.13115449249744415, 0.11122339218854904, -0.5787959098815918, 0.004624115768820047, -0.36610808968544006, -0.1939840167760849, 0.0623706690967083, -0.036843717098236084, 0.18939346075057983, 0.04985719174146652, -0.10358752310276031, 0.09333932399749756, 0.3374357521533966, 0.2163628786802292, -0.23613087832927704, -0.2858027517795563, -0.17387033998966217, 0.050595417618751526, -0.6977003812789917, -0.15645961463451385, -0.39156946539878845, -0.14197885990142822, 0.0876254141330719, -0.019354673102498055, 0.13420319557189941, 0.02201182208955288, -0.16068774461746216, 0.08117110282182693, 0.2296222299337387, 0.21488234400749207, -0.20104971528053284, -0.2686039209365845, -0.16160711646080017, 0.08129261434078217, -0.652517557144165, -0.07218564301729202, -0.5340859889984131, -0.17614369094371796, 0.061576321721076965, 0.05223134905099869, 0.14997364580631256, 0.03325966000556946, -0.18107610940933228, 0.03960691764950752, 0.3192703127861023, 0.24052873253822327, -0.12377236783504486, -0.34431472420692444, -0.20917150378227234, 0.018342090770602226, -0.6219536066055298, -0.09336281567811966, -0.28195321559906006, -0.2937276363372803, 0.0235443077981472, -0.0027033009100705385, 0.10739007592201233, 0.046614643186330795, -0.08115506172180176, 0.12110333144664764, 0.270417720079422, 0.2213689088821411, -0.1299290955066681, -0.32950854301452637, -0.1701372116804123, 0.033308230340480804, -0.5967797636985779, -0.09497448056936264, -0.35363489389419556, -0.19216452538967133, 0.007023653481155634, -0.0426291823387146, 0.2392052859067917, -0.01983903907239437, -0.0494665801525116, 0.19238698482513428, 0.26925376057624817, 0.2507275342941284, -0.1744096875190735, -0.3824688792228699, -0.20669829845428467, -0.0006225851830095053, -0.5958732962608337, -0.08511790633201599, -0.2477802038192749, -0.18481150269508362, 0.029092775657773018, 0.03650638088583946, 0.2056257575750351, 0.047016292810440063, -0.16235673427581787, 0.18911905586719513, 0.3955167829990387, 0.24760809540748596, -0.27040421962738037, -0.32577773928642273, -0.22683127224445343, -0.074549101293087, -0.6467897891998291, -0.04547766223549843, -0.27499082684516907, -0.18518760800361633, 0.07452251762151718, -0.09725073724985123, 0.1731850951910019, 0.03476252034306526, -0.10268740355968475, 0.10422280430793762, 0.39512965083122253, 0.23256869614124298, -0.17364652454853058, -0.3236819803714752, -0.23213306069374084, 0.020025130361318588, -0.6488503217697144, -0.1547556072473526, -0.2911945581436157, -0.21740314364433289, -0.00839796382933855, -0.054822441190481186, 0.18122172355651855, 0.10394743084907532, -0.15409637987613678, 0.12204014509916306, 0.42360469698905945, 0.28445619344711304, -0.13811606168746948, -0.3682508170604706, -0.13109180331230164, 0.008349400013685226, -0.579111635684967, -0.10972919315099716, -0.37809279561042786, -0.13587363064289093, 0.024417340755462646, -0.053904637694358826, 0.17819032073020935, -0.005922918673604727, -0.07920264452695847, 0.035961639136075974, 0.34739580750465393, 0.27512630820274353, -0.19676661491394043, -0.3845396935939789, -0.21306373178958893, 0.032517921179533005, -0.6843562126159668, -0.027239080518484116, -0.41532230377197266, -0.22705936431884766, 0.036036841571331024, -0.09856420010328293, 0.20691224932670593, 0.027625013142824173, -0.19760340452194214, 0.022475358098745346, 0.2766462564468384, 0.26033034920692444, -0.2493857741355896, -0.41266930103302, -0.18120557069778442, 0.12512323260307312, -0.6189551949501038, -0.02140527404844761, -0.32847896218299866, -0.160161092877388, -0.03293289244174957, -0.05748220160603523, 0.18737924098968506, 0.027656113728880882, -0.1553993821144104, 0.0651259645819664, 0.3420065641403198, 0.21906878054141998, -0.3126359283924103, -0.40415072441101074, -0.14586162567138672, 0.08910209685564041, -0.5421531796455383, -0.05471362918615341, -0.2944457232952118, -0.1539113074541092, -0.0003914197732228786, -0.12085359543561935, 0.1758560687303543, -0.05955809727311134, -0.1617961972951889, 0.1288256198167801, 0.2839222550392151, 0.2593934237957001, -0.12954336404800415, -0.326349675655365, -0.14352817833423615, 0.056585218757390976, -0.5461840629577637, -0.05198006331920624, -0.27763983607292175, -0.20251867175102234, 0.006913098506629467, -0.020215556025505066, 0.14402106404304504, -0.00010450674744788557, -0.17618674039840698, -0.01635143905878067, 0.3602007031440735, 0.19458690285682678, -0.19690772891044617, -0.2719268798828125, -0.19569428265094757, -0.01585225760936737, -0.5646367073059082, -0.03223592787981033, -0.2902885377407074, -0.21707476675510406, 0.07162570208311081, -0.070265032351017, 0.140511617064476, 0.05836416780948639, -0.17571838200092316, 0.13677570223808289, 0.47450995445251465, 0.17347103357315063, -0.22915448248386383, -0.2884179949760437, -0.17689818143844604, 0.06930269300937653, -0.6833354234695435, -0.0429840087890625, -0.34664270281791687, -0.19229166209697723, 0.05077734217047691, -0.09695692360401154, 0.12060482800006866, 0.09851415455341339, -0.13113902509212494, 0.10373708605766296, 0.3527219891548157, 0.27560579776763916, -0.24934110045433044, -0.394376665353775, -0.10441885888576508, 0.018811961635947227, -0.5903171300888062, -0.08983441442251205, -0.2702786326408386, -0.18204627931118011, 0.05828383192420006, -0.008520137518644333, 0.13446123898029327, -0.06473408639431, -0.19953985512256622, 0.1799459159374237, 0.4199599027633667, 0.24043259024620056, -0.20662808418273926, -0.2925950288772583, -0.17806476354599, -0.05371479317545891, -0.5695550441741943, -0.03569228574633598, -0.4649318754673004, -0.2149697095155716, 0.041405048221349716, -0.01189426239579916, 0.18191926181316376, -0.0032351359259337187, -0.15158766508102417, 0.11315978318452835, 0.34152552485466003, 0.23918363451957703, -0.12894007563591003, -0.39175570011138916, -0.18711744248867035, -0.09031525254249573, -0.6057837009429932, -0.018137993291020393, -0.3116905391216278, -0.15342658758163452, 0.05625687539577484, -0.018830833956599236, 0.19800905883312225, 0.015186532400548458, -0.08638332039117813, 0.08099116384983063, 0.28180810809135437, 0.3002678453922272, -0.09900912642478943, -0.3924328684806824, -0.17472583055496216, -0.023023901507258415, -0.6518613696098328, -0.05724656581878662, -0.4012053310871124, -0.2131928950548172, -0.03907683119177818, -0.06436798721551895, 0.11500822007656097, -0.02440108172595501, -0.23179525136947632, 0.1593874990940094, 0.3803786635398865, 0.1619054675102234, -0.2213856428861618, -0.37794724106788635, -0.20419147610664368, -0.06109597533941269, -0.5951129198074341, -0.07934440672397614, -0.3157074451446533, -0.1729724109172821, 0.10299380868673325, -0.12670022249221802, 0.2612418234348297, 0.09199272096157074, -0.11714641749858856, 0.21759937703609467, 0.36517465114593506, 0.2988526225090027, -0.22846496105194092, -0.28770744800567627, -0.1612529754638672, -0.011311807669699192, -0.526985228061676, -0.07479120045900345, -0.33867183327674866, -0.20917420089244843, -0.0035620061680674553, -0.009392984211444855, 0.21123218536376953, 0.04627057909965515, -0.2638286352157593, 0.14881962537765503, 0.3374137878417969, 0.19696249067783356, -0.16138021647930145, -0.31628283858299255, -0.22179795801639557, 0.13486997783184052, -0.6933135390281677, -0.14620135724544525, -0.3968234658241272, -0.1282702088356018, 0.02542870305478573, -0.05455943942070007, 0.1712038218975067, -0.0012787414016202092, -0.1803128868341446, 0.06635899841785431, 0.3903234899044037, 0.30880147218704224, -0.24429889023303986, -0.33246392011642456, -0.13023391366004944, -0.04283939674496651, -0.6579605340957642, -0.11145442724227905, -0.40366631746292114, -0.19421705603599548, -0.0025850790552794933, -0.004400149453431368, 0.17284061014652252, 0.04918332025408745, -0.16041386127471924, 0.06666990369558334, 0.35039207339286804, 0.2575288712978363, -0.22683703899383545, -0.32309553027153015, -0.132314532995224, 0.019152335822582245, -0.5471576452255249, -0.09116816520690918, -0.31308507919311523, -0.21974694728851318, 0.04595301300287247, -0.07745406776666641, 0.2300678789615631, 0.037352997809648514, -0.08618748933076859, 0.035114992409944534, 0.33024996519088745, 0.21315038204193115, -0.2181413620710373, -0.37905579805374146, -0.17567132413387299, 0.010385842062532902, -0.6348814368247986, -0.08732741326093674, -0.3297436237335205, -0.1555294245481491, -0.010892238467931747, -0.035912174731492996, 0.17060816287994385, 0.05196968466043472, -0.1489926427602768, 0.09763652831315994, 0.30537092685699463, 0.2588638663291931, -0.23763848841190338, -0.3186401128768921, -0.17480522394180298, 0.013241120614111423, -0.6773917078971863, -0.019363336265087128, -0.30449920892715454, -0.156234011054039, 0.14559084177017212, -0.03858257457613945, 0.2344818413257599, 0.022360563278198242, -0.12609535455703735, 0.11668664216995239, 0.368155300617218, 0.29069146513938904, -0.2067410796880722, -0.342378705739975, -0.14287878572940826, 0.09864089637994766, -0.6749500036239624, -0.14985784888267517, -0.36331915855407715, -0.1948137879371643, 0.04297039285302162, -0.03134571388363838, 0.23345120251178741, 0.07735444605350494, -0.1482631117105484, 0.07494241744279861, 0.33762526512145996, 0.23996345698833466, -0.20282423496246338, -0.4230578541755676, -0.19269630312919617, -0.009995801374316216, -0.6422762274742126, -0.07226621359586716, -0.30551454424858093, -0.2259073704481125, 0.012292654253542423, -0.0088596660643816, 0.16254596412181854, 0.039374690502882004, -0.11233293265104294, 0.11951824277639389, 0.3674338459968567, 0.24795658886432648, -0.15744955837726593, -0.3356478214263916, -0.19488690793514252, 0.0037077402230352163, -0.6937386393547058, -0.10762901604175568, -0.4332602918148041, -0.23330678045749664, 0.10099881142377853, -0.06378104537725449, 0.21368874609470367, 0.02837946079671383, -0.18082715570926666, 0.16533374786376953, 0.38403600454330444, 0.17018863558769226, -0.13947415351867676, -0.39816489815711975, -0.1785399615764618, 0.04465387016534805, -0.6478248834609985, -0.003604013239964843, -0.30105170607566833, -0.2781539857387543, 0.0105998320505023, -0.021852917969226837, 0.14556176960468292, 0.01773812808096409, -0.10581047087907791, 0.11425883322954178, 0.3088947534561157, 0.2665089964866638, -0.22215774655342102, -0.40518492460250854, -0.22614705562591553, 0.025595976039767265, -0.6342378854751587, -0.05927962437272072, -0.39642533659935, -0.20850668847560883, -0.017636841163039207, -0.08227629214525223, 0.166055366396904, -0.04061757028102875, -0.11233635246753693, 0.22276391088962555, 0.3778668940067291, 0.2056335061788559, -0.2051035463809967, -0.42219722270965576, -0.18852509558200836, 0.03661389648914337, -0.6033629775047302, -0.11113835126161575, -0.32415762543678284, -0.14148980379104614, 0.12197612226009369, -0.07099635154008865, 0.13110169768333435, 0.03127274662256241, -0.29220423102378845, 0.07186806946992874, 0.3413769602775574, 0.23481155931949615, -0.2707379162311554, -0.35670286417007446, -0.20847469568252563, 0.04912235215306282, -0.6787559390068054, -0.073668472468853, -0.350762277841568, -0.1709316372871399, -0.09307368844747543, -0.052987851202487946, 0.21229246258735657, 0.010467542335391045, -0.22214657068252563, 0.11474967747926712, 0.3717600703239441, 0.2442542016506195, -0.24953405559062958, -0.3814590573310852, -0.18533208966255188, -0.03252844139933586, -0.7288411259651184, -0.09801555424928665, -0.3422020971775055, -0.22466321289539337, 0.030442897230386734, -0.031773705035448074, 0.21914048492908478, 0.13874316215515137, -0.1092107743024826, 0.028203342109918594, 0.22207847237586975, 0.25305554270744324, -0.18797484040260315, -0.40251898765563965, -0.1560426652431488, -0.027098780497908592, -0.6126987338066101, -0.07582712173461914, -0.3192417621612549, -0.286469966173172, 0.012298195622861385, -0.04338640347123146, 0.10306796431541443, 0.12266886979341507, -0.17283838987350464, 0.1679183542728424, 0.321708619594574, 0.24628010392189026, -0.1323491781949997, -0.32923397421836853, -0.16004155576229095, 0.024545371532440186, -0.7011401057243347, -0.0790288895368576, -0.3869371712207794, -0.18943952023983002, 0.007021632976830006, -0.0381595604121685, 0.17044708132743835, 0.026733435690402985, -0.14753706753253937, 0.06580086797475815, 0.30222558975219727, 0.22456084191799164, -0.13796734809875488, -0.3257172107696533, -0.14029650390148163, 0.08024659752845764, -0.6918037533760071, -0.03405359387397766, -0.39286860823631287, -0.21602682769298553, 0.05843353271484375, 0.04476065933704376, 0.13520345091819763, 0.02381214126944542, -0.08798787742853165, 0.02392936311662197, 0.3730313777923584, 0.242364764213562, -0.23562902212142944, -0.3370937705039978, -0.13989797234535217, -0.03435678780078888, -0.6138327121734619, -0.05088271573185921, -0.40120556950569153, -0.12163956463336945, 0.08871667087078094, -0.07895293831825256, 0.14498016238212585, 0.0670909658074379, -0.1705721616744995, 0.0064338394440710545, 0.40736982226371765, 0.2582177221775055, -0.09619295597076416, -0.3265519440174103, -0.16009464859962463, 0.15497322380542755, -0.6414255499839783, -0.10714200139045715, -0.32765498757362366, -0.14252245426177979, -0.024034468457102776, -0.08136986196041107, 0.19680055975914001, -0.13131532073020935, -0.2004336565732956, 0.1381939798593521, 0.32558873295783997, 0.17645451426506042, -0.0952659323811531, -0.4149520695209503, -0.19581255316734314, -0.007388375233858824, -0.6286749839782715, -0.07802647352218628, -0.3431631624698639, -0.19643548130989075, 0.029887717217206955, -0.03129291161894798, 0.055365417152643204, 0.05060518905520439, -0.2160225510597229, -0.009912505745887756, 0.4239931106567383, 0.19350290298461914, -0.1825508177280426, -0.38305339217185974, -0.20356757938861847, 0.03543093800544739, -0.6852434277534485, -0.04091750085353851, -0.3200802803039551, -0.14108781516551971, 0.017410878092050552, 0.01637250930070877, 0.2603464424610138, 0.10788606852293015, -0.11649587005376816, 0.12087807059288025, 0.4973984658718109, 0.3693874478340149, -0.2298751324415207, -0.2982448637485504, -0.13954630494117737, 0.014861572533845901, -0.6251665949821472, -0.03516196087002754, -0.2922469675540924, -0.1883717179298401, 0.05121122673153877, 0.033650413155555725, 0.19019772112369537, 0.0558030791580677, -0.09628821164369583, 0.18008288741111755, 0.41627776622772217, 0.2766394019126892, -0.18115349113941193, -0.4056054949760437, -0.1550411432981491, -0.040610287338495255, -0.6840341687202454, -0.027025770395994186, -0.34414374828338623, -0.17569972574710846, 0.0028733834624290466, 0.024103596806526184, 0.18150031566619873, 0.08793836832046509, -0.11212769150733948, 0.10668221861124039, 0.33099934458732605, 0.2593540549278259, -0.15014374256134033, -0.3095424175262451, -0.22942350804805756, 0.06253230571746826, -0.6435275673866272, -0.018788030371069908, -0.3739893138408661, -0.16741743683815002, -0.005608054809272289, -0.029488762840628624, 0.15941020846366882, 0.0142733259126544, -0.17412908375263214, 0.09074081480503082, 0.2678939700126648, 0.3016086518764496, -0.14345993101596832, -0.366564005613327, -0.23863200843334198, -0.021087264642119408, -0.5854514837265015, -0.028275176882743835, -0.3541325628757477, -0.15444646775722504, -0.017852498218417168, -0.06658842414617538, 0.19804206490516663, 0.02375628799200058, -0.1752610057592392, -0.027186617255210876, 0.3884865343570709, 0.2739647626876831, -0.2786903977394104, -0.3144090175628662, -0.1703624576330185, 0.02518993802368641, -0.5855725407600403, -0.09654555469751358, -0.3034366965293884, -0.16511750221252441, 0.07229732722043991, 0.04584118351340294, 0.20724844932556152, 0.0004740685108117759, -0.05588049441576004, 0.043054141104221344, 0.3317895233631134, 0.20955517888069153, -0.19668230414390564, -0.29031434655189514, -0.1047549694776535, 0.0344240702688694, -0.6633017659187317, -0.08723224699497223, -0.36927470564842224, -0.15166817605495453, -0.11938408762216568, 0.117679663002491, -0.517853319644928, -0.07974549382925034, 0.44005724787712097, -0.2932074964046478, -1.055469274520874, -0.784866452217102, 0.5621672868728638, 1.034900426864624, 0.5094543099403381, -0.03210799768567085, 1.9144009351730347, 0.1728605479001999, 1.0248578786849976, 0.6017751097679138, -0.032120052725076675, -0.02076210454106331, 0.24170204997062683, 0.010345215909183025, -0.20922769606113434, 0.021755745634436607, 0.32089313864707947, 0.3367558419704437, -0.20193123817443848, -0.39632248878479004, -0.2001178115606308, -0.01292904932051897, -0.6195515394210815, -0.09255203604698181, -0.38361185789108276, -0.20871105790138245, 0.057655055075883865, -0.00010441572521813214, 0.20183829963207245, 0.005677107721567154, -0.23055356740951538, 0.0584939643740654, 0.3691297769546509, 0.19109690189361572, -0.17891331017017365, -0.3739241063594818, -0.19875246286392212, 0.06722354888916016, -0.6218955516815186, 0.008961592800915241, -0.38607755303382874, -0.14277315139770508, 0.07967556267976761, -0.1857067495584488, 0.1698981672525406, 0.10566317290067673, -0.10689684748649597, 0.07635371387004852, 0.3432861864566803, 0.1926262229681015, -0.1812296360731125, -0.4340839087963104, -0.2204096019268036, 0.06720642000436783, -0.6338604688644409, -0.07772251963615417, -0.3444104492664337, -0.06221550330519676, 0.08598093688488007, -0.07473867386579514, 0.25764918327331543, 0.02411220781505108, -0.10881386697292328, 0.06908825039863586, 0.30803292989730835, 0.2978743016719818, -0.22998860478401184, -0.3834912180900574, -0.12475255876779556, -0.04685525223612785, -0.7037947773933411, -0.0322984978556633, -0.43710896372795105, -0.23689648509025574, 0.01593923382461071, -0.10322195291519165, 0.11844217777252197, 0.061845600605010986, -0.13117706775665283, 0.05113419517874718, 0.3813999891281128, 0.33374258875846863, -0.2556651532649994, -0.34136253595352173, -0.07081170380115509, 0.04180619493126869, -0.5921629667282104, -0.06955885142087936, -0.3790130913257599, -0.14214162528514862, 0.06690778583288193, -0.038532763719558716, 0.15104492008686066, -0.05883430689573288, -0.13800440728664398, 0.06383413076400757, 0.3483330011367798, 0.29132744669914246, -0.15054413676261902, -0.3084752559661865, -0.056891053915023804, 0.06950412690639496, -0.6701360940933228, -0.10046631097793579, -0.2227073609828949, -0.23247651755809784, 0.03201477229595184, 0.051286350935697556, 0.26007750630378723, 0.025820352137088776, -0.13370256125926971, 0.016664303839206696, 0.3095082938671112, 0.23951570689678192, -0.19182728230953217, -0.360591322183609, -0.16202956438064575, 0.1208832785487175, -0.6096151471138, -0.044115856289863586, -0.3390235900878906, -0.2079055905342102, 0.09586876630783081, -0.08551967889070511, 0.1680327206850052, -0.02623690664768219, -0.14144323766231537, 0.1043722853064537, 0.4009510576725006, 0.31766173243522644, -0.1583910435438156, -0.26331233978271484, -0.09021911025047302, 0.027016134932637215, -0.6141648292541504, -0.156365305185318, -0.323102742433548, -0.19956478476524353, -0.050203222781419754, -0.05752590671181679, 0.22218289971351624, -0.03338676318526268, -0.08115215599536896, 0.12215973436832428, 0.3435063064098358, 0.25066229701042175, -0.1500479131937027, -0.4159950315952301, -0.06760617345571518, -0.029422776773571968, -0.6710963845252991, 0.001487193861976266, -0.29961228370666504, -0.1330406218767166, 0.08652058988809586, -0.01713680848479271, 0.11049249768257141, 0.004909958690404892, -0.13587774336338043, 0.09756064414978027, 0.2522865831851959, 0.30999892950057983, -0.10498714447021484, -0.2712205648422241, -0.2205372452735901, 0.03304653987288475, -0.7391570806503296, 0.0249496903270483, -0.4361087381839752, -0.17878477275371552, 0.09235052019357681, 0.02860948257148266, 0.19151023030281067, -0.04273054748773575, -0.15085850656032562, 0.08500384539365768, 0.37334367632865906, 0.3071659505367279, -0.15844222903251648, -0.2383444905281067, -0.13659173250198364, -0.008192676119506359, -0.5513332486152649, -0.07527761906385422, -0.3250217139720917, -0.1902713030576706, -0.08044002950191498, -0.02211560681462288, 0.1887837052345276, 0.01806670054793358, -0.15286406874656677, 0.03419959545135498, 0.35904887318611145, 0.3329249322414398, -0.2004077434539795, -0.3705315887928009, -0.20003071427345276, 0.11210259795188904, -0.7005943655967712, -0.017303727567195892, -0.3335443139076233, -0.10326538234949112, 0.09978481382131577, -0.04037250578403473, 0.09723827242851257, 0.1000339463353157, -0.0728975310921669, 0.04317466914653778, 0.3921873867511749, 0.21774034202098846, -0.23035378754138947, -0.42489099502563477, -0.12299900501966476, 0.024804620072245598, -0.6081072092056274, 0.01307253073900938, -0.3056836724281311, -0.17008206248283386, 0.07168523222208023, -0.0891851931810379, 0.17258845269680023, -0.03176112473011017, -0.19079537689685822, 0.093828946352005, 0.40802010893821716, 0.3606893718242645, -0.24580392241477966, -0.3357588052749634, -0.17318978905677795, 0.03760997951030731, -0.603711724281311, -0.04143015667796135, -0.3874737322330475, -0.17957642674446106, 0.07883787155151367, -0.06162614747881889, 0.2732199430465698, 0.01680629514157772, -0.17749272286891937, 0.11323872208595276, 0.34204623103141785, 0.18942002952098846, -0.19285210967063904, -0.2646401822566986, -0.06951765716075897, -0.017103873193264008, -0.587568461894989, -0.052924744784832, -0.32492056488990784, -0.13594163954257965, 0.07682511955499649, -0.028240583837032318, 0.127874493598938, -0.02764708735048771, -0.1490006148815155, 0.11460043489933014, 0.3800843060016632, 0.2968771457672119, -0.14256501197814941, -0.46457913517951965, -0.1244722530245781, -0.04112418740987778, -0.5771287679672241, -0.042718999087810516, -0.3075869083404541, -0.22635649144649506, 0.04835410416126251, -0.1293172389268875, 0.22907403111457825, 0.012010017409920692, -0.2752537429332733, 0.15149395167827606, 0.43876439332962036, 0.21397219598293304, -0.1309087723493576, -0.38191309571266174, -0.2829703092575073, -0.006646424066275358, -0.6165253520011902, -0.05287365987896919, -0.3339349925518036, -0.18448905646800995, 0.04504932090640068, -0.04675851762294769, 0.14424262940883636, 0.015261911787092686, -0.163383349776268, 0.08327364176511765, 0.2997780442237854, 0.2306489497423172, -0.18743295967578888, -0.3289543688297272, -0.172845721244812, 0.0020652038510888815, -0.6061096787452698, -0.018827782943844795, -0.305110901594162, -0.286805659532547, 0.011451284401118755, -0.0006738747470080853, 0.20008157193660736, 0.06254248321056366, -0.1300821453332901, 0.05574293062090874, 0.4160539507865906, 0.1897992193698883, -0.14225637912750244, -0.34524407982826233, -0.24327698349952698, -0.008559740148484707, -0.5726445913314819, -0.08897130191326141, -0.39964500069618225, -0.2137497216463089, -0.006682208739221096, -0.09848660230636597, 0.19618858397006989, 0.06846056878566742, -0.17338629066944122, 0.12460091710090637, 0.33557435870170593, 0.25279295444488525, -0.18537414073944092, -0.2697611451148987, -0.14426632225513458, -0.03870745748281479, -0.6070407629013062, -0.07061062753200531, -0.36243292689323425, -0.24129827320575714, 0.03290852531790733, -0.03896322473883629, 0.15420258045196533, -0.041005413979291916, -0.2311583310365677, 0.10481899231672287, 0.3638359606266022, 0.3669351041316986, -0.20047742128372192, -0.2933541238307953, -0.10988408327102661, 0.008194861933588982, -0.6289921402931213, -0.08104339987039566, -0.2503569424152374, -0.2747318148612976, 0.03223923221230507, 0.033249545842409134, 0.16892769932746887, 5.079049515188672e-06, -0.13415558636188507, 0.1247691735625267, 0.383297860622406, 0.27186083793640137, -0.15403971076011658, -0.2971402704715729, -0.2249884158372879, -0.007696391548961401, -0.5893476605415344, -0.03512021526694298, -0.327384889125824, -0.16568626463413239], "scores": [[-0.9927071697508774, -0.9781303331637462, -0.980726951457204, -0.9686601654444504, -0.9859965111361388, -0.9865717629616055, -0.9872280709373202, -0.9823165548094318, -0.9829072992534935, -0.9878495070628652, -0.9810894835188151, -0.9856951503570026, -0.9774452052979116, -0.9725822976425459, -0.9778063795567629, -0.9730776266008456, -0.9851865014968185, -0.9687661134241308, -0.9923840676188775, -0.9910699275930659, -0.976550872420269, -0.9848451806828293, -0.9876516408567366, -0.9898345110903177, 0.9994412463254624, -0.9770638868134249, -0.9806166377026753, -0.986895191372311, -0.9673911747362711, -0.9820734209775862, -0.9911574853118776, -0.9775900274006717, -0.9815438076069051, -0.9889390243317184, -0.9860499051870153, -0.990346344737256, -0.977599989333287, -0.9757162752791518, -0.9844969925705689, -0.9828237073298414, -0.9836747738809793, -0.9834124515102077, -0.9762736657604103, -0.9863316208064562, -0.9859690212879606, -0.9785105649864247, -0.9719845837543548, -0.9776116742536747], [-0.9924277098064016, -0.9928188631407819, -0.9912300483421472, -0.9923449212553379, -0.9865435859838525, -0.9925936264910195, -0.9919715057932824, -0.9812501550700913, -0.9917742573817456, -0.9778690656937898, -0.9823141793667827, -0.9873755735746267, -0.9731441633092237, -0.984034399574426, -0.9904464797766082, -0.9839821098809262, -0.9761437268630193, -0.9718396732600318, -0.9782555598981743, -0.9750649788330894, -0.9877874211014852, -0.9881886750379459, -0.9871399101894514, -0.9830468506826351, -0.9827622881763346, 0.9996494859150404, -0.987558404612867, -0.9869249809227716, -0.9687118370134803, -0.9871407693269674, -0.9818398625900191, -0.9761055044155703, -0.9832896629231536, -0.9804907391451436, -0.9769434411712624, -0.9704453412490723, -0.9826590242372737, -0.9812950039848866, -0.977409858711416, -0.988268575382572, -0.981042315966165, -0.9816448417852881, -0.9760361575575511, -0.990216513142874, -0.984578168970806, -0.9877420311091998, -0.9770648701114597, -0.9900559255184145]], "kept": [[0, 1, 2, 3, 13, 17, 24, 28, 44, 45, 46, 47], [0, 1, 2, 3, 17, 25, 28, 35, 44, 45, 46, 47]]}, {"name": "small_n_keydiff", "n": 20, "head_dim": 8, "n_kv_heads": 4, "budget": 5, "recent_frac": 0.3, "sink": 4, "keys": [0.06934703141450882, -0.18056292831897736, -0.071917325258255, -0.7295069694519043, 0.5098626017570496, 0.2949908673763275, -0.07241827249526978, 0.22133097052574158, 0.08255092799663544, -0.18323107063770294, -0.11445430666208267, -0.7585928440093994, 0.5683770179748535, 0.3439999520778656, -0.07863762974739075, 0.24681606888771057, 0.00474861916154623, -0.11371274292469025, -0.017960557714104652, -0.7959012985229492, 0.43983322381973267, 0.25935661792755127, -0.053093232214450836, 0.23272664844989777, 0.10920361429452896, -0.11675024777650833, -0.11026711016893387, -0.6997772455215454, 0.5178157687187195, 0.3780212104320526, -0.15165197849273682, 0.20519793033599854, 0.0674334391951561, -0.06280075013637543, -0.15901890397071838, -0.7679321765899658, 0.4981394410133362, 0.38306179642677307, -0.10691644996404648, 0.29250821471214294, -0.03833997994661331, -0.09644563496112823, -0.06905238330364227, -0.7849124073982239, 0.5339828729629517, 0.39538607001304626, -0.09077022969722748, 0.2762759327888489, 0.1740308403968811, -0.1391751766204834, -0.1348148137331009, -0.7525317668914795, 0.5587069988250732, 0.32476168870925903, -0.10410349816083908, 0.22102788090705872, 0.08777997642755508, -0.20618876814842224, -0.19728952646255493, -0.8356729745864868, 0.5862373113632202, 0.3382878601551056, -0.019660208374261856, 0.22584305703639984, 0.018178721889853477, -0.12897533178329468, -0.12462511658668518, -0.7766884565353394, 0.48205047845840454, 0.42293715476989746, -0.07744865119457245, 0.24711018800735474, 0.04145890846848488, -0.18735776841640472, -0.07621290534734726, -0.7192104458808899, 0.4883483946323395, 0.3278917670249939, -0.1404464840888977, 0.23579315841197968, -0.16588741540908813, 0.4600149393081665, 0.3621181845664978, 2.145049571990967, -1.5726239681243896, -1.0108551979064941, 0.28067320585250854, -0.6784366965293884, 0.033552248030900955, -0.18801619112491608, -0.15469001233577728, -0.7550371289253235, 0.44779980182647705, 0.32144972681999207, -0.07510826736688614, 0.27171096205711365, 0.0876416340470314, -0.030004963278770447, -0.10486161708831787, -0.7367958426475525, 0.6198855042457581, 0.28223729133605957, -0.04674229770898819, 0.1785336285829544, 0.07299207895994186, -0.2512916326522827, -0.07583200931549072, -0.7218915224075317, 0.47791972756385803, 0.41851913928985596, -0.05689844489097595, 0.2285812646150589, 0.018014324828982353, -0.1550399214029312, -0.12899214029312134, -0.6777403354644775, 0.5662075281143188, 0.3012103736400604, -0.12263470888137817, 0.19968760013580322, -0.012179368175566196, -0.182450532913208, -0.125416100025177, -0.6794315576553345, 0.5923227667808533, 0.29418089985847473, -0.06761734932661057, 0.20406801998615265, 0.15904588997364044, -0.15530796349048615, -0.09568934887647629, -0.7607872486114502, 0.4857693314552307, 0.3446829617023468, -0.11436890065670013, 0.24395529925823212, -0.022804711014032364, -0.11977154016494751, -0.16576971113681793, -0.6278579831123352, 0.5118804574012756, 0.3894614577293396, -0.16491533815860748, 0.24773818254470825, 0.010885011404752731, -0.17721624672412872, -0.11956588178873062, -0.7298328876495361, 0.5395165681838989, 0.3719809651374817, -0.06252706795930862, 0.2219167947769165, -0.012885031290352345, -0.193104088306427, -0.1305835098028183, -0.8200704455375671, 0.56390780210495, 0.3226162791252136, -0.1077321469783783, 0.2737096846103668, 0.08868201822042465, -0.1418217420578003, -0.1708623766899109, -0.6984580755233765, 0.5436126589775085, 0.2903595268726349, -0.043000586330890656, 0.251685231924057, -0.0431947223842144, -0.1467743217945099, -0.18403059244155884, -0.6571633815765381, 0.5666552186012268, 0.2794186472892761, -0.13940981030464172, 0.2313835322856903, 0.04334595426917076, -0.07661163806915283, -0.0837470069527626, -0.7281005382537842, 0.5551735758781433, 0.23298422992229462, -0.07967294752597809, 0.2696112096309662, 0.04633886739611626, -0.18838250637054443, -0.09250402450561523, -0.6160483956336975, 0.5365472435951233, 0.27856430411338806, -0.022875938564538956, 0.13236349821090698, 0.08044673502445221, -0.1819695234298706, -0.07426577806472778, -0.7564528584480286, 0.5924961566925049, 0.3496118485927582, -0.18150821328163147, 0.12760020792484283, 0.039481718093156815, -0.08816120773553848, -0.11444419622421265, -0.7106230854988098, 0.5145086050033569, 0.3714931905269623, -0.07629983872175217, 0.24030816555023193, 0.006050347350537777, -0.09347912669181824, -0.05264167860150337, -0.7790468335151672, 0.6413785219192505, 0.37072351574897766, -0.11956647783517838, 0.13107307255268097, 0.08812881261110306, -0.12499622255563736, -0.15057729184627533, -0.8035067915916443, 0.5825276374816895, 0.30518224835395813, -0.11875343322753906, 0.3685731291770935, 0.16168510913848877, -0.10518494993448257, -0.14356674253940582, -0.6815060973167419, 0.41223031282424927, 0.3399248421192169, -0.13315708935260773, 0.16201061010360718, -0.0238537285476923, -0.12661470472812653, -0.11633068323135376, -0.7059929370880127, 0.5684226155281067, 0.2922939956188202, -0.15694540739059448, 0.13747259974479675, 0.038526665419340134, -0.11486397683620453, -0.12826687097549438, -0.6819227337837219, 0.46725112199783325, 0.31114211678504944, -0.13346301019191742, 0.16387197375297546, -0.1725168079137802, 0.4599766135215759, 0.36774858832359314, 2.1417202949523926, -1.584946870803833, -1.0115666389465332, 0.3030107915401459, -0.6700875163078308, 0.03676484525203705, -0.2041129171848297, -0.14478811621665955, -0.7805370092391968, 0.44766268134117126, 0.2862911522388458, -0.12916667759418488, 0.211546391248703, 0.0587356761097908, -0.07652698457241058, -0.08018310368061066, -0.7266241312026978, 0.5625997185707092, 0.30015334486961365, -0.022741608321666718, 0.25376471877098083, -0.012505472637712955, -0.11510251462459564, -0.07999107986688614, -0.848258912563324, 0.4888990521430969, 0.3109070658683777, -0.1915917843580246, 0.1729319989681244, 0.007868585176765919, -0.1275169998407364, -0.10577253997325897, -0.5826704502105713, 0.5065908432006836, 0.39225974678993225, -0.09122385084629059, 0.2398371547460556, -0.002694996539503336, -0.24841678142547607, -0.15915310382843018, -0.6758707165718079, 0.605444073677063, 0.3647977411746979, -0.07025213539600372, 0.18435251712799072, 0.13200119137763977, -0.0856814756989479, -0.14733731746673584, -0.7162976861000061, 0.4765011966228485, 0.4079585373401642, -0.074848473072052, 0.22153763473033905, 0.07563149183988571, -0.13084067404270172, -0.14137053489685059, -0.7760574221611023, 0.462116539478302, 0.24767987430095673, -0.07210353016853333, 0.33721795678138733, -0.029666969552636147, -0.14595741033554077, -0.10024650394916534, -0.7632104158401489, 0.5365250706672668, 0.27975136041641235, -0.0351460725069046, 0.19657841324806213, 0.06031451374292374, -0.11865971237421036, -0.14228667318820953, -0.8035407066345215, 0.4997410178184509, 0.3727479577064514, -0.09939280897378922, 0.23778033256530762, 0.0012664040550589561, -0.14571064710617065, -0.08438734710216522, -0.7121567726135254, 0.6217045187950134, 0.324208527803421, -0.14723369479179382, 0.14540790021419525, 0.11288678646087646, -0.15582269430160522, -0.07021460682153702, -0.6910957098007202, 0.607558012008667, 0.39790356159210205, -0.11404390633106232, 0.23998510837554932, 0.10955509543418884, -0.21467554569244385, -0.04825089871883392, -0.653379499912262, 0.57545006275177, 0.3245515823364258, -0.08005912601947784, 0.17042890191078186, 0.05818420276045799, -0.18224076926708221, -0.11646749824285507, -0.7387104630470276, 0.5404939651489258, 0.39356479048728943, -0.16064633429050446, 0.2725604176521301, -0.04430413246154785, -0.26092344522476196, -0.12676413357257843, -0.7136426568031311, 0.5484438538551331, 0.25222891569137573, -0.09816311299800873, 0.2326170951128006, 0.11804674565792084, -0.15806376934051514, -0.09434611350297928, -0.7628504037857056, 0.5169260501861572, 0.31232142448425293, -0.060000959783792496, 0.17684990167617798, 0.013570371083915234, -0.12407238036394119, -0.0313238650560379, -0.6948440074920654, 0.5414817929267883, 0.3172910809516907, -0.04074491560459137, 0.2532867193222046, 0.14194613695144653, -0.13633620738983154, -0.16822955012321472, -0.742476224899292, 0.4769335687160492, 0.2720571458339691, -0.031980305910110474, 0.2666577100753784, -0.006645724177360535, -0.06380575895309448, -0.10856766253709793, -0.7618237733840942, 0.5456148982048035, 0.3311384916305542, -0.13379891216754913, 0.12914414703845978, 0.05712249130010605, -0.1382720023393631, -0.2497495710849762, -0.6862337589263916, 0.5571854710578918, 0.31110841035842896, -0.08956002444028854, 0.21225795149803162, 0.013431366533041, -0.17087453603744507, -0.05321550741791725, -0.7865099906921387, 0.538163423538208, 0.34380409121513367, -0.1305476874113083, 0.20427390933036804, -0.17705869674682617, 0.45020508766174316, 0.3811255395412445, 2.157050371170044, -1.577629566192627, -0.9922161102294922, 0.2706287205219269, -0.6788594722747803, 0.04375749081373215, -0.1728607714176178, -0.05574225261807442, -0.7379197478294373, 0.5930232405662537, 0.3916890025138855, -0.14744943380355835, 0.2780655324459076, 0.02276759035885334, -0.0160488523542881, -0.14008721709251404, -0.7610254883766174, 0.4307499825954437, 0.2699459493160248, -0.12419862300157547, 0.17638947069644928, -0.006147850304841995, -0.1876387745141983, -0.13112308084964752, -0.6415117979049683, 0.4411390423774719, 0.30701613426208496, -0.14286085963249207, 0.15053348243236542, -0.008020338602364063, -0.19815853238105774, -0.08494352549314499, -0.7236741781234741, 0.49940454959869385, 0.36516278982162476, -0.08112770318984985, 0.21512442827224731, 0.07450078427791595, -0.19428934156894684, -0.08942662924528122, -0.6906328797340393, 0.5056886076927185, 0.27362439036369324, -0.019162960350513458, 0.23100091516971588, 0.20784974098205566, -0.1855422407388687, -0.07815355062484741, -0.7356901168823242, 0.5389872193336487, 0.37626776099205017, -0.1351315677165985, 0.221286341547966, 0.06333357095718384, -0.14719778299331665, -0.0909731462597847, -0.6627010107040405, 0.5679852366447449, 0.3477488160133362, -0.11253482848405838, 0.16980668902397156, 0.018511570990085602, -0.1792992353439331, -0.0866599977016449, -0.7321596741676331, 0.4641868472099304, 0.36591553688049316, -0.13532871007919312, 0.1879994422197342, 0.0439729243516922, -0.1292129009962082, -0.2414548695087433, -0.7700518369674683, 0.5350832343101501, 0.31639015674591064, -0.13839340209960938, 0.19921822845935822, 0.12454919517040253, -0.1756581962108612, -0.085162453353405, -0.674385666847229, 0.5037927627563477, 0.3567701578140259, -0.11767806112766266, 0.1957699954509735, -0.0010132236639037728, -0.10912397503852844, -0.14070366322994232, -0.7415862679481506, 0.46993154287338257, 0.42504045367240906, -0.05807589739561081, 0.21965064108371735, 0.020549776032567024, -0.22936178743839264, -0.10620501637458801, -0.6995580792427063, 0.45588377118110657, 0.3926756978034973, -0.05636835843324661, 0.17818717658519745, 0.02750166691839695, -0.17423781752586365, -0.11276175081729889, -0.7212576270103455, 0.4443568289279938, 0.40830308198928833, -0.07935065776109695, 0.28932952880859375, 0.11802761256694794, -0.1632668375968933, -0.10369759052991867, -0.6771458983421326, 0.5647844672203064, 0.3264022469520569, -0.12607544660568237, 0.2280551940202713, 0.01520935632288456, -0.18066848814487457, -0.22230467200279236, -0.7097970247268677, 0.47495922446250916, 0.30680620670318604, -0.09780119359493256, 0.2419826239347458, 0.14974509179592133, -0.1426519751548767, -0.19145894050598145, -0.7074418067932129, 0.49651750922203064, 0.3545674681663513, -0.12944257259368896, 0.1908705234527588, 0.029723305255174637, -0.18424992263317108, -0.21203474700450897, -0.7473927140235901, 0.5256032347679138, 0.3946226239204407, -0.10982508212327957, 0.20770375430583954, 0.0842784121632576, -0.12590166926383972, -0.0408194325864315, -0.6847235560417175, 0.5253312587738037, 0.4015146493911743, -0.018692081794142723, 0.20252195000648499, 0.06319348514080048, -0.2369038164615631, -0.13900206983089447, -0.7581667304039001, 0.5156001448631287, 0.31051647663116455, -0.10346867144107819, 0.22113913297653198, 0.07995989918708801, -0.14132486283779144, -0.14888592064380646, -0.7784373164176941, 0.5880463719367981, 0.405463308095932, -0.1178884282708168, 0.2543615698814392, -0.1584513932466507, 0.4400157034397125, 0.3545571565628052, 2.149765968322754, -1.56759512424469, -1.0097275972366333, 0.2883966267108917, -0.6628654599189758, 0.08759760856628418, -0.12848109006881714, -0.13110923767089844, -0.7045478820800781, 0.6573908925056458, 0.27825602889060974, -0.11086318641901016, 0.21438468992710114, 0.020802440121769905, -0.21996593475341797, -0.041130684316158295, -0.7029743790626526, 0.4384041726589203, 0.29945796728134155, -0.07489614188671112, 0.2464618682861328, -0.01672986149787903, -0.15744313597679138, -0.10775161534547806, -0.69435054063797, 0.5460337996482849, 0.24344748258590698, -0.0028170377481728792, 0.23684754967689514, 0.14558598399162292, -0.009716840460896492, -0.12882356345653534, -0.7756673097610474, 0.5695103406906128, 0.32555651664733887, -0.11773666739463806, 0.17462654411792755, 0.13967572152614594, -0.17500551044940948, -0.19077493250370026, -0.7363577485084534, 0.46844086050987244, 0.3424934148788452, -0.1161344051361084, 0.17341965436935425, 0.08537480235099792, -0.18504348397254944, -0.08460763841867447, -0.6509299874305725, 0.5082061290740967, 0.3022724688053131, -0.08520137518644333, 0.2708633542060852], "scores": [[-0.9975564464117103, -0.9986921566543168, -0.9859543376514985, -0.9939001075479557, -0.992929782327592, -0.9928588310100869, -0.9919402600013656, -0.9933020173155681, -0.9953986103817825, -0.9968082074212746, 0.9996876621868835, -0.9947242876128518, -0.983888899273506, -0.9896373002833608, -0.9949639330609933, -0.9916001626100808, -0.9930081039718504, -0.9859501584404684, -0.9978722657769832, -0.9966405819089261], [-0.9942010490943345, -0.9898516243748926, -0.9924466736738322, -0.9888981549844156, -0.9900840976638492, -0.996489969256307, -0.9898599898733748, -0.9911011997851448, -0.9830094998951321, -0.9925474946692191, -0.9975211775661803, 0.9991339214376107, -0.9910461131356549, -0.9932941354799205, -0.9876273717828549, -0.9875261001896075, -0.986752519443589, -0.9879559941997538, -0.9852625380698514, -0.9939445261239175], [-0.9952231962262665, -0.9926643586088945, -0.9937184686319827, -0.9915817638097764, -0.9960746573516778, -0.9861736641084138, -0.9954020156986553, -0.9929270954414495, -0.9853682477407277, -0.9898161103649217, -0.9874138209697798, -0.9971145378690174, 0.9992159206364314, -0.9957179451724578, -0.9804138929075586, -0.9924890603993186, -0.9956476811416943, -0.9934482215336178, -0.9881305173708601, -0.9970292151347532], [-0.9949033781182776, -0.9922472211840898, -0.9958357446059833, -0.9916387372795002, -0.9922119969835057, -0.9921166884490611, -0.9951436295226932, -0.9928904655710866, -0.9936645247712148, -0.9962474316307899, -0.9895789351606823, -0.9965436340260065, -0.9987064355722328, 0.9995029352261207, -0.9878110181992688, -0.9894351216229182, -0.9856806608257324, -0.9829428712743062, -0.9935500825362488, -0.9945848242230804]], "kept": [[0, 1, 2, 3, 19], [0, 1, 2, 3, 19], [0, 1, 2, 3, 19], [0, 1, 2, 3, 19]]}, {"name": "tight_budget", "n": 40, "head_dim": 8, "n_kv_heads": 2, "budget": 4, "recent_frac": 0.3, "sink": 4, "keys": [0.47380974888801575, -0.4813322126865387, 0.11721540242433548, -0.1614769548177719, -0.12874385714530945, -0.08802423626184464, -0.5645245313644409, -0.07830052077770233, 0.541167676448822, -0.6594098806381226, 0.15381400287151337, -0.15383552014827728, -0.11346650868654251, 0.022669127210974693, -0.48451173305511475, -0.0840219110250473, 0.5079284310340881, -0.6204559206962585, 0.20268046855926514, -0.15732643008232117, -0.12685741484165192, -0.00450623594224453, -0.5560899972915649, -0.07334648072719574, 0.5611973404884338, -0.6184646487236023, 0.11050190776586533, -0.11034020036458969, -0.25608760118484497, -0.003556575160473585, -0.5597492456436157, -0.14219146966934204, 0.5308926701545715, -0.6124549508094788, 0.08368770033121109, -0.19766570627689362, -0.11337324231863022, -0.05725928023457527, -0.44148707389831543, -0.021390071138739586, 0.5267611742019653, -0.5919005274772644, 0.09564992040395737, -0.19014039635658264, -0.08547656983137131, -0.025494994595646858, -0.522508442401886, -0.09790089726448059, 0.528528094291687, -0.7721768617630005, 0.14043231308460236, -0.1192770004272461, -0.1966223418712616, -0.05155424028635025, -0.5599719882011414, -0.020899418741464615, 0.41536203026771545, -0.6932069063186646, 0.14140507578849792, -0.08602536469697952, -0.2225797474384308, -0.07952390611171722, -0.4953659772872925, -0.0892212763428688, 0.5966024398803711, -0.7070435285568237, 0.12365266680717468, -0.19626569747924805, -0.044381339102983475, -0.05570447817444801, -0.5303795337677002, -0.14466971158981323, 0.6011616587638855, -0.6098432540893555, 0.14360426366329193, -0.0869513526558876, -0.09721815586090088, -0.08658424764871597, -0.55177903175354, -0.09877046942710876, 0.5855740308761597, -0.7205012440681458, 0.07610759139060974, -0.15990760922431946, -0.1034485325217247, -0.0258544459939003, -0.5742218494415283, -0.14526113867759705, 0.5168496966362, -0.5868039727210999, 0.1437789797782898, -0.13306288421154022, -0.1305372714996338, -0.03996023163199425, -0.5239337682723999, -0.017900142818689346, 0.4773480296134949, -0.6407701969146729, 0.10038705915212631, -0.11667744815349579, -0.1034475564956665, 0.07287981361150742, -0.4368342459201813, 0.016076387837529182, 0.4150952100753784, -0.6644980311393738, 0.07549554109573364, -0.11720933765172958, -0.22863081097602844, 0.004103017970919609, -0.4584178328514099, -0.12386401742696762, 0.4681429862976074, -0.6875407695770264, 0.10809086263179779, -0.11179686337709427, -0.012285182252526283, -0.06449418514966965, -0.4733918607234955, -0.050989579409360886, 0.60506671667099, -0.6103742718696594, 0.17435358464717865, -0.1977291703224182, -0.12429151684045792, -0.09531053900718689, -0.4365196228027344, -0.025878475978970528, 0.5018131732940674, -0.6701055765151978, 0.13015931844711304, -0.1789201945066452, -0.16120892763137817, -0.030558191239833832, -0.3886103928089142, -0.07106714695692062, 0.4892771244049072, -0.7060400247573853, 0.039175521582365036, -0.11759625375270844, -0.07213933020830154, -0.05416318029165268, -0.4951381981372833, -0.05296464264392853, 0.5240031480789185, -0.7237901091575623, 0.0830201581120491, -0.13827145099639893, -0.15383782982826233, -0.07844340056180954, -0.5527229905128479, -0.07543530315160751, 0.559725821018219, -0.6678112149238586, 0.09823253750801086, -0.10315950214862823, -0.08244097232818604, 0.030138462781906128, -0.6162912249565125, -0.015917522832751274, -1.5467463731765747, 1.9284738302230835, -0.3289775550365448, 0.4299583435058594, 0.33569565415382385, 0.15685981512069702, 1.534585952758789, 0.1749061942100525, 0.5100213289260864, -0.6075146198272705, 0.07835745066404343, -0.03580010309815407, -0.06371915340423584, 0.05415685102343559, -0.5130964517593384, -0.07791490107774734, 0.5254228115081787, -0.6107533574104309, 0.07655468583106995, -0.12486033141613007, -0.1155197024345398, 0.026162171736359596, -0.5449020862579346, -0.006452382542192936, 0.48487722873687744, -0.695513904094696, 0.07041110098361969, -0.20335471630096436, -0.10736196488142014, -0.0030588761437684298, -0.5035512447357178, -0.027544215321540833, 0.5986791253089905, -0.6339808702468872, 0.11568418890237808, -0.15759849548339844, -0.1950916051864624, -0.016634969040751457, -0.599590003490448, -0.026126649230718613, 0.5163543820381165, -0.5911431312561035, 0.10253755748271942, -0.18500542640686035, -0.09678397327661514, -0.08263372629880905, -0.520814836025238, -0.05666693300008774, 0.5103428959846497, -0.6569222807884216, 0.06430382281541824, -0.15330210328102112, -0.22159650921821594, -0.06248852238059044, -0.5716698169708252, -0.002747286343947053, 0.5805667638778687, -0.7450334429740906, 0.11317190527915955, -0.15016360580921173, -0.16701526939868927, -0.02803925797343254, -0.5348514914512634, -0.14714042842388153, 0.5037365555763245, -0.6548948287963867, 0.11124762147665024, -0.2054070681333542, -0.08389543741941452, -0.017849061638116837, -0.569058358669281, -0.09170325100421906, 0.5130535364151001, -0.6757784485816956, 0.1931138038635254, -0.13342681527137756, -0.1652085930109024, -0.09402736276388168, -0.5146405100822449, 0.056028470396995544, 0.5081571936607361, -0.6411083340644836, 0.13162833452224731, -0.1458517163991928, -0.0006464411853812635, -0.08119816333055496, -0.4745568335056305, -0.05073857307434082, 0.548130214214325, -0.7063143253326416, 0.19433170557022095, -0.14992178976535797, -0.11297132074832916, -0.10137072950601578, -0.548251748085022, -0.030548499897122383, 0.5663954019546509, -0.6098194718360901, 0.1663745790719986, -0.10812097042798996, -0.11325639486312866, -0.012795616872608662, -0.48208683729171753, -0.0637892335653305, 0.5533740520477295, -0.583197295665741, 0.11765463650226593, -0.16165566444396973, -0.07874391973018646, 0.04041416943073273, -0.5222951173782349, -0.06337594985961914, 0.5102505087852478, -0.586331307888031, 0.014066156931221485, -0.12555314600467682, -0.05506628751754761, -0.09529107809066772, -0.4373181164264679, -0.0319950208067894, 0.48961859941482544, -0.6372751593589783, 0.029201462864875793, -0.1721239686012268, -0.02333429455757141, -0.09931211918592453, -0.4195384085178375, -0.06283915042877197, 0.5668035745620728, -0.6452873349189758, -0.006148007698357105, -0.17247797548770905, -0.10518680512905121, -0.11216939985752106, -0.5791375041007996, -0.04045117646455765, 0.48459964990615845, -0.732273519039154, 0.07172030955553055, -0.1008085384964943, -0.13836678862571716, -0.007883287034928799, -0.43540334701538086, -0.05713469907641411, 0.4605862498283386, -0.7125604152679443, 0.10461337864398956, -0.09979262948036194, -0.10122872143983841, -0.02725905552506447, -0.5339340567588806, -0.043739333748817444, 0.49350497126579285, -0.6637626886367798, 0.04131374508142471, -0.22245606780052185, -0.13877731561660767, -0.11000188440084457, -0.5597043037414551, -0.02520809695124626, 0.511716365814209, -0.501775860786438, 0.15224316716194153, -0.17550480365753174, -0.07577911764383316, -0.03678172081708908, -0.546051025390625, -0.00363072264008224, 0.4783039689064026, -0.8004012107849121, 0.14562930166721344, -0.17470836639404297, -0.04037760943174362, -0.08710552752017975, -0.5708346962928772, 0.01866261288523674, 0.46329548954963684, -0.6373124718666077, 0.17841242253780365, -0.13812054693698883, -0.11870872974395752, -0.04950074478983879, -0.4698143005371094, -0.09910108149051666, 0.5476024746894836, -0.6186283826828003, 0.22256167232990265, -0.16067169606685638, -0.1604209840297699, -0.017837565392255783, -0.5316248536109924, -0.04430466145277023, 0.5236992239952087, -0.5890624523162842, 0.07308419793844223, -0.14845938980579376, -0.2422851175069809, -0.09912761300802231, -0.434950053691864, -0.09283576905727386, 0.4819965660572052, -0.7083775401115417, 0.13491778075695038, -0.13494591414928436, -0.1740228235721588, -0.021249396726489067, -0.49876171350479126, -0.02105880156159401, 0.4849964380264282, -0.5605817437171936, 0.12641945481300354, -0.14968989789485931, -0.14062120020389557, -0.0668632835149765, -0.46882739663124084, -0.07220880687236786, 0.5709776282310486, -0.6184483766555786, 0.17325082421302795, -0.1031402051448822, -0.117826446890831, -0.15008890628814697, -0.4968384802341461, -0.019306669011712074, 0.5252150297164917, -0.6196606159210205, 0.08234506100416183, -0.16286928951740265, -0.16179893910884857, 0.013473103754222393, -0.5038473010063171, -0.08338477462530136, 0.4504777789115906, -0.6551475524902344, 0.17805762588977814, -0.120749332010746, -0.1509055644273758, 0.016852831467986107, -0.4403415024280548, -0.009721661917865276, 0.5565112233161926, -0.5896995663642883, 0.13308902084827423, -0.10509537905454636, -0.15368075668811798, 0.02035597525537014, -0.5002079606056213, 0.04245011880993843, 0.41495800018310547, -0.6122117042541504, 0.15499915182590485, -0.16099204123020172, -0.09205727279186249, -0.01720559597015381, -0.48289790749549866, -0.08574343472719193, 0.5099735260009766, -0.5965301990509033, 0.1697920709848404, -0.13889656960964203, -0.11082786321640015, -0.04459709674119949, -0.45988374948501587, -0.11146239191293716, 0.45061710476875305, -0.641266405582428, 0.050599630922079086, -0.17320089042186737, -0.11036469787359238, -0.030496669933199883, -0.565371572971344, -0.01868019811809063, 0.4283846616744995, -0.6202113628387451, 0.027039481326937675, -0.20759159326553345, -0.08256838470697403, -0.074940025806427, -0.5515449047088623, -0.08099930733442307, 0.5403220653533936, -0.6501845121383667, 0.10445693135261536, -0.0911065936088562, -0.15049901604652405, -0.07821860164403915, -0.5378884673118591, -0.00031436290009878576, 0.5666406750679016, -0.5831283330917358, 0.10423853993415833, -0.15168555080890656, -0.07204083353281021, -0.07434249669313431, -0.470233678817749, -0.02254583314061165, 0.48215451836586, -0.6516419053077698, 0.09996352344751358, -0.057178396731615067, 0.014711681753396988, -0.059618279337882996, -0.5110637545585632, -0.1728396862745285, 0.5276470184326172, -0.6478338241577148, 0.0522838793694973, -0.10456570237874985, -0.19296717643737793, -0.06536456197500229, -0.504824161529541, -0.12121047079563141, 0.5599074363708496, -0.5947040915489197, 0.036605026572942734, -0.21423734724521637, -0.08456651121377945, -0.0561467669904232, -0.5723371505737305, -0.07352180778980255, 0.5116945505142212, -0.6546341180801392, 0.14730049669742584, -0.12463550269603729, -0.02620629407465458, 0.014096108265221119, -0.477843314409256, -0.004906502086669207, -1.553436279296875, 1.959099292755127, -0.30766746401786804, 0.443833589553833, 0.3437568247318268, 0.15881890058517456, 1.5207619667053223, 0.17280003428459167, 0.5335524082183838, -0.5634208917617798, 0.11478518694639206, -0.15732240676879883, -0.09922489523887634, -0.07659733295440674, -0.503665030002594, -0.05913407355546951, 0.5306515097618103, -0.5513179898262024, 0.0856415405869484, -0.14199675619602203, -0.12465406954288483, -0.020991338416934013, -0.5708457231521606, -0.09339694678783417, 0.5150959491729736, -0.620174765586853, 0.1036120057106018, -0.16885973513126373, -0.15802085399627686, -0.029034284874796867, -0.439657062292099, -0.042616695165634155, 0.5264771580696106, -0.6252885460853577, 0.05879342183470726, -0.1594291627407074, -0.12807580828666687, -0.039859894663095474, -0.5133683085441589, -0.1195804700255394, 0.5733318328857422, -0.5628617405891418, 0.09276621043682098, -0.12390337139368057, -0.10132140666246414, -0.015388735570013523, -0.5857561230659485, 0.02189001441001892, 0.4936777949333191, -0.5842708945274353, 0.10995962470769882, -0.16332319378852844, -0.12170211970806122, -0.009763268753886223, -0.5549308061599731, -0.07751936465501785, 0.49296140670776367, -0.733254611492157, 0.17094732820987701, -0.11490097641944885, -0.11574523895978928, 0.003561219433322549, -0.5595884919166565, -0.06243296340107918, 0.6033405661582947, -0.5984988808631897, -0.0053464509546756744, -0.310449481010437, -0.16657467186450958, -0.001557750510983169, -0.5353893041610718, -0.06964767724275589, 0.5474839806556702, -0.6917423009872437, 0.11656996607780457, -0.10424050688743591, -0.09132882207632065, -0.03482464328408241, -0.5418692827224731, -0.033046115189790726, 0.49568891525268555, -0.582234799861908, 0.2344307005405426, -0.18084071576595306, -0.22340458631515503, -0.07589390873908997, -0.4929247498512268, -0.006253904663026333, 0.5710211992263794, -0.6866797208786011, 0.06626430153846741, -0.08048036694526672, -0.16929887235164642, -0.0001103483882616274, -0.5231114029884338, -0.020440291613340378, 0.5523786544799805, -0.6086495518684387, 0.14694739878177643, -0.19331496953964233, -0.09980343282222748, -0.05392279475927353, -0.4963465929031372, 0.002901948755607009, 0.5035150051116943, -0.6830806732177734, 0.06952029466629028, -0.15938633680343628, -0.07704895734786987, -0.09536270052194595, -0.4538041055202484, -0.0442378968000412, 0.5057648420333862, -0.7140647172927856, 0.11817272752523422, -0.10927195101976395, -0.2029677778482437, 0.004976062569767237, -0.4293796420097351, -0.04183535650372505, 0.5416594743728638, -0.5267974138259888, 0.16752059757709503, -0.1887332797050476, -0.11551174521446228, -0.06246846914291382, -0.5193108320236206, -0.1769738644361496, 0.5549206733703613, -0.6793189644813538, 0.11410864442586899, -0.15065237879753113, -0.07112471759319305, -0.06941235065460205, -0.5515384078025818, -0.03213934600353241, 0.5873146653175354, -0.6634727120399475, 0.1744234710931778, -0.13073626160621643, -0.14131595194339752, 0.02163597382605076, -0.5526983141899109, -0.008058060891926289, 0.4921610951423645, -0.6249712705612183, 0.04421865567564964, -0.18572504818439484, -0.06142687052488327, -0.04521607235074043, -0.5098523497581482, -0.08075088262557983], "scores": [[-0.9839619770618151, -0.9960142629315449, -0.9932822508957411, -0.9858342072312987, -0.9950663032535021, -0.9958637619916275, -0.9947181229770066, -0.9844001127712625, -0.9927854204393161, -0.9918085452625576, -0.9962408771658828, -0.9967041097927791, -0.9885152576299057, -0.9835143090840788, -0.9908167922396489, -0.9869273992042711, -0.9903779649845701, -0.9940814181318186, -0.997739735511437, -0.9925587033472443, 0.9998575519994184, -0.9874461140711148, -0.9947612065349204, -0.9949642105538926, -0.9939586642808791, -0.9964783278178146, -0.991719103187644, -0.9959747083271031, -0.9957478035625418, -0.9877685845711464, -0.9911513141994484, -0.9950708337123442, -0.994805562828454, -0.9930946941833565, -0.9904405896666761, -0.9871365407706127, -0.9895484398579568, -0.9908633866399978, -0.994997447706425, -0.9910984565694915], [-0.9892736539316225, -0.9849231993559078, -0.9945524620214219, -0.9939174900377179, -0.9851288279158523, -0.9942764884765576, -0.9981071672267484, -0.989613203817056, -0.9964438322713656, -0.9897888532585837, -0.9905254065068346, -0.9938789172151583, -0.9953533659744748, -0.9928279529046863, -0.9871805617731844, -0.9955903946012274, -0.994821415673201, -0.9770677089556361, -0.9914684094758843, -0.9913596704587385, -0.9909364978244775, 0.9994344206486792, -0.9973190049848656, -0.9936967090195441, -0.996745386710056, -0.9958203220985364, -0.9902925372682634, -0.9969910179140984, -0.9937343378432435, -0.9776769218179314, -0.9974026196072141, -0.9839894590127014, -0.9930578589079291, -0.9959361207876514, -0.9925630946450104, -0.9876266874962965, -0.9841656554753395, -0.9977305416928992, -0.9954561751873393, -0.9939790711693458]], "kept": [[0, 1, 2, 3], [0, 1, 2, 3]]}, {"name": "reserved_overflow", "n": 30, "head_dim": 8, "n_kv_heads": 3, "budget": 6, "recent_frac": 0.8, "sink": 8, "keys": [-0.3310459852218628, -0.0550963431596756, 0.6515293717384338, 0.3322467803955078, -0.6153426766395569, 0.023526495322585106, -0.3143995404243469, 0.16979050636291504, -0.3464188873767853, -0.012094845995306969, 0.6232651472091675, 0.2094331532716751, -0.6639891862869263, -0.03560157120227814, -0.22073426842689514, 0.051651109009981155, -0.17650803923606873, -0.15866513550281525, 0.6396062970161438, 0.20886267721652985, -0.5923813581466675, -0.10790383815765381, -0.2569284439086914, 0.06766269356012344, -0.32485002279281616, -0.01792415976524353, 0.6486960053443909, 0.3038083016872406, -0.5832157135009766, -0.05099356546998024, -0.279633492231369, 0.046987418085336685, -0.21323861181735992, -0.02463657408952713, 0.6042761206626892, 0.22310560941696167, -0.6710661053657532, -0.03121274895966053, -0.25163987278938293, 0.05056329071521759, -0.14773087203502655, -0.09250511974096298, 0.6253241300582886, 0.2763943672180176, -0.6788286566734314, -0.02043219469487667, -0.23907779157161713, 0.09586133807897568, -0.31643733382225037, 0.001388534321449697, 0.6221374869346619, 0.2619363069534302, -0.5888205170631409, 0.031038807705044746, -0.1867821216583252, 0.06581393629312515, -0.25161728262901306, -0.05136101320385933, 0.589974582195282, 0.3141748309135437, -0.6699442267417908, -0.06501302868127823, -0.13692328333854675, 0.05033179745078087, -0.3095928430557251, 0.025428546592593193, 0.6232746839523315, 0.30656924843788147, -0.6726493835449219, -0.01443983893841505, -0.32417017221450806, -0.03829292580485344, -0.2994583249092102, -0.07108689099550247, 0.6796596050262451, 0.3747765123844147, -0.6827681064605713, 0.04845117777585983, -0.26996076107025146, 0.04952757805585861, -0.3246961832046509, -0.14113423228263855, 0.6008372902870178, 0.19380994141101837, -0.6313526630401611, -0.037126559764146805, -0.17720887064933777, 0.0648050308227539, -0.24237468838691711, -0.010313178412616253, 0.6656659245491028, 0.2515988349914551, -0.635711133480072, -0.008048269897699356, -0.18590761721134186, 0.09630832076072693, -0.2323407083749771, -0.027243472635746002, 0.612533688545227, 0.2790166139602661, -0.7017916440963745, 0.06536269187927246, -0.23524130880832672, 0.06632369011640549, -0.22066740691661835, -0.06767041981220245, 0.6270549297332764, 0.31295469403266907, -0.62538743019104, -0.04779442027211189, -0.21776577830314636, -0.028101978823542595, -0.16785947978496552, -0.0757414922118187, 0.642054557800293, 0.36635398864746094, -0.5235878825187683, 0.0037894961424171925, -0.20850040018558502, 0.11166303604841232, 0.750726580619812, 0.2073666900396347, -1.911268949508667, -0.758678138256073, 1.8813899755477905, 0.0191964004188776, 0.7216143012046814, -0.1669963151216507, -0.2935836613178253, -0.19925187528133392, 0.6267542839050293, 0.32040733098983765, -0.6866352558135986, 0.05968933925032616, -0.18976640701293945, 0.054411932826042175, -0.25101718306541443, -0.06304328143596649, 0.596705973148346, 0.23934894800186157, -0.6335031390190125, 0.02859605848789215, -0.20015306770801544, 0.020017141476273537, -0.20935094356536865, -0.10607846826314926, 0.6313145160675049, 0.2719407379627228, -0.6323075890541077, -0.008348879404366016, -0.22627165913581848, 0.029514091089367867, -0.2832699120044708, -0.0773906409740448, 0.6878238916397095, 0.2483820617198944, -0.5359488725662231, 0.00871182605624199, -0.3884636461734772, 0.0351763591170311, -0.16738489270210266, -0.0668676495552063, 0.6178686618804932, 0.20978248119354248, -0.5649850368499756, 0.04814038798213005, -0.20614197850227356, 0.07855913788080215, -0.1785912960767746, -0.026124853640794754, 0.5782703161239624, 0.29188764095306396, -0.6226935982704163, -0.007516827434301376, -0.20157431066036224, 0.058573368936777115, -0.3973143398761749, 0.010684467852115631, 0.6916463375091553, 0.2473360151052475, -0.6497947573661804, 0.00933158304542303, -0.30396342277526855, 0.07247432321310043, -0.2621464431285858, -0.08483248203992844, 0.5936043858528137, 0.21953366696834564, -0.5758601427078247, 0.06475021690130234, -0.2146018147468567, 0.05487721413373947, -0.29453176259994507, -0.03129725530743599, 0.7203071713447571, 0.17274028062820435, -0.5724619626998901, -0.11028432101011276, -0.2567441761493683, 0.07638527452945709, -0.3188859522342682, -0.14303447306156158, 0.6490440368652344, 0.2973012328147888, -0.6549475789070129, 0.06519376486539841, -0.2787398099899292, 0.14487160742282867, -0.2446163445711136, -0.05541977286338806, 0.6705071330070496, 0.26951682567596436, -0.5388859510421753, 0.10832331329584122, -0.21243782341480255, 0.048868805170059204, -0.30944162607192993, -0.1049036756157875, 0.6402227282524109, 0.2483985275030136, -0.6333585977554321, -0.001748415408656001, -0.21496672928333282, -0.010380316525697708, -0.3019103705883026, -0.011088894680142403, 0.6816749572753906, 0.22667638957500458, -0.5664082169532776, -0.03582040220499039, -0.25233331322669983, -0.034119509160518646, -0.20828048884868622, -0.06762249022722244, 0.693249523639679, 0.28892871737480164, -0.6262013912200928, 0.006299509666860104, -0.17964023351669312, 0.049413442611694336, -0.257750540971756, -0.06317555904388428, 0.6806971430778503, 0.30037471652030945, -0.5344500541687012, -0.0049479347653687, -0.17532044649124146, 0.025862421840429306, -0.21888573467731476, -0.05920157581567764, 0.7283563017845154, 0.2858824133872986, -0.6515471935272217, 0.029023628681898117, -0.22025138139724731, 0.04354151710867882, -0.2630894184112549, -0.11206945776939392, 0.7799769639968872, 0.262610524892807, -0.58004230260849, -0.057850368320941925, -0.2915869653224945, 0.051960334181785583, -0.2556001842021942, -0.02457478828728199, 0.5920740365982056, 0.28296980261802673, -0.5528972744941711, -0.04518313333392143, -0.1917206048965454, 0.011658944189548492, -0.2307884842157364, -0.046475645154714584, 0.7291747331619263, 0.2214740812778473, -0.6202136278152466, 0.05272495746612549, -0.29580506682395935, 0.05923805013298988, -0.2916071116924286, -0.09975580126047134, 0.676382303237915, 0.29101404547691345, -0.4997696280479431, 0.06937193125486374, -0.17277392745018005, 0.051315855234861374, -0.26318249106407166, -0.057883478701114655, 0.5284225344657898, 0.33870387077331543, -0.6624059677124023, -0.0527585931122303, -0.26047980785369873, 0.10299821197986603, -0.18646442890167236, -0.08410150557756424, 0.6634118556976318, 0.2523844540119171, -0.6792827844619751, 0.008132793009281158, -0.23678700625896454, -0.018203649669885635, -0.1941470056772232, -0.14434285461902618, 0.6528239846229553, 0.20228974521160126, -0.6705947518348694, -0.03587159141898155, -0.20328204333782196, -0.00437941076233983, -0.2590215802192688, 0.011525641195476055, 0.6060317158699036, 0.2529059946537018, -0.5969659090042114, -0.03018580749630928, -0.2150578498840332, 0.02500271238386631, -0.21477439999580383, -0.0632137656211853, 0.593782901763916, 0.32474735379219055, -0.6351038813591003, -0.01518693845719099, -0.28518620133399963, 0.0344436950981617, -0.2546752691268921, -0.057819850742816925, 0.6954295635223389, 0.18830366432666779, -0.6479196548461914, 0.058522406965494156, -0.2299536168575287, 0.07825399190187454, -0.17030367255210876, -0.04963633418083191, 0.6637417674064636, 0.2520795166492462, -0.5421420335769653, 0.051138218492269516, -0.2751882076263428, 0.08517579734325409, -0.3074677288532257, -0.012268928810954094, 0.6738637089729309, 0.24595704674720764, -0.6508204936981201, -0.09134197980165482, -0.2011943906545639, 0.05163058266043663, -0.26592034101486206, -0.030760660767555237, 0.6067479848861694, 0.22431117296218872, -0.6657489538192749, -0.0695604607462883, -0.11920707672834396, 0.04744914919137955, -0.26816239953041077, -0.02220253273844719, 0.4822121858596802, 0.25180181860923767, -0.6307142972946167, -0.05081827566027641, -0.21708577871322632, -0.044599998742341995, 0.7538400888442993, 0.18553023040294647, -1.9132479429244995, -0.7667989134788513, 1.8814336061477661, 0.01915026642382145, 0.7163488864898682, -0.16836120188236237, -0.24374057352542877, -0.14035993814468384, 0.6285440325737, 0.18302354216575623, -0.7420589923858643, -0.13002660870552063, -0.19608289003372192, 0.04273872822523117, -0.16097642481327057, -0.14873117208480835, 0.6134938597679138, 0.2618006765842438, -0.6469722390174866, 0.014447957277297974, -0.24098360538482666, 0.017966290935873985, -0.3755180239677429, -0.12292417883872986, 0.7177350521087646, 0.28388041257858276, -0.5700055360794067, -0.03882204368710518, -0.25665003061294556, 0.006166120525449514, -0.23801690340042114, -0.07086054235696793, 0.6557806730270386, 0.3350878059864044, -0.6793904304504395, 0.028575804084539413, -0.3161465525627136, 0.1061536893248558, -0.27888697385787964, -0.07692162692546844, 0.640483021736145, 0.32971587777137756, -0.6388310790061951, -0.032973237335681915, -0.3050508201122284, 0.06487378478050232, -0.2994721233844757, -0.09844140708446503, 0.6323990821838379, 0.2809735834598541, -0.6730248928070068, 0.025441862642765045, -0.1572805941104889, -0.023427637293934822, -0.18857043981552124, 0.03446356952190399, 0.604159951210022, 0.2781519293785095, -0.7058960199356079, 0.009004934690892696, -0.26202622056007385, 0.06640840321779251, -0.30257847905158997, -0.11441409587860107, 0.6111748814582825, 0.21153706312179565, -0.6685976982116699, -0.01873929798603058, -0.30204033851623535, 0.04296685382723808, -0.2335444837808609, -0.1899738758802414, 0.642448365688324, 0.20884981751441956, -0.6946372389793396, -0.07687550038099289, -0.2900691032409668, 0.011915761046111584, -0.24146948754787445, -0.1385239064693451, 0.6836694478988647, 0.20431281626224518, -0.6051236391067505, 0.04437877610325813, -0.3087647557258606, 0.10310962796211243, -0.14843934774398804, -0.09577465802431107, 0.6630820035934448, 0.24173931777477264, -0.6318513751029968, -0.015748335048556328, -0.2687593996524811, 0.08991698175668716, -0.20961256325244904, -0.06996607780456543, 0.7130128741264343, 0.3057379424571991, -0.5979505777359009, -0.039655473083257675, -0.14617504179477692, 0.12187016010284424, -0.3593476414680481, -0.12374424934387207, 0.6148441433906555, 0.29948368668556213, -0.5697513818740845, -0.01714872568845749, -0.22804997861385345, 0.09149337559938431, -0.3148186206817627, -0.03798365965485573, 0.6241517663002014, 0.26248183846473694, -0.6337714791297913, -0.0611870214343071, -0.28960540890693665, 0.059168215841054916, -0.24420303106307983, -0.08039199560880661, 0.6928807497024536, 0.23552612960338593, -0.6920500993728638, -0.01928560435771942, -0.2485109567642212, 0.05059698596596718, -0.19246000051498413, -0.041644781827926636, 0.6924993991851807, 0.2846876084804535, -0.639369547367096, 0.049949634820222855, -0.25267815589904785, 0.06969910115003586, -0.3005616366863251, -0.061948247253894806, 0.6237358450889587, 0.27229756116867065, -0.5400779247283936, 0.11687152087688446, -0.18846097588539124, 0.05605393275618553, -0.20897206664085388, -0.05530593544244766, 0.6097902655601501, 0.25074467062950134, -0.6432484984397888, -0.031773850321769714, -0.1702663004398346, 0.030590442940592766, -0.2750305235385895, -0.03154247626662254, 0.6055077910423279, 0.19644904136657715, -0.6732489466667175, 0.02259703166782856, -0.22041143476963043, 0.07263373583555222, -0.2733134329319, -0.11229307949542999, 0.6856848001480103, 0.18175724148750305, -0.641181230545044, -0.06764116138219833, -0.2593913674354553, 0.050202544778585434, -0.1885284185409546, -0.20983080565929413, 0.7088679671287537, 0.21006044745445251, -0.6155354380607605, 0.03921877220273018, -0.27699360251426697, 0.053785063326358795, -0.22491326928138733, -0.07225742936134338, 0.6040950417518616, 0.26410427689552307, -0.6244378685951233, -0.07564543932676315, -0.17045293748378754, 0.06986087560653687, -0.370292603969574, -0.10720758885145187, 0.7273082733154297, 0.33444979786872864, -0.6160203218460083, -0.08457887172698975, -0.2499302476644516, 0.11186080425977707, -0.22917474806308746, -0.08203081786632538, 0.6047782897949219, 0.2966236472129822, -0.6398770213127136, 0.016130270436406136, -0.21315492689609528, 0.1411631852388382, -0.2394212931394577, -0.05218140408396721, 0.5524348020553589, 0.31115007400512695, -0.5846433043479919, -0.0006337009253911674, -0.35130733251571655, 0.031041521579027176, -0.24478349089622498, 0.039812881499528885, 0.6713612079620361, 0.2952270209789276, -0.5880878567695618, 0.0668412297964096, -0.2945268154144287, 0.016516245901584625, -0.30780884623527527, -0.07954327762126923, 0.6238812804222107, 0.2603961229324341, -0.6807370185852051, 0.004365235101431608, -0.2807947099208832, 0.16469913721084595, -0.2754948139190674, 0.06206158921122551, 0.7231947779655457, 0.19509993493556976, -0.5816571712493896, 0.029972167685627937, -0.31505757570266724, 0.08915407210588455, -0.25365832448005676, -0.04823727160692215, 0.5985330939292908, 0.2518428564071655, -0.6090506911277771, -0.009654578752815723, -0.2687615156173706, 0.05064573511481285, -0.2983059287071228, -0.0486634187400341, 0.5129425525665283, 0.33030396699905396, -0.6266382932662964, 0.023077456280589104, -0.195615753531456, -0.0037319327238947153, 0.7610815763473511, 0.21804533898830414, -1.9373493194580078, -0.7676986455917358, 1.900517225265503, -0.000548365933354944, 0.6865613460540771, -0.16983604431152344, -0.37301695346832275, -0.07613483816385269, 0.6177866458892822, 0.3326479196548462, -0.6442173719406128, -0.023613089695572853, -0.2767672836780548, 0.015145977959036827, -0.2775508165359497, -0.036411553621292114, 0.5744317173957825, 0.24365969002246857, -0.7038586735725403, -0.00978521816432476, -0.2310364842414856, -0.00417251605540514, -0.27891772985458374, -0.06160098686814308, 0.6786713004112244, 0.3013709485530853, -0.6365867257118225, -0.05195815488696098, -0.32109585404396057, 0.09627215564250946, -0.18068858981132507, -0.04883693903684616, 0.7182105183601379, 0.23070798814296722, -0.6508480310440063, 0.07668491452932358, -0.29038310050964355, 0.08743520826101303, -0.23131971061229706, -0.05953969061374664, 0.7265800833702087, 0.20931777358055115, -0.6527152061462402, -0.10694621503353119, -0.29240483045578003, 0.031331777572631836, -0.2701691687107086, -0.07745176553726196, 0.6547117829322815, 0.21987532079219818, -0.7306618690490723, 0.0055902935564517975, -0.21864910423755646, -0.0012396457605063915, -0.2902265787124634, 0.021215219050645828, 0.69129878282547, 0.3080936074256897, -0.59758061170578, 0.005306687671691179, -0.26544302701950073, 0.08585052937269211, -0.2606731057167053, -0.08938014507293701, 0.6391341686248779, 0.24935440719127655, -0.5865384340286255, -0.050299208611249924, -0.11450342833995819, 0.006426911801099777, -0.2721794545650482, -0.019806059077382088, 0.5976122617721558, 0.22917510569095612, -0.6298224925994873, -0.15126919746398926, -0.2653121054172516, 0.008572167716920376, -0.2631056606769562, -0.042113836854696274, 0.6719141006469727, 0.32904064655303955, -0.5585699081420898, 0.05003451555967331, -0.24169503152370453, 0.1097131222486496, -0.30067870020866394, -0.07197252660989761, 0.7123212218284607, 0.29339396953582764, -0.7014450430870056, -0.031699832528829575, -0.19399017095565796, 0.08841320127248764, -0.19609889388084412, -0.15595778822898865, 0.5762574672698975, 0.24120531976222992, -0.679896891117096, -0.0030970433726906776, -0.23857448995113373, 0.07522249966859818], "scores": [[-0.9892935256725782, -0.9922522379310489, -0.982890300588487, -0.9938808736328294, -0.994418372049177, -0.9900764674286718, -0.9945322001277987, -0.9894557092451631, -0.9876943277134009, -0.9962724361889291, -0.9888971718114964, -0.995800346874971, -0.9937726823103912, -0.9932031308133707, -0.98413399895371, 0.9996784564510381, -0.9868099050032573, -0.9975023843239723, -0.9970051172952823, -0.9835919971790351, -0.9932372626793176, -0.9944089237243015, -0.9903506146104045, -0.9964143975908272, -0.9840291696190477, -0.9909333567246519, -0.9900141937779605, -0.9955394377338832, -0.9899034680802837, -0.9954687246852747], [-0.9923996965288949, -0.9966761170015384, -0.9914067772436546, -0.9952588595936873, -0.9926388606381276, -0.984420892447216, -0.9864267456727969, -0.9949472830604766, -0.9919797537263677, -0.995036394445288, -0.9947184685643902, -0.9930090941580365, -0.9896202207403842, -0.9926642281707316, -0.9881948553255564, -0.9829841677779465, 0.9996742282031014, -0.98341440145221, -0.9924723365495485, -0.988193966918481, -0.9937447113410518, -0.9958704060855127, -0.991409124192019, -0.9873870614021487, -0.9935189367805257, -0.9885460023614695, -0.9902045705454154, -0.9932208175384234, -0.9888961130229108, -0.9887246297217169], [-0.9964448309306689, -0.9980468618357503, -0.9946411897546276, -0.9861447262929354, -0.9950795638085934, -0.9949354112423062, -0.9932366781163172, -0.9818554270820504, -0.9941996450454121, -0.9898557138604689, -0.9935745194554446, -0.9884324623803511, -0.9885589326535446, -0.9930802140138858, -0.9838876405327599, -0.9993049958383795, -0.9860393666808165, 0.9995734735315155, -0.991562665125212, -0.9925366858571721, -0.9966618487904201, -0.9899883431321648, -0.9911550568513884, -0.9936261872711777, -0.9940183127812442, -0.9883464729348481, -0.9865483305684745, -0.9915483577412078, -0.9968889376373468, -0.989368284547274]], "kept": [[0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5]]}, {"name": "full_budget", "n": 40, "head_dim": 16, "n_kv_heads": 4, "budget": 40, "recent_frac": 0.3, "sink": 4, "keys": [-0.2033780813217163, -0.3871213495731354, -0.09315542131662369, 0.08003490418195724, 0.24651910364627838, 0.054919105023145676, -0.14042742550373077, -0.22440464794635773, 0.2064642757177353, 0.447559654712677, -0.01439761184155941, -0.31918537616729736, -0.28706270456314087, 0.388774037361145, -0.014076580293476582, -0.4292137622833252, -0.2010875642299652, -0.3441769480705261, -0.11408734321594238, 0.08462567627429962, 0.22761832177639008, -0.040510255843400955, -0.12603384256362915, -0.250400573015213, 0.24449719488620758, 0.4418962597846985, -0.03213727846741676, -0.29274243116378784, -0.2931111454963684, 0.3990846574306488, 0.05257595703005791, -0.5296697616577148, -0.21086440980434418, -0.34174954891204834, -0.01359099056571722, 0.04536285623908043, 0.31908684968948364, -0.0276984553784132, -0.1538376659154892, -0.23695674538612366, 0.2584189474582672, 0.4344775378704071, 0.18934018909931183, -0.2742530405521393, -0.19577568769454956, 0.4394659399986267, 0.02006378024816513, -0.4337497055530548, -0.13167382776737213, -0.3487876057624817, -0.05225104093551636, 0.10337401926517487, 0.3126455843448639, 0.009004744701087475, -0.14489121735095978, -0.18281401693820953, 0.1911335438489914, 0.36281824111938477, 0.11254271119832993, -0.3712729215621948, -0.29808109998703003, 0.3333072066307068, 0.09874296933412552, -0.4482786953449249, -0.2477450966835022, -0.3857611417770386, -0.040634460747241974, 0.05169316381216049, 0.21858085691928864, 0.05794983357191086, -0.19710850715637207, -0.2110549956560135, 0.18564431369304657, 0.38380011916160583, 0.06504885852336884, -0.2394101917743683, -0.2638700306415558, 0.33446642756462097, -0.04144292697310448, -0.4404865801334381, -0.21680612862110138, -0.3157055079936981, -0.08490324765443802, 0.08050327003002167, 0.24611897766590118, 0.001262193312868476, -0.1292617917060852, -0.2139507234096527, 0.19100448489189148, 0.501126229763031, 0.09170946478843689, -0.3851533532142639, -0.15134884417057037, 0.4148223102092743, 0.003323692362755537, -0.38489583134651184, -0.1983104944229126, -0.35972097516059875, -0.09336646646261215, 0.05476360395550728, 0.2845907211303711, 0.08069101721048355, -0.1535257250070572, -0.17389188706874847, 0.27976468205451965, 0.3453373610973358, 0.08062967658042908, -0.3216707110404968, -0.29098811745643616, 0.34613943099975586, 0.049630098044872284, -0.40855512022972107, -0.22577276825904846, -0.32624053955078125, -0.02558058500289917, 0.11926325410604477, 0.32561826705932617, 0.050319962203502655, -0.11287283897399902, -0.10360138863325119, 0.21714098751544952, 0.4125404357910156, 0.10325074940919876, -0.3523050546646118, -0.25491055846214294, 0.4370928406715393, 0.08193369209766388, -0.3528302013874054, -0.19866962730884552, -0.40207645297050476, 0.035671330988407135, 0.15907983481884003, 0.22924789786338806, 0.09604136645793915, -0.13565298914909363, -0.285629540681839, 0.1649646908044815, 0.38062170147895813, 0.1472959667444229, -0.3459523022174835, -0.25070619583129883, 0.38359376788139343, 0.03144571930170059, -0.4759519696235657, -0.1882389336824417, -0.2751198410987854, -0.030492134392261505, 0.058060240000486374, 0.22469398379325867, 0.03319885954260826, -0.17260213196277618, -0.2264413684606552, 0.10199765861034393, 0.5035914778709412, 0.11358451098203659, -0.3550442159175873, -0.19261480867862701, 0.46455615758895874, -0.0690833106637001, -0.45769551396369934, -0.21859194338321686, -0.2965473234653473, -0.0677642896771431, 0.09291362762451172, 0.27926620841026306, 0.11991646885871887, -0.029274098575115204, -0.221174418926239, 0.13967019319534302, 0.540694534778595, 0.01876632496714592, -0.33501896262168884, -0.23619624972343445, 0.42159411311149597, 0.1018461361527443, -0.41069942712783813, -0.24281242489814758, -0.3036409318447113, -0.04919513687491417, 0.19828113913536072, 0.2814347743988037, -0.039598461240530014, -0.1895241141319275, -0.12242543697357178, 0.15897582471370697, 0.30084356665611267, 0.0387185700237751, -0.30634811520576477, -0.1785837858915329, 0.34670838713645935, 0.08372852206230164, -0.3904833197593689, -0.23416268825531006, -0.33833953738212585, 0.026781493797898293, 0.19045943021774292, 0.324960857629776, 0.043847501277923584, -0.0803576409816742, -0.20196695625782013, 0.18122774362564087, 0.36304888129234314, 0.06803172826766968, -0.3104496896266937, -0.09934373944997787, 0.38778963685035706, 0.11393525451421738, -0.3642234206199646, -0.20851720869541168, -0.27049019932746887, -0.17053170502185822, 0.10918155312538147, 0.3250478506088257, -0.09264308214187622, -0.19523167610168457, -0.14212940633296967, 0.17313185334205627, 0.35115694999694824, 0.04907377436757088, -0.33347827196121216, -0.20180295407772064, 0.4201260805130005, 0.03638293966650963, -0.46351540088653564, -0.20188705623149872, -0.26270225644111633, -0.04423971101641655, 0.13638247549533844, 0.2746506929397583, -0.03734062984585762, -0.17281529307365417, -0.16237126290798187, 0.191143199801445, 0.36087489128112793, -0.013139964081346989, -0.30081477761268616, -0.25500187277793884, 0.3799052834510803, 0.03392186015844345, -0.41605302691459656, -0.18447089195251465, -0.28085362911224365, -0.015269937925040722, 0.17103984951972961, 0.32250291109085083, 0.010469780303537846, -0.1399865746498108, -0.17022234201431274, 0.1704677790403366, 0.4335388243198395, 0.1254340261220932, -0.34641116857528687, -0.32654452323913574, 0.4571145474910736, 0.051282353699207306, -0.39481478929519653, -0.2658085525035858, -0.24231043457984924, -0.02476002275943756, 0.09813148528337479, 0.26458537578582764, 0.03294497728347778, -0.14560969173908234, -0.20886759459972382, 0.13000239431858063, 0.4692073464393616, 0.06577467173337936, -0.30261707305908203, -0.233379527926445, 0.2795209288597107, 0.07675142586231232, -0.44670602679252625, -0.21365469694137573, -0.3254706561565399, -0.11031480133533478, 0.1466042399406433, 0.24739153683185577, -0.017932720482349396, -0.1803826093673706, -0.1869954615831375, 0.22327136993408203, 0.3763142228126526, -0.00775577686727047, -0.25322431325912476, -0.29127949476242065, 0.45717331767082214, 0.047679562121629715, -0.46516576409339905, -0.23119527101516724, -0.3058667480945587, -0.01722242683172226, 0.09452668577432632, 0.3295314908027649, 0.13159909844398499, -0.14768853783607483, -0.25240543484687805, 0.13326238095760345, 0.33655011653900146, 0.03577028587460518, -0.3425378203392029, -0.30428048968315125, 0.4239410161972046, -0.0801725909113884, -0.4962882399559021, -0.09843068569898605, -0.27706992626190186, -0.03555143252015114, 0.12729814648628235, 0.3707316219806671, 0.05403166264295578, -0.1333942711353302, -0.2564410865306854, 0.18229904770851135, 0.3693726658821106, 0.0324447937309742, -0.34206080436706543, -0.24025766551494598, 0.38094624876976013, 0.09394988417625427, -0.496557354927063, 0.611250638961792, 0.9917982816696167, 0.16194097697734833, -0.31679877638816833, -0.836624801158905, -0.08202638477087021, 0.3980172574520111, 0.57875657081604, -0.5551645755767822, -1.2063647508621216, -0.209880992770195, 0.9194701910018921, 0.7040749192237854, -1.2050305604934692, -0.15706437826156616, 1.3037043809890747, -0.23358401656150818, -0.3764893710613251, -0.08875469118356705, 0.08473067730665207, 0.261779248714447, 0.024614794179797173, -0.15905015170574188, -0.17558449506759644, 0.21212594211101532, 0.4239603579044342, 0.03459838032722473, -0.3371741771697998, -0.21628211438655853, 0.38757964968681335, 0.048236772418022156, -0.5548480153083801, -0.3341984748840332, -0.31210264563560486, -0.045213084667921066, 0.17410679161548615, 0.3182429373264313, 0.03833712264895439, -0.1364549994468689, -0.26627010107040405, 0.23014119267463684, 0.37588757276535034, 0.09287778288125992, -0.2963467836380005, -0.2244204729795456, 0.442065566778183, 0.025901051238179207, -0.44961002469062805, -0.24198634922504425, -0.34781765937805176, 0.021606018766760826, 0.08626925945281982, 0.3414655923843384, 0.06855075061321259, -0.10080903023481369, -0.15478233993053436, 0.13526856899261475, 0.424739271402359, 0.1471741646528244, -0.34113866090774536, -0.2403806746006012, 0.3629738390445709, 0.0662449300289154, -0.48417478799819946, -0.11870928853750229, -0.3478018045425415, -0.13882409036159515, 0.19397473335266113, 0.33071091771125793, 0.053175754845142365, -0.11064374446868896, -0.21282298862934113, 0.17481760680675507, 0.44226783514022827, 0.11363480240106583, -0.25971823930740356, -0.22739776968955994, 0.37411239743232727, 0.04078999161720276, -0.37476828694343567, -0.12719736993312836, -0.346025675535202, -0.08555621653795242, 0.16681267321109772, 0.31836721301078796, 0.062113091349601746, -0.2028898000717163, -0.21362096071243286, 0.16023261845111847, 0.41646212339401245, -0.01649354211986065, -0.24276122450828552, -0.22301873564720154, 0.4191736578941345, -0.008812662214040756, -0.43156930804252625, -0.13246075809001923, -0.4074448347091675, -0.04640261456370354, 0.1393754482269287, 0.2475874125957489, 0.06798724085092545, -0.15377657115459442, -0.13344475626945496, 0.21235798299312592, 0.41728517413139343, 0.168199360370636, -0.2647540271282196, -0.17259353399276733, 0.3379558026790619, 0.018064003437757492, -0.38841983675956726, -0.15391267836093903, -0.33174705505371094, -0.02270469069480896, 0.051517508924007416, 0.29287654161453247, -0.030114011839032173, -0.120244599878788, -0.2192416787147522, 0.15393251180648804, 0.4088818430900574, 0.15862655639648438, -0.3569369614124298, -0.2681128680706024, 0.45081499218940735, 0.029216023162007332, -0.3417850732803345, -0.18806679546833038, -0.3244284689426422, -0.04384681209921837, 0.1914941817522049, 0.3149528503417969, 0.05308578908443451, -0.12468023598194122, -0.2730540335178375, 0.141718789935112, 0.4802340865135193, 0.108847476541996, -0.2810456156730652, -0.2061721831560135, 0.3702521324157715, 0.06729257106781006, -0.4752880036830902, -0.17962516844272614, -0.36466681957244873, -0.07238201051950455, 0.1271059364080429, 0.265845388174057, 0.059629276394844055, -0.11447121202945709, -0.17696018517017365, 0.25220295786857605, 0.33237263560295105, -0.01206055749207735, -0.27553677558898926, -0.19098562002182007, 0.49801382422447205, 0.0882072001695633, -0.3887375295162201, -0.24334728717803955, -0.29132896661758423, -0.027999071404337883, 0.05164193734526634, 0.2373688966035843, 0.021698622032999992, -0.12286043167114258, -0.3421723544597626, 0.1894976645708084, 0.4176096022129059, 0.05891174077987671, -0.29300761222839355, -0.21523001790046692, 0.34684988856315613, 0.07120336592197418, -0.4428136348724365, -0.24648721516132355, -0.3185030221939087, 0.01269119419157505, 0.03142941743135452, 0.1995292752981186, -0.05540812388062477, -0.03840753436088562, -0.2075122743844986, 0.18300805985927582, 0.36673808097839355, 0.020731167867779732, -0.4076918661594391, -0.2295050024986267, 0.4104097783565521, -0.04843234643340111, -0.48240575194358826, -0.23639896512031555, -0.33026018738746643, -0.049484558403491974, 0.16072484850883484, 0.35040757060050964, 0.08067290484905243, -0.07019669562578201, -0.2532539367675781, 0.15825779736042023, 0.4632473587989807, 0.08218660950660706, -0.30541276931762695, -0.33697474002838135, 0.30446964502334595, 0.08693882822990417, -0.4131694734096527, -0.21399615705013275, -0.3496086597442627, -0.14799614250659943, 0.12299595028162003, 0.33390581607818604, 0.10935872793197632, -0.1348162293434143, -0.20514293015003204, 0.13118253648281097, 0.46722906827926636, 0.15309633314609528, -0.3370073735713959, -0.22348739206790924, 0.40196529030799866, 0.02170298993587494, -0.5551207661628723, -0.2174932360649109, -0.2985878586769104, -0.13385611772537231, 0.06249113380908966, 0.11800674349069595, 0.09147656708955765, -0.11225930601358414, -0.1926591694355011, 0.1724587231874466, 0.3742027282714844, 0.06381324678659439, -0.32186460494995117, -0.25044316053390503, 0.3957412540912628, 0.05667690560221672, -0.35279935598373413, -0.27166488766670227, -0.31586164236068726, -0.09291905909776688, 0.04746113717556, 0.163776233792305, 0.05759478732943535, -0.18679209053516388, -0.23291738331317902, 0.20897632837295532, 0.3513932228088379, 0.0685558170080185, -0.3105928897857666, -0.19843316078186035, 0.44746384024620056, 0.006551764439791441, -0.3848893940448761, -0.20453056693077087, -0.3233201801776886, -0.07281669974327087, 0.14271217584609985, 0.22107474505901337, 0.03975439444184303, -0.07898413389921188, -0.20961658656597137, 0.2868710160255432, 0.36877158284187317, 0.07793836295604706, -0.33097928762435913, -0.26946285367012024, 0.34114325046539307, 0.022891033440828323, -0.43336012959480286, -0.23984813690185547, -0.285340279340744, 0.013825314119458199, 0.027490388602018356, 0.3234269320964813, 0.03615318611264229, -0.12560708820819855, -0.07240812480449677, 0.24569863080978394, 0.4222823679447174, 0.07640926539897919, -0.41463470458984375, -0.19940577447414398, 0.3599974513053894, 0.059180986136198044, -0.42536675930023193, -0.15117570757865906, -0.2274349182844162, -0.012510904110968113, 0.16808262467384338, 0.27045759558677673, 0.050038859248161316, -0.09881067276000977, -0.2669144868850708, 0.2880595326423645, 0.3703234791755676, 0.003876391565427184, -0.2246466428041458, -0.13210339844226837, 0.36227649450302124, 0.019728858023881912, -0.2746526300907135, -0.19356241822242737, -0.42816296219825745, -0.09230445325374603, 0.2045411318540573, 0.315140962600708, 0.06485513597726822, -0.07604525238275528, -0.16114415228366852, 0.1693221628665924, 0.3263512849807739, 0.09007209539413452, -0.3621506094932556, -0.2408372461795807, 0.4853803515434265, 0.07742687314748764, -0.427273690700531, -0.1959092766046524, -0.2924204170703888, -0.015629328787326813, 0.038594506680965424, 0.29145872592926025, 0.11303253471851349, -0.09955468773841858, -0.2331893891096115, 0.1520010083913803, 0.3029707670211792, 0.028084615245461464, -0.2260005921125412, -0.24150152504444122, 0.35468125343322754, 0.02158833108842373, -0.43578556180000305, -0.204520583152771, -0.34665873646736145, -0.10014574229717255, 0.13626369833946228, 0.359111487865448, -0.001147668226622045, -0.16433671116828918, -0.13546544313430786, 0.11279521137475967, 0.37649714946746826, 0.048848703503608704, -0.2979597747325897, -0.2517722547054291, 0.3531016409397125, 0.0659196749329567, -0.48963379859924316, -0.28631895780563354, -0.4303729236125946, -0.024256089702248573, 0.01759241707623005, 0.3045376241207123, 0.06078563630580902, -0.09692815691232681, -0.20366722345352173, 0.15857239067554474, 0.4103504419326782, 0.0380668118596077, -0.3706589639186859, -0.17525333166122437, 0.35270991921424866, 0.1401888132095337, -0.4045100510120392, -0.22192111611366272, -0.30707767605781555, -0.0001835436705732718, 0.03551173210144043, 0.2358461171388626, -0.025213195011019707, -0.129021555185318, -0.17472054064273834, 0.18743081390857697, 0.36965638399124146, 0.04133098945021629, -0.2469967007637024, -0.18217818439006805, 0.40399956703186035, -0.015136090107262135, -0.3988419473171234, -0.1848103255033493, -0.31632354855537415, 0.024810563772916794, 0.14546218514442444, 0.23346050083637238, -0.020987845957279205, -0.06825742870569229, -0.18464726209640503, 0.2303352952003479, 0.4007469117641449, 0.020279014483094215, -0.29013878107070923, -0.1570795476436615, 0.41738858819007874, -0.0028838810976594687, -0.4425412118434906, -0.22061391174793243, -0.31659287214279175, -0.006788310594856739, 0.2071496844291687, 0.29790931940078735, 0.010425851680338383, -0.12869320809841156, -0.18938978016376495, 0.16949257254600525, 0.41199707984924316, 0.07251813262701035, -0.3603748083114624, -0.20583410561084747, 0.3840966820716858, 0.006007153075188398, -0.4009491205215454, -0.21877293288707733, -0.23704729974269867, -0.03314568102359772, 0.11286703497171402, 0.24889595806598663, 0.011330622248351574, -0.09049671143293381, -0.22505739331245422, 0.2281644493341446, 0.30516868829727173, 0.08177618682384491, -0.26158374547958374, -0.21721062064170837, 0.4010688066482544, 0.10471136122941971, -0.4692128300666809, -0.24458922445774078, -0.31994935870170593, 0.002190167782828212, 0.1297602653503418, 0.27621564269065857, 0.046003762632608414, -0.11141499131917953, -0.2534559965133667, 0.04912732541561127, 0.35853636264801025, -0.003230542642995715, -0.3315998315811157, -0.294401079416275, 0.357868492603302, 0.07583323866128922, -0.4215727746486664, -0.1965247392654419, -0.24195487797260284, -0.0806174948811531, 0.08959110826253891, 0.3497423231601715, -0.024017293006181717, -0.2144019901752472, -0.1250666081905365, 0.20740626752376556, 0.44076424837112427, 0.10437370836734772, -0.2563232183456421, -0.2566963732242584, 0.39867010712623596, -0.00013521780783776194, -0.4181036353111267, -0.18232056498527527, -0.34627944231033325, -0.022862492129206657, 0.2576664090156555, 0.2302061915397644, 0.07449796795845032, -0.08422788232564926, -0.13560457527637482, 0.15190744400024414, 0.4721370339393616, 0.10682749003171921, -0.30611106753349304, -0.24853837490081787, 0.41644126176834106, 0.10397060960531235, -0.37903955578804016, -0.0775502547621727, -0.3392834961414337, -0.02319188043475151, 0.09056295454502106, 0.32899513840675354, 0.04156595095992088, -0.15939147770404816, -0.25346967577934265, 0.1553259938955307, 0.42592623829841614, 0.14910195767879486, -0.25980859994888306, -0.2900937795639038, 0.3807511627674103, 0.059597600251436234, -0.4296819865703583, -0.18968446552753448, -0.2843948006629944, -0.04599762335419655, 0.07437726110219955, 0.18995042145252228, 0.013965575024485588, -0.2179102897644043, -0.20015625655651093, 0.2041540890932083, 0.34107309579849243, 0.06196111440658569, -0.3227071762084961, -0.14272138476371765, 0.41987019777297974, -0.04464486613869667, -0.4092249572277069, -0.179878368973732, -0.3407067060470581, -0.08237920701503754, 0.1321585774421692, 0.31287726759910583, 0.046351686120033264, -0.18987946212291718, -0.2106444388628006, 0.1047094315290451, 0.3277224004268646, 0.0360138900578022, -0.36297792196273804, -0.26280584931373596, 0.5170704126358032, 0.07052148878574371, -0.4792933464050293, -0.3122662603855133, -0.3761017620563507, -0.13885916769504547, -0.011141303926706314, 0.3124369978904724, -0.0028771108482033014, -0.12145055830478668, -0.046621181070804596, 0.1933111846446991, 0.40569594502449036, 0.07896185666322708, -0.32592374086380005, -0.2856254279613495, 0.3805570900440216, 0.10130608826875687, -0.4499166011810303, -0.2737957239151001, -0.39738792181015015, -0.11917819827795029, 0.09907472133636475, 0.30047017335891724, 0.045667123049497604, -0.08321964740753174, -0.21026574075222015, 0.22295346856117249, 0.41790926456451416, 0.09897581487894058, -0.4021351933479309, -0.1790688931941986, 0.3171345293521881, -0.008666079491376877, -0.3938212990760803, -0.22077594697475433, -0.24251244962215424, -0.05533837527036667, 0.051571402698755264, 0.2886044383049011, 0.005950505845248699, -0.06077045947313309, -0.11447407305240631, 0.22198878228664398, 0.3849419355392456, 0.0712193176150322, -0.2545858919620514, -0.2787080705165863, 0.35709530115127563, 0.0923263281583786, -0.39394867420196533, -0.21129393577575684, -0.34287336468696594, -0.15974727272987366, 0.16311763226985931, 0.2758795917034149, -0.044563353061676025, -0.19402417540550232, -0.23994813859462738, 0.2708304822444916, 0.4403197467327118, 0.0006814416265115142, -0.21378464996814728, -0.1712663620710373, 0.3443697392940521, -0.08265525102615356, -0.3821733593940735, -0.22933217883110046, -0.335774302482605, -0.08793642371892929, 0.167399600148201, 0.3561336100101471, 0.06775607168674469, -0.1448049694299698, -0.1818409115076065, 0.21704146265983582, 0.32733505964279175, 0.12992262840270996, -0.3222838342189789, -0.22487179934978485, 0.40443187952041626, 0.0958101749420166, -0.37667420506477356, -0.08853328973054886, -0.34633398056030273, -0.10651861876249313, 0.11167580634355545, 0.2500934302806854, 0.07100532203912735, -0.17681923508644104, -0.18741492927074432, 0.23392361402511597, 0.4577832520008087, 0.07907512038946152, -0.23464204370975494, -0.17248116433620453, 0.3136821389198303, 0.06966018676757812, -0.4198661148548126, -0.21268360316753387, -0.3593237102031708, -0.057937122881412506, 0.12537217140197754, 0.29761749505996704, 0.0972445160150528, -0.08586741983890533, -0.22259606420993805, 0.14985840022563934, 0.4502783715724945, 0.034755971282720566, -0.296159565448761, -0.21234169602394104, 0.45082417130470276, 0.07813788205385208, -0.3560667335987091, -0.22018885612487793, -0.2934580147266388, -0.09258690476417542, 0.1157781332731247, 0.2904062867164612, -0.017145320773124695, -0.2654370367527008, -0.14507846534252167, 0.20619384944438934, 0.41517892479896545, 0.034796327352523804, -0.3209218680858612, -0.21317945420742035, 0.40050965547561646, 0.1104365661740303, -0.42233818769454956, 0.6041936278343201, 0.9834110140800476, 0.1841108649969101, -0.3086074888706207, -0.8485358953475952, -0.08689276874065399, 0.40801122784614563, 0.5904675722122192, -0.5627331137657166, -1.2111330032348633, -0.22287589311599731, 0.9105051755905151, 0.7088791728019714, -1.1872601509094238, -0.1614018976688385, 1.2843945026397705, -0.27448272705078125, -0.3009287714958191, -0.0767802819609642, 0.17524300515651703, 0.2796403765678406, 0.020103909075260162, -0.08448360115289688, -0.17975753545761108, 0.2675274610519409, 0.35457196831703186, 0.09111794084310532, -0.2709790766239166, -0.20880091190338135, 0.4565051794052124, 0.10342954099178314, -0.42674461007118225, -0.22527824342250824, -0.2779861092567444, -0.057042643427848816, 0.15474481880664825, 0.3524038791656494, 0.04017507657408714, -0.17923006415367126, -0.20436812937259674, 0.22887079417705536, 0.3978812098503113, 0.13052144646644592, -0.3002472221851349, -0.227105513215065, 0.3705945312976837, 0.04998756945133209, -0.37093454599380493, -0.21173247694969177, -0.3411175310611725, -0.09623625129461288, 0.147746741771698, 0.21028125286102295, -0.001407671021297574, -0.08789922297000885, -0.12006731331348419, 0.178500235080719, 0.457766056060791, -0.02799593098461628, -0.2651572823524475, -0.22016140818595886, 0.3394674062728882, -0.035443659871816635, -0.45242419838905334, -0.18318510055541992, -0.3075323700904846, -0.05317457392811775, 0.14593666791915894, 0.32014450430870056, -0.03900753706693649, -0.15355680882930756, -0.21766717731952667, 0.24011072516441345, 0.3842264711856842, 0.12079337239265442, -0.29900485277175903, -0.11800196021795273, 0.3977190852165222, 0.05337022244930267, -0.4330871105194092, -0.18910221755504608, -0.20481616258621216, -0.08417020738124847, 0.06352395564317703, 0.2749355435371399, 0.06343144178390503, -0.18411953747272491, -0.15263020992279053, 0.22661767899990082, 0.4002237021923065, 0.05750631168484688, -0.34343138337135315, -0.24946808815002441, 0.3596886098384857, 0.11471668630838394, -0.3467825949192047, -0.21127423644065857, -0.32893994450569153, -0.1545799821615219, 0.079970583319664, 0.3010447919368744, 0.042764853686094284, -0.12928511202335358, -0.15088313817977905, 0.19230809807777405, 0.38842788338661194, 0.1494685262441635, -0.40046876668930054, -0.2941804826259613, 0.3621293902397156, 0.015449367463588715, -0.4675217568874359, -0.20912498235702515, -0.36334145069122314, -0.06341984122991562, 0.13491728901863098, 0.2845233976840973, 0.034739360213279724, -0.18316100537776947, -0.25085559487342834, 0.1720854789018631, 0.45287463068962097, 0.08710277080535889, -0.25158828496932983, -0.18796803057193756, 0.4539403021335602, 0.010730315931141376, -0.36802101135253906, -0.1609240174293518, -0.22509601712226868, -0.03180551156401634, 0.12120821326971054, 0.2763131260871887, 0.0360245555639267, -0.21890030801296234, -0.2067749947309494, 0.18467910587787628, 0.4558931589126587, 0.0357263945043087, -0.2991633713245392, -0.14664778113365173, 0.371730238199234, 0.09288816899061203, -0.4561516344547272, -0.2035520374774933, -0.30546069145202637, -0.017230400815606117, 0.0805962011218071, 0.2320798635482788, 0.1031932458281517, -0.1273736208677292, -0.1549200415611267, 0.2319113314151764, 0.4442823529243469, 0.05544568970799446, -0.312444269657135, -0.314398854970932, 0.40699851512908936, 0.02779550850391388, -0.430728554725647, -0.1060057282447815, -0.2942725718021393, -0.05841744691133499, 0.037797961384058, 0.31887033581733704, 0.10315784811973572, -0.1772964894771576, -0.17330436408519745, 0.1610788106918335, 0.39923548698425293, 0.05035717785358429, -0.3851725161075592, -0.2626788914203644, 0.3181690573692322, 0.11930567771196365, -0.3783038258552551, -0.19643989205360413, -0.3146211802959442, -0.058254558593034744, 0.11481605470180511, 0.29243004322052, 0.016070645302534103, -0.1448669284582138, -0.2604547142982483, 0.13733065128326416, 0.3768734037876129, 0.07857494056224823, -0.32401618361473083, -0.14373111724853516, 0.388744592666626, 0.03350164368748665, -0.4083513915538788, -0.14796598255634308, -0.23729072511196136, -0.013146992772817612, 0.07626575231552124, 0.2745839059352875, 0.07958868891000748, -0.12382309883832932, -0.15486027300357819, 0.18318986892700195, 0.48547837138175964, 0.13963699340820312, -0.25019127130508423, -0.22930878400802612, 0.3195803165435791, -0.014559482224285603, -0.41312846541404724, -0.16600225865840912, -0.4702707827091217, -0.08364641666412354, 0.1620451956987381, 0.3301841616630554, 0.004258008673787117, -0.1047697439789772, -0.19207535684108734, 0.17670322954654694, 0.462240993976593, 0.12490507960319519, -0.3305232524871826, -0.2827174961566925, 0.33504414558410645, -0.017547324299812317, -0.36350324749946594, -0.10595196485519409, -0.31693556904792786, -0.008544891141355038, 0.07278812676668167, 0.2805974781513214, 0.07381589710712433, -0.12002414464950562, -0.19480624794960022, 0.10666961222887039, 0.33702754974365234, 0.049434952437877655, -0.3314598798751831, -0.23972363770008087, 0.3682199716567993, 0.04169166833162308, -0.4253652095794678, -0.17214219272136688, -0.3238458037376404, -0.08654269576072693, 0.09275387227535248, 0.32814887166023254, 0.01904619298875332, -0.1825771927833557, -0.24223528802394867, 0.20154032111167908, 0.43505150079727173, 0.1290709227323532, -0.32033148407936096, -0.16934846341609955, 0.3821328282356262, 0.040661513805389404, -0.4563368260860443, -0.17213565111160278, -0.27645257115364075, -0.061441246420145035, 0.08150717616081238, 0.20853830873966217, 0.03585664927959442, -0.15259115397930145, -0.20338453352451324, 0.15660692751407623, 0.3647880256175995, 0.04812183976173401, -0.28337135910987854, -0.19029287993907928, 0.3979960083961487, 0.06719718873500824, -0.43875032663345337, -0.20418475568294525, -0.39431506395339966, -0.017791928723454475, 0.1343982219696045, 0.3539063334465027, -0.0014776684110984206, -0.1300404965877533, -0.23873035609722137, 0.1986803114414215, 0.40111804008483887, 0.10159097611904144, -0.3123689889907837, -0.2057761400938034, 0.3746993839740753, 0.08030203729867935, -0.44383013248443604, -0.21929006278514862, -0.3236044943332672, -0.042741402983665466, 0.037216030061244965, 0.20982590317726135, 0.037534523755311966, -0.1388782262802124, -0.29228299856185913, 0.10461556911468506, 0.3700263798236847, 0.08442366868257523, -0.29527807235717773, -0.1880338042974472, 0.45539599657058716, 0.16639599204063416, -0.4988662898540497, -0.19988253712654114, -0.2760991156101227, -0.12126436084508896, 0.03740314766764641, 0.2930256128311157, 0.08163459599018097, -0.075240358710289, -0.27982398867607117, 0.12839166820049286, 0.40269896388053894, 0.043414633721113205, -0.28937602043151855, -0.2301875501871109, 0.48422735929489136, 0.04872223362326622, -0.4199996292591095, -0.19409513473510742, -0.2738778591156006, -0.11416985839605331, 0.124625563621521, 0.2462013065814972, -0.04001658782362938, -0.07845877856016159, -0.16779892146587372, 0.2555058002471924, 0.42020514607429504, 0.10110907256603241, -0.2978802025318146, -0.20843790471553802, 0.3793863356113434, 0.13547448813915253, -0.47125813364982605, -0.11390525847673416, -0.33638936281204224, -0.017082171514630318, 0.11347543448209763, 0.30629903078079224, 0.10220170766115189, -0.1278361976146698, -0.23842786252498627, 0.2002224326133728, 0.4403132498264313, -0.04943016171455383, -0.25607284903526306, -0.1820949912071228, 0.31364336609840393, 0.04550129175186157, -0.3874346613883972, -0.253156840801239, -0.34689992666244507, -0.07566612213850021, 0.13445816934108734, 0.3393654525279999, 0.02839767001569271, -0.13150469958782196, -0.14683590829372406, 0.21525044739246368, 0.4033694267272949, 0.13568075001239777, -0.3731134831905365, -0.22792205214500427, 0.3794115483760834, 0.028269445523619652, -0.417930543422699, -0.19471605122089386, -0.20561492443084717, -0.02747289650142193, 0.12428320944309235, 0.36591678857803345, 0.07971885055303574, -0.22633635997772217, -0.18641060590744019, 0.22507555782794952, 0.3787747919559479, 0.08474762737751007, -0.32642850279808044, -0.21721799671649933, 0.31955331563949585, -0.043318185955286026, -0.45511487126350403, -0.18594719469547272, -0.2613030672073364, -0.07331009954214096, 0.06952988356351852, 0.3292858898639679, 0.03089030645787716, -0.046502210199832916, -0.1974700391292572, 0.09175650775432587, 0.3685072362422943, 0.022498033940792084, -0.3733445107936859, -0.2746497690677643, 0.3649957478046417, 0.03786872327327728, -0.45583146810531616, -0.17761649191379547, -0.30716514587402344, -0.05158202722668648, 0.11554570496082306, 0.19047266244888306, -0.05382528901100159, -0.17946064472198486, -0.1810963749885559, 0.1149037703871727, 0.38649696111679077, 0.08123975992202759, -0.20701348781585693, -0.22393688559532166, 0.3721354901790619, 0.10551440715789795, -0.4190313220024109, -0.21297787129878998, -0.27296844124794006, -0.10622544586658478, 0.11470688879489899, 0.2868722677230835, 0.072612464427948, -0.19348378479480743, -0.2585776448249817, 0.16706483066082, 0.39274731278419495, 0.07003480195999146, -0.3247874081134796, -0.2165699452161789, 0.43166157603263855, 0.0614546500146389, -0.48257043957710266, -0.1421343833208084, -0.4051640033721924, -0.07213867455720901, 0.14830540120601654, 0.251274973154068, 0.09098481386899948, -0.042733993381261826, -0.11717372387647629, 0.2071380615234375, 0.454317569732666, 0.04183407127857208, -0.28693294525146484, -0.2158699631690979, 0.4016067683696747, 0.057614222168922424, -0.441005676984787, -0.2970859408378601, -0.3095528483390808, -0.04658045619726181, 0.20590046048164368, 0.31707054376602173, 0.030155586078763008, -0.0942993089556694, -0.16443833708763123, 0.1847044825553894, 0.3866993486881256, -0.005646397825330496, -0.30067867040634155, -0.15765391290187836, 0.3042490780353546, 0.0027121894527226686, -0.3930843472480774, -0.19001330435276031, -0.30849117040634155, -0.15457969903945923, 0.07470567524433136, 0.30691176652908325, 0.05614512041211128, -0.12298250198364258, -0.23923732340335846, 0.21243402361869812, 0.3462260365486145, 0.020551588386297226, -0.36473530530929565, -0.16672676801681519, 0.3764616549015045, 0.05269456282258034, -0.5401694774627686, -0.1451280415058136, -0.3063507676124573, -0.14459216594696045, 0.023944087326526642, 0.364330917596817, -0.02030963823199272, -0.04607115313410759, -0.21882817149162292, 0.06169058009982109, 0.46073395013809204, 0.09222786873579025, -0.27294033765792847, -0.2639707624912262, 0.3861652612686157, 0.0975017249584198, -0.47189706563949585, -0.2979547381401062, -0.2843427360057831, -0.0933033898472786, 0.16733062267303467, 0.25740328431129456, 0.005212489515542984, -0.15057934820652008, -0.313248872756958, 0.243914395570755, 0.3739044666290283, 0.05721741542220116, -0.23401685059070587, -0.23302710056304932, 0.4413854777812958, 0.06640651077032089, -0.39241284132003784, -0.1860785335302353, -0.2720203697681427, -0.07289796322584152, 0.12676090002059937, 0.24611368775367737, 0.08581887185573578, -0.24696072936058044, -0.18629536032676697, 0.20337769389152527, 0.4225189685821533, 0.06676401197910309, -0.24004609882831573, -0.13295388221740723, 0.3670620024204254, -0.04561002925038338, -0.4852072298526764, -0.2404124140739441, -0.3363651633262634, -0.06507286429405212, 0.15770846605300903, 0.26021960377693176, 0.024768711999058723, -0.13241645693778992, -0.2394670844078064, 0.16946563124656677, 0.543196439743042, 0.06249691918492317, -0.35511475801467896, -0.3475351333618164, 0.448637992143631, 0.03361285850405693, -0.46262094378471375, -0.07765110582113266, -0.2580461800098419, -0.14795753359794617, 0.08636405318975449, 0.33137646317481995, 0.06828777492046356, -0.19362811744213104, -0.10682837665081024, 0.2777727544307709, 0.349928617477417, -0.021262945607304573, -0.37663328647613525, -0.2054862380027771, 0.45054107904434204, 0.05876521021127701, -0.4608965218067169, -0.20475083589553833, -0.301697313785553, -0.12332485616207123, 0.018593408167362213, 0.27690407633781433, 0.08791559189558029, -0.12878108024597168, -0.18630149960517883, 0.1511702984571457, 0.45095422863960266, -0.018756501376628876, -0.3107703626155853, -0.26889175176620483, 0.38849806785583496, 0.002784817712381482, -0.3941170275211334, -0.20442074537277222, -0.36018338799476624, -0.0376681312918663, 0.10035545378923416, 0.25467950105667114, -0.0038257238920778036, -0.09996145963668823, -0.24353013932704926, 0.1731978803873062, 0.3909427225589752, 0.0816948339343071, -0.3177717924118042, -0.13196344673633575, 0.30831408500671387, 0.07820998877286911, -0.43369802832603455, -0.14559532701969147, -0.41203591227531433, -0.0027279651258140802, 0.03341447561979294, 0.31481772661209106, 0.0508992075920105, -0.13766464591026306, -0.23373334109783173, 0.14320793747901917, 0.4089975953102112, 0.028227975592017174, -0.40586718916893005, -0.33037880063056946, 0.43546608090400696, 0.08966939151287079, -0.40631213784217834, -0.1985803246498108, -0.31164708733558655, -0.03858457878232002, 0.11542900651693344, 0.24105818569660187, -0.010892611928284168, -0.19630464911460876, -0.18874862790107727, 0.13506247103214264, 0.39680635929107666, 0.1322333663702011, -0.3263871967792511, -0.19064027070999146, 0.47426411509513855, -0.04191602021455765, -0.5029984712600708, -0.17651435732841492, -0.3653380274772644, -0.03581719473004341, 0.13897904753684998, 0.25573286414146423, 0.06786730140447617, -0.05072829872369766, -0.14770899713039398, 0.1417189985513687, 0.39833584427833557, 0.040523454546928406, -0.22828173637390137, -0.2479555308818817, 0.3627106845378876, 0.052577946335077286, -0.4046248495578766, -0.2180098444223404, -0.3236071765422821, -0.06156642735004425, 0.14151428639888763, 0.33107975125312805, 0.041234441101551056, -0.16029101610183716, -0.1742352545261383, 0.14185340702533722, 0.46030426025390625, 0.08577185869216919, -0.2934767007827759, -0.35854023694992065, 0.5166410803794861, 0.12395232170820236, -0.36582455039024353, 0.5998181104660034, 0.9848651885986328, 0.17731162905693054, -0.3176513612270355, -0.856867253780365, -0.07464529573917389, 0.41521739959716797, 0.5924631953239441, -0.5362080335617065, -1.2064534425735474, -0.19904282689094543, 0.9173902273178101, 0.7362604141235352, -1.190348744392395, -0.1537628471851349, 1.303437352180481, -0.17079107463359833, -0.3755992650985718, 0.025383910164237022, 0.12138788402080536, 0.26245102286338806, -0.03901265561580658, -0.13964083790779114, -0.20115308463573456, 0.22228559851646423, 0.46235617995262146, 0.07315469533205032, -0.2794031798839569, -0.16287216544151306, 0.43227046728134155, 0.041441578418016434, -0.3953332304954529, -0.22131244838237762, -0.3023952841758728, -0.05517324432730675, 0.0534956194460392, 0.3163362443447113, 0.03541158139705658, -0.09674002975225449, -0.25736382603645325, 0.24529118835926056, 0.4675074815750122, 0.05404563248157501, -0.307471364736557, -0.23189875483512878, 0.40005308389663696, 0.09576348215341568, -0.4624016284942627, -0.2532382309436798, -0.33776089549064636, 0.03645756468176842, 0.08805475383996964, 0.23515833914279938, -0.015868160873651505, -0.12622667849063873, -0.13352340459823608, 0.15167023241519928, 0.417184054851532, 0.08077769726514816, -0.2808014452457428, -0.2285775989294052, 0.4521402418613434, 0.0868821069598198, -0.45891526341438293, -0.07583095878362656, -0.39505285024642944, -0.12555402517318726, 0.15821021795272827, 0.29348552227020264, 0.016803212463855743, -0.1252984255552292, -0.1587088704109192, 0.1667054146528244, 0.37615036964416504, 0.017125830054283142, -0.34767767786979675, -0.27632424235343933, 0.45637428760528564, 0.06597466766834259, -0.40146544575691223, -0.22579655051231384, -0.27500930428504944, -0.1382339894771576, 0.12962017953395844, 0.31835708022117615, -0.0003113810671493411, -0.18722237646579742, -0.16760021448135376, 0.1517646163702011, 0.36255311965942383, 0.06785448640584946, -0.36113467812538147, -0.2437237948179245, 0.49988678097724915, 0.02964806742966175, -0.38375821709632874, -0.25075411796569824, -0.32369348406791687, -0.07416468858718872, 0.09715337306261063, 0.32109755277633667, 0.13574187457561493, -0.14258837699890137, -0.1286976933479309, 0.2045031636953354, 0.44389164447784424, 0.045318711549043655, -0.3407101631164551, -0.24710379540920258, 0.3597487211227417, 0.05085054039955139, -0.4821251332759857, -0.19803600013256073, -0.3192283511161804, 0.030004063621163368, 0.1915350705385208, 0.30860063433647156, 0.003065337659791112, -0.1059333011507988, -0.15314070880413055, 0.17714837193489075, 0.3645983040332794, -0.020810270681977272, -0.307298868894577, -0.14094147086143494, 0.3880976140499115, 0.07067868113517761, -0.4109644293785095, -0.1739368587732315, -0.2892330288887024, -0.16087457537651062, 0.09799724072217941, 0.3021266460418701, 0.02389765903353691, -0.07533769309520721, -0.21233610808849335, 0.2333298623561859, 0.4956689476966858, 0.056483134627342224, -0.30346086621284485, -0.25114086270332336, 0.43730470538139343, -0.055005308240652084, -0.5017740726470947, -0.2121972143650055, -0.2835860848426819, -0.0303016547113657, 0.06534315645694733, 0.36442282795906067, 0.03479272127151489, -0.16652598977088928, -0.16631387174129486, 0.2533561885356903, 0.40483763813972473, -0.0060460008680820465, -0.3317396640777588, -0.23112379014492035, 0.44674152135849, 0.0024904748424887657, -0.4148085415363312, -0.2561659514904022, -0.3227948248386383, -0.12603291869163513, 0.08360908925533295, 0.20830687880516052, -0.01939675584435463, -0.2145155668258667, -0.29111218452453613, 0.24967937171459198, 0.4675038158893585, 0.09425651282072067, -0.32169800996780396, -0.1619095802307129, 0.38594216108322144, 0.07439548522233963, -0.37068021297454834, -0.18524254858493805, -0.25211861729621887, -0.06602195650339127, 0.09969332814216614, 0.2917725741863251, -0.02754661999642849, -0.18800626695156097, -0.15295590460300446, 0.10265369713306427, 0.4658210277557373, 0.08470674604177475, -0.243751659989357, -0.1964603066444397, 0.4308331310749054, 0.14105086028575897, -0.4305792450904846, -0.2008884847164154, -0.2689777910709381, -0.09121542423963547, 0.16475598514080048, 0.32024165987968445, 0.0056508989073336124, -0.19754835963249207, -0.2927328646183014, 0.2832861840724945, 0.33479782938957214, 0.010687632486224174, -0.25972676277160645, -0.21108025312423706, 0.41115760803222656, 0.0576993003487587, -0.4485335648059845, -0.24242804944515228, -0.36322110891342163, -0.029045501723885536, 0.16110824048519135, 0.34412330389022827, 0.027970021590590477, -0.2472756952047348, -0.16695183515548706, 0.18101394176483154, 0.2740170657634735, 0.1396368145942688, -0.29273849725723267, -0.2381930649280548, 0.40990149974823, 0.059067681431770325, -0.4156300127506256, -0.1906481832265854, -0.32497477531433105, -0.0301629900932312, 0.12479739636182785, 0.4062124490737915, 0.037458065897226334, -0.15963508188724518, -0.22155670821666718, 0.20877385139465332, 0.415823757648468, 0.08385773003101349, -0.3125944137573242, -0.2432725727558136, 0.43910878896713257, 0.12265326082706451, -0.4090023636817932, -0.20060265064239502, -0.38318848609924316, -0.11597209423780441, 0.11045939475297928, 0.27341699600219727, 0.025775272399187088, -0.18963384628295898, -0.19805997610092163, 0.09132017195224762, 0.3490760326385498, 0.030643928796052933, -0.2743838131427765, -0.26633742451667786, 0.35223984718322754, 0.06590498983860016, -0.4334043860435486, -0.15130163729190826, -0.31125789880752563, -0.049651138484478, 0.09611926227807999, 0.31637176871299744, -0.008549125865101814, -0.21305249631404877, -0.1489909291267395, 0.16273005306720734, 0.4088519811630249, 0.09939847886562347, -0.255053848028183, -0.38318416476249695, 0.3450883626937866, 0.1120818480849266, -0.51552414894104, -0.17538008093833923, -0.3470951020717621, -0.10629603266716003, 0.09787841141223907, 0.19735923409461975, 0.047544971108436584, -0.07844780385494232, -0.2665565609931946, 0.152296781539917, 0.4661307632923126, 0.08616528660058975, -0.32274532318115234, -0.1662660837173462, 0.3130757510662079, 0.03634442761540413, -0.4257930815219879, -0.21540382504463196, -0.2892109751701355, -0.1458984613418579, 0.1065322756767273, 0.24242936074733734, 0.08718529343605042, -0.08504100888967514, -0.2326369732618332, 0.13043329119682312, 0.40344491600990295, 0.02427947334945202, -0.3352871537208557, -0.21684005856513977, 0.37400415539741516, 0.1410892754793167, -0.4272722601890564, -0.2258094847202301, -0.3854678273200989, -0.0882495790719986, 0.15953749418258667, 0.3092785179615021, 0.09851527959108353, -0.07195556163787842, -0.16749899089336395, 0.15056414902210236, 0.3980419337749481, 0.07071996480226517, -0.38980162143707275, -0.22194258868694305, 0.3448399305343628, 0.04560184106230736, -0.4566132426261902, -0.17051059007644653, -0.19730384647846222, -0.12203644961118698, 0.10307586938142776, 0.38448289036750793, -0.01965836063027382, -0.14894504845142365, -0.20061638951301575, 0.20131570100784302, 0.32458850741386414, 0.08290289342403412, -0.2761755883693695, -0.1103191077709198, 0.393420547246933, 0.02135624550282955, -0.42363640666007996, -0.3303997814655304, -0.3120095133781433, 0.016573315486311913, 0.10277999192476273, 0.28048449754714966, 0.01511877216398716, -0.13018633425235748, -0.25721991062164307, 0.2637902498245239, 0.3832698166370392, 0.04525318741798401, -0.22276271879673004, -0.23645615577697754, 0.37689438462257385, 0.13392846286296844, -0.42028096318244934, -0.151095911860466, -0.3565910756587982, -0.14941498637199402, 0.11432085931301117, 0.35518351197242737, 0.005279575940221548, -0.11773890256881714, -0.1573372632265091, 0.2005527913570404, 0.4110836684703827, 0.10801876336336136, -0.2941443920135498, -0.2632811963558197, 0.4520716071128845, 0.09241267293691635, -0.40962105989456177, -0.1612161546945572, -0.45965954661369324, -0.04639843851327896, 0.10830893367528915, 0.18418057262897491, 0.1333393156528473, -0.11771950870752335, -0.21504426002502441, 0.1690390557050705, 0.4214667081832886, 0.06897932291030884, -0.3851994276046753, -0.2560465633869171, 0.4220035970211029, 0.0833786204457283, -0.38457196950912476, -0.2502979338169098, -0.35846439003944397, -0.040688756853342056, 0.0762258991599083, 0.29538917541503906, -0.012012055143713951, -0.10522784292697906, -0.18915122747421265, 0.2179969847202301, 0.4107581079006195, 0.03125566616654396, -0.36457258462905884, -0.2326136976480484, 0.36999133229255676, -0.021017203107476234, -0.45442941784858704, -0.15599575638771057, -0.26323962211608887, -0.019779127091169357, 0.17647898197174072, 0.26736655831336975, 0.023608900606632233, -0.18118001520633698, -0.22981126606464386, 0.27097630500793457, 0.43202850222587585, 0.10454560816287994, -0.17112071812152863, -0.13687245547771454, 0.358327180147171, 0.10399852693080902, -0.4637880325317383, -0.14499835669994354, -0.41896870732307434, -0.10439497977495193, 0.06286318600177765, 0.38752874732017517, -0.057550426572561264, -0.07681818306446075, -0.1362069696187973, 0.1965440809726715, 0.4525921046733856, 0.20227332413196564, -0.28898611664772034, -0.24732175469398499, 0.4702710807323456, 0.09448374807834625, -0.4836080074310303, -0.185119166970253, -0.31002217531204224, -0.11481142044067383, 0.06098815053701401, 0.2532883584499359, -0.02903241664171219, -0.10677707195281982, -0.17535194754600525, 0.07806041836738586, 0.38896793127059937, 0.07977231591939926, -0.37058180570602417, -0.248116597533226, 0.4160495698451996, -0.03180548921227455, -0.44203630089759827, -0.31541621685028076, -0.28760573267936707, -0.0956311747431755, 0.03402029350399971, 0.19150984287261963, -0.053681109100580215, -0.06995641440153122, -0.2353920042514801, 0.15904495120048523, 0.3879251182079315, -0.006982677616178989, -0.26614582538604736, -0.3637540936470032, 0.4439294934272766, 0.013009890913963318, -0.42713499069213867, -0.18337416648864746, -0.28572121262550354, -0.12395074218511581, 0.13601996004581451, 0.28005802631378174, 0.019764013588428497, -0.15122564136981964, -0.1379181146621704, 0.2101961225271225, 0.47093096375465393, 0.11453862488269806, -0.4068065881729126, -0.24311156570911407, 0.448782354593277, 0.004605366848409176, -0.5157157778739929, -0.3158889412879944, -0.22797204554080963, -0.10880935937166214, 0.11289295554161072, 0.2272017002105713, 0.04855785891413689, -0.0532381534576416, -0.2691318392753601, 0.18241439759731293, 0.36674049496650696, -0.02834472805261612, -0.3522498905658722, -0.2369077503681183, 0.4601310193538666, 0.1153860092163086, -0.43130627274513245, -0.2838796079158783, -0.2953682541847229, -0.06211871653795242, 0.10345693677663803, 0.2235412746667862, 0.019118869677186012, -0.037025775760412216, -0.11885110288858414, 0.25105249881744385, 0.4402233064174652, 0.047435831278562546, -0.3062537610530853, -0.23029740154743195, 0.417389452457428, 0.03258209675550461, -0.4744325280189514, -0.1419612318277359, -0.3479670584201813, -0.018573304638266563, 0.0955919548869133, 0.31695660948753357, 0.05370272323489189, -0.09926535934209824, -0.15958257019519806, 0.13769957423210144, 0.3368576467037201, 0.0655662789940834, -0.22163161635398865, -0.28039613366127014, 0.35017114877700806, 0.1669141948223114, -0.5090839862823486, -0.22827687859535217, -0.3497098386287689, -0.09620840102434158, 0.071586973965168, 0.3373768925666809, 0.03787441924214363, -0.1278783231973648, -0.2794034779071808, 0.22665125131607056, 0.41457399725914, 0.07558891922235489, -0.26414239406585693, -0.22191263735294342, 0.3961030840873718, 0.0718713030219078, -0.4661678671836853, -0.24517899751663208, -0.39780768752098083, -0.012248339131474495, 0.07133909314870834, 0.2913811206817627, 0.0840335413813591, -0.04488297551870346, -0.21298575401306152, 0.19472286105155945, 0.4112149178981781, 0.001485165674239397, -0.33043938875198364, -0.09536581486463547, 0.3945406973361969, 0.1409100592136383, -0.39100006222724915, -0.10924763977527618, -0.2610088586807251, -0.04975946620106697, 0.04173450544476509, 0.3555298149585724, 0.06368056684732437, -0.11221693456172943, -0.15036195516586304, 0.14589616656303406, 0.413007527589798, 0.06787043809890747, -0.38177812099456787, -0.2989017367362976, 0.39129021763801575, -0.062321651726961136, -0.39257413148880005, -0.1597926914691925, -0.33268705010414124, -0.04733836650848389, 0.011898837052285671, 0.2908995747566223, 0.0654301717877388, -0.11161574721336365, -0.14820200204849243, 0.18256226181983948, 0.5073899030685425, 0.0896231010556221, -0.26615655422210693, -0.1831558644771576, 0.3582175374031067, -0.007080461829900742, -0.39070209860801697, -0.26319098472595215, -0.35976290702819824, -0.01170752476900816, 0.04853270202875137, 0.23193693161010742, -0.03622785210609436, -0.1906583607196808, -0.09991233050823212, 0.21357183158397675, 0.454681396484375, -0.0313291922211647, -0.42922812700271606, -0.2969815731048584, 0.45024651288986206, 0.10972274094820023, -0.5314770340919495, -0.2887853682041168, -0.29640787839889526, -0.04967743530869484, 0.1259177029132843, 0.3150472044944763, -0.025074316188693047, -0.1318928301334381, -0.21028830111026764, 0.2475702315568924, 0.38927632570266724, -0.029606489464640617, -0.24313615262508392, -0.14714768528938293, 0.44580331444740295, 0.032448891550302505, -0.47971686720848083, -0.14920417964458466, -0.3571913242340088, -0.03641892969608307, 0.1429402083158493, 0.2739860415458679, -0.07604369521141052, -0.11320454627275467, -0.2474195808172226, 0.1049872562289238, 0.40216147899627686, 0.07313674688339233, -0.3126246929168701, -0.20739756524562836, 0.29854828119277954, 0.07836858183145523, -0.34903350472450256, -0.2049950212240219, -0.35615089535713196, -0.07211073487997055, 0.16519850492477417, 0.27105697989463806, -0.014689976349473, -0.22506926953792572, -0.1245942935347557, 0.2572370767593384, 0.405086487531662, 0.17201004922389984, -0.32776162028312683, -0.20238421857357025, 0.3294866979122162, -0.002069035079330206, -0.43179357051849365, 0.6016976833343506, 0.9842420220375061, 0.18951523303985596, -0.2935035824775696, -0.8448853492736816, -0.07756523787975311, 0.40349283814430237, 0.580612063407898, -0.5483757257461548, -1.2184139490127563, -0.20128624141216278, 0.9308857917785645, 0.7116462588310242, -1.1758184432983398, -0.15674158930778503, 1.2840869426727295, -0.27763602137565613, -0.28542226552963257, -0.11945952475070953, 0.12354537844657898, 0.29269060492515564, -0.05144540220499039, -0.13689133524894714, -0.20821478962898254, 0.15155401825904846, 0.5088478326797485, 0.07560724765062332, -0.32538673281669617, -0.14972876012325287, 0.39567920565605164, 0.04332858696579933, -0.5322296023368835, -0.16889461874961853, -0.2783319354057312, 0.03515410050749779, 0.05284569039940834, 0.3350231349468231, 0.09830668568611145, -0.10071077197790146, -0.19491121172904968, 0.1020435094833374, 0.41546449065208435, 0.039417434483766556, -0.3949292302131653, -0.20761781930923462, 0.3911170959472656, 0.038274914026260376, -0.3302488625049591, -0.22652168571949005, -0.43465593457221985, -0.17472413182258606, 0.1307458132505417, 0.27077314257621765, -0.011579778976738453, -0.24544119834899902, -0.23756760358810425, 0.128316268324852, 0.3997161090373993, 0.023323675617575645, -0.3516266345977783, -0.24130897223949432, 0.3563641905784607, 0.06573490798473358, -0.604702889919281, -0.2633722722530365, -0.268771231174469, -0.05272718146443367, 0.04788961634039879, 0.2711973488330841, -0.0035924201365560293, -0.15758371353149414, -0.12399689853191376, 0.163223996758461, 0.4150059223175049, 0.022867852821946144, -0.34575867652893066, -0.3182411789894104, 0.36239781975746155, 0.027707288041710854, -0.44772064685821533, -0.1982058882713318, -0.32590726017951965, -0.03014744445681572, 0.0918646901845932, 0.3469720184803009, 0.056308235973119736, -0.06894198805093765, -0.1991652548313141, 0.24594050645828247, 0.4093293249607086, -0.01747276820242405, -0.3089260458946228, -0.24839572608470917, 0.32221871614456177, -0.0010235850932076573, -0.46242713928222656, -0.08522891998291016, -0.3146669864654541, 0.028752854093909264, 0.15037088096141815, 0.3356929421424866, 0.06711122393608093, -0.2136775404214859, -0.13270455598831177, 0.13422222435474396, 0.4272220730781555, 0.0616479367017746, -0.2253902405500412, -0.1783800721168518, 0.48922282457351685, 0.0744004026055336, -0.4304756820201874, -0.16089428961277008, -0.20856565237045288, -0.035922929644584656, 0.10622873157262802, 0.21981751918792725, -0.010761572048068047, -0.1418769806623459, -0.16124045848846436, 0.2169945240020752, 0.40080130100250244, 0.07417958229780197, -0.22937703132629395, -0.1453515738248825, 0.4575325548648834, 0.12718364596366882, -0.367739200592041, -0.10746193677186966, -0.3922877609729767, -0.0873807743191719, 0.11630319803953171, 0.3017379641532898, 0.09461399912834167, -0.2296888530254364, -0.18742772936820984, 0.19945749640464783, 0.3722393810749054, 0.060915205627679825, -0.25584885478019714, -0.2144097089767456, 0.39137211441993713, 0.06106087937951088, -0.43614354729652405, -0.15174651145935059, -0.3336452841758728, -0.011944407597184181, 0.07693923264741898, 0.2810065448284149, -0.028455542400479317, -0.14698584377765656, -0.15215449035167694, 0.13566817343235016, 0.389078825712204, 0.07372301071882248, -0.32348185777664185, -0.19517327845096588, 0.33583495020866394, 0.09436215460300446, -0.3630167543888092, -0.11494162678718567, -0.3312731385231018, -0.1011166125535965, 0.0260999146848917, 0.317465603351593, 0.0695667564868927, -0.12161128968000412, -0.1765650361776352, 0.22085486352443695, 0.38901636004447937, -0.00520017696544528, -0.3771441578865051, -0.2765578031539917, 0.3523435890674591, -0.02852076292037964, -0.4508914053440094, -0.2502971589565277, -0.2947980761528015, -0.06424500793218613, 0.14553265273571014, 0.21658840775489807, -0.026963092386722565, -0.21647340059280396, -0.1794300079345703, 0.3439711332321167, 0.418337881565094, 0.04710027575492859, -0.2944469749927521, -0.23841041326522827, 0.4471856951713562, 0.05138498917222023, -0.473398357629776, -0.29372939467430115, -0.30968961119651794, -0.009993530809879303, 0.1131434291601181, 0.2387079894542694, 0.0066177877597510815, -0.15234027802944183, -0.16445431113243103, 0.18411670625209808, 0.4662500321865082, 0.05885253846645355, -0.33766570687294006, -0.1437128335237503, 0.3482740819454193, 0.01197112537920475, -0.3240886926651001, -0.18380744755268097, -0.23782169818878174, -0.18626336753368378, 0.16889864206314087, 0.2833963930606842, 0.03370708227157593, -0.11121556907892227, -0.3326375186443329, 0.1419636309146881, 0.4298076629638672, 0.23177777230739594, -0.3430720269680023, -0.11791174858808517, 0.4307663142681122, 0.10342731326818466, -0.4774751365184784, -0.1671520620584488, -0.37102213501930237, -0.12150812149047852, 0.17004890739917755, 0.24795660376548767, 0.002747882390394807, -0.10112576186656952, -0.234691321849823, 0.12342139333486557, 0.4186013638973236, 0.08258213847875595, -0.2286045253276825, -0.2936885356903076, 0.4345916509628296, 0.08747749775648117, -0.45132094621658325, -0.20666876435279846, -0.32690635323524475, -0.11276627331972122, 0.03863993287086487, 0.31869301199913025, -0.025732578709721565, -0.18570460379123688, -0.15425963699817657, 0.15927517414093018, 0.38624486327171326, -0.002347861882299185, -0.36821648478507996, -0.3075789511203766, 0.3281269967556, 0.11332294344902039, -0.47905123233795166, -0.31634846329689026, -0.3852788209915161, -0.026853231713175774, 0.04490399360656738, 0.2787010669708252, -0.02159462310373783, -0.2273687720298767, -0.22429083287715912, 0.1324663758277893, 0.41295742988586426, 0.10144368559122086, -0.31181812286376953, -0.10703931003808975, 0.34190401434898376, 0.11583908647298813, -0.41121336817741394], "scores": [[-0.9900531624389137, -0.9827313967889009, -0.9833436290631039, -0.9868309944389874, -0.9822332089676672, -0.9863759730940074, -0.9863742868285197, -0.9866738569138636, -0.9838407056419323, -0.9771370158293124, -0.9787839682153481, -0.9783151399116897, -0.9758214938677477, -0.9754788806258986, -0.9889305425230899, -0.9872026420211696, -0.9824592587832607, -0.9854225110600596, -0.9783685026446226, -0.9827458629412629, 0.9995756041877734, -0.9913784234040237, -0.9884143803672711, -0.9862168113279217, -0.9823312653771906, -0.9828884516321728, -0.9788213548524198, -0.9824534154294642, -0.9884392869159719, -0.9805294958035897, -0.9838130082617104, -0.9712822060818689, -0.9813203240493309, -0.9860262504834726, -0.9779580449212484, -0.9809826760380893, -0.9888629541195051, -0.9754875410457768, -0.9632892836185954, -0.981257378059502], [-0.981064245384258, -0.9874490167453631, -0.977206144650754, -0.9870825757463242, -0.9827300748254172, -0.9904349834926158, -0.9833593434079635, -0.9767579900630751, -0.9834725752632627, -0.9760186647183402, -0.9821489130047512, -0.9799255042295144, -0.9836036552266529, -0.9700090204692248, -0.9811324221271057, -0.983343750485874, -0.9643228384766567, -0.9873577588018465, -0.9804485909235797, -0.9883563027122855, -0.9868208790563022, 0.9997284045613235, -0.9842719371391817, -0.9908176099040273, -0.9769011560191202, -0.9874606411388951, -0.9821765040376712, -0.9848265519738197, -0.9892680962431732, -0.9844692043141747, -0.9873466707787226, -0.9779764694907066, -0.9925476969948329, -0.9794832866978929, -0.9778120836506345, -0.986694377881929, -0.9936471360031063, -0.9937827418554035, -0.9933507529733832, -0.9741714407609564], [-0.9838247179044004, -0.9840345465555529, -0.9783885165767597, -0.9897456428550659, -0.9749405720916309, -0.9825629019529302, -0.9828155045429215, -0.9934934234939207, -0.981764729207952, -0.9766466623489054, -0.9837609448630618, -0.97323539894024, -0.9788759105324093, -0.9767928113738867, -0.9902738091545793, -0.9693413374875643, -0.9852144978427172, -0.985621756003213, -0.9801824885776339, -0.9829645327520973, -0.9867466134891829, -0.9819918721943937, 0.9994321151563219, -0.9843988935872974, -0.992455559407602, -0.9840986328596163, -0.9815636703915287, -0.9853698486431275, -0.988694544547271, -0.9815562284521864, -0.9848026088607633, -0.9880423018334099, -0.9766081157021469, -0.9824721718542451, -0.9812044900517125, -0.9754243801588423, -0.9910638260923281, -0.9865177555276556, -0.975558939528402, -0.9808014246728878], [-0.9850488440244606, -0.986509862342904, -0.9729684442421597, -0.9759756743728931, -0.9880296617412785, -0.9743176482062564, -0.991905836922575, -0.9699835005049942, -0.9767303180192917, -0.9831926611134594, -0.9652876158079903, -0.9888206856621442, -0.9719805740000709, -0.9844014597407569, -0.974919125770806, -0.9926541333245144, -0.9744647601477228, -0.9718289451722217, -0.9816719681457671, -0.9786084680272797, -0.9817639685021871, -0.978948027525884, -0.9776183756057706, 0.9995353270873023, -0.9862930882256842, -0.9731889144070676, -0.9773390670761821, -0.9853953687566839, -0.984698584051192, -0.9687517207635873, -0.9773527031458551, -0.9819355913844239, -0.9905704945041488, -0.9793476576429958, -0.9800817789079357, -0.9791489738340817, -0.9594279531407077, -0.9831424315495938, -0.9827962561805812, -0.9729055006724165]], "kept": [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]]}, {"name": "big_dim_keydiff", "n": 40, "head_dim": 32, "n_kv_heads": 3, "budget": 10, "recent_frac": 0.3, "sink": 4, "keys": [-0.048693448305130005, 0.016022924333810806, 0.0012311515165492892, -0.20870418846607208, -0.0875617265701294, -0.14320795238018036, -0.01781250163912773, 0.2477240115404129, -0.08750692009925842, -0.11408457159996033, 0.03133044019341469, 0.07126089930534363, 0.08786512911319733, -0.2532215416431427, 0.037440355867147446, 0.13738438487052917, -0.28613853454589844, 0.013528568670153618, -0.32123008370399475, -0.3036952018737793, -0.3443737030029297, -0.015599223785102367, -0.24899446964263916, 0.08541618287563324, 0.026301074773073196, -0.0019687467720359564, -0.4037573039531708, -0.13559946417808533, 0.0009899466531351209, -0.0017492674523964524, -0.28284186124801636, -0.14965809881687164, -0.028732573613524437, 0.04665498808026314, -0.00687558576464653, -0.11106636375188828, -0.15211202204227448, -0.2271595597267151, 0.043712690472602844, 0.15368811786174774, -0.11618862301111221, -0.12213800847530365, 0.15543398261070251, 0.10192408412694931, 0.0035632853396236897, -0.19429311156272888, -0.018038548529148102, 0.2075931429862976, -0.27546626329421997, -0.10167627781629562, -0.3417136073112488, -0.24976927042007446, -0.3579637110233307, -0.10013706237077713, -0.2401314377784729, 0.0290916096419096, 0.08793333172798157, -0.0026766995433717966, -0.47689059376716614, -0.06839729845523834, -0.026160461828112602, 0.07402242720127106, -0.2894752621650696, -0.06112925708293915, -0.0643121525645256, 0.07379879057407379, -0.13622398674488068, -0.2700939178466797, -0.1011594831943512, -0.2324238121509552, 0.019570158794522285, 0.36554694175720215, -0.13461628556251526, -0.14847083389759064, 0.10285342484712601, 0.09210455417633057, 0.011103653348982334, -0.1861608326435089, 0.02959436923265457, 0.1574120670557022, -0.3057487905025482, -0.09045132249593735, -0.35757872462272644, -0.2964549660682678, -0.33510756492614746, -0.08733152598142624, -0.19095201790332794, 0.06090796738862991, 0.034092266112565994, -0.0648825392127037, -0.4816139042377472, -0.20170366764068604, -0.06573735922574997, 0.03955810144543648, -0.39563363790512085, -0.04796794801950455, -0.08707232028245926, 0.09430170804262161, -0.09408862888813019, -0.12937790155410767, -0.0793880820274353, -0.2642691731452942, 0.07382495701313019, 0.32539448142051697, -0.09632037580013275, -0.13096947968006134, 0.084589883685112, 0.018696274608373642, 0.07485329359769821, -0.20300890505313873, -0.008088299073278904, 0.09175185859203339, -0.2853686511516571, -0.1503785103559494, -0.2964895963668823, -0.25143513083457947, -0.2998034358024597, -0.043767474591732025, -0.274275541305542, 0.03493640944361687, 0.0016153891338035464, -0.034933168441057205, -0.4944467544555664, -0.11681243032217026, -0.07809571921825409, -0.018926184624433517, -0.20650240778923035, -0.12386003881692886, -0.052472181618213654, 0.07333110272884369, 0.018549829721450806, -0.24102868139743805, -0.09636173397302628, -0.21903006732463837, -0.07668345421552658, 0.29005545377731323, -0.09420232474803925, -0.11370156705379486, 0.05496765673160553, 0.09019310027360916, -0.007040911354124546, -0.18300947546958923, -0.06094181910157204, 0.07061154395341873, -0.18728841841220856, -0.11184749007225037, -0.3447590470314026, -0.24542026221752167, -0.37015676498413086, -0.06983175128698349, -0.20805123448371887, 0.03617728874087334, 0.02205475978553295, -0.034220047295093536, -0.4168580174446106, -0.06779079884290695, 0.009963029064238071, -0.006762458477169275, -0.3583037257194519, -0.04282188042998314, 0.04855486378073692, 0.04942936822772026, -0.024719586595892906, -0.12925530970096588, -0.04437641799449921, -0.14135827124118805, -0.011413362808525562, 0.3290577530860901, -0.15535977482795715, -0.0741875097155571, 0.11727983504533768, 0.1111348494887352, 0.11387436836957932, -0.10164204239845276, -0.06278762221336365, 0.04698310047388077, -0.21322055160999298, -0.1372428834438324, -0.3599633276462555, -0.20174437761306763, -0.4302893877029419, -0.14993272721767426, -0.22659040987491608, 0.053489964455366135, 0.017336787655949593, -0.0334043875336647, -0.5187091827392578, -0.17749106884002686, -0.01749972626566887, -0.027169322595000267, -0.3713793456554413, -0.0650143250823021, -0.0028374246321618557, 0.07679121196269989, -0.10127852857112885, -0.20123040676116943, -0.13588778674602509, -0.23175953328609467, 0.021137915551662445, 0.21416038274765015, -0.0752268135547638, -0.10028586536645889, 0.19384127855300903, -0.0021855621598660946, 0.06431907415390015, -0.18033871054649353, -0.006230264902114868, 0.05892348662018776, -0.2770746946334839, -0.049332395195961, -0.3634669780731201, -0.2396780252456665, -0.3626353442668915, 0.013294784352183342, -0.24062898755073547, -0.05875011160969734, -0.0049766856245696545, -0.13377095758914948, -0.6382553577423096, -0.12832210958003998, 0.05751100927591324, 0.023772107437253, -0.3478325605392456, -0.13733336329460144, 0.056763168424367905, 0.06434611976146698, -0.04941381886601448, -0.17100055515766144, -0.0840156227350235, -0.14715716242790222, 0.038995884358882904, 0.26409435272216797, -0.1451735496520996, -0.09171821922063828, 0.05837087705731392, 0.12214617431163788, -0.04362858459353447, -0.18274536728858948, -0.005896692629903555, 0.06518439948558807, -0.16796642541885376, -0.013471920043230057, -0.38252225518226624, -0.20514468848705292, -0.3291656970977783, -0.17511168122291565, -0.22703546285629272, 0.048203468322753906, 0.0337878093123436, -0.08917487412691116, -0.4891507625579834, -0.11072928458452225, 0.05023772642016411, 0.038137465715408325, -0.28948304057121277, -0.013849874958395958, -0.027529889717698097, 0.036993272602558136, -0.1426515281200409, -0.08987216651439667, -0.03771902248263359, -0.14158503711223602, 0.04481244087219238, 0.25881654024124146, -0.08225568383932114, -0.12987399101257324, 0.08240320533514023, 0.07016906887292862, 0.0955154299736023, -0.14807991683483124, -0.008451784960925579, 0.10244707763195038, -0.28581494092941284, -0.006357007659971714, -0.33400869369506836, -0.24035321176052094, -0.36540859937667847, -0.09988637268543243, -0.24289843440055847, 0.09495357424020767, 0.010000078938901424, -0.04669325426220894, -0.48673513531684875, -0.09633662551641464, -0.0888175293803215, 0.00964619405567646, -0.33192506432533264, -0.04606912285089493, -0.03829742595553398, 0.08531712740659714, 0.024408092722296715, -0.18400727212429047, -0.11601445078849792, -0.17785581946372986, 0.011266067624092102, 0.2036283016204834, -0.06998416036367416, -0.016497857868671417, 0.07967762649059296, 0.0573100820183754, -0.03232264518737793, -0.15990988910198212, -0.06787759065628052, 0.07607012987136841, -0.1900816410779953, -0.13176491856575012, -0.30527517199516296, -0.16751284897327423, -0.3351331949234009, -0.01676413230597973, -0.1419428139925003, 0.04143425077199936, -2.3348162358161062e-05, -0.10299267619848251, -0.47359806299209595, -0.02785913646221161, 0.0388127863407135, -0.025688450783491135, -0.331974059343338, -0.11550693958997726, 0.014845910482108593, 0.046199213713407516, -0.041091080754995346, -0.15349049866199493, -0.10087431967258453, -0.189436137676239, 0.021697137504816055, 0.2491106241941452, -0.06785408407449722, -0.023729871958494186, 0.12218184024095535, 0.07024441659450531, -0.06438197195529938, -0.15646645426750183, -0.10286270081996918, 0.060964975506067276, -0.21133305132389069, -0.05118050426244736, -0.3668400049209595, -0.32923129200935364, -0.3666669428348541, -0.07837061583995819, -0.207713320851326, 0.1641571968793869, 0.040473464876413345, -0.07429668307304382, -0.5342109799385071, -0.10462106019258499, -0.018006643280386925, -0.03615984693169594, -0.28338751196861267, -0.14784404635429382, 0.05583786964416504, 0.10959731787443161, 0.002423783764243126, -0.19202998280525208, -0.06020960956811905, -0.19403091073036194, -0.008073084056377411, 0.2363518327474594, -0.1580158919095993, -0.18946684896945953, 0.13229897618293762, 0.057892121374607086, 0.03074519895017147, -0.12577879428863525, -0.09218540042638779, 0.09221017360687256, -0.24529826641082764, -0.0668875202536583, -0.3781967759132385, -0.19227178394794464, -0.33757975697517395, -0.10510280728340149, -0.2860935628414154, 0.09154422581195831, 0.05281851440668106, -0.13028419017791748, -0.4082978069782257, -0.07191499322652817, 0.05800016596913338, 0.002231067279353738, -0.30399060249328613, -0.1466212272644043, 0.12707898020744324, 0.04769207537174225, 0.027562880888581276, -0.2006920874118805, -0.07774358242750168, -0.27099505066871643, -0.007775838021188974, 0.30249685049057007, -0.15561732649803162, -0.06366228312253952, 0.10944511741399765, 0.015256344340741634, -0.005155907012522221, -0.19881759583950043, -0.008004745468497276, 0.10460948199033737, -0.29542940855026245, -0.10172164440155029, -0.41068753600120544, -0.30820706486701965, -0.35050833225250244, -0.0002899917308241129, -0.31602394580841064, 0.05144607648253441, -0.0028708679601550102, -0.08418839424848557, -0.433011531829834, -0.12772482633590698, 0.06574810296297073, -0.017575856298208237, -0.26988017559051514, -0.1016625240445137, -0.03746858239173889, 0.08584854006767273, -0.05956290662288666, -0.13816693425178528, -0.08830021321773529, -0.24171824753284454, 0.006264057010412216, 0.255906879901886, -0.045106980949640274, -0.16258710622787476, 0.09061674028635025, -0.018640046939253807, 0.05249860882759094, -0.22993846237659454, -0.0958467572927475, 0.12845583260059357, -0.19878076016902924, -0.16271460056304932, -0.41374483704566956, -0.2808944880962372, -0.40457433462142944, -0.025462308898568153, -0.2799237370491028, 0.015194933861494064, 0.05879221111536026, -0.07310685515403748, -0.4540444016456604, -0.15038560330867767, -0.06977391242980957, -0.0703563541173935, -0.19614514708518982, -0.10631132870912552, 0.012429174967110157, 0.05491231009364128, -0.04381569102406502, -0.1658431440591812, 0.009475212544202805, -0.23937377333641052, -0.06650558859109879, 0.20271113514900208, -0.1597655862569809, -0.07992555946111679, 0.13360214233398438, 0.019388113170862198, -0.049597833305597305, -0.19360414147377014, 0.06402741372585297, -0.009561792016029358, -0.22773367166519165, -0.14028002321720123, -0.30732470750808716, -0.29762694239616394, -0.3623703122138977, -0.11974766105413437, -0.2884182631969452, 0.12056262791156769, 0.07065587490797043, -0.055414531379938126, -0.5191920399665833, -0.19650670886039734, -0.02884981967508793, 0.019870948046445847, -0.29340800642967224, -0.09498797357082367, -0.05585797503590584, 0.05315109342336655, -0.053747449070215225, -0.10379935801029205, 0.00740103330463171, -0.19427688419818878, -0.026947777718305588, 0.2500600218772888, -0.12341119349002838, -0.15439535677433014, 0.0896507203578949, 0.015288668684661388, 0.05022929236292839, -0.1810600906610489, 0.006971636787056923, 0.12226967513561249, -0.2904277443885803, -0.13389024138450623, -0.3712068498134613, -0.2711687684059143, -0.3364044427871704, -0.04465528950095177, -0.3076706826686859, 0.05462673678994179, -0.03751607984304428, -0.06615500897169113, -0.49040108919143677, -0.20558001101016998, -0.0045916116796433926, 0.028965279459953308, -0.2971065044403076, -0.11151424050331116, -0.018447676673531532, 0.007637812756001949, -0.06529879570007324, -0.19595804810523987, -0.0813513696193695, -0.24763424694538116, 0.023147014901041985, 0.26047006249427795, -0.10010827332735062, -0.13923390209674835, 0.12020022422075272, -0.015776194632053375, 0.042946621775627136, -0.16371244192123413, 0.00864003598690033, 0.15058231353759766, -0.2867472171783447, -0.0994652584195137, -0.3274901509284973, -0.22219878435134888, -0.33776387572288513, -0.12014971673488617, -0.2126609981060028, 0.10974422097206116, 0.08011051267385483, -0.02363722212612629, -0.5535673499107361, -0.05468910560011864, -0.016529716551303864, -0.10520987212657928, -0.27034491300582886, -0.16490696370601654, -0.06458952277898788, 0.02471684105694294, 0.011815778911113739, -0.18686972558498383, -0.07238605618476868, -0.10002904385328293, 0.09106900542974472, 0.24814143776893616, -0.1051061823964119, -0.18031719326972961, 0.057860326021909714, 0.08872180432081223, 0.03971049562096596, -0.17035239934921265, 0.04421130567789078, 0.09279827773571014, -0.256868839263916, -0.049930110573768616, -0.3301357626914978, -0.19018346071243286, -0.3282485008239746, -0.0599038302898407, -0.22144575417041779, 0.0011409108992666006, -0.05234602466225624, -0.006297535728663206, -0.47844043374061584, -0.08639586716890335, -0.09405120462179184, 0.0031604100950062275, -0.3191981315612793, -0.13350898027420044, -0.11251824349164963, 0.039721112698316574, -0.006948565132915974, -0.14927756786346436, -0.11598297208547592, -0.18817123770713806, 0.049207523465156555, 0.11528822034597397, -0.0992569550871849, -0.09011365473270416, 0.12668965756893158, 0.15249040722846985, 0.07667624205350876, -0.1602361798286438, 0.009570022113621235, 0.17076393961906433, -0.28103137016296387, -0.08850166201591492, -0.31403112411499023, -0.14585848152637482, -0.3560628890991211, -0.04685373604297638, -0.22963134944438934, 0.11843085289001465, 0.028111277148127556, 0.03813682869076729, -0.524016261100769, -0.11111816018819809, -0.019075240939855576, 0.060741398483514786, -0.23694226145744324, -0.16577059030532837, -0.045523375272750854, 0.07330740988254547, -0.08474846184253693, -0.24444885551929474, -0.03401163965463638, -0.1627289354801178, 0.03602639585733414, 0.22953683137893677, -0.04158320650458336, -0.12927165627479553, 0.1474049985408783, 0.021870631724596024, -0.022782815620303154, -0.16563116014003754, -0.04064538702368736, 0.1651892364025116, -0.24106061458587646, -0.13264082372188568, -0.3556997776031494, -0.2612946331501007, -0.3023078739643097, -0.07606135308742523, -0.26151400804519653, 0.11183251440525055, 0.14155660569667816, 0.06461931020021439, -0.4725222587585449, -0.0908736139535904, 0.06750594824552536, 0.015194371342658997, -0.33802273869514465, -0.08445459604263306, 0.0069738104939460754, -0.15671013295650482, 0.1513977348804474, 0.5019180178642273, 0.2609904408454895, 0.5592715740203857, -0.039173852652311325, -0.7434058785438538, 0.2770785689353943, 0.3421608507633209, -0.27287572622299194, -0.18400372564792633, -0.05896377936005592, 0.5399066805839539, 0.012207858264446259, -0.3991631865501404, 0.7552968263626099, 0.25804266333580017, 1.0963926315307617, 0.7196372747421265, 1.0606709718704224, 0.12321663647890091, 0.716327965259552, -0.14545409381389618, -0.08257816731929779, 0.10881367325782776, 1.4105565547943115, 0.3090713918209076, 0.017277304083108902, -0.03855924308300018, 0.8654815554618835, 0.2727823257446289, 0.03379916399717331, 0.031053666025400162, 0.044396378099918365, -0.08284792304039001, -0.057638250291347504, -0.15321294963359833, -0.08999552577733994, 0.28519460558891296, -0.10273555666208267, -0.09557837247848511, 0.12670619785785675, 0.05038930103182793, -0.0646001324057579, -0.15747058391571045, -0.04259796068072319, 0.11490697413682938, -0.28429049253463745, -0.10357096791267395, -0.4748190939426422, -0.1828833967447281, -0.3354322016239166, 0.011136259883642197, -0.1405632644891739, 0.05240032821893692, -0.06049486994743347, -0.08002546429634094, -0.5360286235809326, -0.12691007554531097, -0.005184584762901068, -0.07869324833154678, -0.27208152413368225, -0.16581620275974274, 0.015100769698619843, 0.05099022760987282, -0.06749533861875534, -0.171979621052742, -0.11292453855276108, -0.218051016330719, -0.07277029752731323, 0.25182774662971497, -0.0007534435717388988, -0.0182489063590765, 0.15867410600185394, 0.1027439683675766, -0.013900049962103367, -0.10372677445411682, -0.008335156366229057, 0.12797290086746216, -0.2686263620853424, -0.0818941667675972, -0.3811038136482239, -0.24791985750198364, -0.4023294746875763, -0.0631168857216835, -0.1255275309085846, 0.04779097065329552, 0.017693912610411644, -0.008712410926818848, -0.4402773380279541, -0.15750479698181152, -0.019524922594428062, 0.06732990592718124, -0.27571341395378113, -0.0842006728053093, 0.07793339341878891, 0.022576147690415382, -0.047676485031843185, -0.1943303942680359, -0.011420227587223053, -0.28502047061920166, -0.022139566019177437, 0.22685126960277557, -0.059853147715330124, -0.08695988357067108, 0.16234293580055237, -0.011247752234339714, 0.057447705417871475, -0.19052830338478088, -0.038810040801763535, 0.15839511156082153, -0.30007198452949524, -0.1903379261493683, -0.37785398960113525, -0.3186192810535431, -0.3805452883243561, -0.025855964049696922, -0.22395537793636322, 0.13061563670635223, 0.019644204527139664, -0.11199770122766495, -0.5134882926940918, -0.14778555929660797, -0.07005824148654938, 0.043184470385313034, -0.3214668929576874, -0.18916702270507812, 0.03507864847779274, 0.05089345946907997, -0.033971916884183884, -0.1630413830280304, -0.05435290187597275, -0.185526505112648, 0.07317764312028885, 0.27455824613571167, -0.0734267458319664, -0.09678031504154205, 0.01975134201347828, 0.05922210216522217, 0.006958131678402424, -0.16548697650432587, -0.07346348464488983, 0.2131224125623703, -0.24884557723999023, -0.1470581293106079, -0.4447874128818512, -0.257795125246048, -0.35258328914642334, -0.08034830540418625, -0.23494966328144073, 0.01921188272535801, 0.05720985308289528, -0.07155821472406387, -0.47760844230651855, -0.05287439003586769, 0.11941646039485931, -0.02896604873239994, -0.3124306797981262, -0.13229015469551086, 0.039448924362659454, -0.0009401789284311235, -0.0760316327214241, -0.1698075234889984, -0.1348695605993271, -0.2352936863899231, -0.012413624674081802, 0.1482871025800705, -0.165304034948349, -0.13792537152767181, 0.09999476373195648, 0.058165889233350754, -0.06877447664737701, -0.19905363023281097, 0.034393101930618286, 0.1592099517583847, -0.2580024003982544, -0.13086074590682983, -0.3277822434902191, -0.2726771831512451, -0.406561940908432, -0.08454301208257675, -0.16713809967041016, 0.06227967143058777, 0.08758929371833801, -0.059297941625118256, -0.42877596616744995, -0.13189154863357544, -0.01703835278749466, 0.14573362469673157, -0.25084784626960754, -0.11535689234733582, -0.004008445423096418, 0.0727759599685669, 0.00870910007506609, -0.1927841156721115, -0.17300158739089966, -0.20140953361988068, 0.012136791832745075, 0.25878602266311646, -0.027195623144507408, -0.10143930464982986, 0.1332290768623352, 0.012398205697536469, 0.06331902742385864, -0.07105332612991333, 0.03311668708920479, 0.14409446716308594, -0.24636559188365936, 0.002864853711798787, -0.405702143907547, -0.24928586184978485, -0.32508912682533264, -0.007233647629618645, -0.2614217698574066, 0.06663674116134644, 0.01573086716234684, -0.029307568445801735, -0.4822880029678345, -0.15889602899551392, -0.010222663171589375, 0.06527368724346161, -0.33755621314048767, -0.10235296189785004, 0.03347150981426239, 0.0029715311247855425, -0.04268186166882515, -0.22133395075798035, -0.02920411340892315, -0.07178636640310287, 0.1124802827835083, 0.24235008656978607, -0.05602017417550087, -0.11122377216815948, 0.09768852591514587, 0.1448417454957962, -0.04603502154350281, -0.12309958785772324, -0.007977577857673168, 0.20184370875358582, -0.24470333755016327, -0.12012585997581482, -0.345486044883728, -0.20693239569664001, -0.3463113307952881, -0.02003178931772709, -0.26563912630081177, -0.05542352423071861, 0.07462812215089798, -0.0003731379401870072, -0.4682745039463043, -0.09839581698179245, 0.04264780506491661, -0.0014392249286174774, -0.32453200221061707, -0.0997258722782135, 0.05968736484646797, -0.012890858575701714, 0.007777720224112272, -0.20029009878635406, -0.1409728080034256, -0.12442434579133987, 0.006523014511913061, 0.18829739093780518, -0.11096677929162979, -0.07072103768587112, 0.15218694508075714, 0.04609895497560501, 0.04023999348282814, -0.14016008377075195, -0.03776010498404503, 0.1491684764623642, -0.255658894777298, -0.11329347640275955, -0.38395047187805176, -0.24037902057170868, -0.3466068506240845, -0.07270820438861847, -0.2608526051044464, 0.10693753510713577, 0.04031829908490181, 0.008923593908548355, -0.4155923128128052, -0.07237423211336136, 0.10437732934951782, -0.01984908990561962, -0.24878481030464172, -0.10620557516813278, 0.09302166849374771, 0.14148679375648499, -0.14952434599399567, -0.21677076816558838, -0.05271999537944794, -0.14793407917022705, 0.048195730894804, 0.24976584315299988, -0.07027408480644226, -0.0846225917339325, 0.0886896625161171, 0.1188189908862114, -0.09305090457201004, -0.14417463541030884, -0.05723140761256218, 0.1793505698442459, -0.2654992341995239, -0.13093161582946777, -0.34064796566963196, -0.2892974317073822, -0.39373791217803955, -0.12279842048883438, -0.2408907413482666, 0.07611184567213058, 0.08077824115753174, -0.04241751879453659, -0.4232906699180603, -0.10091830044984818, -0.013831407763063908, 0.05009425804018974, -0.23628564178943634, -0.10737010091543198, -0.01195557415485382, 0.04842321574687958, -0.047675322741270065, -0.21334786713123322, -0.03453533723950386, -0.20744791626930237, 0.03448969125747681, 0.21203549206256866, -0.07508993148803711, -0.09769400954246521, 0.07154916226863861, 0.16849836707115173, 0.03847595304250717, -0.08701787143945694, 0.04242816939949989, 0.0983029380440712, -0.27317434549331665, -0.06468715518712997, -0.3562844395637512, -0.2412576824426651, -0.362409770488739, -0.13485752046108246, -0.25082698464393616, -0.05963540077209473, 0.04821791872382164, -0.07163401693105698, -0.5114542841911316, -0.11278153955936432, 0.004466633312404156, -0.05018419772386551, -0.37663534283638, -0.1436016708612442, -0.1018538698554039, 0.008122306317090988, 0.027730371803045273, -0.2211567610502243, -0.05336537957191467, -0.25600919127464294, 0.026323527097702026, 0.23732241988182068, -0.09602076560258865, -0.08883477747440338, 0.1803952306509018, 0.07718920707702637, 0.026141904294490814, -0.22453269362449646, 0.023638203740119934, 0.11911306530237198, -0.21246466040611267, -0.08867757767438889, -0.27230027318000793, -0.3428765833377838, -0.36292949318885803, -0.0003596210735850036, -0.2570899724960327, 0.011662017554044724, 0.01633291132748127, -0.10432755947113037, -0.4697357416152954, 0.020206745713949203, 0.04808459058403969, -0.03403434529900551, -0.33287233114242554, -0.11053476482629776, 0.0504533126950264, 0.015390436165034771, -0.08632534742355347, -0.12408998608589172, -0.04270301014184952, -0.2061181664466858, -0.044520650058984756, 0.17582187056541443, -0.12797963619232178, -0.22880010306835175, 0.13007421791553497, 0.035952355712652206, 0.04398864135146141, -0.0824480727314949, 0.05312100797891617, 0.07385993748903275, -0.2106025516986847, -0.028599407523870468, -0.3966608941555023, -0.2913956046104431, -0.3535841107368469, -0.12450487166643143, -0.16601860523223877, -0.06899751722812653, -0.025713428854942322, -0.04880938306450844, -0.4870377779006958, -0.09351013600826263, 0.004405444487929344, 0.010735510848462582, -0.23236097395420074, -0.19726718962192535, 0.0002242810296593234, 0.020735565572977066, -0.04518810659646988, -0.15728947520256042, -0.13152708113193512, -0.2194748818874359, 0.05099685490131378, 0.2707619369029999, -0.12704254686832428, -0.015279112383723259, 0.20804211497306824, -0.005669233854860067, 0.03501370549201965, -0.050414592027664185, 0.03366615250706673, 0.14246968924999237, -0.26446783542633057, -0.11355197429656982, -0.36996880173683167, -0.27126720547676086, -0.31085410714149475, -0.06434138864278793, -0.2616129219532013, -0.008837363682687283, 0.0271474476903677, -0.0800372064113617, -0.484720915555954, -0.049725569784641266, 0.009129260666668415, 0.046652376651763916, -0.2714007496833801, -0.08733958750963211, -0.006133904680609703, 0.0410744771361351, -0.013857929036021233, -0.22253945469856262, -0.01889950968325138, -0.18572352826595306, -0.026067500934004784, 0.2288518100976944, -0.12687964737415314, -0.1092633381485939, 0.05669107660651207, 0.12456045299768448, -0.01915799267590046, -0.28949040174484253, -0.04207870364189148, 0.030990619212388992, -0.2560584545135498, -0.03353435918688774, -0.32695508003234863, -0.31061023473739624, -0.3862122893333435, 0.04449797421693802, -0.22362978756427765, 0.051476605236530304, 0.08243007212877274, 0.0873059332370758, -0.4108547568321228, -0.09484558552503586, 0.00856742449104786, 0.05227009952068329, -0.3198402225971222, -0.14323434233665466, 0.002854960970580578, 0.009104224853217602, -0.05488689988851547, -0.16352130472660065, 0.031116226688027382, -0.22997993230819702, 0.005278072785586119, 0.24509397149085999, -0.07116460055112839, -0.06493744254112244, 0.0680590569972992, 0.026386288926005363, -0.06211157515645027, -0.2221745401620865, 0.021293263882398605, 0.13369902968406677, -0.3048367202281952, -0.105130635201931, -0.3577693998813629, -0.2185467928647995, -0.37776923179626465, -0.05695570632815361, -0.3304770886898041, -0.006298619322478771, 0.11051272600889206, -0.14473815262317657, -0.4925735592842102, -0.09085643291473389, -0.02788444608449936, -0.01899176463484764, -0.29225921630859375, -0.09054664522409439, -0.0038733636029064655, -0.03742082044482231, -0.059183962643146515, -0.21108309924602509, -0.1133224293589592, -0.1760787069797516, 0.04340395703911781, 0.298147976398468, -0.11785078048706055, -0.07107352465391159, 0.1512695997953415, 0.12428578734397888, 0.08941235393285751, -0.18314844369888306, -0.014235374517738819, 0.1726692169904709, -0.32239121198654175, -0.07602080702781677, -0.385871022939682, -0.2621701955795288, -0.43517524003982544, -0.0889609158039093, -0.24058005213737488, 0.09569332748651505, 0.07912827283143997, -0.03934744745492935, -0.4851478338241577, -0.14335030317306519, 0.010950841009616852, 0.009037211537361145, -0.2589110732078552, -0.002714035101234913, -0.00138624117244035, -0.018402937799692154, -0.09485629945993423, -0.24123483896255493, -0.14579112827777863, -0.12216389924287796, 0.02270049788057804, 0.17730039358139038, -0.060348521918058395, -0.054254915565252304, 0.07458949834108353, 0.0336543470621109, 0.003055070759728551, -0.16163398325443268, 0.026469729840755463, 0.1905275136232376, -0.19344109296798706, -0.02722206525504589, -0.2912469804286957, -0.21066135168075562, -0.4246448278427124, -0.05098191276192665, -0.22385933995246887, 0.03211435675621033, 0.0751899853348732, -0.0533418245613575, -0.5219707489013672, -0.02960139699280262, 0.02413323149085045, 0.032378800213336945, -0.24286983907222748, -0.03717166185379028, 0.017284296452999115, -0.06641828268766403, -0.08559325337409973, -0.19115641713142395, -0.13514567911624908, -0.17749357223510742, 0.07097762823104858, 0.22906383872032166, -0.1497717648744583, -0.015871919691562653, 0.07001134753227234, 0.005805643741041422, 0.03180781751871109, -0.15458019077777863, -0.040781281888484955, 0.1709393560886383, -0.2784309685230255, -0.1326398104429245, -0.3504135310649872, -0.20519229769706726, -0.37979209423065186, -0.027902424335479736, -0.24422994256019592, 0.19249235093593597, -0.005801042541861534, 0.033877454698085785, -0.4783007502555847, -0.10815444588661194, 0.026916271075606346, 0.06611564010381699, -0.22585450112819672, -0.07386578619480133, -0.02978731319308281, 0.02958248183131218, -0.026331119239330292, -0.1392824649810791, -0.016033092513680458, -0.16650591790676117, 0.06437592953443527, 0.32914766669273376, -0.08479010313749313, -0.1915515959262848, 0.03361937403678894, -0.0044362531043589115, 0.09950347244739532, -0.21823474764823914, 0.056063298135995865, 0.16070327162742615, -0.1683015525341034, -0.03633756563067436, -0.3644615709781647, -0.25373226404190063, -0.3438411355018616, -0.03567065671086311, -0.26609253883361816, 0.0496392548084259, 0.11009294539690018, -0.12043803930282593, -0.46275901794433594, -0.147163987159729, 0.00018487028137315065, 0.06340890377759933, -0.29208579659461975, -0.051649030297994614, -0.0792594626545906, 0.11181424558162689, -0.08197024464607239, -0.1372070610523224, -0.0768132358789444, -0.3167055547237396, -0.02643568627536297, 0.2643345296382904, -0.015331325121223927, -0.10291024297475815, 0.1054680272936821, -0.0031803143210709095, 0.0914812907576561, -0.08551046252250671, -0.004100026562809944, 0.12042882293462753, -0.33501750230789185, -0.04232620447874069, -0.224229097366333, -0.2078750878572464, -0.2849143445491791, -0.017329338937997818, -0.28880614042282104, 0.11810430139303207, -0.03210180252790451, -0.045864369720220566, -0.4617408215999603, -0.05767086520791054, 0.011632299050688744, 0.040757834911346436, -0.3272796869277954, -0.15476694703102112, 0.0033065401948988438, 0.020955046638846397, -0.11720306426286697, -0.2316247820854187, -0.11044206470251083, -0.2800563871860504, -0.056005917489528656, 0.17155227065086365, -0.08391715586185455, -0.09687444567680359, 0.19294027984142303, -0.007422599941492081, -0.014055891893804073, -0.1302337497472763, -0.016367468982934952, 0.1150667816400528, -0.16860651969909668, -0.10341256856918335, -0.4171527922153473, -0.30957773327827454, -0.3313148021697998, -0.029171623289585114, -0.30814048647880554, 0.0020008578430861235, 0.0563625730574131, -0.006467786151915789, -0.5078445076942444, -0.07070554047822952, 0.01942574605345726, -0.06773797422647476, -0.30480846762657166, -0.06954209506511688, -0.028421279042959213, -0.05048180744051933, -0.06596451252698898, -0.1307228058576584, -0.006915360223501921, -0.2969665825366974, 0.14203359186649323, 0.19695229828357697, -0.04518158733844757, -0.055563755333423615, 0.14173409342765808, 0.07493481785058975, -0.03814692795276642, -0.14453637599945068, -0.04129043594002724, 0.005281931720674038, -0.11262007802724838, -0.0510871559381485, -0.2684392035007477, -0.28931188583374023, -0.2747243642807007, 0.03505975380539894, -0.16182787716388702, 0.05021035298705101, -0.030496161431074142, -0.04419676586985588, -0.5858218669891357, -0.18691696226596832, -0.004570044577121735, -0.028153812512755394, -0.29742762446403503, -0.09574836492538452, 0.09676885604858398, 0.121346116065979, -0.05405740812420845, -0.10788248479366302, -0.1249212995171547, -0.2042722851037979, 0.05687873438000679, 0.1912260800600052, -0.10697916895151138, -0.1833159476518631, 0.09845685213804245, 0.14907968044281006, -0.01768088899552822, -0.16457034647464752, -0.050394874066114426, 0.0739392340183258, -0.315857470035553, -0.013754653744399548, -0.24122129380702972, -0.2195785641670227, -0.38391417264938354, -0.009189121425151825, -0.2562715411186218, 0.09134058654308319, 0.0441766232252121, -0.021345917135477066, -0.44531962275505066, -0.13079102337360382, -0.02938537485897541, -0.0646461695432663, -0.31129875779151917, -0.1614150106906891, -0.007001553196460009, 0.008583239279687405, -0.04675205796957016, -0.1456654965877533, -0.15694177150726318, -0.22809138894081116, -0.035798076540231705, 0.29063379764556885, -0.00771202752366662, -0.07469683140516281, 0.07511228322982788, 0.1407042145729065, -0.05578143149614334, -0.10089641064405441, -0.038339488208293915, -0.0075979819521307945, -0.12152477353811264, -0.0075911046005785465, -0.41093724966049194, -0.23528032004833221, -0.3588244915008545, -0.039322663098573685, -0.3137196898460388, 0.017195971682667732, 0.053025007247924805, -0.024196315556764603, -0.41572150588035583, -0.09782883524894714, 0.10829409956932068, -0.014370466582477093, -0.27623867988586426, -0.18611426651477814, -0.06242414936423302, 0.12289108335971832, -0.09895391762256622, -0.14179369807243347, -0.08902417868375778, -0.23965488374233246, -0.006044944282621145, 0.2275974303483963, -0.11739842593669891, -0.0795932486653328, 0.12520045042037964, 0.03858260437846184, -0.025431184098124504, -0.16174745559692383, -0.010529548861086369, 0.1828307956457138, -0.12130033224821091, -0.04434603825211525, -0.3610795736312866, -0.2523329555988312, -0.3909342288970947, -0.08976734429597855, -0.2897261381149292, 0.031503912061452866, 0.00826976913958788, 0.001855410635471344, -0.4956333637237549, -0.11168882995843887, -0.07008068263530731, 0.10364489257335663, -0.2638559639453888, -0.0008547168108634651, 0.04007679596543312, 0.1309465765953064, -0.06423815339803696, -0.13246941566467285, 0.04933967813849449, -0.2486041635274887, 0.0701809674501419, 0.33820632100105286, -0.07197374850511551, -0.07901304215192795, 0.15656495094299316, 0.03393913060426712, 0.04079454392194748, -0.22026456892490387, -0.0400911308825016, 0.09983853995800018, -0.2613784074783325, -0.0790034756064415, -0.33619141578674316, -0.3162761926651001, -0.24676074087619781, 0.014742745086550713, -0.20237721502780914, 0.032944150269031525, 0.025061609223484993, -0.007272803690284491, -0.4546114206314087, -0.18786947429180145, 0.028796980157494545, 0.17082849144935608, -0.38386979699134827, -0.03931449353694916, 0.01915760524570942, 0.05074402689933777, 0.0185370072722435, -0.2290072739124298, -0.07758089154958725, -0.11232970654964447, 0.0012625920353457332, 0.23184596002101898, -0.08713126927614212, -0.14011143147945404, 0.17204166948795319, 0.02086150087416172, 0.06477051973342896, -0.2401985079050064, 0.02884984202682972, 0.1260267049074173, -0.3868231177330017, -0.08679941296577454, -0.41418224573135376, -0.2665698230266571, -0.2690219283103943, -0.10186275839805603, -0.2940528690814972, 0.12531481683254242, 0.03558678925037384, 0.04390542209148407, -0.45905303955078125, -0.1471695750951767, -0.012575352564454079, 0.08462253212928772, -0.24083001911640167, -0.09025280177593231, 0.0027396902441978455, 0.0506797656416893, -0.08154714107513428, -0.06903347373008728, -0.08556598424911499, -0.24557289481163025, -0.013153358362615108, 0.2906500995159149, -0.06005215644836426, -0.12432361394166946, 0.058220382779836655, 0.06868299841880798, -0.06611042469739914, -0.2427864819765091, 0.03554120287299156, 0.10677824169397354, -0.23479081690311432, -0.023862631991505623, -0.32594743371009827, -0.24267126619815826, -0.23870041966438293, -0.010108763352036476, -0.2577783763408661, 0.08890550583600998, 0.0505942665040493, -0.07933042198419571, -0.47193264961242676, -0.1102975532412529, -0.11024460196495056, 0.01255353819578886, -0.30375805497169495, -0.11283034086227417, -0.08432233333587646, 0.038232285529375076, -0.05385388433933258, -0.16308768093585968, -0.14430361986160278, -0.15491610765457153, -0.05111604928970337, 0.24180705845355988, -0.025947192683815956, -0.15286152064800262, 0.06633314490318298, 0.12262154370546341, 0.00018828161410056055, -0.19112713634967804, 0.005839155521243811, 0.185337632894516, -0.3672963082790375, -0.1290508210659027, -0.4110887348651886, -0.29945141077041626, -0.39198869466781616, -0.08751111477613449, -0.22409455478191376, 0.005664285738021135, 0.03590615466237068, -0.014760095626115799, -0.5130860805511475, -0.09512253850698471, 0.0747089684009552, 0.0379188098013401, -0.32185426354408264, -0.09362440556287766, -0.04666459560394287, 0.07032159715890884, -0.007691322825849056, -0.2102246731519699, -0.09663283079862595, -0.13056263327598572, -0.014056766405701637, 0.26291424036026, -0.14533667266368866, -0.15911294519901276, 0.06641007214784622, 0.17626988887786865, -0.0005285507068037987, -0.20236225426197052, -0.03368845582008362, 0.12313655763864517, -0.22126850485801697, -0.07701773941516876, -0.2564523220062256, -0.23304101824760437, -0.27572929859161377, -0.030598146840929985, -0.20927628874778748, -0.020909326151013374, 0.023273969069123268, -0.03877850994467735, -0.4878312647342682, -0.2094602733850479, 0.034864239394664764, 0.0020585947204381227, -0.31110885739326477, -0.10305694490671158, -0.011171452701091766, 0.07681971043348312, -0.1153653934597969, -0.21487955749034882, -0.07140397280454636, -0.18014830350875854, -0.0074652209877967834, 0.2246134877204895, -0.1347513645887375, -0.12019141763448715, 0.15161985158920288, 0.014551352709531784, 0.09608986228704453, -0.13366524875164032, -0.028176965191960335, 0.17057041823863983, -0.3273601830005646, -0.1730710119009018, -0.4238321781158447, -0.2642180621623993, -0.3643700182437897, -0.07019082456827164, -0.23458422720432281, -0.055487025529146194, 0.05439441278576851, -0.10629615187644958, -0.5087259411811829, -0.12245886772871017, -0.06697751581668854, -0.04713387042284012, -0.33950936794281006, -0.09089455753564835, 0.029731616377830505, 0.08467654883861542, -0.10153833031654358, -0.2079063206911087, -0.13196580111980438, -0.164936825633049, -0.09050349146127701, 0.31522607803344727, -0.11727037280797958, -0.15644989907741547, 0.1095559224486351, 0.11908484250307083, 0.03705311939120293, -0.1232270896434784, 0.001189162372611463, 0.19289861619472504, -0.19180700182914734, -0.09146619588136673, -0.28738275170326233, -0.23549406230449677, -0.3437548577785492, -0.0905073881149292, -0.253994345664978, 0.09014980494976044, -0.036005403846502304, -0.0028499243780970573, -0.4582526385784149, -0.08793827891349792, -0.1194915920495987, 0.016253964975476265, -0.256792277097702, -0.12844611704349518, 0.004747841507196426, -0.06243887543678284, -0.018244128674268723, -0.20144909620285034, -0.04236779361963272, -0.27956676483154297, 0.04598088935017586, 0.24146215617656708, -0.05264993757009506, -0.16355915367603302, 0.06166037917137146, 0.17957645654678345, -0.02157365158200264, -0.2111656814813614, -0.01935788057744503, 0.08028874546289444, -0.19454216957092285, -0.00024159254098776728, -0.39454835653305054, -0.3300323486328125, -0.2895006537437439, 0.010985822416841984, -0.19847239553928375, 0.10909624397754669, -0.013749321922659874, -0.043755944818258286, -0.5448889136314392, -0.17272448539733887, 0.03988808020949364, -0.012179551646113396, -0.17522670328617096, -0.08658936619758606, -0.03178873285651207, 0.09635904431343079, 0.038249339908361435, -0.1440209150314331, -0.14679290354251862, -0.16887730360031128, 0.08014742285013199, 0.22864706814289093, -0.13171222805976868, -0.06645126640796661, 0.09958161413669586, 0.012326785363256931, 0.0009366262820549309, -0.21364744007587433, 0.009284654632210732, 0.1369009017944336, -0.1965426504611969, -0.051009006798267365, -0.43751975893974304, -0.22338122129440308, -0.3032529056072235, -0.021639740094542503, -0.18805237114429474, 0.02379182167351246, -0.017914649099111557, 0.011172376573085785, -0.5289566516876221, -0.20979322493076324, 0.04209801182150841, -0.014793135225772858, -0.30450859665870667, -0.16019734740257263, -0.06604387611150742, -0.0013646557927131653, -0.0698462724685669, -0.1498275101184845, -0.20082619786262512, -0.1498187780380249, -0.0477323904633522, 0.19999352097511292, -0.059137359261512756, -0.1210053563117981, 0.015761541202664375, 0.15968352556228638, -0.023107124492526054, -0.16349852085113525, 0.0044657387770712376, 0.2019830346107483, -0.2768484354019165, -0.036087773740291595, -0.39643752574920654, -0.18762430548667908, -0.38624873757362366, -0.06254415959119797, -0.14475703239440918, 0.06711924076080322, 0.035114891827106476, 0.0767713412642479, -0.45761558413505554, -0.14262321591377258, 0.027564415708184242, -0.037667933851480484, -0.23278945684432983, -0.029999487102031708, -0.02841545268893242, 0.025069693103432655, -0.04298856481909752, -0.1676197052001953, -0.13482001423835754, -0.15902304649353027, -0.09072626382112503, 0.23166614770889282, -0.11322717368602753, -0.13093693554401398, 0.10794442147016525, 0.006052768789231777, 0.04934971034526825, -0.18487201631069183, -0.03803958371281624, 0.06307537108659744, -0.31322047114372253, -0.06847602128982544, -0.40753552317619324, -0.24052080512046814, -0.2950003743171692, 0.007198427803814411, -0.16153551638126373, 0.03645600378513336, -0.024630511179566383, 0.015695150941610336, -0.46020323038101196, -0.04561881348490715, -0.008614180609583855, 0.08119840919971466, -0.24688591063022614, -0.054477330297231674, 0.025408649817109108, 0.07634973526000977, -0.040161989629268646, -0.15703214704990387, -0.036425910890102386, -0.21302641928195953, 0.09815177321434021, 0.247996985912323, -0.04376101866364479, -0.1217857226729393, 0.08024992793798447, 0.16906596720218658, 0.004456717055290937, -0.23972515761852264, -0.0412328876554966, 0.1155552789568901, -0.1463054120540619, -0.07615446299314499, -0.31260737776756287, -0.29381075501441956, -0.3304691016674042, -0.010859785601496696, -0.14947400987148285, 0.016457147896289825, 0.05980141460895538, -0.020201928913593292, -0.5267691612243652, -0.10278915613889694, -0.02234072983264923, -0.09485215693712234, -0.2592718303203583, -0.0632862076163292, -0.021714232861995697, -0.010463310405611992, 0.022180262953042984, -0.15502749383449554, -0.03510057553648949, -0.11691135168075562, -0.014677819795906544, 0.29391685128211975, -0.10508757829666138, -0.12040109187364578, 0.08830244839191437, 0.06738794595003128, 0.07562196999788284, -0.16967293620109558, -0.013215416111052036, 0.11758981645107269, -0.2369993031024933, -0.13123565912246704, -0.384475976228714, -0.24634477496147156, -0.3763919174671173, -0.05035750940442085, -0.27009421586990356, 0.15146856009960175, 0.09607997536659241, -0.012833517044782639, -0.47836434841156006, 0.007721861358731985, 0.006980661768466234, 0.01707383617758751, -0.27404895424842834, -0.07500121742486954, 0.07074838876724243, 0.055516552180051804, 0.05315246060490608, -0.2124285250902176, -0.05123312771320343, -0.237518772482872, 0.10222256928682327, 0.23678083717823029, -0.09593959152698517, -0.07106181234121323, 0.15535026788711548, 0.011860048398375511, 0.0026160068809986115, -0.24561047554016113, 0.0013364790938794613, 0.12634804844856262, -0.2715117037296295, -0.12071532756090164, -0.3786383867263794, -0.28105050325393677, -0.321651428937912, 0.031363800168037415, -0.3360915780067444, 0.045472294092178345, 0.008718352764844894, -0.010712695308029652, -0.5232717990875244, -0.1091381385922432, -0.06114659458398819, 0.056251004338264465, -0.28424155712127686, -0.09837210178375244, 0.018741903826594353, 0.04414062201976776, -0.12950482964515686, -0.13854093849658966, -0.07152213156223297, -0.19513142108917236, 0.06061376631259918, 0.3142338991165161, -0.1476442664861679, -0.15632747113704681, 0.16994749009609222, 0.1383964866399765, -0.01699570007622242, -0.23489981889724731, -0.03740715608000755, 0.10651085525751114, -0.33326828479766846, -0.08493389934301376, -0.4233520030975342, -0.30425024032592773, -0.30229419469833374, -0.0785931646823883, -0.2431289255619049, 0.08272548764944077, 0.07028814405202866, -0.05218213051557541, -0.4734775424003601, -0.14308664202690125, 0.07820200175046921, 0.053913600742816925, -0.34436726570129395, -0.09009547531604767, -0.012164534069597721, -0.17418316006660461, 0.1425309181213379, 0.5147976875305176, 0.25518718361854553, 0.5679089426994324, -0.04094843193888664, -0.7649387717247009, 0.29197219014167786, 0.34210404753685, -0.29576411843299866, -0.19829927384853363, -0.07118520140647888, 0.5231983065605164, 0.02570216916501522, -0.4024359881877899, 0.7709795236587524, 0.2526645064353943, 1.0929359197616577, 0.7444432973861694, 1.0395846366882324, 0.13116882741451263, 0.7103466987609863, -0.15181198716163635, -0.07738912105560303, 0.09152580797672272, 1.4362246990203857, 0.2922096848487854, 0.018903745338320732, -0.08212537318468094, 0.8886653184890747, 0.2714821398258209, 0.06574640423059464, 0.11151730269193649, -0.0400388240814209, -0.25608140230178833, -0.1382337510585785, -0.13859228789806366, 0.09702357649803162, 0.26813989877700806, -0.03066418319940567, -0.1476617455482483, 0.08311387151479721, 0.06751128286123276, 0.04107879474759102, -0.2523047924041748, 0.04150456190109253, 0.19754400849342346, -0.31150177121162415, -0.15425923466682434, -0.3418150544166565, -0.2764463424682617, -0.46664100885391235, -0.004004527349025011, -0.23177564144134521, 0.027438320219516754, 0.13529916107654572, -0.028385912999510765, -0.5133548974990845, -0.09675875306129456, -0.06117044761776924, -0.016669226810336113, -0.26988714933395386, -0.11800828576087952, -0.02231619693338871, 0.13991202414035797, -0.008831340819597244, -0.1281861662864685, -0.059494804590940475, -0.16321736574172974, 0.08419676125049591, 0.25233182311058044, -0.0002544228336773813, -0.16484373807907104, 0.07983134686946869, -0.0014759278856217861, 0.002735305344685912, -0.14994218945503235, 0.0670335665345192, 0.09825029969215393, -0.25379979610443115, -0.10445278137922287, -0.39627304673194885, -0.3031997084617615, -0.34506645798683167, -0.13103695213794708, -0.23402820527553558, 0.0523182675242424, -0.003260573372244835, -0.08587630093097687, -0.5448988080024719, -0.008249926380813122, -0.07626558840274811, 0.09930122643709183, -0.25919705629348755, -0.1084994226694107, -0.05762133374810219, 0.10858946293592453, 0.034668780863285065, -0.21502868831157684, -0.10087289661169052, -0.1348462551832199, 0.028156686574220657, 0.35330715775489807, -0.12153735011816025, -0.0901591032743454, 0.03003683127462864, 0.1722484827041626, -0.0165930837392807, -0.2862626612186432, -0.011816979385912418, 0.12444514781236649, -0.1582864373922348, -0.10343324393033981, -0.35997524857521057, -0.2140588015317917, -0.2687859833240509, -0.04759518429636955, -0.18758146464824677, 0.07717490196228027, 0.014815174974501133, -0.03145340085029602, -0.4742497205734253, -0.14761757850646973, 0.051517054438591, -0.04786490276455879, -0.23064684867858887, -0.042577408254146576, -0.0044199563562870026, 0.01819256879389286, -0.06313503533601761, -0.14468872547149658, -0.04878271743655205, -0.17308148741722107, 0.06097433716058731, 0.22543738782405853, -0.08130130916833878, -0.19569094479084015, 0.13076455891132355, 0.07511270046234131, -0.000854752492159605, -0.1177871972322464, 0.0290475282818079, -0.009403033182024956, -0.2520708441734314, -0.15924346446990967, -0.30601006746292114, -0.11927700787782669, -0.37402617931365967, -0.040704745799303055, -0.25210702419281006, 0.0688113272190094, 0.061854392290115356, 0.029028572142124176, -0.5250545144081116, -0.02323460578918457, -0.05574267730116844, 0.03527054563164711, -0.23127645254135132, -0.05463843792676926, -0.046696241945028305, 0.02136860601603985, -0.07493032515048981, -0.17088592052459717, -0.028587976470589638, -0.252018004655838, 0.034702595323324203, 0.2838077247142792, -0.06511569023132324, -0.09395834058523178, 0.1684991419315338, 0.09184430539608002, 0.06277191638946533, -0.14768698811531067, 0.08412685990333557, 0.036426443606615067, -0.1899595856666565, -0.09954379498958588, -0.3733789026737213, -0.2922045886516571, -0.44153428077697754, -0.06352844834327698, -0.2739200294017792, 0.07597936689853668, -0.043986182659864426, 0.03624953329563141, -0.4620867371559143, -0.11421148478984833, -0.04463815316557884, -0.01794424094259739, -0.3457096517086029, -0.1175164133310318, 0.0005350274732336402, 0.04759661480784416, -0.12253817170858383, -0.18161886930465698, -0.13059478998184204, -0.17943333089351654, 0.10460011661052704, 0.28456026315689087, -0.10868612676858902, -0.1985178142786026, 0.021847013384103775, -0.01688227616250515, 0.056222379207611084, -0.21411778032779694, -0.0023987656459212303, 0.10031597316265106, -0.2495051771402359, -0.01640840247273445, -0.39359036087989807, -0.2607785165309906, -0.38841670751571655, -0.0007049578707665205, -0.2908412218093872, -0.005378491710871458, -0.039756204932928085, -0.1057548001408577, -0.4521386921405792, 0.0266399048268795, -0.058935798704624176, 0.06899340450763702, -0.27664992213249207, -0.14539951086044312, -0.013602470979094505, 0.044818952679634094, -0.08634887635707855, -0.07179711759090424, -0.18124166131019592, -0.16837561130523682, 0.027943743392825127, 0.22432830929756165, -0.08555498719215393, -0.07339341193437576, 0.10021998733282089, 0.08832015842199326, 0.0545695535838604, -0.27746710181236267, 0.0683901309967041, 0.24533188343048096, -0.20502525568008423, -0.035809025168418884, -0.42975541949272156, -0.24965216219425201, -0.3887017071247101, -0.07379341870546341, -0.1912035197019577, 0.01107564102858305, 0.023121876642107964, -0.13076016306877136, -0.4836050271987915, -0.09731551259756088, -0.04440649598836899, -0.014578228816390038, -0.19488413631916046, -0.15078559517860413, -0.05229834467172623, 0.016014263033866882, 0.00812717154622078, -0.0632503405213356, -0.06932684779167175, -0.22329333424568176, -0.010019098408520222, 0.30654972791671753, -0.13858352601528168, -0.044626329094171524, 0.16341213881969452, 0.07096429169178009, 0.05223045498132706, -0.13837701082229614, 0.07218172401189804, 0.0788440927863121, -0.19500477612018585, -0.1052536591887474, -0.39773109555244446, -0.23859712481498718, -0.3588167130947113, -0.09380835294723511, -0.3194218873977661, 0.0041659255512058735, 0.1022448018193245, 0.06279735267162323, -0.44317761063575745, -0.04252007231116295, -0.03500863164663315, 0.01755852811038494, -0.2750670611858368, -0.14436285197734833, -0.04926470294594765, 0.06132365018129349, -0.06782897561788559, -0.2215052992105484, -0.06660903245210648, -0.20059140026569366, 0.06243521720170975, 0.2530500590801239, -0.10804545134305954, -0.11089000105857849, 0.08398405462503433, 0.03405158594250679, -0.00017384793318342417, -0.1383933573961258, 0.026797058060765266, 0.19572806358337402, -0.33871471881866455, -0.10614679008722305, -0.38007181882858276, -0.23514212667942047, -0.37070736289024353, -0.07009285688400269, -0.20040661096572876, 0.06017504632472992, -0.028691260144114494, 0.03139419108629227, -0.42584583163261414, -0.13051432371139526, 0.032397035509347916, -0.015342050231993198, -0.23292036354541779, -0.10118993371725082, -0.03990961238741875, 0.03780956566333771, -0.09010529518127441, -0.1587204784154892, -0.07718037813901901, -0.18563729524612427, -0.016463253647089005, 0.07025502622127533, -0.0883883461356163, -0.21208538115024567, 0.10479264706373215, 0.12399357557296753, 0.03361823782324791, -0.19533614814281464, -0.014522658661007881, 0.12476354837417603, -0.2815271019935608, -0.1156817302107811, -0.405537873506546, -0.18432264029979706, -0.3488771617412567, 0.003379866713657975, -0.25205790996551514, 0.05487905815243721, 0.021107809618115425, -0.012189322151243687, -0.4343516528606415, -0.08983111381530762, -0.04285779222846031, -0.017581580206751823, -0.2062467634677887, -0.10430026054382324, -0.014322600327432156, 0.12971515953540802, 0.002579278079792857, -0.14163556694984436, -0.10590546578168869, -0.0697675570845604, 0.031016208231449127, 0.28065839409828186, -0.10158543288707733, -0.1879890263080597, 0.025845127180218697, 0.08780279755592346, 0.039515428245067596, -0.18786422908306122, -0.06411296874284744, 0.059333786368370056, -0.27045753598213196, -0.15929865837097168, -0.31439730525016785, -0.18609750270843506, -0.20664779841899872, -0.0005178304272703826, -0.26919785141944885, 0.007687421515583992, -0.14295946061611176, -0.06507558375597, -0.48453211784362793, -0.05342598259449005, 0.02141302451491356, 0.05083359777927399, -0.27852392196655273, -0.05725287273526192, 0.07261250913143158, 0.003220941871404648, -0.07467880100011826, -0.23775404691696167, -0.00500262388959527, -0.12258398532867432, 0.04898172616958618, 0.19338154792785645, -0.08438289165496826, -0.12900562584400177, 0.09662458300590515, 0.058047983795404434, 0.04592159762978554, -0.1956063061952591, 0.03719974681735039, 0.14348642528057098, -0.26871880888938904, -0.09208492934703827, -0.3813724219799042, -0.20303654670715332, -0.3616727292537689, -0.029689183458685875, -0.2591755986213684, -0.022984422743320465, 0.07493216544389725, -0.0882374495267868, -0.44172537326812744, -0.08080870658159256, -0.09196865558624268, -0.02581796608865261, -0.31491032242774963, -0.13133427500724792, -0.05887103080749512, 0.09346956759691238, -0.06378904730081558, -0.14904293417930603, -0.10746106505393982, -0.1721610128879547, -0.026190025731921196, 0.25198328495025635, -0.06393784284591675, -0.1150200217962265, 0.06972771883010864, 0.013120565563440323, 0.028305605053901672, -0.1586170345544815, -0.018446750938892365, 0.09295066446065903, -0.22693584859371185, 0.0375942625105381, -0.3744661509990692, -0.22689376771450043, -0.32922986149787903, -0.17275694012641907, -0.23336224257946014, 0.01594473607838154, 0.09886134415864944, -0.048030417412519455, -0.5145602226257324, -0.11274131387472153, -0.0008389309514313936, -0.0065791672095656395, -0.2918384373188019, -0.15800637006759644, -0.01103016547858715, 0.09524843096733093, 0.04664483293890953, -0.13300171494483948, -0.07233894616365433, -0.12823913991451263, 0.0623229518532753, 0.33650124073028564, -0.03889733552932739, -0.12316683679819107, 0.10550560802221298, 0.07641195505857468, 0.03503091633319855, -0.1658482551574707, -0.04465021938085556, 0.026234883815050125, -0.2876489460468292, -0.18779398500919342, -0.3559451103210449, -0.24350392818450928, -0.4318782389163971, -0.010025294497609138, -0.19621245563030243, 0.05606326088309288, 0.11301025748252869, 0.05825215205550194, -0.535983145236969, -0.04998109117150307, 0.012875409796833992, 0.04233210161328316, -0.24248291552066803, -0.1262567639350891, -0.029386114329099655, 0.015131181105971336, -0.08955889940261841, -0.17325304448604584, -0.15852685272693634, -0.23971380293369293, 0.04024393483996391, 0.24976098537445068, -0.07388238608837128, -0.21173667907714844, 0.053390566259622574, 0.03968275710940361, 0.021550696343183517, -0.20517852902412415, 0.04113195464015007, 0.12850287556648254, -0.2532142996788025, -0.08581534773111343, -0.3239437937736511, -0.26731178164482117, -0.2919798195362091, -0.02143077366054058, -0.23185351490974426, 0.05335749685764313, 0.05564213916659355, 0.015438452363014221, -0.42840704321861267, -0.08302333950996399, 0.034668952226638794, 0.027334021404385567, -0.22658078372478485, -0.09162870049476624, -0.10352445393800735, 0.12335891276597977, -0.04642122611403465, -0.22688138484954834, -0.09584682434797287, -0.23161911964416504, 0.11637798696756363, 0.2781933546066284, -0.1714656949043274, -0.08996543288230896, 0.07264623045921326, 0.16329306364059448, -0.03271087259054184, -0.2882157862186432, -0.0022353194653987885, 0.11401531100273132, -0.2748942971229553, -0.19355422258377075, -0.33800259232521057, -0.1600634902715683, -0.4435519576072693, 0.01412748172879219, -0.26799121499061584, 0.16872093081474304, 0.051427338272333145, -0.04472334310412407, -0.4204143285751343, -0.07551492750644684, -0.0387897826731205, 0.05692695826292038, -0.3147400915622711, -0.17972277104854584, 0.10181211680173874, 0.03898563235998154, -0.0822644829750061, -0.15139135718345642, -0.16391722857952118, -0.2551063597202301, -0.013614815659821033, 0.2273230254650116, -0.08943566679954529, -0.06114577502012253, 0.06888188421726227, 0.0920938104391098, 0.05000120773911476, -0.23006808757781982, 0.0058940076269209385, 0.08428330719470978, -0.2008700668811798, -0.06373776495456696, -0.3585178554058075, -0.3107157349586487, -0.3422999978065491, -0.08278507739305496, -0.20425723493099213, 0.045304346829652786, -0.010990286245942116, 0.010944769717752934, -0.4396066665649414, -0.13148120045661926, 0.04893440753221512, 0.006496377754956484, -0.3046628534793854, -0.08410832285881042, -0.037399180233478546, 0.034382790327072144, -0.059837743639945984, -0.09181331098079681, -0.010953152552247047, -0.20037095248699188, 0.0918894037604332, 0.25114476680755615, -0.07400109618902206, -0.07309477031230927, 0.10718562453985214, 0.18821300566196442, 0.03209127113223076, -0.23574377596378326, 0.05974477156996727, 0.11081350594758987, -0.269279420375824, -0.23607590794563293, -0.3092937767505646, -0.20280829071998596, -0.3081485331058502, 0.004486592020839453, -0.1550053507089615, 0.048970576375722885, 0.07650776952505112, -0.07252311706542969, -0.49337348341941833, -0.04897833988070488, 0.010389207862317562, -0.045254647731781006, -0.2711388170719147, -0.0994439497590065, -0.04070080816745758, 0.07817913591861725, -0.07203906774520874, -0.24685195088386536, -0.14149340987205505, -0.09896893799304962, -0.06437892466783524, 0.26444920897483826, -0.06252733618021011, -0.08531420677900314, 0.031309645622968674, 0.051376983523368835, 0.026966292411088943, -0.1358538568019867, -0.03777682036161423, 0.07324295490980148, -0.20393630862236023, -0.03123457543551922, -0.3408394455909729, -0.2741701006889343, -0.3277614414691925, -0.03896816819906235, -0.19767393171787262, -0.011184977367520332, 0.035439979285001755, -0.04328431561589241, -0.5311676263809204, -0.091033935546875, -0.007421092130243778, 0.0008981779683381319, -0.2694038450717926, -0.16123822331428528, -0.0024495830293744802, 0.07803549617528915, -0.03761288523674011, -0.1145060732960701, -0.1459912657737732, -0.2628214657306671, 0.02332090586423874, 0.27606675028800964, 0.012961060740053654, -0.08644303679466248, 0.04888581484556198, 0.1077553778886795, -0.022357551380991936, -0.25015610456466675, 0.12492626905441284, 0.1477079540491104, -0.21831747889518738, -0.058314744383096695, -0.2687392234802246, -0.22952541708946228, -0.28721460700035095, -0.09579789638519287, -0.3091634213924408, 0.12259852141141891, 0.10336992889642715, -0.009794834069907665, -0.46438008546829224, -0.09350752085447311, -0.04357505589723587, 0.09834800660610199, -0.25626125931739807, -0.06500310450792313, 0.019038770347833633, 0.10886042565107346, -0.02656351402401924, -0.131059467792511, -0.14438217878341675, -0.1619257926940918, 0.12072759121656418, 0.22738295793533325, -0.0994715690612793, -0.07267647236585617, 0.14044839143753052, 0.09867524355649948, -0.06095605716109276, -0.15077780187129974, -0.06471250206232071, 0.09018158167600632, -0.22666232287883759, -0.10128346085548401, -0.3188619017601013, -0.0863238126039505, -0.30924052000045776, -0.008291070349514484, -0.2843417227268219, 0.01912245713174343, -0.02355615235865116, -0.061960916966199875, -0.530271053314209, -0.13480085134506226, -0.019603075459599495, -0.02861258201301098, -0.36016014218330383, -0.15837155282497406, 0.02292856201529503, 0.0594521127641201, -0.0005167631898075342, -0.11610549688339233, -0.10296758264303207, -0.15392525494098663, -0.06065850704908371, 0.3674602806568146, -0.03325991332530975, -0.12802532315254211, 0.034423258155584335, 0.08645632117986679, -0.07357972115278244, -0.2148294299840927, 0.03190910071134567, 0.13518913090229034, -0.27861323952674866, -0.03895527869462967, -0.4112747609615326, -0.22949571907520294, -0.3729937970638275, -0.08680222928524017, -0.21638557314872742, 0.001626698998734355, 0.124727264046669, -0.064566969871521, -0.3761924207210541, -0.15651488304138184, 0.02113058790564537, -0.038562674075365067, -0.35626116394996643, -0.08231310546398163, 0.03411044925451279, -0.007248361129313707, 0.007393573876470327, -0.2082074135541916, -0.07937450706958771, -0.16843849420547485, 0.0411016047000885, 0.1634185016155243, -0.012101998552680016, -0.08918400853872299, 0.07130355387926102, 0.016246195882558823, -0.0647796168923378, -0.2439354807138443, 0.02171814627945423, 0.11436301469802856, -0.310640811920166, -0.11283900588750839, -0.2811374366283417, -0.26026448607444763, -0.37732192873954773, 0.0230966005474329, -0.21646077930927277, 0.0534847117960453, -0.0007621530094183981, -0.11295166611671448, -0.5342105627059937, -0.12583942711353302, -0.020148267969489098, 0.039228226989507675, -0.2616170644760132, -0.07915230095386505, 0.011252932250499725, 0.006088551599532366, -0.15955153107643127, -0.19437125325202942, -0.13364310562610626, -0.22005878388881683, -0.0023601793218404055, 0.19112876057624817, -0.14855565130710602, -0.1222895160317421, 0.09837175905704498, 0.03465397655963898, 0.044838208705186844, -0.18683704733848572, -0.05652443319559097, 0.13345320522785187, -0.1599443256855011, -0.12023856490850449, -0.3010360598564148, -0.1748419553041458, -0.2731914222240448, -0.08556510508060455, -0.1387866586446762, 0.05708814412355423, 0.01693749614059925, -0.04684852063655853, -0.44560474157333374, -0.09925997257232666, -0.01801181770861149, 0.08559555560350418, -0.32534343004226685, -0.1424555778503418, 0.03988775983452797, 0.09164465218782425, 0.002885097172111273, -0.22193744778633118, -0.03186606615781784, -0.16332636773586273, 0.09949629008769989, 0.1908947378396988, -0.08211810886859894, -0.13115203380584717, 0.013094933703541756, 0.031709492206573486, 0.07627716660499573, -0.19931234419345856, -0.04165865480899811, 0.17482969164848328, -0.20462118089199066, -0.03715306147933006, -0.3988277316093445, -0.2466311901807785, -0.29656654596328735, -0.07723540812730789, -0.2770219147205353, 0.09268227219581604, 0.10774437338113785, -0.09293486177921295, -0.6055618524551392, -0.1743321716785431, -0.05978960543870926, 0.014532276429235935, -0.25841283798217773, -0.09762343764305115, -0.0347844697535038, 0.06413327157497406, -0.09990736842155457, -0.17067347466945648, -0.11801405996084213, -0.053410280495882034, 0.052206866443157196, 0.21155838668346405, -0.14287683367729187, -0.13918256759643555, 0.010339473374187946, 0.030929166823625565, -0.016987400129437447, -0.13166803121566772, -0.026858793571591377, 0.091889388859272, -0.3210883140563965, -0.14532418549060822, -0.36266085505485535, -0.25448721647262573, -0.2913869321346283, -0.07589023560285568, -0.18659614026546478, 0.06412698328495026, 0.02939515747129917, -0.13574634492397308, -0.54018634557724, -0.06474490463733673, 0.060724008828401566, 0.03173855319619179, -0.25107893347740173, -0.11068448424339294, -0.018335038796067238, 0.023316655308008194, -0.0030788411386311054, -0.15118759870529175, -0.09764410555362701, -0.2038470059633255, 0.0391927994787693, 0.2694648802280426, -0.13463222980499268, -0.031474366784095764, 0.10543467849493027, 0.06797784566879272, 0.09378823637962341, -0.12466346472501755, -0.009927120991051197, 0.1582159399986267, -0.3075639307498932, -0.1185344010591507, -0.47235962748527527, -0.1827123910188675, -0.4430200755596161, -0.010369941592216492, -0.2810892164707184, 0.002322097308933735, 0.01987539790570736, -0.02201051637530327, -0.4450781047344208, -0.013234762474894524, 0.008763719350099564, 0.06547588109970093, -0.31912973523139954, -0.054612308740615845, 0.007895944640040398, 0.0333055779337883, -0.05661274492740631, -0.17143021523952484, -0.0763796791434288, -0.16735787689685822, -0.014076999388635159, 0.21627044677734375, -0.055656548589468, -0.2021012157201767, 0.06167289614677429, 0.14508476853370667, -0.11543592810630798, -0.21051691472530365, 0.028857190161943436, 0.04890061914920807, -0.12415789812803268, -0.21591243147850037, -0.2875150740146637, -0.16586367785930634, -0.3022818863391876, -0.05551298335194588, -0.22169645130634308, 0.006408176384866238, 0.07009317725896835, -0.09132436662912369, -0.48937711119651794, -0.0520038828253746, -0.06488212943077087, 0.003202683525159955, -0.2809920608997345, -0.15738779306411743, 0.058451130986213684, 0.07350349426269531, -0.1275932788848877, -0.17067591845989227, -0.12470240145921707, -0.22030533850193024, 0.015204759314656258, 0.2178245633840561, -0.04480454698204994, -0.15250658988952637, 0.14166872203350067, 0.06526591628789902, 0.06271239370107651, -0.20293104648590088, -0.030792975798249245, 0.07379341125488281, -0.22516606748104095, -0.03960249572992325, -0.2533836364746094, -0.17920437455177307, -0.3356104791164398, -0.05498223006725311, -0.22012019157409668, 0.04988378658890724, 0.10280076414346695, 0.011866074055433273, -0.5175215005874634, -0.13619787991046906, 0.059197112917900085, 0.03682011738419533, -0.30079326033592224, -0.03098156675696373, -0.019147198647260666, 0.04247600957751274, -0.07487235963344574, -0.26641109585762024, -0.04272279143333435, -0.2478816658258438, -0.017067115753889084, 0.21611551940441132, -0.056680597364902496, -0.12070279568433762, 0.09335405379533768, -0.016519099473953247, 0.015552103519439697, -0.24968096613883972, 0.009831070899963379, 0.14975282549858093, -0.2549348771572113, 0.015664925798773766, -0.33295661211013794, -0.21393953263759613, -0.34098395705223083, -0.05407160520553589, -0.1912594437599182, 0.10634315013885498, -0.03543107584118843, 0.017393356189131737, -0.44999226927757263, -0.05378134176135063, 0.01414195541292429, -0.04079737514257431, -0.34785109758377075, 0.0009311511530540884, 0.01285369973629713, -0.014397988095879555, -0.034407179802656174, -0.18643364310264587, -0.17810633778572083, -0.3019368350505829, -0.06876581907272339, 0.2668987512588501, -0.09543374180793762, -0.12973162531852722, 0.10681713372468948, 0.09751041233539581, -0.08520141243934631, -0.1909218728542328, -0.04692451283335686, 0.06975693255662918, -0.2814250588417053, -0.0926358625292778, -0.37266087532043457, -0.3164641261100769, -0.3722569942474365, 0.05635794252157211, -0.2922830879688263, -0.03308397904038429, 0.04338901862502098, -0.0054997168481349945, -0.452481210231781, -0.15780222415924072, 0.04073776304721832, -0.017850497737526894, -0.34594911336898804, -0.12072224915027618, -0.005202518776059151, 0.09216935932636261, -0.020622188225388527, -0.16945242881774902, -0.03929036110639572, -0.18335296213626862, 0.056166715919971466, 0.18311567604541779, -0.17173883318901062, -0.07848763465881348, 0.023079106584191322, 0.05237793177366257, 0.015018818899989128, -0.20026186108589172, -0.02820325270295143, 0.20104800164699554, -0.38054507970809937, -0.0422985777258873, -0.24458444118499756, -0.2425515204668045, -0.3490482568740845, -0.008115163072943687, -0.14811263978481293, -0.008131776936352253, 0.03401140868663788, -0.044641949236392975, -0.44771435856819153, -0.09385701268911362, 0.02381567284464836, 0.0003344772267155349, -0.28675517439842224, -0.12381549179553986, -0.03298201784491539, 0.049387793987989426, -0.05891427397727966, -0.22695164382457733, -0.00743272015824914, -0.1941816359758377, -0.05781751871109009, 0.30838584899902344, -0.13044427335262299, -0.0918595939874649, 0.09173238277435303, 0.07998470216989517, 0.02847501076757908, -0.20462270081043243, -0.0034814642276614904, 0.23307128250598907, -0.36594530940055847, -0.08319440484046936, -0.3375939726829529, -0.2963443100452423, -0.41661691665649414, 0.06351209431886673, -0.2581305503845215, 0.15522043406963348, 0.05641510710120201, -0.03152407333254814, -0.4518848955631256, -0.07006607204675674, 0.04981577768921852, 0.04320690408349037, -0.28197476267814636, -0.08148399740457535, 0.06782817095518112, 0.09731319546699524, -0.10191486030817032, -0.20010890066623688, -0.0693981945514679, -0.21857351064682007, -0.007456831634044647, 0.205513134598732, -0.1061122715473175, -0.09609227627515793, 0.14234131574630737, 0.06791543960571289, 0.12366645783185959, -0.17968498170375824, 0.1541423797607422, 0.18149928748607635, -0.2292288839817047, -0.11144393682479858, -0.5005677342414856, -0.2514873147010803, -0.3391886055469513, 0.05168895795941353, -0.3085428476333618, 0.14656499028205872, 0.017064722254872322, 0.0008717191522009671, -0.60267573595047, -0.09486483037471771, -0.026353586465120316, 0.05553758144378662, -0.2846440374851227, -0.14741036295890808, 0.04758188873529434, 0.0911962240934372, -0.02197537012398243, -0.11788169294595718, -0.030887266620993614, -0.18851801753044128, -0.05651535466313362, 0.18406100571155548, -0.025542566552758217, -0.08175128698348999, 0.09768469631671906, 0.04572147876024246, 0.02150181122124195, -0.17462460696697235, -0.022366713732481003, 0.09646543115377426, -0.14584916830062866, -0.0750642642378807, -0.38963329792022705, -0.3088338375091553, -0.3843841850757599, -0.008447378873825073, -0.2016361802816391, 0.040352992713451385, 0.032212477177381516, -0.05058375000953674, -0.5310752391815186, -0.036732036620378494, 0.009464781731367111, 0.1452338695526123, -0.3270137906074524, -0.09717504680156708, 0.04438101872801781, 0.044210050255060196, -0.12119852751493454, -0.21577094495296478, -0.08089813590049744, -0.17025907337665558, 0.006637252867221832, 0.30348336696624756, -0.04339185357093811, -0.10113470256328583, 0.09392782300710678, 0.05480927228927612, 0.04747521132230759, -0.1298830211162567, -0.0024985377676784992, 0.153916597366333, -0.270280659198761, -0.15706974267959595, -0.40939077734947205, -0.20544244349002838, -0.32574936747550964, 0.03507697954773903, -0.23675043880939484, 0.09450935572385788, 0.09577778726816177, 0.014391383156180382, -0.5061107873916626, -0.11104463785886765, -0.0026783468201756477, -0.0278773233294487, -0.265490859746933, -0.03909895196557045, -0.023810239508748055, 0.1550924926996231, -0.06634997576475143, -0.17847536504268646, -0.09784958511590958, -0.22284124791622162, 0.03226042911410332, 0.25431039929389954, -0.009961536154150963, -0.08690816164016724, 0.11946770548820496, 0.056376997381448746, -0.050787679851055145, -0.1803394854068756, -0.048043373972177505, 0.1790088415145874, -0.2555871605873108, -0.0032051156740635633, -0.3225775361061096, -0.2019539475440979, -0.33656665682792664, -0.03400008752942085, -0.2068917155265808, 0.0974041149020195, -0.005901571363210678, -0.010717091150581837, -0.34462904930114746, -0.10630084574222565, -0.021944154053926468, -0.013207385316491127, -0.3616335093975067, -0.030854053795337677, 0.0032807777170091867, 0.07144616544246674, 0.040427934378385544, -0.0428679995238781, -0.13036292791366577, -0.1346995085477829, 0.04255979135632515, 0.2732887268066406, -0.06903119385242462, -0.118350550532341, 0.17214739322662354, 0.09828630834817886, 0.037275444716215134, -0.11339310556650162, 0.03942517191171646, 0.15455634891986847, -0.23758725821971893, 0.0014304360374808311, -0.3337279260158539, -0.24808190762996674, -0.39081692695617676, -0.061493903398513794, -0.2919003963470459, 0.05679355934262276, 0.042618781328201294, -0.13772684335708618, -0.47462761402130127, -0.06922087073326111, -0.0007025949307717383, -0.02618127316236496, -0.22563402354717255, -0.12498524785041809, -0.06105490401387215, 0.0025561146903783083, -0.052413210272789, -0.11905821412801743, -0.02753675915300846, -0.1768241971731186, -0.04820264130830765, 0.22057494521141052, -0.13781872391700745, -0.12610021233558655, 0.003964561969041824, 0.07309139519929886, 0.0522511750459671, -0.1832476556301117, 0.01754085347056389, 0.08454432338476181, -0.32674798369407654, -0.11829306930303574, -0.30144011974334717, -0.19218552112579346, -0.3708975911140442, 0.015797650441527367, -0.2557596266269684, 0.1269042193889618, 0.053794875741004944, -0.05789397656917572, -0.43588632345199585, -0.14052048325538635, -0.055929481983184814, -0.05111166462302208, -0.2762760519981384, -0.09295632690191269, -0.0036231563426554203, 0.03597217798233032, 0.020080454647541046, -0.18950775265693665, -0.12210626155138016, -0.1251218169927597, 0.08243171125650406, 0.2702520489692688, -0.18003152310848236, -0.1472592055797577, 0.12322838604450226, 0.06834006309509277, 0.09762773662805557, -0.24421003460884094, 0.028390269726514816, 0.10006501525640488, -0.1879129409790039, -0.09024810791015625, -0.30781078338623047, -0.27610719203948975, -0.4185744524002075, -0.09995285421609879, -0.26989030838012695, 2.998867057613097e-05, 0.08965000510215759, 0.03161439672112465, -0.4620310962200165, -0.15352609753608704, -0.015085521154105663, -0.012882659211754799, -0.21650156378746033, -0.07450120151042938, 0.0017500064568594098, -0.17615550756454468, 0.16611100733280182, 0.4839506149291992, 0.2580735683441162, 0.5843495726585388, -0.03792309761047363, -0.7597558498382568, 0.28581270575523376, 0.34370648860931396, -0.2775212228298187, -0.1927517205476761, -0.05782795324921608, 0.531302809715271, 0.013091991655528545, -0.39109402894973755, 0.7627526521682739, 0.27402743697166443, 1.0852731466293335, 0.7287554144859314, 1.0198304653167725, 0.14045833051204681, 0.7262187600135803, -0.1629810929298401, -0.10886450856924057, 0.09457530081272125, 1.436312198638916, 0.2948012351989746, 0.022905699908733368, -0.06204439327120781, 0.8798595666885376, 0.27071356773376465, -0.002340150997042656, 0.0063135651871562, -0.09331214427947998, -0.22082781791687012, -0.07630321383476257, -0.22310210764408112, -0.04170187562704086, 0.2652144134044647, -0.06677854061126709, -0.20531411468982697, 0.1770668774843216, 0.06906794756650925, 0.014665588736534119, -0.12252555787563324, -0.0020381591748446226, 0.10122372955083847, -0.1443362832069397, -0.1402120441198349, -0.3642241060733795, -0.20137101411819458, -0.4015817940235138, -0.049535103142261505, -0.22022710740566254, 0.1713530421257019, -0.01501291710883379, -0.03771420568227768, -0.4427480697631836, -0.09776408225297928, -0.06930934637784958, 0.041822560131549835, -0.2838277816772461, -0.06801681220531464, 0.0568058118224144, 0.10712460428476334, -0.03142566233873367, -0.1527981460094452, -0.1408592313528061, -0.1941431760787964, 0.02238359861075878, 0.23325835168361664, -0.08814969658851624, -0.15853965282440186, 0.16929954290390015, 0.17505784332752228, 0.0004466482496354729, -0.15584638714790344, -0.01723674312233925, 0.1945597380399704, -0.23231516778469086, -0.08262558281421661, -0.3705262243747711, -0.23742437362670898, -0.3981180489063263, -0.010657078586518764, -0.27525511384010315, 0.07801476120948792, 0.036108698695898056, 0.01941603235900402, -0.47653305530548096, -0.050662796944379807, -0.05528461933135986, 0.051888346672058105, -0.30041012167930603, -0.0805874764919281, 0.003819983219727874, 0.10327934473752975, -0.15279440581798553, -0.15217337012290955, -0.16066056489944458, -0.1270865947008133, -0.08726964145898819, 0.20312002301216125, -0.17672409117221832, -0.035311855375766754, 0.028133302927017212, 0.07314961403608322, -0.0068656764924526215, -0.2658524215221405, -0.022393178194761276, 0.1645534485578537, -0.26327842473983765, -0.03943825885653496, -0.3774392902851105, -0.24338781833648682, -0.36733946204185486, -0.027255287393927574, -0.14460976421833038, 0.0693826973438263, 0.03934900462627411, -0.06625185906887054, -0.5134574174880981, -0.028612881898880005, -0.06477890908718109, -0.02424098178744316, -0.2940778434276581, -0.11161389946937561, 0.09804195910692215, -0.061846811324357986, -0.033745672553777695, -0.13269193470478058, -0.14196032285690308, -0.21965628862380981, 0.032998017966747284, 0.33478477597236633, -0.13905580341815948, -0.1962347775697708, 0.14120671153068542, 0.14364105463027954, 0.0031723768915981054, -0.28655365109443665, 0.06706444174051285, 0.12149015814065933, -0.19787009060382843, -0.08866634964942932, -0.37812539935112, -0.266326367855072, -0.27442726492881775, 0.027288861572742462, -0.3464478552341461, 0.0044579217210412025, 0.11848088353872299, -0.05369579792022705, -0.5437070727348328, -0.10159306228160858, -0.014585323631763458, 0.01654885895550251, -0.31318342685699463, -0.10580222308635712, -0.02256288193166256, 0.07316011935472488, -0.09245077520608902, -0.18280968070030212, -0.09313933551311493, -0.12214039266109467, 0.030038075521588326, 0.24782410264015198, -0.0662207305431366, -0.09696774929761887, 0.14697496592998505, 0.09277955442667007, 0.10961949080228806, -0.10863346606492996, -0.007578848861157894, 0.11069119721651077, -0.2099372148513794, -0.06066080555319786, -0.3640715181827545, -0.3175783157348633, -0.43971821665763855, -0.02702077478170395, -0.2010132521390915, 0.09144244343042374, -0.03473164513707161, 0.01653606817126274, -0.5023912787437439, -0.0823952704668045, 0.011667585000395775, -0.018708210438489914, -0.3187715411186218, -0.19395101070404053, -0.04560451954603195, -0.03366231545805931, -0.029945923015475273, -0.16525661945343018, -0.04023303464055061, -0.16960032284259796, -0.0203703660517931, 0.27061915397644043, -0.11255138367414474, -0.0868205726146698, 0.10835187882184982, 0.01802522875368595, 0.020033910870552063, -0.1730976104736328, -0.02091432549059391, 0.1761620044708252, -0.1684124618768692, -0.10487894713878632, -0.3768779933452606, -0.35613390803337097, -0.3545454740524292, -0.06624283641576767, -0.3476143181324005, 0.06569535285234451, 0.05917760357260704, -0.07203646004199982, -0.494314044713974, -0.15014053881168365, 0.028960121795535088, -0.054456088691949844, -0.30486834049224854, 0.007255232892930508, 0.002811903366819024, 0.07986138015985489, -0.05914648622274399, -0.22704875469207764, -0.0875692069530487, -0.16710375249385834, -0.039764564484357834, 0.2703897953033447, -0.03078462742269039, -0.06378918886184692, 0.013445965945720673, 0.01868990994989872, -0.0607600100338459, -0.12230655550956726, 0.024987688288092613, 0.16657370328903198, -0.29798251390457153, -0.14482556283473969, -0.35549768805503845, -0.23868156969547272, -0.3908121883869171, -0.03732490912079811, -0.2073269635438919, 0.06267146021127701, -0.022342035546898842, -0.14096423983573914, -0.4886274039745331, -0.09359860420227051, -0.022171173244714737, -1.3955080930827535e-06, -0.32377755641937256, -0.05451647564768791, -0.002386227250099182, 0.06262581795454025, -0.10038995742797852, -0.1993485391139984, -0.07774574309587479, -0.23769833147525787, -0.05159743130207062, 0.2368515580892563, -0.03298158943653107, -0.1077260747551918, 0.1655632108449936, 0.060877759009599686, 0.00442068837583065, -0.24547618627548218, -0.03434000536799431, 0.2031758725643158, -0.23768749833106995, -0.04983620345592499, -0.3240881562232971, -0.24420802295207977, -0.3583398163318634, -0.03847452253103256, -0.2059902548789978, 0.07416398078203201, 0.05422555282711983, -0.027623916044831276, -0.5141801238059998, -0.11489611864089966, -0.05623963847756386, 0.07164900749921799, -0.3943074941635132, -0.10119157284498215, 0.11959308385848999, 0.039388060569763184, -0.10855300724506378, -0.16922520101070404, -0.021466821432113647, -0.19544996321201324, 0.04203495755791664, 0.2583067715167999, -0.11167377978563309, 0.040240731090307236, 0.09614076465368271, 0.04059600457549095, -0.004575210157781839, -0.15683558583259583, 0.014932091347873211, 0.17001725733280182, -0.2470078021287918, -0.15807317197322845, -0.3327469527721405, -0.25917890667915344, -0.3624045252799988, -0.11857922375202179, -0.268231600522995, 0.014928760007023811, 0.027803748846054077, -0.10800207406282425, -0.4731506109237671, -0.09559954702854156, -0.06515537947416306, -0.009816707111895084, -0.33208295702934265, -0.044917650520801544, -0.12618069350719452, 0.039298541843891144, -0.13484011590480804, -0.18991103768348694, -0.04359859600663185, -0.21895800530910492, 0.018286949023604393, 0.21107037365436554, 0.003944001160562038, -0.0716138407588005, 0.14894279837608337, 0.0010448184330016375, -0.035129986703395844, -0.12187644094228745, -0.015247187577188015, 0.1185205727815628, -0.23737584054470062, -0.14925110340118408, -0.3206997215747833, -0.2428145557641983, -0.32413750886917114, -0.0779644027352333, -0.213893860578537, 0.016026590019464493, 0.07734597474336624, 0.021151185035705566, -0.43008914589881897, -0.07599316537380219, 0.022523829713463783, -0.11388026177883148, -0.24746108055114746, -0.10400036722421646, 0.004979756195098162, 0.042279478162527084, -0.12137062847614288, -0.19525912404060364, -0.053008899092674255, -0.11655896157026291, -0.019778594374656677, 0.20811054110527039, -0.020442014560103416, -0.1614021211862564, 0.19626502692699432, 0.06998632103204727, -0.024246495217084885, -0.12664742767810822, 0.07913866639137268, 0.0596792995929718, -0.17611689865589142, -0.10056252032518387, -0.28502658009529114, -0.262753963470459, -0.42695915699005127, -0.019335348159074783, -0.19322645664215088, 0.156068816781044, 0.11580682545900345, -0.0005690217949450016, -0.48756879568099976, -0.12784892320632935, -0.023910630494356155, -0.020130500197410583, -0.30293288826942444, -0.21752475202083588, 0.00917590968310833, 0.05501803010702133, 0.0764799416065216, -0.13039040565490723, -0.10994665324687958, -0.15919071435928345, -0.049920786172151566, 0.22317451238632202, -0.1864098608493805, -0.04546068236231804, 0.12225022912025452, 0.008674676530063152, 0.016899576410651207, -0.12837590277194977, 0.019226649776101112, 0.15053053200244904, -0.2969571053981781, -0.11835451424121857, -0.3545854389667511, -0.19771796464920044, -0.3895428478717804, -0.10050370544195175, -0.18235386908054352, 0.004755054134875536, 0.08348502963781357, -0.023178381845355034, -0.43163105845451355, -0.014597336761653423, 0.025266192853450775, 0.06021770462393761, -0.3350970447063446, -0.125057652592659, 0.032429832965135574, 0.13317783176898956, 0.017791323363780975, -0.05249419063329697, -0.07039110362529755, -0.1772882491350174, 0.061677612364292145, 0.2542547881603241, -0.09615354984998703, -0.16964191198349, 0.14214423298835754, 0.013854001648724079, 0.045536961406469345, -0.2468230277299881, 0.049288053065538406, 0.07997629046440125, -0.21189089119434357, -0.16595903038978577, -0.3813338577747345, -0.1848134696483612, -0.38836678862571716, -0.041074153035879135, -0.2634769082069397, -0.06420084089040756, 0.0450298972427845, -0.013718199916183949, -0.5310720205307007, -0.07313203066587448, -0.04895173758268356, -0.02875373139977455, -0.2888626158237457, -0.12436912208795547, 0.11485418677330017, -0.021281294524669647, -0.01489177905023098, -0.11888209730386734, -0.1848991960287094, -0.27405163645744324, 0.03948706388473511, 0.2292335331439972, -0.07385198771953583, -0.06030130758881569, 0.0739758163690567, 0.014255605638027191, -0.1165480986237526, -0.16563433408737183, -0.0519595593214035, 0.14708594977855682, -0.24248388409614563, -0.15571564435958862, -0.2731492519378662, -0.18956783413887024, -0.30568593740463257, -0.10365734994411469, -0.2599268853664398, 0.023924194276332855, 0.008888917043805122, -0.0915365144610405, -0.45395106077194214, -0.09035467356443405, 0.00910162553191185, -0.0038813776336610317, -0.22465437650680542, -0.0809113010764122, -0.13665850460529327, 0.024459095671772957, -0.05466387793421745, -0.12006116658449173, -0.10097158700227737, -0.19993171095848083, -0.021773293614387512, 0.28658679127693176, -0.1154094859957695, -0.1474461704492569, 0.05547543242573738, 0.09528296440839767, -0.006724776700139046, -0.25965920090675354, 0.025604059919714928, 0.09634987264871597, -0.23621487617492676, -0.06780300289392471, -0.29916271567344666, -0.19150422513484955, -0.37685099244117737, -0.05275293067097664, -0.3040756583213806, 0.13513730466365814, 0.05170830339193344, 0.034389812499284744, -0.49163323640823364, -0.20427356660366058, 0.0674186497926712, 0.02607199177145958, -0.2825460433959961, -0.08752521872520447, 0.03246621415019035, 0.08312144875526428, -0.1313406080007553, -0.11398278921842575, -0.11238332837820053, -0.2107798457145691, 0.09773895889520645, 0.2993370592594147, -0.10993940383195877, -0.21009327471256256, 0.10066509246826172, 0.09438668191432953, 0.015897180885076523, -0.23816627264022827, -0.04506317526102066, 0.06865562498569489, -0.24329517781734467, -0.045521412044763565, -0.30807310342788696, -0.28034213185310364, -0.3363482356071472, -0.016730213537812233, -0.26511281728744507, 0.04384529963135719, 0.13582025468349457, -0.025074973702430725, -0.4315354824066162, -0.12355805933475494, -0.0070791118778288364, -0.06430033594369888, -0.2904587686061859, -0.03068447671830654, 0.10517743229866028, 0.11629683524370193, -0.017470981925725937, -0.16567924618721008, -0.1159340962767601, -0.19544100761413574, 0.042009320110082626, 0.2140519618988037, -0.0463738776743412, -0.1664794683456421, 0.003621034324169159, 0.016848938539624214, -0.06334507465362549, -0.24448110163211823, 0.051077891141176224, 0.12140908092260361, -0.2431359887123108, -0.028586579486727715, -0.2963879704475403, -0.26896053552627563, -0.28167128562927246, -0.02520381473004818, -0.23938903212547302, 0.07337561994791031, -0.04165438935160637, -0.07463618367910385, -0.4810905456542969, -0.0713309645652771, -0.027768535539507866, 0.015050173737108707, -0.2407335638999939, -0.01971329376101494], "scores": [[-0.9694600783213314, -0.9752350769014719, -0.973758606385061, -0.966433897194902, -0.9734132196262224, -0.9659014678632705, -0.9686524463000995, -0.9669882938189318, -0.9711593935656206, -0.9659494805834077, -0.9733545479202754, -0.9700626343538978, -0.9720618181756813, -0.965633132187245, -0.9632746621569854, -0.9779176000293075, -0.9768870964106867, -0.9663532918801406, -0.9568715846779827, -0.970036526517887, 0.9985416425392378, -0.953014913879445, -0.9692310226185358, -0.9731955438080574, -0.9741229112928698, -0.9640142376101226, -0.9722142829786493, -0.9622092195616353, -0.972610201227328, -0.9630844469985969, -0.9730115811721509, -0.9623584745215201, -0.9511971638060007, -0.9709920411785178, -0.9562070605689867, -0.9725104847968804, -0.9792861693355663, -0.963894374672333, -0.9600769919889249, -0.9619309527586393], [-0.9451211077581689, -0.9624914406232582, -0.928466839645338, -0.9584819294009858, -0.9431688799500514, -0.9630497131120129, -0.9449949140364544, -0.9619267458340851, -0.9684092459145898, -0.9769459078666201, -0.9668266188837207, -0.9697750298257446, -0.9633083171605605, -0.9551538623336018, -0.9680244296374251, -0.9509572002975658, -0.967667452869065, -0.9673899467178897, -0.9709678328587845, -0.9734796421599841, -0.9776098248793852, 0.9983989460907801, -0.9702580770268087, -0.9639109054908185, -0.956084754733136, -0.960224942235697, -0.9723209551288166, -0.9597877652244731, -0.9572348641178676, -0.9605301488044723, -0.9765557616484043, -0.9654048791940326, -0.9415620910109193, -0.9681822565203615, -0.969526037375213, -0.9642371850925143, -0.9803193364876406, -0.9543638803708816, -0.9732130503747424, -0.9576702818899628], [-0.9697407023258322, -0.9558892615252941, -0.9574164524036887, -0.960595090237812, -0.9697689444666063, -0.9619511064781203, -0.9624800312578798, -0.9581831284691824, -0.96537683368794, -0.9496645594351372, -0.9708727779731668, -0.9628236397298706, -0.9669042883056691, -0.9585915205987804, -0.9681327992263484, -0.9624453006115443, -0.9651770900349347, -0.9773293324508712, -0.9628865060798997, -0.9614761463664547, -0.9671587003269966, -0.9628376376333152, 0.9986904404283573, -0.9649126843306594, -0.978835332295634, -0.9616748640937511, -0.9624178719263404, -0.9691334178728641, -0.9679234094530359, -0.9737758951016443, -0.9814342315139092, -0.9644800018631186, -0.9542614346294989, -0.9543213793366342, -0.9590872175249663, -0.963177084416209, -0.952439643007878, -0.9646595539138472, -0.9664968905571389, -0.9626895310570158]], "kept": [[0, 1, 2, 3, 20, 21, 32, 37, 38, 39], [0, 1, 2, 3, 4, 21, 32, 37, 38, 39], [0, 1, 2, 3, 9, 22, 36, 37, 38, 39]]}], "rope": {"head_dim": 16, "rope_theta": 1000000.0, "abs_pos": 137, "rank": 5, "pre_rope": [0.08249430428370294, -0.46441841495421887, 0.05051506297463688, 0.6862308196812632, -1.7567905055789348, 1.6844316011395088, -0.4578428392637714, -0.5964200946055478, -1.046967562282426, 0.9317920227947954, 0.6749804835796053, 1.2444412018021018, 0.893087422223549, 0.26300494250388173, 0.3285178485491658, 0.9352436940748697], "post_rope_at_abs": [-0.9592153201489803, 0.3155531917495639, 0.6080730435707898, -0.3742057991623469, -1.862300270732207, 1.6775249421526817, -0.45926178276742863, -0.5971404364025743, -0.4276123902582183, 0.9921426414932828, -0.2973173363314788, 1.3709546536980524, 0.6447912265663527, 0.30395968014610286, 0.32653124991303284, 0.9347839299907986], "post_rope_at_rank": [-0.9805620956161258, -1.0161853371605156, -0.056394719209451605, 0.6509741554351701, -1.761233964248518, 1.6841970872033156, -0.45789477677319124, -0.5964463906787036, -0.37609089772522947, 0.2264689789630685, 0.6745146851225527, 1.2632415808460964, 0.8842923427258904, 0.26450253336168084, 0.32844545313387075, 0.9352269241218659]}} \ No newline at end of file diff --git a/experiments/niah/gen_kv_compress_fixtures.py b/experiments/niah/gen_kv_compress_fixtures.py new file mode 100644 index 000000000..24c2965c5 --- /dev/null +++ b/experiments/niah/gen_kv_compress_fixtures.py @@ -0,0 +1,136 @@ +"""Generate cross-check fixtures for the C++ KeyDiff KV-compression port. + +Produces a JSON fixture file consumed by cactus-engine/tests/test_kv_compress.cpp. +Each fixture holds a random PRE-RoPE key matrix (n_kv_heads, n, head_dim) plus the +reference outputs from the Python source of truth so the C++ port can be checked +bit-for-bit: + + * keydiff_score (diagnostic_separability.py) -> per-head [n] scores + * build_geom_keepset (geom_keepset.py) -> per-head kept index lists + +A separate "rope" fixture provides a single pre-RoPE vector + the rotate_half RoPE +applied at a chosen rank position, mirroring the cache's post-RoPE storage convention +(`out[i] = x[i]*cos - x[i+half]*sin`, `out[i+half] = x[i+half]*cos + x[i]*sin`, +`angle = pos * theta^(-2i/head_dim)`), so the C++ Route-B renumber math can be checked. + +The generated fixture JSON (~360 KB) is checked into the repo, so the C++ test runs +with no Python step. Regenerating it requires the untracked MLX KeyDiff harness +(diagnostic_separability.py / geom_keepset.py) to be importable on the path. + +Run: python experiments/niah/gen_kv_compress_fixtures.py +Writes: experiments/niah/fixtures/kv_compress_fixtures.json +""" +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np + +from diagnostic_separability import keydiff_score +from geom_keepset import build_geom_keepset + +OUT = Path(__file__).parent / "fixtures" / "kv_compress_fixtures.json" + +# (name, n, head_dim, budget_frac, recent_frac, sink, seed) +# Kept small (n<=48, head_dim<=32) so the fixture covers the budget/recent/sink edge cases, +# the reserved-overflow fallback, and the planted-outlier geometry pick the tests assert on. +GRID = [ + ("base_keydiff", 48, 16, 0.25, 0.30, 4, 0), + ("small_n_keydiff", 20, 8, 0.25, 0.30, 4, 2), + ("tight_budget", 40, 8, 0.10, 0.30, 4, 3), + ("reserved_overflow", 30, 8, 0.20, 0.80, 8, 4), + ("full_budget", 40, 16, 1.00, 0.30, 4, 5), + ("big_dim_keydiff", 40, 32, 0.25, 0.30, 4, 7), +] + + +def _outlier_keys(n_kv_heads, n, d, rng): + """Tight cluster around one centroid direction, with planted mid-context outliers + so the geometry selection is exercised (not just sink/recent).""" + base = rng.normal(size=(d,)) + base /= np.linalg.norm(base) + keys = base[None, None, :] + 0.05 * rng.normal(size=(n_kv_heads, n, d)) + mid = n // 2 + for h in range(n_kv_heads): + keys[h, mid + h] = -3.0 * base + 0.01 * rng.normal(size=(d,)) + return keys.astype(np.float32) + + +def _budget(budget_frac, n): + # Mirror the live harness: B = round(budget_frac * n), matching build_geom_keepset's + # `budget` argument and keepset_for_head's B = min(budget, n). + return max(1, int(round(budget_frac * n))) + + +def make_fixture(name, n, d, bf, rf, sink, seed): + rng = np.random.default_rng(seed) + keys = _outlier_keys(2 + (seed % 3), n, d, rng) + n_kv_heads = keys.shape[0] + budget = _budget(bf, n) + + scores = [] + for h in range(n_kv_heads): + kh = np.asarray(keys[h], dtype=np.float64) + scores.append([float(x) for x in keydiff_score(kh)]) + + kept = build_geom_keepset(keys, budget=budget, signal="keydiff", sink=sink, + recent_frac=rf) + return { + "name": name, + "n": n, + "head_dim": d, + "n_kv_heads": n_kv_heads, + "budget": budget, + "recent_frac": rf, + "sink": sink, + "keys": keys.reshape(-1).astype(np.float32).tolist(), + "scores": scores, + "kept": kept, + } + + +def _rope_at(vec, pos, theta): + """rotate_half RoPE applied to a single (head_dim,) vector at position `pos`, + matching cactus-kernels/src/norms_rope.cpp.""" + d = vec.shape[0] + half = d // 2 + out = np.empty_like(vec) + for i in range(half): + inv = theta ** (-(2.0 * i) / d) + ang = pos * inv + c, s = np.cos(ang), np.sin(ang) + x1, x2 = vec[i], vec[i + half] + out[i] = x1 * c - x2 * s + out[i + half] = x2 * c + x1 * s + return out + + +def make_rope_fixture(): + rng = np.random.default_rng(99) + d, theta = 16, 1000000.0 + vec = rng.normal(size=(d,)).astype(np.float64) + abs_pos, rank = 137, 5 + post_rope_at_abs = _rope_at(vec, abs_pos, theta) + post_rope_at_rank = _rope_at(vec, rank, theta) + return { + "head_dim": d, + "rope_theta": theta, + "abs_pos": abs_pos, + "rank": rank, + "pre_rope": vec.tolist(), + "post_rope_at_abs": post_rope_at_abs.tolist(), + "post_rope_at_rank": post_rope_at_rank.tolist(), + } + + +def main(): + OUT.parent.mkdir(parents=True, exist_ok=True) + fixtures = [make_fixture(*spec) for spec in GRID] + payload = {"fixtures": fixtures, "rope": make_rope_fixture()} + OUT.write_text(json.dumps(payload)) + print(f"wrote {len(fixtures)} fixtures + rope to {OUT}") + + +if __name__ == "__main__": + main() From 10cbac88f81562041c77d7264486cf30d1208422 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 11:00:28 +0100 Subject: [PATCH 03/55] Precompute RoPE cos/sin tables to fix long-context precision The transpiled graph computed the RoPE angle (position * inv_freq) and ran it through fp16, which cannot represent absolute positions > 2048 exactly. Past ~6k context the angle error reaches several radians and randomises cos/sin, corrupting generation (verified against the MLX reference, which stays correct through 16k). Add an optimize_graph pass that precomputes cos/sin offline in fp64, materialises them as fp16 constant tables, and gathers them by position id, keeping the precision-critical angle off the runtime fp16 path with no new kernels. Gated to the text decoder so it does not touch vision/audio encoder rope. Handles models with multiple rope thetas (e.g. Gemma sliding vs global layers) by resolving each cos/sin node's own inv_freq. Signed-off-by: Noah Cylich --- python/cactus/transpile/optimize_graph.py | 122 ++++++ python/tests/test_precompute_rope_tables.py | 390 ++++++++++++++++++++ 2 files changed, 512 insertions(+) create mode 100644 python/tests/test_precompute_rope_tables.py diff --git a/python/cactus/transpile/optimize_graph.py b/python/cactus/transpile/optimize_graph.py index 430fd0e7d..9a2a495eb 100644 --- a/python/cactus/transpile/optimize_graph.py +++ b/python/cactus/transpile/optimize_graph.py @@ -20,8 +20,10 @@ from cactus.transpile.fusion import match_rope from cactus.transpile.fusion import match_self_attention_block from cactus.transpile.fusion.common import producer +from cactus.transpile.fusion.common import strip_layout_passthrough from cactus.transpile.fusion.common import strip_passthrough from cactus.transpile.fusion.linear import match_linear +from cactus.transpile.fusion.rope import _extract_rope_angle_source, _find_constant_ancestor from cactus.transpile.graph_ir import IRGraph from cactus.transpile.graph_ir import IRNode from cactus.transpile.graph_ir import IRValue @@ -106,6 +108,9 @@ def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig break canonicalize_exported_graph(graph) + if precompute_rope_tables(graph): + canonicalize_exported_graph(graph) + annotate_gold_patterns(graph) _prune_unused_inputs(graph) verify_ir(graph) @@ -361,6 +366,123 @@ def fuse_rope(graph: IRGraph) -> bool: return changed +def precompute_rope_tables(graph: IRGraph) -> bool: + """Replace the runtime ``cos/sin(position * inv_freq)`` rope angle with an + fp64-precomputed fp16 table gathered by position id (the fp16 angle matmul + cannot represent positions > 2048, randomising cos/sin past long context).""" + + changed = False + for node_id in list(graph.order): + node = graph.nodes.get(node_id) + if node is None or node.op not in {"scalar_cos", "scalar_sin"} or len(node.inputs) != 1: + continue + # Only the text decoder's rope. Vision/audio encoders carry cos/sin with + # the same shape but an axial/2D position this scalar table cannot model. + if node.meta.get("component") != "decoder": + continue + + output_id = node.outputs[0] + output_value = graph.values.get(output_id) + if output_value is None or output_value.shape is None: + continue + index_shape = tuple(int(dim) for dim in output_value.shape[:-1]) + + match = _match_rope_angle_table_source(graph, node.inputs[0], index_shape) + if match is None: + continue + inv_freq_value_id, position_value_id = match + + inv_freq = graph.constants[inv_freq_value_id].detach().cpu().to(torch.float64).reshape(-1) + head_dim = int(inv_freq.numel()) * 2 + max_seq = _rope_table_max_seq(graph) + positions = torch.arange(max_seq, dtype=torch.float64).reshape(max_seq, 1) + freqs = positions * inv_freq.reshape(1, -1) + emb = torch.cat((freqs, freqs), dim=-1) + table = (torch.cos(emb) if node.op == "scalar_cos" else torch.sin(emb)).to(torch.float16) + assert table.shape[-1] == head_dim + + table_value_id = _materialize_rope_table_constant(graph, node.id, table) + output_value.dtype = "fp16" + node.op = "embedding" + node.inputs = [table_value_id, position_value_id] + node.attrs = {} + node.kind = "generic" + changed = True + + if changed: + rebuild_graph(graph) + return changed + + +def _match_rope_angle_table_source(graph: IRGraph, value_id: str, index_shape: tuple[int, ...]) -> tuple[str, str] | None: + cat_node = producer(graph, strip_passthrough(graph, value_id)) + if cat_node is None or cat_node.op != "cat" or len(cat_node.inputs) != 2: + return None + if strip_layout_passthrough(graph, cat_node.inputs[0]) != strip_layout_passthrough(graph, cat_node.inputs[1]): + return None + angle_info = _extract_rope_angle_source(graph, cat_node.inputs[0]) + if angle_info is None: + return None + angle_node = angle_info["matmul_node"] + if len(angle_node.inputs) != 2: + return None + + inv_freq_value_id: str | None = None + position_value_id: str | None = None + for input_id in angle_node.inputs: + const_id = _find_constant_ancestor(graph, input_id) + if const_id is not None: + inv_freq_value_id = const_id + continue + position_value_id = _trace_integer_position_source(graph, input_id, index_shape) + + if inv_freq_value_id is None or position_value_id is None: + return None + return inv_freq_value_id, position_value_id + + +def _trace_integer_position_source(graph: IRGraph, value_id: str, index_shape: tuple[int, ...]) -> str | None: + current = value_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + value = graph.values.get(current) + if ( + value is not None + and str(value.dtype) in {"int32", "int64", "i32", "i64"} + and value.shape is not None + and tuple(int(dim) for dim in value.shape) == index_shape + ): + return current + node = producer(graph, current) + if node is None or node.op not in {"precision_cast", "view", "reshape", "unsqueeze", "squeeze", "contiguous"} or len(node.inputs) != 1: + return None + current = node.inputs[0] + return None + + +def _rope_table_max_seq(graph: IRGraph) -> int: + max_seq = _coerce_optional_int(graph.meta.get("max_cache_seq_len")) + if max_seq is None or max_seq <= 0: + max_seq = _coerce_optional_int(graph.meta.get("max_position_embeddings")) + if max_seq is None or max_seq <= 0: + raise ValueError("rope table precompute requires max_cache_seq_len or max_position_embeddings in graph.meta") + return int(max_seq) + + +def _materialize_rope_table_constant(graph: IRGraph, node_id: str, table: torch.Tensor) -> str: + value_id = f"c_rope_table_{node_id}" + graph.constants[value_id] = table + graph.values[value_id] = IRValue( + id=value_id, + shape=tuple(int(dim) for dim in table.shape), + dtype=dtype_to_ir(table.dtype), + producer=None, + users=[], + ) + return value_id + + def fuse_attention(graph: IRGraph) -> bool: changed = False for node_id in list(graph.order): diff --git a/python/tests/test_precompute_rope_tables.py b/python/tests/test_precompute_rope_tables.py new file mode 100644 index 000000000..ae8af1153 --- /dev/null +++ b/python/tests/test_precompute_rope_tables.py @@ -0,0 +1,390 @@ +import json +from pathlib import Path + +import pytest +import torch + +from cactus.transpile.canonicalize.utils import rebuild_graph +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.optimize_graph import precompute_rope_tables + + +WEIGHTS = Path(__file__).resolve().parents[2] / "weights" +GEMMA_BUNDLE = WEIGHTS / "gemma-4-e2b-it" +GEMMA_16K_BUNDLE = WEIGHTS / "gemma-4-e2b-16k" +QWEN_BUNDLE = WEIGHTS / "qwen3-0.6b" + +TEST_MAX_SEQ = 16384 + +# inv_freq = 1 / theta ** (arange(0, dim, 2) / dim), exactly the constant the +# real graph carries. The serialized optimized_ir drops multi-element tensor +# payloads, so we re-materialize the genuine per-theta inv_freq. The (theta, +# head_dim) spec is read from the bundle's config.json at test time (not +# hardcoded) so a model shipped with the wrong rope_theta / head_dim fails here +# instead of being matched against a self-referential reconstruction. +def _gemma_inv_freq_spec(bundle: Path) -> dict[str, tuple[float, int]]: + text = json.loads((bundle / "config.json").read_text())["text_config"] + rope = text["rope_parameters"] + return { + "v_module_backbone_rotary_emb_sliding_attention_inv_freq": ( + float(rope["sliding_attention"]["rope_theta"]), + int(text["head_dim"]), + ), + "v_module_backbone_rotary_emb_full_attention_inv_freq": ( + float(rope["full_attention"]["rope_theta"]), + int(text["global_head_dim"]), + ), + } + + +def _qwen_inv_freq_spec(bundle: Path) -> dict[str, tuple[float, int]]: + cfg = json.loads((bundle / "config.json").read_text()) + return { + "v_module_backbone_rotary_emb_inv_freq": ( + float(cfg["rope_theta"]), + int(cfg["head_dim"]), + ), + } + + +# The IR these tests load lives in transpiled weight bundles under weights/ +# (gitignored, ~9MB), so skip when they are absent (e.g. CI) rather than failing +# collection; run locally after transpiling the bundles. +_BUNDLES_PRESENT = (GEMMA_BUNDLE / "config.json").exists() and (QWEN_BUNDLE / "config.json").exists() +pytestmark = pytest.mark.skipif( + not _BUNDLES_PRESENT, reason="requires transpiled weight bundles under weights/ (gitignored)" +) + +GEMMA_INV_FREQ = _gemma_inv_freq_spec(GEMMA_BUNDLE) if _BUNDLES_PRESENT else {} +GEMMA_16K_INV_FREQ = _gemma_inv_freq_spec(GEMMA_16K_BUNDLE) if (GEMMA_16K_BUNDLE / "config.json").exists() else {} +QWEN_INV_FREQ = _qwen_inv_freq_spec(QWEN_BUNDLE) if _BUNDLES_PRESENT else {} + + +def _inv_freq(theta: float, head_dim: int) -> torch.Tensor: + return 1.0 / (theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim)) + + +def test_gemma_inv_freq_spec_matches_config() -> None: + """Guard the reconstruction source: the (theta, head_dim) the parity tests + feed the pass must equal the bundle's real config.json rope_parameters, so a + wrong-theta or wrong-head_dim bundle fails instead of self-referentially + passing.""" + text = json.loads((GEMMA_BUNDLE / "config.json").read_text())["text_config"] + rope = text["rope_parameters"] + assert GEMMA_INV_FREQ == { + "v_module_backbone_rotary_emb_sliding_attention_inv_freq": ( + float(rope["sliding_attention"]["rope_theta"]), + int(text["head_dim"]), + ), + "v_module_backbone_rotary_emb_full_attention_inv_freq": ( + float(rope["full_attention"]["rope_theta"]), + int(text["global_head_dim"]), + ), + } + sliding = GEMMA_INV_FREQ["v_module_backbone_rotary_emb_sliding_attention_inv_freq"] + full = GEMMA_INV_FREQ["v_module_backbone_rotary_emb_full_attention_inv_freq"] + assert sliding[0] != full[0], "sliding and full thetas must be distinct (dual-theta)" + assert sliding[1] != full[1], "sliding and full head_dims must be distinct" + + +def _load_ir(path: Path, inv_freq_spec: dict[str, tuple[float, int]], *, max_seq: int = TEST_MAX_SEQ) -> IRGraph: + payload = json.loads(path.read_text()) + graph = payload["graph"] + values = { + vid: IRValue( + id=vid, + shape=tuple(v["shape"]) if v.get("shape") is not None else None, + dtype=v.get("dtype"), + ) + for vid, v in graph["values"].items() + } + nodes: dict[str, IRNode] = {} + order: list[str] = [] + for n in graph["nodes"]: + nodes[n["id"]] = IRNode( + id=n["id"], + op=n["op"], + inputs=list(n["inputs"]), + outputs=list(n["outputs"]), + attrs=dict(n.get("attrs", {})), + meta=dict(n.get("meta", {})), + kind=n.get("kind", "generic"), + ) + order.append(n["id"]) + constants = { + cid: _inv_freq(*inv_freq_spec[cid]) + for cid in graph["constants"] + if cid in inv_freq_spec + } + meta = dict(graph["meta"]) + meta["max_cache_seq_len"] = max_seq + ir = IRGraph( + values=values, + nodes=nodes, + order=order, + inputs=list(graph["inputs"]), + outputs=list(graph["outputs"]), + constants=constants, + meta=meta, + ) + rebuild_graph(ir) + return ir + + +def _cos_sin_inputs(payload_path: Path) -> dict[str, str]: + """Map each scalar_cos/scalar_sin output id -> the int position graph input the + original angle matmul consumed, mirroring the pass's index_shape constraint + (the int value whose shape == cos/sin output shape minus its last dim).""" + graph = json.loads(payload_path.read_text())["graph"] + producer = {o: n for n in graph["nodes"] for o in n["outputs"]} + values = graph["values"] + + def find_position(value_id: str, index_shape: tuple[int, ...]) -> str | None: + seen: set[str] = set() + stack = [value_id] + while stack: + cur = stack.pop() + if cur in seen: + continue + seen.add(cur) + val = values.get(cur, {}) + dtype = str(val.get("dtype")) + shape = val.get("shape") + if ( + dtype in {"int32", "int64", "i32", "i64"} + and shape is not None + and tuple(int(d) for d in shape) == index_shape + ): + return cur + node = producer.get(cur) + if node is None: + continue + stack.extend(node["inputs"]) + return None + + result: dict[str, str] = {} + for n in graph["nodes"]: + if n["op"] in {"scalar_cos", "scalar_sin"}: + out_shape = values[n["outputs"][0]]["shape"] + index_shape = tuple(int(d) for d in out_shape[:-1]) + result[n["outputs"][0]] = find_position(n["inputs"][0], index_shape) + return result + + +def _rope_nodes(ir: IRGraph) -> tuple[list[IRNode], list[str]]: + embeddings = [ + n + for n in ir.nodes.values() + if n.op == "embedding" and n.inputs and n.inputs[0].startswith("c_rope_table_") + ] + tables = [c for c in ir.constants if c.startswith("c_rope_table_")] + return embeddings, tables + + +# --------------------------------------------------------------------------- # +# fires +# --------------------------------------------------------------------------- # + + +def test_gemma_decoder_step_rewrites_four_cos_sin() -> None: + ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_step.json", GEMMA_INV_FREQ) + assert precompute_rope_tables(ir) is True + embeddings, tables = _rope_nodes(ir) + assert len(embeddings) == 4 + assert len(tables) == 4 + assert all(n.op != "scalar_cos" and n.op != "scalar_sin" for n in ir.nodes.values()) + + +def test_gemma_prefill_chunk_rewrites_eight_cos_sin() -> None: + ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_prefill_chunk.json", GEMMA_INV_FREQ) + assert precompute_rope_tables(ir) is True + embeddings, tables = _rope_nodes(ir) + assert len(embeddings) == 8 + assert len(tables) == 8 + + +def test_gemma_media_step_rewrites_its_rope_tables() -> None: + path = GEMMA_16K_BUNDLE / "components" / "decoder_media_step" / "optimized_ir.json" + if not path.exists(): + path = GEMMA_16K_BUNDLE / "optimized_ir_decoder_media_step.json" + if not path.exists(): + pytest.skip( + "gemma4 component pipeline emits no decoder_media_step component " + "(only lm_encoder_media_step, which carries no rope angle path); " + "this guards future models that do emit a decoder media step" + ) + ir = _load_ir(path, GEMMA_16K_INV_FREQ) + assert precompute_rope_tables(ir) is True + embeddings, tables = _rope_nodes(ir) + assert len(embeddings) >= 4 + assert len(tables) >= 4 + widths = {int(ir.constants[t].shape[-1]) for t in tables} + assert 256 in widths and 512 in widths + + +# --------------------------------------------------------------------------- # +# two thetas: both widths exist and each table is the correct fp64 cos/sin +# --------------------------------------------------------------------------- # + + +def _table_by_width(ir: IRGraph) -> dict[int, dict[str, torch.Tensor]]: + out: dict[int, dict[str, torch.Tensor]] = {} + for node in ir.nodes.values(): + if node.op != "embedding" or not node.inputs[0].startswith("c_rope_table_"): + continue + table = ir.constants[node.inputs[0]] + width = int(table.shape[-1]) + kind = "cos" if "cos" in node.inputs[0] else "sin" + out.setdefault(width, {})[kind] = table + return out + + +def test_gemma_two_thetas_tables_match_fresh_fp64_cos_sin() -> None: + ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_step.json", GEMMA_INV_FREQ) + precompute_rope_tables(ir) + by_width = _table_by_width(ir) + assert set(by_width.keys()) == {256, 512} + + expected = { + head_dim: _inv_freq(theta, head_dim).to(torch.float64) + for theta, head_dim in GEMMA_INV_FREQ.values() + } + assert set(expected.keys()) == {256, 512} + probe_positions = [0, 1, 2000, 2049, TEST_MAX_SEQ - 1] + for width, inv in expected.items(): + positions = torch.tensor(probe_positions, dtype=torch.float64).reshape(-1, 1) + freqs = positions * inv.reshape(1, -1) + emb = torch.cat((freqs, freqs), dim=-1) + exp_cos = torch.cos(emb).to(torch.float16) + exp_sin = torch.sin(emb).to(torch.float16) + got_cos = by_width[width]["cos"][probe_positions] + got_sin = by_width[width]["sin"][probe_positions] + assert torch.equal(got_cos, exp_cos), f"cos width {width}" + assert torch.equal(got_sin, exp_sin), f"sin width {width}" + + +def test_gemma_width256_uses_sliding_not_global_inv_freq() -> None: + """Cross-check the per-theta binding: the width-256 table must NOT match the + global theta and vice versa.""" + ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_step.json", GEMMA_INV_FREQ) + precompute_rope_tables(ir) + by_width = _table_by_width(ir) + + wrong = _inv_freq(1000000.0, 256).to(torch.float64) # global theta on sliding width + positions = torch.tensor([2049.0], dtype=torch.float64).reshape(-1, 1) + freqs = positions * wrong.reshape(1, -1) + emb = torch.cat((freqs, freqs), dim=-1) + wrong_cos = torch.cos(emb).to(torch.float16) + assert not torch.equal(by_width[256]["cos"][[2049]], wrong_cos) + + +# --------------------------------------------------------------------------- # +# layout +# --------------------------------------------------------------------------- # + + +def test_gemma_table_width_matches_output_and_halves_are_byte_equal() -> None: + payload = json.loads((GEMMA_BUNDLE / "optimized_ir_decoder_step.json").read_text()) + out_width = { + n["outputs"][0]: int(payload["graph"]["values"][n["outputs"][0]]["shape"][-1]) + for n in payload["graph"]["nodes"] + if n["op"] in {"scalar_cos", "scalar_sin"} + } + ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_step.json", GEMMA_INV_FREQ) + precompute_rope_tables(ir) + for node in ir.nodes.values(): + if node.op != "embedding" or not node.inputs[0].startswith("c_rope_table_"): + continue + out_id = node.outputs[0] + table = ir.constants[node.inputs[0]] + width = int(table.shape[-1]) + assert width == out_width[out_id] + half = width // 2 + assert torch.equal(table[:, :half], table[:, half:]) + + +# --------------------------------------------------------------------------- # +# position binding +# --------------------------------------------------------------------------- # + + +def test_gemma_embedding_index_is_original_position_input() -> None: + path = GEMMA_BUNDLE / "optimized_ir_decoder_step.json" + pos_for_output = _cos_sin_inputs(path) + ir = _load_ir(path, GEMMA_INV_FREQ) + precompute_rope_tables(ir) + for node in ir.nodes.values(): + if node.op != "embedding" or not node.inputs[0].startswith("c_rope_table_"): + continue + out_id = node.outputs[0] + assert node.inputs[1] == pos_for_output[out_id] + assert set(pos_for_output.values()) == {"v_args_2"} + + +# --------------------------------------------------------------------------- # +# Qwen regression: single width table +# --------------------------------------------------------------------------- # + + +def test_qwen_single_width_table_still_correct() -> None: + path = QWEN_BUNDLE / "optimized_ir_decoder_step.json" + pos_for_output = _cos_sin_inputs(path) + ir = _load_ir(path, QWEN_INV_FREQ) + assert precompute_rope_tables(ir) is True + embeddings, tables = _rope_nodes(ir) + assert len(embeddings) == 2 + assert len(tables) == 2 + + widths = {int(ir.constants[t].shape[-1]) for t in tables} + qwen_theta, qwen_head_dim = QWEN_INV_FREQ["v_module_backbone_rotary_emb_inv_freq"] + assert widths == {qwen_head_dim} + + inv = _inv_freq(qwen_theta, qwen_head_dim).to(torch.float64) + positions = torch.tensor([0.0, 1.0, 2049.0, TEST_MAX_SEQ - 1], dtype=torch.float64).reshape(-1, 1) + freqs = positions * inv.reshape(1, -1) + emb = torch.cat((freqs, freqs), dim=-1) + for node in ir.nodes.values(): + if node.op != "embedding" or not node.inputs[0].startswith("c_rope_table_"): + continue + table = ir.constants[node.inputs[0]] + ref = (torch.cos(emb) if "cos" in node.inputs[0] else torch.sin(emb)).to(torch.float16) + idx = [0, 1, 2049, TEST_MAX_SEQ - 1] + assert torch.equal(table[idx], ref) + assert node.inputs[1] == pos_for_output[node.outputs[0]] + + +# --------------------------------------------------------------------------- # +# inert: lm_encoder has no rope angle path +# --------------------------------------------------------------------------- # + + +def test_lm_encoder_is_inert() -> None: + ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_lm_encoder_step.json", GEMMA_INV_FREQ) + assert precompute_rope_tables(ir) is False + embeddings, tables = _rope_nodes(ir) + assert tables == [] + + +def test_vision_encoder_is_inert() -> None: + """The vision tower's axial rope has the same cat(halves) <- matmul(inv_freq, + position) shape as decoder rope, but its position is a mid-graph ``select`` + (not a graph input). The pass must NOT fire, or it corrupts vision RoPE. The + inv_freq constant is materialized so the pass is rejected by the position + graph-input check, not by a missing constant.""" + path = GEMMA_BUNDLE / "optimized_ir_vision_encoder.json" + if not path.exists(): + pytest.skip("no vision encoder component for this bundle") + graph = json.loads(path.read_text())["graph"] + spec = { + cid: (10000.0, int(graph["values"][cid]["shape"][0]) * 2) + for cid in graph["constants"] + if "inv_freq" in cid + } + assert spec, "expected a vision rope inv_freq constant" + assert any(n["op"] in {"scalar_cos", "scalar_sin"} for n in graph["nodes"]), "vision encoder should have rope trig" + ir = _load_ir(path, spec) + assert precompute_rope_tables(ir) is False + _, tables = _rope_nodes(ir) + assert tables == [] From 135a584d060e3a9ca4f57d5d552a413d2217dfc6 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 11:00:28 +0100 Subject: [PATCH 04/55] Fix chunked-prefill sliding-window KV cache corruption run_chunked_prefill pads the final partial chunk up to chunk_size to fit the fixed-size chunk graph. Global-attention layers trim the padding tokens from the KV cache, but sliding-window layers do not: the padded chunk evicts real recent tokens and shifts the entire sliding window by the padding count, corrupting deep-context attention (e.g. Gemma needle retrieval past ~2k). Disable tail padding when the prefill graph has a sliding-window KV cache and process the tail token-by-token via run_step, which already produces an exact cache. Detect via a new get_node_window_size accessor reading window_size off the KV_CACHE_STATE nodes. Models without sliding-window caches (e.g. Qwen) are unaffected. Signed-off-by: Noah Cylich --- cactus-engine/src/model.cpp | 17 +++++++++++------ cactus-graph/cactus_graph.h | 1 + cactus-graph/src/builder.cpp | 4 ++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 78eb4b481..6f1fc8c2d 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -1077,26 +1077,31 @@ Model::ChunkedPrefillResult Model::run_chunked_prefill(const std::vector 0 ? std::min(chunk_size, component_tokens) : component_tokens; if (effective_chunk != component_tokens) effective_chunk = component_tokens; size_t whole_chunks_end = (tokens.size() / effective_chunk) * effective_chunk; - const bool has_recurrent_state = [&]() { + auto any_cache_node = [&](auto predicate) { if (!decoder_prefill_->graph) return false; for (const auto& state : decoder_prefill_->cache_states) { for (int node_id : {state.key_node_id, state.value_node_id}) { if (node_id < 0) continue; - if (decoder_prefill_->graph->get_node_op_type(static_cast(node_id)) - == OpType::RECURRENT_CACHE_STATE) { - return true; - } + if (predicate(static_cast(node_id))) return true; } } return false; - }(); + }; + const bool has_recurrent_state = any_cache_node([&](size_t id) { + return decoder_prefill_->graph->get_node_op_type(id) == OpType::RECURRENT_CACHE_STATE; + }); if (has_recurrent_state && whole_chunks_end > effective_chunk) { whole_chunks_end = effective_chunk; } + const bool has_sliding_window_cache = any_cache_node([&](size_t id) { + return decoder_prefill_->graph->get_node_op_type(id) == OpType::KV_CACHE_STATE + && decoder_prefill_->graph->get_node_window_size(id) > 0; + }); const size_t tail_tokens = tokens.size() - whole_chunks_end; const size_t padding_cutoff = std::max(1, effective_chunk / 16); const bool pad_tail = family_ != "lfm2_vl" && !has_recurrent_state + && !has_sliding_window_cache && tail_tokens >= padding_cutoff; const size_t executable_tokens = whole_chunks_end + (pad_tail ? effective_chunk : 0); if (executable_tokens == 0) { diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 7c77adf30..acb88baba 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -707,6 +707,7 @@ class CactusGraph { const std::vector& output_shape, const OpParams& params = {}); const BufferDesc& get_output_buffer(size_t node_id) const; OpType get_node_op_type(size_t node_id) const; + size_t get_node_window_size(size_t node_id) const; void allocate_buffers(); size_t get_node_count() const; diff --git a/cactus-graph/src/builder.cpp b/cactus-graph/src/builder.cpp index 3f513d0d5..b54b0a92a 100644 --- a/cactus-graph/src/builder.cpp +++ b/cactus-graph/src/builder.cpp @@ -1257,6 +1257,10 @@ OpType CactusGraph::get_node_op_type(size_t node_id) const { return nodes_[node_index_map_.at(node_id)]->op_type; } +size_t CactusGraph::get_node_window_size(size_t node_id) const { + return nodes_[node_index_map_.at(node_id)]->params.window_size; +} + size_t CactusGraph::persistent(size_t source_node) { const auto& source_buffer = get_output_buffer(source_node); OpParams params; From 6820973e3ece8c6782d9ed293458d6dcfff0130b Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 13:02:41 +0100 Subject: [PATCH 05/55] Make rolling KV compaction the default for causal LLMs KeyDiff compaction now defaults on (trigger 4096 -> target 2048) for causal-LM generation, bounding the KV cache without an env var. CACTUS_KV_COMPRESS_AT (trigger) and CACTUS_KV_COMPRESS_TO (target) override the defaults independently; CACTUS_KV_COMPRESS_AT=0 disables. The one-shot budget-fraction mode (never shipped) is removed. STT models are unaffected -- they transcribe via a separate API path that never reaches the compaction hook; sliding-window models cleanly no-op via the existing all-layers guard. Signed-off-by: Noah Cylich --- cactus-engine/src/engine.h | 38 +-- cactus-engine/src/model.cpp | 118 +++------ .../tests/test_kv_compress_free_functions.cpp | 245 ++++++------------ 3 files changed, 127 insertions(+), 274 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index ca4d72e1e..e8c8253a8 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -173,17 +173,13 @@ struct Config { std::vector layer_types; size_t conv_L_cache = 0; - bool kv_compress = false; // master switch (default OFF -> exact no-op) - float kv_compress_budget_frac = 0.25f; + // Rolling bounded KV compaction (default ON, 4096 -> 2048). Override at runtime with + // CACTUS_KV_COMPRESS_AT (trigger) / CACTUS_KV_COMPRESS_TO (target); CACTUS_KV_COMPRESS_AT=0 disables. + bool kv_compress = true; float kv_compress_recent_frac = 0.30f; uint32_t kv_compress_sink = 4; - std::string kv_compress_signal = "keydiff"; - int32_t kv_compress_manifold_window = 512; - // Rolling bounded cache: when trigger_len > 0, whenever the live KV seq-len reaches - // trigger_len, KeyDiff-compact to the best target_len tokens (absolute budget) and continue. - // 0/0 keeps the legacy one-shot-at-end-of-prefill behavior driven by budget_frac. - int32_t kv_compress_trigger_len = 0; - int32_t kv_compress_target_len = 0; + int32_t kv_compress_trigger_len = 4096; + int32_t kv_compress_target_len = 2048; uint32_t altup_num_inputs = 4; uint32_t laurel_rank = 64; @@ -241,14 +237,10 @@ struct Config { bool from_json(const std::string& json_path); std::string to_json() const; - // Degenerate-config guard for rolling compaction: if kv_compress_trigger_len > 0, require - // 0 < kv_compress_target_len < kv_compress_trigger_len. Otherwise warn and disable rolling - // (treat as trigger_len = 0). Called after from_json and after any runtime override. + // Disable rolling unless 0 < kv_compress_target_len < kv_compress_trigger_len (when trigger > 0). void validate_kv_compress(); - // Parse runtime override strings onto this config (nullptr = unset). `oneshot` is - // ":" (e.g. "keydiff:0.25"); `roll` is ":" - // (e.g. "4096:2048"). roll takes precedence. Returns true if any override was applied. - bool parse_kv_compress_override(const char* oneshot, const char* roll); + // Apply CACTUS_KV_COMPRESS_AT / CACTUS_KV_COMPRESS_TO overrides (nullptr = unset). Returns true if applied. + bool parse_kv_compress_override(const char* trigger_env, const char* target_env); }; @@ -642,18 +634,14 @@ class Model { void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} - // KeyDiff KV compression: at end of prefill, per compressible (layer, kv-head) keep a - // budget of survivors (sink + recent + top KeyDiff geometry), physically compact, and - // RoPE-renumber to contiguous positions 0..B-1. Default OFF via Config::kv_compress. + // Rolling KeyDiff compaction: keep abs_budget survivors per (layer, kv-head), renumber to + // 0..B-1; refuses unless every layer is compressible. See impl for the full contract. void compress_kv_cache_keydiff(const cactus::kvcompress::Params& params); - // Rolling bounded compaction: if kv_compress_trigger_len > 0 and the live KV seq-len has - // reached it, KeyDiff-compact to kv_compress_target_len (absolute budget) and renumber. - // No-op when kv_compress is off or trigger_len <= 0. Called after prefill and each decode. + // Compact to kv_compress_target_len once the live KV seq-len reaches kv_compress_trigger_len. void maybe_roll_compact(); - // Layer indices (cache_states order) that are physically compressible: Qwen = all; - // Gemma = full/global layers that are not KV-shared sources/consumers. + // Layer indices (cache_states order) that are physically compressible. std::vector compressible_layers() const; - // Apply CACTUS_KV_COMPRESS / CACTUS_KV_COMPRESS_ROLL env overrides onto config_ (test/eval). + // Apply CACTUS_KV_COMPRESS_AT / CACTUS_KV_COMPRESS_TO env overrides onto config_. void apply_kv_compress_env_override(); void set_tool_constraints(const std::vector& tools); diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 6f1fc8c2d..a5c4d8bc6 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -362,10 +362,9 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_ERROR("model", "Failed to load config.txt from: " << bundle_dir); return false; } - // Runtime KV-compression override (test/eval only; bundle config unchanged). Unset => OFF - // (current behavior). CACTUS_KV_COMPRESS=: for one-shot-at-prefill - // (e.g. keydiff:0.25); CACTUS_KV_COMPRESS_ROLL=: for rolling - // bounded (e.g. 4096:2048). ROLL takes precedence when both are set. + // Runtime KV-compression override (bundle config unchanged). Rolling bounded compaction is the + // default (4096 -> 2048); CACTUS_KV_COMPRESS_AT (trigger) / CACTUS_KV_COMPRESS_TO (target) tune + // it, and CACTUS_KV_COMPRESS_AT=0 disables it. apply_kv_compress_env_override(); if (!load_manifest()) { CACTUS_LOG_ERROR("model", "Failed to load bundle manifest from: " << bundle_dir); @@ -1600,25 +1599,10 @@ void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, } last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; - if (prepare_decode && config_.kv_compress) { - if (config_.kv_compress_trigger_len > 0) { - // NOTE: rolling compaction does NOT fire mid-chunk. It runs here at - // prefill-completion and once per decode step, so a single prompt grows to its - // full n tokens before this completion-time compaction bounds it back to target_len. - // Rolling bounded mode: compact if the prefilled prompt crossed trigger_len. - maybe_roll_compact(); - } else { - cactus::kvcompress::Params p; - p.enabled = true; - p.budget_frac = config_.kv_compress_budget_frac; - p.recent_frac = config_.kv_compress_recent_frac; - p.sink = config_.kv_compress_sink; - p.signal = (config_.kv_compress_signal == "manifold") - ? cactus::kvcompress::Signal::MANIFOLD - : cactus::kvcompress::Signal::KEYDIFF; - p.manifold_window = config_.kv_compress_manifold_window; - compress_kv_cache_keydiff(p); - } + if (prepare_decode) { + // Rolling bounded compaction (self-guards on kv_compress + trigger_len). Does NOT fire + // mid-chunk: the prompt grows to its full length here, then this bounds it to target_len. + maybe_roll_compact(); } } @@ -2977,9 +2961,8 @@ void Model::remove_thinking_tokens(const std::vector>& } void Model::apply_kv_compress_env_override() { - const char* oneshot = std::getenv("CACTUS_KV_COMPRESS"); - const char* roll = std::getenv("CACTUS_KV_COMPRESS_ROLL"); - config_.parse_kv_compress_override(oneshot, roll); + config_.parse_kv_compress_override(std::getenv("CACTUS_KV_COMPRESS_AT"), + std::getenv("CACTUS_KV_COMPRESS_TO")); } std::vector Model::compressible_layers() const { @@ -2989,19 +2972,15 @@ std::vector Model::compressible_layers() const { config_.layer_types, config_.num_layers, shared); } +// Rolling KeyDiff compaction: keep params.abs_budget survivors per (layer, kv-head) -- attention +// sink + recent window + the most distinctive middle tokens by KeyDiff key geometry -- physically +// compact them and renumber their RoPE positions to a contiguous window 0..B-1. Refuses (no-op +// with a warning) unless EVERY layer is compressible, since the engine tracks a single global +// position counter and a subset compaction would desync the non-compacted layers. void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) { - if (!params.enabled || !decoder_) return; - - struct CacheHeader { - uint64_t current_seq_len; - uint64_t max_seq_len; - uint64_t num_kv_heads; - uint64_t head_dim; - uint64_t sink_size; - uint64_t reserved[3]; - }; - constexpr size_t kHeaderBytes = 64; - static_assert(sizeof(CacheHeader) == kHeaderBytes, "CacheHeader layout mismatch"); + if (!decoder_) return; + using cactus::kvcompress::CacheHeader; + constexpr size_t kHeaderBytes = sizeof(CacheHeader); std::vector layers = compressible_layers(); // Subset compression (e.g. Gemma physical set {4,9}) renumbers only some layers, but the @@ -3015,7 +2994,6 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) "subset compression unsupported (needs per-layer positions); disabled for this model"); return; } - std::set compressible(layers.begin(), layers.end()); const double rope_theta = static_cast(config_.rope_theta); Component& comp = *decoder_; @@ -3025,7 +3003,6 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) size_t new_seq_len = 0; bool have_new_seq_len = false; for (size_t li = 0; li < comp.cache_states.size(); ++li) { - if (!compressible.count(li)) continue; const auto& cs = comp.cache_states[li]; if (cs.key_node_id < 0 || cs.value_node_id < 0) continue; @@ -3041,7 +3018,7 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) size_t n = khdr->current_seq_len; size_t kv_heads = khdr->num_kv_heads; size_t head_dim = khdr->head_dim; - if (n == 0 || kv_heads == 0 || head_dim == 0) continue; + if (kv_heads == 0 || head_dim == 0) continue; if (kdesc.precision == Precision::FP16) { auto* kbase = reinterpret_cast(static_cast(kraw) + kHeaderBytes); @@ -3083,21 +3060,15 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } void Model::maybe_roll_compact() { + // Only reached from the causal-LM generation path (complete.cpp -> prefill/decode). STT models + // transcribe via transcribe.cpp with their own decode loops and never call this, and multimodal + // vision/audio encoder passes aren't on this path either -- so no model-type gate is needed. if (!config_.kv_compress || config_.kv_compress_trigger_len <= 0) return; if (cache_total_seq_len_ < static_cast(config_.kv_compress_trigger_len)) return; cactus::kvcompress::Params p; - p.enabled = true; - p.budget_frac = config_.kv_compress_budget_frac; p.recent_frac = config_.kv_compress_recent_frac; p.sink = config_.kv_compress_sink; - p.signal = (config_.kv_compress_signal == "manifold") - ? cactus::kvcompress::Signal::MANIFOLD - : cactus::kvcompress::Signal::KEYDIFF; - p.manifold_window = config_.kv_compress_manifold_window; - // Absolute target: KeyDiff-keep the best target_len tokens (sink + recent + distinctive - // middle), renumber survivors to 0..target_len-1. compress_kv_cache_keydiff sets the global - // position counter to that B, so the cache stays bounded at <= trigger_len across cycles. p.abs_budget = config_.kv_compress_target_len; compress_kv_cache_keydiff(p); } @@ -3289,11 +3260,8 @@ bool Config::from_json(const std::string& config_path) { } else if (key == "conv_L_cache") conv_L_cache = static_cast(std::stoul(value)); else if (key == "kv_compress") kv_compress = (value == "true" || value == "1"); - else if (key == "kv_compress_budget_frac") kv_compress_budget_frac = std::stof(value); else if (key == "kv_compress_recent_frac") kv_compress_recent_frac = std::stof(value); else if (key == "kv_compress_sink") kv_compress_sink = static_cast(std::stoul(value)); - else if (key == "kv_compress_signal") kv_compress_signal = value; - else if (key == "kv_compress_manifold_window") kv_compress_manifold_window = static_cast(std::stol(value)); else if (key == "kv_compress_trigger_len") kv_compress_trigger_len = static_cast(std::stol(value)); else if (key == "kv_compress_target_len") kv_compress_target_len = static_cast(std::stol(value)); else if (key == "layer_types") { @@ -3468,40 +3436,24 @@ bool Config::from_json(const std::string& config_path) { return true; } -bool Config::parse_kv_compress_override(const char* oneshot, const char* roll) { - if (roll && *roll) { - std::string s(roll); - size_t colon = s.find(':'); - if (colon == std::string::npos) { - CACTUS_LOG_WARN("kv_compress", "CACTUS_KV_COMPRESS_ROLL must be :, got: " << s); - return false; - } - kv_compress = true; - kv_compress_signal = "keydiff"; - kv_compress_trigger_len = static_cast(std::stol(s.substr(0, colon))); - kv_compress_target_len = static_cast(std::stol(s.substr(colon + 1))); - validate_kv_compress(); - CACTUS_LOG_INFO("kv_compress", "rolling override: trigger_len=" << kv_compress_trigger_len - << " target_len=" << kv_compress_target_len); - return true; - } - if (oneshot && *oneshot) { - std::string s(oneshot); - size_t colon = s.find(':'); - if (colon == std::string::npos) { - CACTUS_LOG_WARN("kv_compress", "CACTUS_KV_COMPRESS must be :, got: " << s); - return false; - } - kv_compress = true; - kv_compress_signal = s.substr(0, colon); - kv_compress_budget_frac = std::stof(s.substr(colon + 1)); +bool Config::parse_kv_compress_override(const char* trigger_env, const char* target_env) { + const bool has_trigger = trigger_env && *trigger_env; + const bool has_target = target_env && *target_env; + if (!has_trigger && !has_target) return false; + if (has_trigger) kv_compress_trigger_len = static_cast(std::stol(trigger_env)); + if (has_target) kv_compress_target_len = static_cast(std::stol(target_env)); + if (kv_compress_trigger_len <= 0) { + kv_compress = false; kv_compress_trigger_len = 0; kv_compress_target_len = 0; - CACTUS_LOG_INFO("kv_compress", "one-shot override: signal=" << kv_compress_signal - << " budget_frac=" << kv_compress_budget_frac); + CACTUS_LOG_INFO("kv_compress", "rolling compaction disabled (CACTUS_KV_COMPRESS_AT <= 0)"); return true; } - return false; + kv_compress = true; + validate_kv_compress(); + CACTUS_LOG_INFO("kv_compress", "rolling override: trigger_len=" << kv_compress_trigger_len + << " target_len=" << kv_compress_target_len); + return true; } void Config::validate_kv_compress() { diff --git a/cactus-engine/tests/test_kv_compress_free_functions.cpp b/cactus-engine/tests/test_kv_compress_free_functions.cpp index 0240421ad..676ccf364 100644 --- a/cactus-engine/tests/test_kv_compress_free_functions.cpp +++ b/cactus-engine/tests/test_kv_compress_free_functions.cpp @@ -15,33 +15,14 @@ using namespace cactus::kvcompress; namespace { -constexpr size_t kHeaderBytes = 64; +using Header = cactus::kvcompress::CacheHeader; +constexpr size_t kHeaderBytes = sizeof(Header); constexpr size_t kGroupSize = 32; // KV_QUANT_GROUP_SIZE -struct Header { - uint64_t current_seq_len; - uint64_t max_seq_len; - uint64_t num_kv_heads; - uint64_t head_dim; - uint64_t sink_size; - uint64_t reserved[3]; -}; - float f16_to_f32(uint16_t u) { __fp16 v; std::memcpy(&v, &u, 2); return static_cast(v); } uint16_t f32_to_f16(float f) { __fp16 v = static_cast<__fp16>(f); uint16_t u; std::memcpy(&u, &v, 2); return u; } -// rotate_half RoPE on a float row, matching the reference / norms_rope.cpp. -std::vector rope_row(const std::vector& v, double pos, double theta) { - size_t d = v.size(), half = d / 2; - std::vector o(d); - for (size_t i = 0; i < half; ++i) { - double inv = std::pow(theta, -(2.0 * (double)i) / (double)d); - double a = pos * inv, c = std::cos(a), s = std::sin(a); - o[i] = (float)(v[i] * c - v[i + half] * s); - o[i + half] = (float)(v[i + half] * c + v[i] * s); - } - return o; -} +using EngineTestUtils::rope_reference; // Build an FP16 cache buffer: 64B header + [max_seq][kv_heads][head_dim] fp16. Returns the // raw buffer; `pre_rope[h][t][d]` is the planted pre-RoPE key, stored post-RoPE at position t. @@ -63,13 +44,9 @@ std::vector make_fp16_cache(size_t n, size_t max_seq, size_t kv_heads, siz v[d] = std::cos(0.07f * (hh + 1) * (t + 2) + 0.2f * d); } pre_rope[hh][t] = k; values[hh][t] = v; - std::vector kr = rope_row(k, (double)t, theta); - for (size_t d = 0; d < head_dim; ++d) { + std::vector kr = rope_reference(k, (double)t, theta); + for (size_t d = 0; d < head_dim; ++d) rows[(t * kv_heads + hh) * head_dim + d] = f32_to_f16(kr[d]); - // value stored at the SAME slot but in a separate value buffer; reuse k buffer - // layout for V in a parallel buffer (built by caller). Here we only fill K. - (void)v; - } } return buf; } @@ -110,7 +87,7 @@ bool test_compact_fp16_cache() { for (size_t rank = 0; rank < B; ++rank) { int abs_pos = kept[h][rank]; // Expected K at slot `rank` = rope(pre_rope[h][abs_pos], rank). - std::vector expK = rope_row(pre[h][abs_pos], (double)rank, theta); + std::vector expK = rope_reference(pre[h][abs_pos], (double)rank, theta); const uint16_t* kdst = krows + (rank * kv_heads + h) * head_dim; const uint16_t* vdst = vrows + (rank * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) { @@ -123,7 +100,7 @@ bool test_compact_fp16_cache() { } bool test_dense_check_full_budget() { - // budget_frac=1.0 -> keep all indices in order; renumber maps rank==abs_pos so RoPE delta + // abs_budget == n -> keep all indices in order; renumber maps rank==abs_pos so RoPE delta // is 0 -> the K rows are byte-identical and V unchanged. const size_t n = 40, max_seq = 64, kv_heads = 2, head_dim = 16; const double theta = 1000000.0; @@ -134,8 +111,7 @@ bool test_dense_check_full_budget() { auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); - Params p; p.enabled = true; p.budget_frac = 1.0f; p.recent_frac = 0.3f; p.sink = 4; - p.signal = Signal::KEYDIFF; + Params p; p.recent_frac = 0.3f; p.sink = 4; p.abs_budget = (int)n; auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); for (auto& k : kept) if (k.size() != n) return false; // B == n compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); @@ -153,8 +129,8 @@ bool test_rope_renumber_contiguous() { std::vector pre(head_dim); for (size_t d = 0; d < head_dim; ++d) pre[d] = std::sin(0.4 * d) + 0.1 * d; int abs_pos = 137; size_t rank = 5; - std::vector at_abs = rope_row(pre, (double)abs_pos, theta); - std::vector at_rank = rope_row(pre, (double)rank, theta); + std::vector at_abs = rope_reference(pre, (double)abs_pos, theta); + std::vector at_rank = rope_reference(pre, (double)rank, theta); std::vector delta = at_abs; rope_rotate_row(delta.data(), head_dim, theta, (double)rank - (double)abs_pos); for (size_t d = 0; d < head_dim; ++d) @@ -162,84 +138,6 @@ bool test_rope_renumber_contiguous() { return true; } -// Mirror Model::compress_kv_cache_keydiff's master gate: only touch the cache when enabled. -// Returns true if compaction ran. Operates on an FP16 K/V cache pair in place. -bool run_compress_if_enabled(uint16_t* krows, uint16_t* vrows, size_t n, size_t kv_heads, - size_t head_dim, double theta, const Params& p) { - if (!p.enabled) return false; // the Model gate early-returns here -> exact no-op. - auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); - compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); - return true; -} - -bool test_compress_noop_when_disabled() { - // Strengthened: assert the cache bytes are ACTUALLY unchanged when disabled, and ACTUALLY - // changed when enabled with a real (<1.0 budget) keep-set. Not a tautology on the flag. - const size_t n = 50, max_seq = 96, kv_heads = 2, head_dim = 16; - const double theta = 1000000.0; - std::vector>> pre, vals; - auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); - auto vbuf = make_fp16_value_cache(n, max_seq, kv_heads, head_dim, vals); - std::vector kbuf0 = kbuf, vbuf0 = vbuf; - auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); - auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); - size_t live = n * kv_heads * head_dim * sizeof(uint16_t); - - Params off; off.enabled = false; off.budget_frac = 0.25f; off.recent_frac = 0.3f; off.sink = 4; - bool ran_off = run_compress_if_enabled(krows, vrows, n, kv_heads, head_dim, theta, off); - if (ran_off) return false; - // Disabled => byte-identical. - if (std::memcmp(kbuf.data(), kbuf0.data(), kbuf.size()) != 0) return false; - if (std::memcmp(vbuf.data(), vbuf0.data(), vbuf.size()) != 0) return false; - - Params on; on.enabled = true; on.budget_frac = 0.25f; on.recent_frac = 0.3f; on.sink = 4; - on.signal = Signal::KEYDIFF; - bool ran_on = run_compress_if_enabled(krows, vrows, n, kv_heads, head_dim, theta, on); - if (!ran_on) return false; - // Enabled with budget < 1.0 => the live region must actually change (real keep-set applied). - bool kchanged = std::memcmp(kbuf.data() + kHeaderBytes, kbuf0.data() + kHeaderBytes, live) != 0; - bool vchanged = std::memcmp(vbuf.data() + kHeaderBytes, vbuf0.data() + kHeaderBytes, live) != 0; - return kchanged && vchanged; -} - -bool test_subset_compression_rejected() { - // BLOCKER guard: Model::compress_kv_cache_keydiff only proceeds when EVERY layer is - // compressible (compressible_layers().size() == num_layers). A strict subset (Gemma {4,9}) - // must be rejected as a no-op, because the single global position counter can't represent - // per-layer lengths. Here we exercise the exact predicate the guard uses + show the cache - // stays untouched on the reject path. - const size_t num_layers = 35; - std::vector gemma; - for (int i = 0; i < (int)num_layers; ++i) gemma.push_back(((i + 1) % 5 == 0) ? "global" : "sliding"); - auto gset = physical_compressible_layers(gemma, num_layers, 20); - bool gemma_is_subset = gset.size() != num_layers; // guard => reject (no-op) - if (!gemma_is_subset) return false; - - std::vector qwen(28, "full_attention"); - auto qset = physical_compressible_layers(qwen, 28, 0); - bool qwen_full = qset.size() == 28; // guard => accept (compress) - if (!qwen_full) return false; - - // Behavioral: on the reject path the cache must be byte-identical (compaction skipped). - const size_t n = 40, max_seq = 64, kv_heads = 2, head_dim = 16; - const double theta = 1000000.0; - std::vector>> pre, vals; - auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); - auto vbuf = make_fp16_value_cache(n, max_seq, kv_heads, head_dim, vals); - std::vector kbuf0 = kbuf, vbuf0 = vbuf; - auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); - auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); - Params p; p.enabled = true; p.budget_frac = 0.25f; p.recent_frac = 0.3f; p.sink = 4; - if (gemma_is_subset) { - // guard fires -> we do NOT compact (mirrors Model's early return). - } else { - run_compress_if_enabled(krows, vrows, n, kv_heads, head_dim, theta, p); - } - if (std::memcmp(kbuf.data(), kbuf0.data(), kbuf.size()) != 0) return false; - if (std::memcmp(vbuf.data(), vbuf0.data(), vbuf.size()) != 0) return false; - return true; -} - bool test_fp16_storage_round_trip() { // Write pre-RoPE -> post-RoPE into an fp16 cache, compact+renumber, read back, and compare // each survivor against the reference rope(pre, rank) within fp16 tolerance. Exercises the @@ -252,8 +150,7 @@ bool test_fp16_storage_round_trip() { auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); - Params p; p.enabled = true; p.budget_frac = 0.25f; p.recent_frac = 0.3f; p.sink = 4; - p.signal = Signal::KEYDIFF; + Params p; p.recent_frac = 0.3f; p.sink = 4; p.abs_budget = 16; auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); @@ -261,7 +158,7 @@ bool test_fp16_storage_round_trip() { for (size_t h = 0; h < kv_heads; ++h) { for (size_t rank = 0; rank < kept[h].size(); ++rank) { int abs_pos = kept[h][rank]; - std::vector expK = rope_row(pre[h][abs_pos], (double)rank, theta); + std::vector expK = rope_reference(pre[h][abs_pos], (double)rank, theta); const uint16_t* kdst = krows + (rank * kv_heads + h) * head_dim; const uint16_t* vdst = vrows + (rank * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) { @@ -318,7 +215,7 @@ bool test_compact_int8_cache() { std::vector k(head_dim), v(head_dim); for (size_t d = 0; d < head_dim; ++d) { k[d] = std::sin(0.1 * (h + 1) * (t + 1) + 0.2 * d); v[d] = std::cos(0.05 * (t + 1) + 0.1 * d); } pre[h][t] = k; vals[h][t] = v; - std::vector kr = rope_row(k, (double)t, theta); + std::vector kr = rope_reference(k, (double)t, theta); quant_row(kr, k_i8.data() + t * int8_stride + h * head_dim, k_sc.data() + t * scale_stride + h * groups); quant_row(v, v_i8.data() + t * int8_stride + h * head_dim, v_sc.data() + t * scale_stride + h * groups); } @@ -332,7 +229,7 @@ bool test_compact_int8_cache() { for (size_t h = 0; h < kv_heads; ++h) for (size_t rank = 0; rank < B; ++rank) { int abs_pos = kept[h][rank]; - std::vector expK = rope_row(pre[h][abs_pos], (double)rank, theta); + std::vector expK = rope_reference(pre[h][abs_pos], (double)rank, theta); const int8_t* kd = k_i8.data() + rank * int8_stride + h * head_dim; const float* ks = k_sc.data() + rank * scale_stride + h * groups; const int8_t* vd = v_i8.data() + rank * int8_stride + h * head_dim; @@ -353,7 +250,7 @@ bool test_compact_int8_cache() { size_t roll_compact_once(uint16_t* krows, uint16_t* vrows, Header* khdr, Header* vhdr, size_t kv_heads, size_t head_dim, double theta, int target_len) { size_t n = khdr->current_seq_len; - Params p; p.enabled = true; p.recent_frac = 0.30f; p.sink = 4; p.signal = Signal::KEYDIFF; + Params p; p.recent_frac = 0.30f; p.sink = 4; p.abs_budget = target_len; // absolute budget -> keep exactly min(target_len, n) per head auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); @@ -397,7 +294,7 @@ bool test_rolling_bounded_compaction() { for (size_t d = 0; d < head_dim; ++d) k[d] = std::sin(0.05 * (h + 1) + 0.3 * d) + 0.01f * std::sin(0.01 * t); } - std::vector kr = rope_row(k, (double)t, theta); + std::vector kr = rope_reference(k, (double)t, theta); std::vector vr(head_dim); for (size_t d = 0; d < head_dim; ++d) vr[d] = std::cos(0.07 * (t + 1) + 0.2 * d); for (size_t d = 0; d < head_dim; ++d) { @@ -453,10 +350,10 @@ bool test_rolling_bounded_compaction() { } bool test_config_parse_rolling_fields() { - // Config::from_json parses a key=value file. Defaults are 0/0 (legacy one-shot, no rolling). + // Config::from_json parses a key=value file. Rolling is the default (4096 -> 2048). cactus::engine::Config def; - if (def.kv_compress_trigger_len != 0 || def.kv_compress_target_len != 0) return false; - if (def.kv_compress) return false; + if (!def.kv_compress) return false; + if (def.kv_compress_trigger_len != 4096 || def.kv_compress_target_len != 2048) return false; char tmpl[] = "/tmp/cactus_kvcfg_XXXXXX"; int fd = mkstemp(tmpl); @@ -478,30 +375,46 @@ bool test_config_parse_rolling_fields() { return cfg.kv_compress && cfg.kv_compress_trigger_len == 4096 && cfg.kv_compress_target_len == 2048; } -bool test_trigger_zero_no_rolling() { - // trigger_len == 0 (default) => the rolling gate never fires, mirroring Model::maybe_roll_compact's - // early return. A cache grown well past any threshold stays byte-identical (no compaction). - cactus::engine::Config cfg; // defaults: kv_compress=false, trigger_len=0, target_len=0 - bool gate_open = cfg.kv_compress && cfg.kv_compress_trigger_len > 0; - if (gate_open) return false; - +bool test_trigger_zero_gates_rolling() { + // The rolling gate (Model::maybe_roll_compact): fire iff kv_compress && trigger_len > 0 && + // current_seq_len >= trigger_len. Drive the SAME compactor under both arms on identical caches: + // trigger_len == 0 must skip (cache byte-identical); a reached trigger must compact (len -> target, + // bytes change). The skip assertion is meaningful only against this proven mutation. const size_t n = 5000, kv_heads = 2, head_dim = 16, max_seq = 5008; const double theta = 1000000.0; - std::vector kbuf(kHeaderBytes + max_seq * kv_heads * head_dim * sizeof(uint16_t), 0); - std::vector vbuf(kbuf.size(), 0); - auto* khdr = reinterpret_cast(kbuf.data()); - *khdr = Header{n, max_seq, kv_heads, head_dim, 4, {0, 0, 0}}; - auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); - for (size_t i = 0; i < n * kv_heads * head_dim; ++i) krows[i] = f32_to_f16(0.01f * (i % 97)); - std::vector kbuf0 = kbuf, vbuf0 = vbuf; - // Gate closed => no compaction call at all => bytes unchanged and length unchanged. - if (gate_open) roll_compact_once(krows, reinterpret_cast(vbuf.data() + kHeaderBytes), - khdr, reinterpret_cast(vbuf.data()), - kv_heads, head_dim, theta, 2048); - if (khdr->current_seq_len != n) return false; - if (std::memcmp(kbuf.data(), kbuf0.data(), kbuf.size()) != 0) return false; - if (std::memcmp(vbuf.data(), vbuf0.data(), vbuf.size()) != 0) return false; + auto fresh_cache = [&]() { + std::vector buf(kHeaderBytes + max_seq * kv_heads * head_dim * sizeof(uint16_t), 0); + *reinterpret_cast(buf.data()) = Header{n, max_seq, kv_heads, head_dim, 4, {0, 0, 0}}; + auto* rows = reinterpret_cast(buf.data() + kHeaderBytes); + for (size_t i = 0; i < n * kv_heads * head_dim; ++i) rows[i] = f32_to_f16(0.01f * (i % 97)); + return buf; + }; + auto roll_if_gated = [&](cactus::engine::Config& cfg, std::vector& kbuf, std::vector& vbuf) { + auto* khdr = reinterpret_cast(kbuf.data()); + bool fire = cfg.kv_compress && cfg.kv_compress_trigger_len > 0 && + khdr->current_seq_len >= (size_t)cfg.kv_compress_trigger_len; + if (fire) + roll_compact_once(reinterpret_cast(kbuf.data() + kHeaderBytes), + reinterpret_cast(vbuf.data() + kHeaderBytes), khdr, + reinterpret_cast(vbuf.data()), kv_heads, head_dim, theta, + cfg.kv_compress_target_len); + }; + + // Gate closed (trigger_len == 0): caller skips => cache byte-identical, length unchanged. + cactus::engine::Config off; off.kv_compress_trigger_len = 0; off.kv_compress_target_len = 0; + std::vector k_off = fresh_cache(), v_off(k_off.size(), 0), k_off0 = k_off, v_off0 = v_off; + roll_if_gated(off, k_off, v_off); + if (reinterpret_cast(k_off.data())->current_seq_len != n) return false; + if (std::memcmp(k_off.data(), k_off0.data(), k_off.size()) != 0) return false; + if (std::memcmp(v_off.data(), v_off0.data(), v_off.size()) != 0) return false; + + // Gate open (trigger reached): the same compactor compacts to target_len and rewrites bytes. + cactus::engine::Config on; on.kv_compress_trigger_len = 4096; on.kv_compress_target_len = 2048; + std::vector k_on = fresh_cache(), v_on(k_on.size(), 0), k_on0 = k_on; + roll_if_gated(on, k_on, v_on); + if (reinterpret_cast(k_on.data())->current_seq_len != 2048) return false; + if (std::memcmp(k_on.data(), k_on0.data(), k_on.size()) == 0) return false; return true; } @@ -530,45 +443,45 @@ bool test_degenerate_rolling_config_disabled() { } bool test_env_override_parse() { - // parse_kv_compress_override maps the env strings onto Config Params exactly. - // Unset => no-op (OFF preserved). + // parse_kv_compress_override maps CACTUS_KV_COMPRESS_AT / CACTUS_KV_COMPRESS_TO onto Config. + // Unset => no override; the rolling default (4096 -> 2048) is preserved. cactus::engine::Config off; if (off.parse_kv_compress_override(nullptr, nullptr)) return false; - if (off.kv_compress) return false; - - // One-shot: ":". - cactus::engine::Config os; - if (!os.parse_kv_compress_override("keydiff:0.25", nullptr)) return false; - if (!os.kv_compress || os.kv_compress_signal != "keydiff") return false; - if (std::abs(os.kv_compress_budget_frac - 0.25f) > 1e-6f) return false; - if (os.kv_compress_trigger_len != 0 || os.kv_compress_target_len != 0) return false; - - // Rolling: ":" (takes precedence over one-shot). - cactus::engine::Config roll; - if (!roll.parse_kv_compress_override("keydiff:0.25", "4096:2048")) return false; - if (!roll.kv_compress) return false; - if (roll.kv_compress_trigger_len != 4096 || roll.kv_compress_target_len != 2048) return false; - - // Rolling with degenerate values is disabled by the validate guard inside the parser. + if (!off.kv_compress || off.kv_compress_trigger_len != 4096 || off.kv_compress_target_len != 2048) return false; + + // Both vars override. + cactus::engine::Config both; + if (!both.parse_kv_compress_override("3000", "1000")) return false; + if (!both.kv_compress || both.kv_compress_trigger_len != 3000 || both.kv_compress_target_len != 1000) return false; + + // Setting only one keeps the other's default. + cactus::engine::Config one; + if (!one.parse_kv_compress_override(nullptr, "1500")) return false; + if (one.kv_compress_trigger_len != 4096 || one.kv_compress_target_len != 1500) return false; + + // CACTUS_KV_COMPRESS_AT=0 disables. + cactus::engine::Config disabled; + if (!disabled.parse_kv_compress_override("0", nullptr)) return false; + if (disabled.kv_compress) return false; + + // Degenerate (target >= trigger) disabled by the validate guard. cactus::engine::Config bad; - if (!bad.parse_kv_compress_override(nullptr, "2048:4096")) return false; + if (!bad.parse_kv_compress_override("2048", "4096")) return false; if (bad.kv_compress_trigger_len != 0 || bad.kv_compress_target_len != 0) return false; return true; } int main() { - TestUtils::TestRunner runner("KV Compress Integration Tests"); + TestUtils::TestRunner runner("KV Compress Free-Function Tests"); runner.run_test("compact_fp16_cache", test_compact_fp16_cache()); runner.run_test("dense_check_full_budget", test_dense_check_full_budget()); runner.run_test("rope_renumber_contiguous", test_rope_renumber_contiguous()); - runner.run_test("compress_noop_when_disabled", test_compress_noop_when_disabled()); - runner.run_test("subset_compression_rejected", test_subset_compression_rejected()); runner.run_test("fp16_storage_round_trip", test_fp16_storage_round_trip()); runner.run_test("gemma_layer_selection", test_gemma_layer_selection()); runner.run_test("compact_int8_cache", test_compact_int8_cache()); runner.run_test("rolling_bounded_compaction", test_rolling_bounded_compaction()); runner.run_test("config_parse_rolling_fields", test_config_parse_rolling_fields()); - runner.run_test("trigger_zero_no_rolling", test_trigger_zero_no_rolling()); + runner.run_test("trigger_zero_gates_rolling", test_trigger_zero_gates_rolling()); runner.run_test("degenerate_rolling_config_disabled", test_degenerate_rolling_config_disabled()); runner.run_test("env_override_parse", test_env_override_parse()); runner.print_summary(); From d84b08bcf78a78d5ac053569e970502deb36ccbd Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 16:28:51 +0100 Subject: [PATCH 06/55] Compact global + re-rope sliding-window layers at compaction (engine-only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make rolling KV compaction work for Gemma-style hybrid models (interleaved sliding-window + global layers) entirely in the engine, on a single position frame, with no graph/transpiler change (works on existing bundles). At each compaction, compress_kv_cache_keydiff now runs two passes over the decoder cache states: - Pass 1 (global/full-attention layers): KeyDiff compact + renumber to 0..B-1, as before. - Pass 2 (sliding-window layers): rotate the recent K rows [sink_size, current_seq_len) in place by -Δ (Δ = old frontier - B) using the layer's LOCAL rope theta, so the window tracks the renumbered global frontier. Sink rows stay fixed; V is never rotated. current_seq_len is untouched (graph eviction already bounds it). RoPE is relative, so a uniform shift preserves all query·key offsets; a single shared position_ids = B then serves global (renumbered) and sliding (shifted) caches alike. The old all-global guard is replaced by global-subset compaction. New pure free functions in kv_compress: rerope_recent_fp16/int8 and the int8 helper rotate_int8_row; compact_int8's dequant->rotate->requant is factored into a shared requant_row (no behavior change). Tests T1-T6 cover the fp16/int8 re-rope, the int8 refactor (bitwise), the combined sliding+global rolling invariant over 3 cycles, the local-theta requirement, and the no-op guards. Verified: 17/17 free-function tests; model_loading/llm/vlm green; live Gemma gemma-4-e2b-it stays coherent across ~8 in-decode compaction cycles (no re-transpile); Qwen all-global path unchanged. Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.cpp | 74 ++++-- cactus-engine/src/kv_compress.h | 16 ++ cactus-engine/src/model.cpp | 60 +++-- .../tests/test_kv_compress_free_functions.cpp | 222 +++++++++++++++++- 4 files changed, 330 insertions(+), 42 deletions(-) diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index 1e9061f45..a3fef6ad6 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -134,6 +134,27 @@ inline double renumber_delta(int abs_pos, size_t rank) { return static_cast(rank) - static_cast(abs_pos); } +// Quantize one float row into int8 + per-group scales, matching the engine's KV quant convention +// (quantize_group_fp16_to_int8 in cactus-kernels/src/quants.cpp): scale floored at 1e-10, +// round-to-nearest via roundf, clamp to [-128, 127]. groups = ceil(head_dim/group_size). +void requant_row(const float* row, int8_t* dst, float* dsc, size_t head_dim, size_t group_size) { + size_t groups = (head_dim + group_size - 1) / group_size; + for (size_t g = 0; g < groups; ++g) { + size_t lo = g * group_size, hi = std::min(head_dim, lo + group_size); + float amax = 0.0f; + for (size_t d = lo; d < hi; ++d) amax = std::max(amax, std::fabs(row[d])); + float scale = amax / 127.0f; + if (scale < 1e-10f) scale = 1e-10f; + dsc[g] = scale; + float inv = 1.0f / scale; + for (size_t d = lo; d < hi; ++d) { + int32_t q = static_cast(std::roundf(row[d] * inv)); + q = std::max(-128, std::min(127, q)); + dst[d] = static_cast(q); + } + } +} + // Per-head KeyDiff keep-set pipeline shared by the fp16 and int8 entry points. `fill_post(h, post)` // gathers head h's post-RoPE rows into `post` ([n][head_dim]); the rest (un-RoPE -> score -> keepset) // is identical regardless of storage precision. @@ -198,28 +219,45 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, rope_rotate_row(row.data(), head_dim, rope_theta, renumber_delta(abs_pos, rank)); int8_t* dst = int8_rows + rank * int8_stride + h * head_dim; float* dsc = scale_rows + rank * scale_stride + h * groups; - for (size_t g = 0; g < groups; ++g) { - size_t lo = g * group_size, hi = std::min(head_dim, lo + group_size); - // Match the engine's KV quantization convention (quantize_group_fp16_to_int8 in - // cactus-kernels/src/quants.cpp): scale floored at 1e-10, round-to-nearest via - // roundf, clamp to [-128, 127]. Keeps re-quantized rows consistent with how the - // rest of the engine writes the int8 cache. - float amax = 0.0f; - for (size_t d = lo; d < hi; ++d) amax = std::max(amax, std::fabs(row[d])); - float scale = amax / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - dsc[g] = scale; - float inv = 1.0f / scale; - for (size_t d = lo; d < hi; ++d) { - int32_t q = static_cast(std::roundf(row[d] * inv)); - q = std::max(-128, std::min(127, q)); - dst[d] = static_cast(q); - } - } + requant_row(row.data(), dst, dsc, head_dim, group_size); } } } +void rotate_int8_row(int8_t* int8, float* scale, size_t head_dim, size_t group_size, + double rope_theta, double delta_pos) { + std::vector row(head_dim); + for (size_t d = 0; d < head_dim; ++d) row[d] = static_cast(int8[d]) * scale[d / group_size]; + rope_rotate_row(row.data(), head_dim, rope_theta, delta_pos); + requant_row(row.data(), int8, scale, head_dim, group_size); +} + +void rerope_recent_fp16(uint16_t* key_rows_u, size_t kv_heads, size_t head_dim, + size_t lo, size_t hi, double rope_theta, double delta_pos) { + if (delta_pos == 0.0 || hi <= lo) return; + __fp16* key_rows = reinterpret_cast<__fp16*>(key_rows_u); + std::vector row(head_dim); + for (size_t t = lo; t < hi; ++t) + for (size_t h = 0; h < kv_heads; ++h) { + __fp16* r = key_rows + (t * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) row[d] = r[d]; + rope_rotate_row(row.data(), head_dim, rope_theta, delta_pos); + for (size_t d = 0; d < head_dim; ++d) r[d] = static_cast<__fp16>(row[d]); + } +} + +void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, + size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos) { + if (delta_pos == 0.0 || hi <= lo) return; + size_t groups = (head_dim + group_size - 1) / group_size; + size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; + for (size_t t = lo; t < hi; ++t) + for (size_t h = 0; h < kv_heads; ++h) + rotate_int8_row(int8_rows + t * int8_stride + h * head_dim, + scale_rows + t * scale_stride + h * groups, + head_dim, group_size, rope_theta, delta_pos); +} + std::vector> keepsets_from_fp16(const uint16_t* key_rows_u, size_t n, size_t kv_heads, size_t head_dim, double rope_theta, const Params& p) { diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index c8335a14c..b3acd9b74 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -49,6 +49,12 @@ std::vector keepset_for_head(const float* scores, size_t n, const Params& p // In-place delta rotation of one key row of length head_dim by `delta_pos`. void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos); +// INT8 analog of rope_rotate_row: dequantize one quantized key row (int8 * per-group scale), +// rotate by `delta_pos`, and re-quantize in place. `scale` is the row's [groups] scales, +// groups = ceil(head_dim/group_size). +void rotate_int8_row(int8_t* int8, float* scale, size_t head_dim, size_t group_size, + double rope_theta, double delta_pos); + // Recover the pre-RoPE keys for one kv-head from post-RoPE rows. `post_rope` is // [n][head_dim], rows at absolute positions 0..n-1. Writes pre_rope [n][head_dim]. void unrope_head(const float* post_rope, size_t n, size_t head_dim, double rope_theta, @@ -77,6 +83,16 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, const std::vector>& kept_per_head, double rope_theta, bool renumber); +// Sliding-window re-rope (single position frame). Rotate the recent K rows [lo, hi) in place by +// `delta_pos` (typically negative) so a windowed cache tracks the renumbered global frontier after +// compaction. Rows [0, lo) (the attention sink) are left fixed; V is never rotated. `rope_theta` is +// the LOCAL theta for sliding layers. No-op when delta_pos == 0 or hi <= lo. The int8 variant takes +// the K int8 region + its per-group scales (same layout as compact_int8). +void rerope_recent_fp16(uint16_t* key_rows, size_t kv_heads, size_t head_dim, + size_t lo, size_t hi, double rope_theta, double delta_pos); +void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, + size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos); + // Build per-head keep-sets for one layer from POST-RoPE FP16 key rows: un-RoPE -> score -> // keepset_for_head. Returns n_kv_heads lists, each length B. std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index a5c4d8bc6..71cb3a5db 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2983,26 +2983,22 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) constexpr size_t kHeaderBytes = sizeof(CacheHeader); std::vector layers = compressible_layers(); - // Subset compression (e.g. Gemma physical set {4,9}) renumbers only some layers, but the - // engine has a SINGLE global position counter (cache_total_seq_len_). If only a subset is - // compacted, the non-compacted layers still hold n tokens while the counter would be set to - // B, so they RoPE the next decode query at the wrong offset -> silent corruption. Physical - // subset-renumbering needs per-layer position tracking, which this engine cannot represent. - // Only proceed when EVERY layer is compressible (the Qwen all-global case). - if (layers.size() != config_.num_layers) { - CACTUS_LOG_WARN("kv_compress", - "subset compression unsupported (needs per-layer positions); disabled for this model"); - return; - } + if (layers.empty()) return; // no global layers to anchor the compressed frame + std::set compressible(layers.begin(), layers.end()); const double rope_theta = static_cast(config_.rope_theta); + // Sliding-window layers re-rope into the compressed frame with their LOCAL theta (pass 2). + const double rope_local_theta = (config_.rope_local_base_freq == Config::UNSET_F32) + ? rope_theta : static_cast(config_.rope_local_base_freq); + const size_t old_total = cache_total_seq_len_; // pre-compaction compressed frontier (for Δ) Component& comp = *decoder_; if (!comp.graph) return; - // Capture the compacted length (B, identical across all layers) from the keep-set itself, - // not from a header we just mutated, so the global position counter stays consistent. + // Pass 1: compact + renumber the global (full-attention) layers to 0..B-1. B is identical across + // them; capture it from the keep-set, not a header we just mutated. size_t new_seq_len = 0; bool have_new_seq_len = false; for (size_t li = 0; li < comp.cache_states.size(); ++li) { + if (!compressible.count(li)) continue; // global layers only const auto& cs = comp.cache_states[li]; if (cs.key_node_id < 0 || cs.value_node_id < 0) continue; @@ -3054,8 +3050,42 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } } - // Every layer was compacted to the same B (guarded above). Decode appends at B: subsequent - // positions are B, B+1, ... (renumbered window). + // Pass 2: re-rope the sliding-window layers' recent K rows by -Δ (Δ = old frontier − B) so they + // track the renumbered global frontier in the same (compressed) position frame. RoPE is relative, + // so a uniform shift preserves all query·key offsets. Sink rows [0, sink_size) stay fixed. + const size_t Delta = (have_new_seq_len && old_total >= new_seq_len) ? old_total - new_seq_len : 0; + if (Delta > 0) { + const double dpos = -static_cast(Delta); + for (size_t li = 0; li < comp.cache_states.size(); ++li) { + if (compressible.count(li)) continue; // sliding (non-compressible) layers only + const auto& cs = comp.cache_states[li]; + if (cs.key_node_id < 0) continue; + const auto& kdesc = comp.graph->get_output_buffer(static_cast(cs.key_node_id)); + if (kdesc.byte_size <= kHeaderBytes) continue; + void* kraw = comp.graph->get_output(static_cast(cs.key_node_id)); + if (!kraw) continue; + auto* khdr = static_cast(kraw); + size_t kv_heads = khdr->num_kv_heads, head_dim = khdr->head_dim; + if (kv_heads == 0 || head_dim == 0) continue; + size_t hi = khdr->current_seq_len; + size_t lo = std::min(khdr->sink_size, hi); + if (kdesc.precision == Precision::FP16) { + auto* kbase = reinterpret_cast(static_cast(kraw) + kHeaderBytes); + cactus::kvcompress::rerope_recent_fp16(kbase, kv_heads, head_dim, lo, hi, + rope_local_theta, dpos); + } else if (kdesc.precision == Precision::INT8) { + size_t max_seq = khdr->max_seq_len; + auto* k_i8 = reinterpret_cast(static_cast(kraw) + kHeaderBytes); + auto* k_sc = reinterpret_cast(static_cast(kraw) + kHeaderBytes + + max_seq * kv_heads * head_dim); + cactus::kvcompress::rerope_recent_int8(k_i8, k_sc, kv_heads, head_dim, + KV_QUANT_GROUP_SIZE, lo, hi, rope_local_theta, dpos); + } + } + } + + // Global layers were compacted to B; the next decode query uses position_ids = B for ALL layers + // (global renumbered, sliding shifted into the same frame). if (have_new_seq_len) cache_total_seq_len_ = new_seq_len; } diff --git a/cactus-engine/tests/test_kv_compress_free_functions.cpp b/cactus-engine/tests/test_kv_compress_free_functions.cpp index 676ccf364..7c6a01129 100644 --- a/cactus-engine/tests/test_kv_compress_free_functions.cpp +++ b/cactus-engine/tests/test_kv_compress_free_functions.cpp @@ -65,6 +65,21 @@ std::vector make_fp16_value_cache(size_t n, size_t max_seq, size_t kv_head return buf; } +// Independent oracle quantizer (mirrors quants.cpp / compact_int8's convention: scale floored +// 1e-10, roundf, clamp [-128,127]). Kept separate from production requant so the tests don't +// validate it against itself. +void quantize_row(const std::vector& row, int8_t* dst, float* sc, + size_t head_dim, size_t group_size) { + size_t groups = (head_dim + group_size - 1) / group_size; + for (size_t g = 0; g < groups; ++g) { + size_t lo = g * group_size, hi = std::min(head_dim, lo + group_size); + float amax = 0.f; for (size_t d = lo; d < hi; ++d) amax = std::max(amax, std::fabs(row[d])); + float scale = amax / 127.f; if (scale < 1e-10f) scale = 1e-10f; sc[g] = scale; + float inv = 1.f / scale; + for (size_t d = lo; d < hi; ++d) { int32_t q = (int32_t)std::roundf(row[d] * inv); q = std::max(-128, std::min(127, q)); dst[d] = (int8_t)q; } + } +} + } // namespace bool test_compact_fp16_cache() { @@ -199,16 +214,8 @@ bool test_compact_int8_cache() { std::vector k_i8(max_seq * int8_stride, 0), v_i8(max_seq * int8_stride, 0); std::vector k_sc(max_seq * scale_stride, 0.f), v_sc(max_seq * scale_stride, 0.f); - // Match the engine KV quant convention (quants.cpp): scale floored at 1e-10, roundf, - // clamp [-128,127] -- same as compact_int8's re-quant path. auto quant_row = [&](const std::vector& row, int8_t* dst, float* sc) { - for (size_t g = 0; g < groups; ++g) { - size_t lo = g * kGroupSize, hi = std::min(head_dim, lo + kGroupSize); - float amax = 0.f; for (size_t d = lo; d < hi; ++d) amax = std::max(amax, std::fabs(row[d])); - float scale = amax / 127.f; if (scale < 1e-10f) scale = 1e-10f; sc[g] = scale; - float inv = 1.f / scale; - for (size_t d = lo; d < hi; ++d) { int32_t q = (int32_t)std::roundf(row[d] * inv); q = std::max(-128, std::min(127, q)); dst[d] = (int8_t)q; } - } + quantize_row(row, dst, sc, head_dim, kGroupSize); }; for (size_t h = 0; h < kv_heads; ++h) for (size_t t = 0; t < n; ++t) { @@ -471,6 +478,197 @@ bool test_env_override_parse() { return true; } +// --- Sliding-window re-rope (single position frame) ------------------------------------------- + +bool test_rerope_recent_fp16_uniform() { + // Rotate recent rows [sink, n) by a uniform (negative) delta: each recent row now reads as + // rope(pre, t+delta); sink rows [0, sink) stay byte-identical. Uses the LOCAL theta. + const size_t n = 40, max_seq = 64, kv_heads = 2, head_dim = 16, sink = 4; + const double theta = 10000.0, delta = -7.0; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); // header sink_size=4 + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + std::vector sink_before(krows, krows + sink * kv_heads * head_dim); + + rerope_recent_fp16(krows, kv_heads, head_dim, sink, n, theta, delta); + + if (std::memcmp(krows, sink_before.data(), sink_before.size() * sizeof(uint16_t)) != 0) return false; + for (size_t t = sink; t < n; ++t) + for (size_t h = 0; h < kv_heads; ++h) { + std::vector exp = rope_reference(pre[h][t], (double)t + delta, theta); + const uint16_t* r = krows + (t * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) + if (std::abs(f16_to_f32(r[d]) - exp[d]) > 5e-2f) return false; + } + return true; +} + +namespace { +// Build an int8 K cache storing rope(pre, t, theta) per group for t in [0,n). +void build_int8_key_cache(size_t n, size_t kv_heads, size_t head_dim, double theta, + std::vector>>& pre, + std::vector& k_i8, std::vector& k_sc) { + size_t groups = (head_dim + kGroupSize - 1) / kGroupSize; + size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; + pre.assign(kv_heads, std::vector>(n)); + k_i8.assign(n * int8_stride, 0); k_sc.assign(n * scale_stride, 0.f); + for (size_t h = 0; h < kv_heads; ++h) + for (size_t t = 0; t < n; ++t) { + std::vector k(head_dim); + for (size_t d = 0; d < head_dim; ++d) k[d] = std::sin(0.1 * (h + 1) * (t + 1) + 0.2 * d); + pre[h][t] = k; + std::vector kr = rope_reference(k, (double)t, theta); + quantize_row(kr, k_i8.data() + t * int8_stride + h * head_dim, + k_sc.data() + t * scale_stride + h * groups, head_dim, kGroupSize); + } +} +} // namespace + +bool test_rerope_recent_int8() { + const size_t n = 40, kv_heads = 2, head_dim = 32, sink = 4; + const double theta = 10000.0, delta = -9.0; + size_t groups = (head_dim + kGroupSize - 1) / kGroupSize; + size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; + std::vector>> pre; + std::vector k_i8; std::vector k_sc; + build_int8_key_cache(n, kv_heads, head_dim, theta, pre, k_i8, k_sc); + + std::vector sink_i8(k_i8.begin(), k_i8.begin() + sink * int8_stride); + std::vector sink_sc(k_sc.begin(), k_sc.begin() + sink * scale_stride); + rerope_recent_int8(k_i8.data(), k_sc.data(), kv_heads, head_dim, kGroupSize, sink, n, theta, delta); + + if (std::memcmp(k_i8.data(), sink_i8.data(), sink_i8.size()) != 0) return false; + if (std::memcmp(k_sc.data(), sink_sc.data(), sink_sc.size() * sizeof(float)) != 0) return false; + for (size_t h = 0; h < kv_heads; ++h) + for (size_t t = sink; t < n; ++t) { + std::vector exp = rope_reference(pre[h][t], (double)t + delta, theta); + const int8_t* kd = k_i8.data() + t * int8_stride + h * head_dim; + const float* ks = k_sc.data() + t * scale_stride + h * groups; + for (size_t d = 0; d < head_dim; ++d) + if (std::abs((float)kd[d] * ks[d / kGroupSize] - exp[d]) > 0.05f) return false; + } + return true; +} + +bool test_rotate_int8_row_matches_inline() { + // Refactor safety: the factored rotate_int8_row must be bitwise-identical to the old inline + // dequant -> rope_rotate_row -> per-group requant. + const size_t head_dim = 32; const double theta = 10000.0, delta = -9.0; + size_t groups = (head_dim + kGroupSize - 1) / kGroupSize; + std::vector pre(head_dim); + for (size_t d = 0; d < head_dim; ++d) pre[d] = std::sin(0.3 * d) + 0.1 * d; + std::vector stored = rope_reference(pre, 21.0, theta); + std::vector a_i8(head_dim), b_i8(head_dim); + std::vector a_sc(groups), b_sc(groups); + quantize_row(stored, a_i8.data(), a_sc.data(), head_dim, kGroupSize); + b_i8 = a_i8; b_sc = a_sc; + + // Path A: literal old inline sequence. + std::vector row(head_dim); + for (size_t d = 0; d < head_dim; ++d) row[d] = (float)a_i8[d] * a_sc[d / kGroupSize]; + rope_rotate_row(row.data(), head_dim, theta, delta); + quantize_row(row, a_i8.data(), a_sc.data(), head_dim, kGroupSize); + // Path B: factored helper. + rotate_int8_row(b_i8.data(), b_sc.data(), head_dim, kGroupSize, theta, delta); + + return std::memcmp(a_i8.data(), b_i8.data(), head_dim) == 0 + && std::memcmp(a_sc.data(), b_sc.data(), groups * sizeof(float)) == 0; +} + +bool test_sliding_plus_global_rolling() { + // Cross-layer single-frame invariant over 3 cycles: a GLOBAL cache compacts+renumbers to B and + // a SLIDING cache re-ropes its recent rows by -Δ (Δ = old_total - B), so a shared query at + // position B matches both. Global distinctive token survives; sliding sink stays; sliding + // recent rows track to (t - sum Δ) within fp16 tolerance across cycles. + const size_t kv_heads = 2, head_dim = 16, sink = 4; + const double gtheta = 1000000.0, ltheta = 10000.0; + + // --- Global: one keydiff compaction, distinctive token must survive + renumber to 0..B-1. --- + { + const size_t n = 60, max_seq = 96; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, gtheta, pre, vals); + auto vbuf = make_fp16_value_cache(n, max_seq, kv_heads, head_dim, vals); + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); + Params p; p.recent_frac = 0.3f; p.sink = sink; p.abs_budget = 16; + auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, gtheta, p); + size_t B = kept[0].size(); + compact_fp16(krows, vrows, kv_heads, head_dim, kept, gtheta); + for (auto& k : kept) if (k.size() != B) return false; // uniform B across heads + for (size_t h = 0; h < kv_heads; ++h) // global survivors renumber to 0..B-1 + for (size_t rank = 0; rank < B; ++rank) { + std::vector exp = rope_reference(pre[h][kept[h][rank]], (double)rank, gtheta); + const uint16_t* r = krows + (rank * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) + if (std::abs(f16_to_f32(r[d]) - exp[d]) > 5e-2f) return false; + } + } + + // --- Sliding: re-rope recent rows by -Δ over 3 cycles, with Δ derived from a global cycle. --- + const size_t n = 16, max_seq = 32; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, ltheta, pre, vals); + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + std::vector sink_before(krows, krows + sink * kv_heads * head_dim); + + const size_t old_total = 30, B = 12; // a representative global cycle + const double Delta = (double)(old_total - B); + double accum = 0.0; + for (int cycle = 0; cycle < 3; ++cycle) { + rerope_recent_fp16(krows, kv_heads, head_dim, sink, n, ltheta, -Delta); + accum -= Delta; + } + if (std::memcmp(krows, sink_before.data(), sink_before.size() * sizeof(uint16_t)) != 0) return false; + for (size_t t = sink; t < n; ++t) + for (size_t h = 0; h < kv_heads; ++h) { + std::vector exp = rope_reference(pre[h][t], (double)t + accum, ltheta); + const uint16_t* r = krows + (t * kv_heads + h) * head_dim; + for (size_t d = 0; d < head_dim; ++d) + if (std::abs(f16_to_f32(r[d]) - exp[d]) > 8e-2f) return false; // looser: 3-cycle drift + } + return true; +} + +bool test_rerope_local_theta_used() { + // The sliding re-rope must use the LOCAL theta: rotating with the wrong (global) theta does NOT + // recover the expected position. Pins the orchestration's theta choice as load-bearing. + const size_t n = 24, max_seq = 32, kv_heads = 1, head_dim = 16, sink = 4; + const double ltheta = 10000.0, gtheta = 1000000.0, delta = -5.0; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, ltheta, pre, vals); + std::vector kbuf2 = kbuf; + auto* a = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* b = reinterpret_cast(kbuf2.data() + kHeaderBytes); + + rerope_recent_fp16(a, kv_heads, head_dim, sink, n, ltheta, delta); // correct + rerope_recent_fp16(b, kv_heads, head_dim, sink, n, gtheta, delta); // wrong theta + + bool correct_ok = true, wrong_differs = false; + for (size_t t = sink; t < n; ++t) { + std::vector exp = rope_reference(pre[0][t], (double)t + delta, ltheta); + const uint16_t* ra = a + (t * kv_heads) * head_dim; + const uint16_t* rb = b + (t * kv_heads) * head_dim; + for (size_t d = 0; d < head_dim; ++d) { + if (std::abs(f16_to_f32(ra[d]) - exp[d]) > 5e-2f) correct_ok = false; + if (std::abs(f16_to_f32(rb[d]) - exp[d]) > 5e-2f) wrong_differs = true; + } + } + return correct_ok && wrong_differs; +} + +bool test_rerope_zero_delta_noop() { + const size_t n = 20, max_seq = 32, kv_heads = 2, head_dim = 16, sink = 4; + const double theta = 10000.0; + std::vector>> pre, vals; + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); + std::vector before = kbuf; + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + rerope_recent_fp16(krows, kv_heads, head_dim, sink, n, theta, 0.0); // zero delta + rerope_recent_fp16(krows, kv_heads, head_dim, n, n, theta, -3.0); // empty range hi<=lo + return kbuf == before; +} + int main() { TestUtils::TestRunner runner("KV Compress Free-Function Tests"); runner.run_test("compact_fp16_cache", test_compact_fp16_cache()); @@ -479,6 +677,12 @@ int main() { runner.run_test("fp16_storage_round_trip", test_fp16_storage_round_trip()); runner.run_test("gemma_layer_selection", test_gemma_layer_selection()); runner.run_test("compact_int8_cache", test_compact_int8_cache()); + runner.run_test("rerope_recent_fp16_uniform", test_rerope_recent_fp16_uniform()); + runner.run_test("rerope_recent_int8", test_rerope_recent_int8()); + runner.run_test("rotate_int8_row_matches_inline", test_rotate_int8_row_matches_inline()); + runner.run_test("sliding_plus_global_rolling", test_sliding_plus_global_rolling()); + runner.run_test("rerope_local_theta_used", test_rerope_local_theta_used()); + runner.run_test("rerope_zero_delta_noop", test_rerope_zero_delta_noop()); runner.run_test("rolling_bounded_compaction", test_rolling_bounded_compaction()); runner.run_test("config_parse_rolling_fields", test_config_parse_rolling_fields()); runner.run_test("trigger_zero_gates_rolling", test_trigger_zero_gates_rolling()); From 80a129c38baa2eb321300acd317e608320e29e05 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 18:13:01 +0100 Subject: [PATCH 07/55] Default chunked prefill for text qwen3 conversions Builds on the merged #687 text chunked-prefill adapters: make the chunked component pipeline (lm_encoder_step, lm_encoder_text_chunk, decoder_prefill_chunk, decoder_media_step, decoder_step) the DEFAULT for text-only Qwen3ForCausalLM conversions, instead of the monolithic decoder/decoder_step. Gated on the Qwen3ForCausalLM architecture (the chunk adapters are qwen3-only); other families and multimodal paths are unchanged. ~5x faster prefill, identical output, and avoids the stale-bundle FP16-legalization crash on re-convert. Signed-off-by: Noah Cylich --- python/cactus/transpile/component_plan.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python/cactus/transpile/component_plan.py b/python/cactus/transpile/component_plan.py index 5c57ac9f3..11eb06f5f 100644 --- a/python/cactus/transpile/component_plan.py +++ b/python/cactus/transpile/component_plan.py @@ -28,6 +28,18 @@ def _plan_from_profile( if config is not None and profile.needs_image: has_vision = _has_dict_config(config, "vision_config", "visual_config", "image_config") if not has_vision: + # Text-only causal LM (no vision). qwen3 defaults to the chunked-prefill component + # pipeline (#687: ~5x faster prefill, identical output); the chunk adapters are qwen3 + # only, so other families stay on the monolithic decoder. Key on the architecture + # ("Qwen3ForCausalLM") since the bundle config's model_type collapses to "qwen". + architectures = [str(a).lower() for a in (config.get("architectures") or ())] + if any("qwen3forcausallm" in a for a in architectures): + return ComponentPlan( + task="causal_lm_logits", + components=("decoder_step", "lm_encoder_step", "lm_encoder_text_chunk", + "decoder_prefill_chunk", "decoder_media_step"), + force_component_pipeline=True, + ) return ComponentPlan( task="causal_lm_logits", components=("decoder", "decoder_step"), From 20b596aa8aa892c06c9cd787973bd418a1b0aca9 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 18:36:19 +0100 Subject: [PATCH 08/55] Skip non-attention caches in KV compaction (fix LFM2/hybrid corruption) compress_kv_cache_keydiff iterated all of decoder_->cache_states and classified layers as compressible attention purely by the "sliding" substring in layer_types. Hybrid models (LFM2: conv + gated-deltanet) bind their conv and recurrent cache states into the SAME list, so those were KeyDiff-compacted as if they were attention KV -> state corruption and out-of-bounds heap r/w (the kv_heads/head_dim==0 guard is bypassed because conv metadata lands nonzero in those header slots). Compaction is default-on, so this corrupted any hybrid model past the trigger. Fix: in both compaction passes, skip any cache state whose key node op type is not KV_CACHE_STATE (the same op-type discrimination the engine already uses for recurrent/conv caches in run_chunked_prefill). Conv and gated-deltanet caches are now left untouched; pure-attention and sliding+global models are unaffected. Verified: LFM2 (lfm2.5-vl-1.6b) now generates coherently with compaction firing (was <|pad|> garbage at the trigger); Qwen all-global and Gemma sliding+global unchanged; kv unit tests 17/17. Signed-off-by: Noah Cylich --- cactus-engine/src/model.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 71cb3a5db..0a11eae5e 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -3001,6 +3001,10 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) if (!compressible.count(li)) continue; // global layers only const auto& cs = comp.cache_states[li]; if (cs.key_node_id < 0 || cs.value_node_id < 0) continue; + // Only attention KV caches. Hybrid models (LFM2) put conv + gated-deltanet recurrent cache + // states in the same list (different / header-less layout); KeyDiff would corrupt them and + // read out of bounds, so skip anything that is not a KV_CACHE_STATE. + if (comp.graph->get_node_op_type(static_cast(cs.key_node_id)) != OpType::KV_CACHE_STATE) continue; const auto& kdesc = comp.graph->get_output_buffer(static_cast(cs.key_node_id)); const auto& vdesc = comp.graph->get_output_buffer(static_cast(cs.value_node_id)); @@ -3060,6 +3064,7 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) if (compressible.count(li)) continue; // sliding (non-compressible) layers only const auto& cs = comp.cache_states[li]; if (cs.key_node_id < 0) continue; + if (comp.graph->get_node_op_type(static_cast(cs.key_node_id)) != OpType::KV_CACHE_STATE) continue; // skip conv/recurrent caches const auto& kdesc = comp.graph->get_output_buffer(static_cast(cs.key_node_id)); if (kdesc.byte_size <= kHeaderBytes) continue; void* kraw = comp.graph->get_output(static_cast(cs.key_node_id)); From 5463bd51c219a2e5fbdb67751d06762cf506b3f9 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 19:19:14 +0100 Subject: [PATCH 09/55] Re-rope KV-shared global source layers with global theta Compaction pass 2 re-roped every non-compressible layer's recent keys with the local (sliding) theta, assuming the complement of the compressible set is all sliding-window layers. That is false for KV-sharing hybrids (Gemma): a global layer that is a KV-shared source is excluded from compaction yet is not sliding, so it was re-roped with the wrong theta -- desyncing the cache that the shared global layers attend to. Pick the re-rope theta from each layer's own type via is_sliding_layer (shared with physical_compressible_layers' is_full check) instead of from compressibility: sliding -> local theta, full-attention -> global theta. Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.cpp | 9 +++++---- cactus-engine/src/kv_compress.h | 6 ++++++ cactus-engine/src/model.cpp | 14 +++++++++----- .../tests/test_kv_compress_free_functions.cpp | 11 ++++++++++- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index a3fef6ad6..b575a84df 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -287,12 +287,13 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const }); } +bool is_sliding_layer(const std::vector& layer_types, size_t li) { + return li < layer_types.size() && layer_types[li].find("sliding") != std::string::npos; +} + std::vector physical_compressible_layers(const std::vector& layer_types, size_t num_layers, size_t num_kv_shared) { - auto is_full = [&](size_t i) { - if (i >= layer_types.size()) return true; - return layer_types[i].find("sliding") == std::string::npos; - }; + auto is_full = [&](size_t i) { return !is_sliding_layer(layer_types, i); }; std::vector full; for (size_t i = 0; i < num_layers; ++i) if (is_full(i)) full.push_back(i); diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index b3acd9b74..0a596d2d2 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -107,6 +107,12 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const size_t group_size, double rope_theta, const Params& p); +// True if layer `li` is a sliding-window (local) layer. Its key cache re-ropes with the LOCAL +// theta during compaction; full-attention layers -- including KV-shared global *source* layers +// that are excluded from compaction yet still ride the renumbered frame -- return false and +// re-rope with the GLOBAL theta. Out-of-range (empty layer_types -> Qwen) is full/global. +bool is_sliding_layer(const std::vector& layer_types, size_t li); + // Physically-compressible layer indices, mirroring MLX compressible_layers(physical=True). // `layer_types` token contains "sliding" for local layers; everything else is full/global. // Empty layer_types -> all layers full (Qwen). num_kv_shared drops the shared consumer diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 0cda2a101..55df9743e 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -3016,14 +3016,16 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } } - // Pass 2: re-rope the sliding-window layers' recent K rows by -Δ (Δ = old frontier − B) so they + // Pass 2: re-rope the non-compacted layers' recent K rows by -Δ (Δ = old frontier − B) so they // track the renumbered global frontier in the same (compressed) position frame. RoPE is relative, - // so a uniform shift preserves all query·key offsets. Sink rows [0, sink_size) stay fixed. + // so a uniform shift preserves all query·key offsets. Sink rows [0, sink_size) stay fixed. Each + // layer shifts with its OWN theta: sliding layers use the local theta; full-attention layers that + // were excluded from compaction (KV-shared global *source* layers, e.g. Gemma) keep the global theta. const size_t Delta = (have_new_seq_len && old_total >= new_seq_len) ? old_total - new_seq_len : 0; if (Delta > 0) { const double dpos = -static_cast(Delta); for (size_t li = 0; li < comp.cache_states.size(); ++li) { - if (compressible.count(li)) continue; // sliding (non-compressible) layers only + if (compressible.count(li)) continue; // skip the layers compacted in pass 1 const auto& cs = comp.cache_states[li]; if (cs.key_node_id < 0) continue; if (comp.graph->get_node_op_type(static_cast(cs.key_node_id)) != OpType::KV_CACHE_STATE) continue; // skip conv/recurrent caches @@ -3036,17 +3038,19 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) if (kv_heads == 0 || head_dim == 0) continue; size_t hi = khdr->current_seq_len; size_t lo = std::min(khdr->sink_size, hi); + const double layer_theta = cactus::kvcompress::is_sliding_layer(config_.layer_types, li) + ? rope_local_theta : rope_theta; if (kdesc.precision == Precision::FP16) { auto* kbase = reinterpret_cast(static_cast(kraw) + kHeaderBytes); cactus::kvcompress::rerope_recent_fp16(kbase, kv_heads, head_dim, lo, hi, - rope_local_theta, dpos); + layer_theta, dpos); } else if (kdesc.precision == Precision::INT8) { size_t max_seq = khdr->max_seq_len; auto* k_i8 = reinterpret_cast(static_cast(kraw) + kHeaderBytes); auto* k_sc = reinterpret_cast(static_cast(kraw) + kHeaderBytes + max_seq * kv_heads * head_dim); cactus::kvcompress::rerope_recent_int8(k_i8, k_sc, kv_heads, head_dim, - KV_QUANT_GROUP_SIZE, lo, hi, rope_local_theta, dpos); + KV_QUANT_GROUP_SIZE, lo, hi, layer_theta, dpos); } } } diff --git a/cactus-engine/tests/test_kv_compress_free_functions.cpp b/cactus-engine/tests/test_kv_compress_free_functions.cpp index 7c6a01129..d27a4199a 100644 --- a/cactus-engine/tests/test_kv_compress_free_functions.cpp +++ b/cactus-engine/tests/test_kv_compress_free_functions.cpp @@ -197,7 +197,16 @@ bool test_gemma_layer_selection() { for (int i = 0; i < 35; ++i) g.push_back(((i + 1) % 5 == 0) ? "global" : "sliding"); auto gg = physical_compressible_layers(g, 35, 20); std::vector expect = {4, 9}; - return gg == expect; + if (gg != expect) return false; + + // Pass-2 re-rope theta is chosen per layer via is_sliding_layer (NOT compressibility): sliding + // layers shift with the local theta, full-attention layers with the global theta. Layer 14 is a + // global KV-shared *source* -- excluded from compaction (not in {4,9}) yet must keep global theta. + if (is_sliding_layer(g, 14)) return false; // global source -> global theta (the fix) + if (is_sliding_layer(g, 4)) return false; // compacted global -> global theta + if (!is_sliding_layer(g, 3)) return false; // sliding -> local theta + for (size_t i = 0; i < 28; ++i) if (is_sliding_layer(qwen, i)) return false; // all-global Qwen + return true; } bool test_compact_int8_cache() { From de45aaeeee780bc7492b423819ed43c7b863d824 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 19:23:37 +0100 Subject: [PATCH 10/55] Disable kv_compress flag on invalid rolling config validate_kv_compress zeroed trigger/target on an invalid config but left kv_compress=true, inconsistent with the explicit-disable path in parse_kv_compress_override. Set kv_compress=false here too so both disable paths leave the same state. Signed-off-by: Noah Cylich --- cactus-engine/src/model.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 55df9743e..043d6d1b5 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -3463,6 +3463,7 @@ void Config::validate_kv_compress() { CACTUS_LOG_WARN("kv_compress", "invalid rolling config (target_len=" << kv_compress_target_len << ", trigger_len=" << kv_compress_trigger_len << "): require 0 < target_len < trigger_len; disabling rolling compaction"); + kv_compress = false; kv_compress_trigger_len = 0; kv_compress_target_len = 0; } From 5dbe28846a9c0494f6d36dcc4e1fd7593ab92e11 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 19:32:43 +0100 Subject: [PATCH 11/55] Remove NIAH KV-compress test fixtures and consumer test The MLX-fixture consumer test (test_kv_compress.cpp) and its generated fixtures were research artifacts. The self-contained free-functions test covers the KV-compress math without them. Signed-off-by: Noah Cylich --- .gitignore | 10 - cactus-engine/tests/test_kv_compress.cpp | 364 ------------------ .../niah/fixtures/kv_compress_fixtures.json | 1 - experiments/niah/gen_kv_compress_fixtures.py | 136 ------- 4 files changed, 511 deletions(-) delete mode 100644 cactus-engine/tests/test_kv_compress.cpp delete mode 100644 experiments/niah/fixtures/kv_compress_fixtures.json delete mode 100644 experiments/niah/gen_kv_compress_fixtures.py diff --git a/.gitignore b/.gitignore index 60f91e82b..e0918fde3 100644 --- a/.gitignore +++ b/.gitignore @@ -50,13 +50,3 @@ vision_activations.json gemma-4-eap-extras/ .githooks/ ruy - -# The MLX research harness under experiments/ stays untracked, but the C++ KV-compress -# math tests depend on the fixture generator + one small fixture, so track exactly those. -experiments/* -!experiments/niah/ -experiments/niah/* -!experiments/niah/gen_kv_compress_fixtures.py -!experiments/niah/fixtures/ -experiments/niah/fixtures/* -!experiments/niah/fixtures/kv_compress_fixtures.json diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp deleted file mode 100644 index 8bb63c0cd..000000000 --- a/cactus-engine/tests/test_kv_compress.cpp +++ /dev/null @@ -1,364 +0,0 @@ -#include "test_utils.h" -#include "../src/kv_compress.h" - -#include -#include -#include -#include -#include -#include -#include - -using namespace TestUtils; -using namespace cactus::kvcompress; - -// --------------------------------------------------------------------------- // -// Minimal JSON reader for the fixture file produced by gen_kv_compress_fixtures // -// --------------------------------------------------------------------------- // -namespace { - -struct Fixture { - std::string name; - int n = 0, head_dim = 0, n_kv_heads = 0, budget = 0, sink = 0; - float recent_frac = 0.f; - std::vector keys; // [n_kv_heads * n * head_dim] - std::vector> scores; // [n_kv_heads][n] - std::vector> kept; // [n_kv_heads][B] -}; - -struct RopeFixture { - int head_dim = 0, abs_pos = 0, rank = 0; - float rope_theta = 0.f; - std::vector pre_rope, post_rope_at_abs, post_rope_at_rank; -}; - -// Tiny tokenizing scanner. The fixture file is machine-generated with a known shape, so a -// streaming number/string/structure scanner is enough (no general-purpose JSON needed). -struct Scanner { - const std::string& s; - size_t i = 0; - explicit Scanner(const std::string& str) : s(str) {} - void skip_ws() { while (i < s.size() && (s[i] == ' ' || s[i] == '\n' || s[i] == '\t' || s[i] == '\r')) ++i; } - char peek() { skip_ws(); return i < s.size() ? s[i] : '\0'; } - void expect(char c) { skip_ws(); if (i >= s.size() || s[i] != c) throw std::runtime_error(std::string("expected ") + c); ++i; } - bool accept(char c) { skip_ws(); if (i < s.size() && s[i] == c) { ++i; return true; } return false; } - std::string parse_string() { - skip_ws(); expect('"'); - std::string out; - while (i < s.size() && s[i] != '"') out.push_back(s[i++]); - expect('"'); - return out; - } - double parse_number() { - skip_ws(); - size_t start = i; - while (i < s.size() && (std::isdigit((unsigned char)s[i]) || s[i] == '-' || s[i] == '+' || - s[i] == '.' || s[i] == 'e' || s[i] == 'E')) ++i; - return std::stod(s.substr(start, i - start)); - } - void skip_value() { // skip an arbitrary JSON value (only used to skip unknown keys) - char c = peek(); - if (c == '"') { parse_string(); return; } - if (c == '{') { parse_object_skip(); return; } - if (c == '[') { expect('['); if (peek() != ']') { do { skip_value(); } while (accept(',')); } expect(']'); return; } - parse_number(); - } - void parse_object_skip() { - expect('{'); - if (peek() != '}') { do { parse_string(); expect(':'); skip_value(); } while (accept(',')); } - expect('}'); - } - template - void parse_object(F&& on_key) { // on_key(key) -> bool (true if consumed value) - expect('{'); - if (peek() != '}') { - do { - std::string key = parse_string(); - expect(':'); - if (!on_key(key)) skip_value(); - } while (accept(',')); - } - expect('}'); - } - std::vector parse_float_array() { - std::vector v; - expect('['); - if (peek() != ']') do { v.push_back(static_cast(parse_number())); } while (accept(',')); - expect(']'); - return v; - } - std::vector parse_int_array() { - std::vector v; - expect('['); - if (peek() != ']') do { v.push_back(static_cast(std::llround(parse_number()))); } while (accept(',')); - expect(']'); - return v; - } -}; - -Fixture parse_fixture(Scanner& sc) { - Fixture f; - sc.parse_object([&](const std::string& k) -> bool { - if (k == "name") { f.name = sc.parse_string(); return true; } - if (k == "n") { f.n = (int)sc.parse_number(); return true; } - if (k == "head_dim") { f.head_dim = (int)sc.parse_number(); return true; } - if (k == "n_kv_heads") { f.n_kv_heads = (int)sc.parse_number(); return true; } - if (k == "budget") { f.budget = (int)sc.parse_number(); return true; } - if (k == "recent_frac") { f.recent_frac = (float)sc.parse_number(); return true; } - if (k == "sink") { f.sink = (int)sc.parse_number(); return true; } - if (k == "keys") { f.keys = sc.parse_float_array(); return true; } - if (k == "scores") { - sc.expect('['); - if (sc.peek() != ']') do { f.scores.push_back(sc.parse_float_array()); } while (sc.accept(',')); - sc.expect(']'); - return true; - } - if (k == "kept") { - sc.expect('['); - if (sc.peek() != ']') do { f.kept.push_back(sc.parse_int_array()); } while (sc.accept(',')); - sc.expect(']'); - return true; - } - return false; - }); - return f; -} - -struct FixtureFile { - std::vector fixtures; - RopeFixture rope; -}; - -FixtureFile load_fixtures(const std::string& path) { - std::ifstream in(path); - if (!in) throw std::runtime_error("cannot open fixture file: " + path); - std::stringstream ss; ss << in.rdbuf(); - std::string data = ss.str(); - Scanner sc(data); - FixtureFile out; - sc.parse_object([&](const std::string& k) -> bool { - if (k == "fixtures") { - sc.expect('['); - if (sc.peek() != ']') do { out.fixtures.push_back(parse_fixture(sc)); } while (sc.accept(',')); - sc.expect(']'); - return true; - } - if (k == "rope") { - RopeFixture r; - sc.parse_object([&](const std::string& rk) -> bool { - if (rk == "head_dim") { r.head_dim = (int)sc.parse_number(); return true; } - if (rk == "rope_theta") { r.rope_theta = (float)sc.parse_number(); return true; } - if (rk == "abs_pos") { r.abs_pos = (int)sc.parse_number(); return true; } - if (rk == "rank") { r.rank = (int)sc.parse_number(); return true; } - if (rk == "pre_rope") { r.pre_rope = sc.parse_float_array(); return true; } - if (rk == "post_rope_at_abs") { r.post_rope_at_abs = sc.parse_float_array(); return true; } - if (rk == "post_rope_at_rank") { r.post_rope_at_rank = sc.parse_float_array(); return true; } - return false; - }); - out.rope = r; - return true; - } - return false; - }); - return out; -} - -std::string fixture_path() { - if (const char* env = std::getenv("CACTUS_KV_FIXTURES")) return env; - return "experiments/niah/fixtures/kv_compress_fixtures.json"; -} - -Params params_for(const Fixture& f) { - Params p; - p.recent_frac = f.recent_frac; - p.sink = (size_t)f.sink; - p.abs_budget = f.budget; // fixture's budget = round(budget_frac * n) - return p; -} - -using EngineTestUtils::rope_reference; - -FixtureFile g_ff; -const float* head_ptr(const Fixture& f, int h) { - return f.keys.data() + (size_t)h * f.n * f.head_dim; -} - -} // namespace - -// --------------------------------------------------------------------------- // -// Tests // -// --------------------------------------------------------------------------- // - -bool test_keydiff_score_matches_python() { - bool ok = true; - for (const auto& f : g_ff.fixtures) { - for (int h = 0; h < f.n_kv_heads; ++h) { - std::vector out(f.n); - keydiff_score(head_ptr(f, h), f.n, f.head_dim, out.data()); - for (int i = 0; i < f.n; ++i) { - if (std::abs(out[i] - f.scores[h][i]) > 1e-4f) { - std::cerr << "keydiff mismatch " << f.name << " h" << h << " i" << i - << " cpp=" << out[i] << " py=" << f.scores[h][i] << "\n"; - ok = false; - } - } - } - } - return ok; -} - -bool test_keepset_matches_python_exactly() { - bool ok = true; - for (const auto& f : g_ff.fixtures) { - Params p = params_for(f); - for (int h = 0; h < f.n_kv_heads; ++h) { - std::vector sc(f.n); - keydiff_score(head_ptr(f, h), f.n, f.head_dim, sc.data()); - std::vector kept = keepset_for_head(sc.data(), f.n, p); - if (kept != f.kept[h]) { - std::cerr << "keepset mismatch " << f.name << " h" << h - << " cpp_size=" << kept.size() << " py_size=" << f.kept[h].size() << "\n"; - ok = false; - } - } - } - return ok; -} - -bool test_exactly_B_per_cell_rectangular() { - bool ok = true; - for (const auto& f : g_ff.fixtures) { - Params p = params_for(f); - long B = std::min(std::max(1, f.budget), f.n); - for (int h = 0; h < f.n_kv_heads; ++h) { - std::vector sc(f.n); - keydiff_score(head_ptr(f, h), f.n, f.head_dim, sc.data()); - std::vector kept = keepset_for_head(sc.data(), f.n, p); - if ((long)kept.size() != B) { ok = false; continue; } - for (size_t k = 1; k < kept.size(); ++k) - if (kept[k] <= kept[k - 1]) ok = false; // strictly sorted, unique - for (int idx : kept) if (idx < 0 || idx >= f.n) ok = false; - } - } - return ok; -} - -bool test_sink_always_included() { - bool ok = true; - for (const auto& f : g_ff.fixtures) { - long B = std::min(std::max(1, f.budget), f.n); - long sink = std::min(std::max(0, f.sink), f.n); - if (sink > B) continue; // budget too small to hold the full sink - for (const auto& kept : f.kept) { - for (int s = 0; s < sink; ++s) { - if (std::find(kept.begin(), kept.end(), s) == kept.end()) ok = false; - } - } - } - return ok; -} - -bool test_recent_window_included() { - bool ok = true; - for (const auto& f : g_ff.fixtures) { - long B = std::min(std::max(1, f.budget), f.n); - long sink = std::min(std::max(0, f.sink), f.n); - long n_recent = std::min(std::lround((double)f.recent_frac * B), f.n); - // The full recent window is guaranteed kept only when reserved fits in the budget. - if (sink + n_recent > B) continue; - for (const auto& kept : f.kept) { - for (long r = f.n - n_recent; r < f.n; ++r) { - if (r < 0) continue; - if (std::find(kept.begin(), kept.end(), (int)r) == kept.end()) ok = false; - } - } - } - return ok; -} - -bool test_outlier_kept_filler_dropped() { - // Fixtures plant a centroid-opposite outlier at mid+h; the adjacent filler stays on the - // centroid. KeyDiff must keep the outlier (a mid-context geometry pick). - bool ok = true; - for (const auto& f : g_ff.fixtures) { - if (f.name != std::string("base_keydiff")) continue; - int mid = f.n / 2; - for (int h = 0; h < f.n_kv_heads; ++h) { - int outlier = mid + h; - const auto& kept = f.kept[h]; - if (std::find(kept.begin(), kept.end(), outlier) == kept.end()) { - std::cerr << "outlier not kept " << f.name << " h" << h << " idx" << outlier << "\n"; - ok = false; - } - } - } - return ok; -} - -bool test_padding_when_budget_exceeds_reserves() { - // reserved_overflow: large recent_frac + sink on a tight budget exercises the |reserved|>B - // fallback. Still exactly B unique sorted indices. - bool ok = false; - for (const auto& f : g_ff.fixtures) { - if (f.name != std::string("reserved_overflow")) continue; - ok = true; - Params p = params_for(f); - long B = std::min(std::max(1, f.budget), f.n); - for (int h = 0; h < f.n_kv_heads; ++h) { - std::vector sc(f.n); - keydiff_score(head_ptr(f, h), f.n, f.head_dim, sc.data()); - std::vector kept = keepset_for_head(sc.data(), f.n, p); - if ((long)kept.size() != B) ok = false; - if (kept != f.kept[h]) ok = false; - } - } - return ok; -} - -bool test_renumber_mapping() { - // RoPE-at-rank for a survivor equals the reference rope of that vector at its rank position. - // Also check Route-B delta rotation: rotating the post-RoPE-at-abs vector by (rank - abs) - // recovers post-RoPE-at-rank. - const RopeFixture& r = g_ff.rope; - if (r.head_dim == 0) return false; - - // RoPE-at-rank matches reference. - std::vector at_rank = rope_reference(r.pre_rope, r.rank, r.rope_theta); - for (int i = 0; i < r.head_dim; ++i) - if (std::abs(at_rank[i] - r.post_rope_at_rank[i]) > 1e-3f) return false; - - // Route B via the PRODUCTION rope_rotate_row (shipped code, not a local reimplementation): - // rotating post_rope_at_abs by (rank - abs) in place must yield post_rope_at_rank. - std::vector delta = r.post_rope_at_abs; - rope_rotate_row(delta.data(), (size_t)r.head_dim, r.rope_theta, - (double)(r.rank - r.abs_pos)); - for (int i = 0; i < r.head_dim; ++i) - if (std::abs(delta[i] - r.post_rope_at_rank[i]) > 1e-3f) return false; - - return true; -} - -int main() { - TestUtils::TestRunner runner("KV Compress Math Tests"); - try { - g_ff = load_fixtures(fixture_path()); - } catch (const std::exception& e) { - std::cerr << "Failed to load fixtures: " << e.what() << "\n" - << "Run: python experiments/niah/gen_kv_compress_fixtures.py\n" - << "Or set CACTUS_KV_FIXTURES to the fixture json path.\n"; - return 1; - } - std::cout << "Loaded " << g_ff.fixtures.size() << " fixtures\n"; - - runner.run_test("keydiff_score_matches_python", test_keydiff_score_matches_python()); - runner.run_test("keepset_matches_python_exactly", test_keepset_matches_python_exactly()); - runner.run_test("exactly_B_per_cell_rectangular", test_exactly_B_per_cell_rectangular()); - runner.run_test("sink_always_included", test_sink_always_included()); - runner.run_test("recent_window_included", test_recent_window_included()); - runner.run_test("outlier_kept_filler_dropped", test_outlier_kept_filler_dropped()); - runner.run_test("padding_when_budget_exceeds_reserves", test_padding_when_budget_exceeds_reserves()); - runner.run_test("renumber_mapping", test_renumber_mapping()); - - runner.print_summary(); - return runner.all_passed() ? 0 : 1; -} diff --git a/experiments/niah/fixtures/kv_compress_fixtures.json b/experiments/niah/fixtures/kv_compress_fixtures.json deleted file mode 100644 index 98684028e..000000000 --- a/experiments/niah/fixtures/kv_compress_fixtures.json +++ /dev/null @@ -1 +0,0 @@ -{"fixtures": [{"name": "base_keydiff", "n": 48, "head_dim": 16, "n_kv_heads": 2, "budget": 12, "recent_frac": 0.3, "sink": 4, "keys": [0.007008590269833803, -0.0517716109752655, 0.1948932260274887, 0.08067762106657028, -0.15222644805908203, 0.1667429506778717, 0.3216659724712372, 0.27535417675971985, -0.1463707536458969, -0.33972468972206116, -0.20681923627853394, -0.03483806923031807, -0.6557185053825378, -0.04854146018624306, -0.38959580659866333, -0.20976898074150085, 0.026260288432240486, -0.008914324454963207, 0.18504466116428375, 0.046320591121912, -0.17849114537239075, 0.09193909168243408, 0.39412447810173035, 0.3324502110481262, -0.25449755787849426, -0.2687291204929352, -0.10235049575567245, 0.05031377077102661, -0.6196094155311584, -0.07524735480546951, -0.26621386408805847, -0.1012972891330719, 0.12430328130722046, 0.029798584058880806, 0.19218072295188904, -0.0318639762699604, -0.14602242410182953, 0.1312435120344162, 0.29050764441490173, 0.27753475308418274, -0.17005108296871185, -0.30962318181991577, -0.22885017096996307, -0.02183692902326584, -0.6546539664268494, -0.11804131418466568, -0.25214651226997375, -0.22410574555397034, 0.050670020282268524, -0.04888522997498512, 0.2534853518009186, 0.09457000344991684, -0.11413208395242691, -0.011755724437534809, 0.3575271666049957, 0.29196298122406006, -0.14134618639945984, -0.3753206729888916, -0.07854370027780533, -0.054773349314928055, -0.6659085750579834, -0.012798716314136982, -0.3366621732711792, -0.09919057786464691, 0.04364749789237976, -0.067616306245327, 0.1554335206747055, -0.026005351915955544, -0.20968373119831085, 0.12994034588336945, 0.3839839994907379, 0.3225066065788269, -0.22927455604076385, -0.25996994972229004, -0.18401364982128143, 0.08996861428022385, -0.6544715166091919, -0.09632538259029388, -0.32662561535835266, -0.1477375477552414, 0.042272016406059265, -0.06523304432630539, 0.10725072026252747, -0.04152405634522438, -0.12066557258367538, 0.14790542423725128, 0.3467109799385071, 0.2040604203939438, -0.14789216220378876, -0.40844500064849854, -0.20529767870903015, 0.042299091815948486, -0.7453392744064331, -0.040232736617326736, -0.36819693446159363, -0.1938462257385254, 0.03043646365404129, -0.02585088461637497, 0.20902030169963837, -0.0093665337190032, -0.074750617146492, 0.1347244530916214, 0.39711233973503113, 0.3160218596458435, -0.15216486155986786, -0.3022213876247406, -0.16586458683013916, -0.06009049341082573, -0.639584481716156, -0.09802694618701935, -0.4102519750595093, -0.1863875687122345, 0.005794066935777664, -0.08744682371616364, 0.12216164916753769, 0.0419728085398674, -0.12786611914634705, 0.16454264521598816, 0.35422998666763306, 0.3098706603050232, -0.12143102288246155, -0.2869170308113098, -0.28790947794914246, 0.07268238812685013, -0.6158512234687805, -0.03836264833807945, -0.3205535113811493, -0.18017235398292542, 0.050192251801490784, -0.053902268409729004, 0.079229936003685, 0.02310621738433838, -0.18598631024360657, 0.15242794156074524, 0.34048739075660706, 0.2619524300098419, -0.23402456939220428, -0.36995643377304077, -0.17022092640399933, -0.06302055716514587, -0.6177979707717896, -0.06485483050346375, -0.3984008729457855, -0.3192218542098999, 0.059874147176742554, -0.050835806876420975, 0.14781127870082855, 0.0167442224919796, -0.05497591942548752, 0.09592971950769424, 0.35925668478012085, 0.18342502415180206, -0.10917731374502182, -0.29855090379714966, -0.11629752814769745, 0.013631836511194706, -0.5869994759559631, -0.04100387543439865, -0.3084554374217987, -0.2069198489189148, -0.03947285935282707, 0.015486113727092743, 0.07756371796131134, 0.016555121168494225, -0.15602584183216095, 0.0462767630815506, 0.3855818808078766, 0.24776217341423035, -0.21338768303394318, -0.3184332251548767, -0.193473219871521, 0.080697201192379, -0.6152594685554504, -0.08326786756515503, -0.43632814288139343, -0.2646978795528412, 0.08856307715177536, -0.03848680853843689, 0.16015544533729553, 0.11071453243494034, -0.20993217825889587, 0.06913688033819199, 0.33129632472991943, 0.2870955169200897, -0.2247210294008255, -0.3750962018966675, -0.24990174174308777, 0.047715671360492706, -0.5925252437591553, -0.08337005227804184, -0.3309479057788849, -0.26394250988960266, 0.010630881413817406, 0.03294094279408455, 0.18109823763370514, 0.14407013356685638, -0.1851593554019928, 0.12743398547172546, 0.3451504409313202, 0.2860695719718933, -0.19190484285354614, -0.3724852204322815, -0.21302510797977448, 0.1645500361919403, -0.6366994380950928, -0.16038425266742706, -0.3715449273586273, -0.16540822386741638, 0.009221117943525314, 0.032065704464912415, 0.2244316190481186, 0.020935021340847015, -0.16941051185131073, 0.04817971959710121, 0.3199273943901062, 0.1841215044260025, -0.13132445514202118, -0.26489028334617615, -0.2324511706829071, -0.047835949808359146, -0.7212578058242798, -0.10774392634630203, -0.494431734085083, -0.2564241588115692, 0.0990673080086708, -0.05324023216962814, 0.21704091131687164, 0.004103500861674547, -0.057766351848840714, 0.10838066786527634, 0.3358255922794342, 0.38539987802505493, -0.20776785910129547, -0.40548649430274963, -0.1595487743616104, 0.009306447580456734, -0.5795159935951233, -0.10563290864229202, -0.29887905716896057, -0.15667277574539185, 0.0008371748263016343, -0.027794403955340385, 0.13277409970760345, 0.14584235846996307, -0.18100669980049133, 0.07576604932546616, 0.3016338050365448, 0.24047259986400604, -0.1918380707502365, -0.3060358464717865, -0.20016860961914062, 0.001959501765668392, -0.7036566734313965, -0.10092132538557053, -0.20132452249526978, -0.1472480446100235, -0.004849662538617849, -0.10282646864652634, 0.12553256750106812, 0.027467411011457443, -0.1440633237361908, 0.06120174005627632, 0.29059699177742004, 0.3288975954055786, -0.16895999014377594, -0.36315372586250305, -0.18067732453346252, -0.015228323638439178, -0.7796344757080078, -0.05376812443137169, -0.3926421105861664, -0.24944442510604858, 0.002208419144153595, 0.000658481614664197, 0.1157851591706276, -0.04316211864352226, -0.11380711197853088, 0.1361382156610489, 0.3069790303707123, 0.2858985364437103, -0.2061258852481842, -0.329360693693161, -0.23269228637218475, 0.05289292335510254, -0.572669267654419, -0.027697553858160973, -0.3111979067325592, -0.3879239559173584, 0.04725302755832672, -0.03722887113690376, 0.16695943474769592, -0.0029769453685730696, -0.14303146302700043, 0.11902564018964767, 0.3417363166809082, 0.23461246490478516, -0.13005657494068146, -0.39969366788864136, -0.11813651025295258, 0.02008882351219654, -0.6730474829673767, -0.07405030727386475, -0.3851145803928375, -0.16555698215961456, 0.05161662772297859, -0.06379641592502594, 0.11920078843832016, 0.04363775998353958, -0.0979304388165474, 0.09272795170545578, 0.375843346118927, 0.238977313041687, -0.1881658136844635, -0.3589894473552704, -0.15494216978549957, -0.06424042582511902, -0.6006226539611816, -0.071047842502594, -0.3211844861507416, -0.21632571518421173, 0.050239600241184235, -0.08958474546670914, 0.23376941680908203, -0.05662527680397034, -0.1977616548538208, 0.11020307242870331, 0.4280678331851959, 0.2716849744319916, -0.20393969118595123, -0.41567981243133545, -0.17920809984207153, 0.010254349559545517, -0.5483037829399109, -0.028444955125451088, -0.415569543838501, -0.09797125309705734, 0.014471045695245266, -0.07992959022521973, 0.24805283546447754, 0.02606416679918766, -0.16416984796524048, 0.10935907065868378, 0.39717015624046326, 0.3074454665184021, -0.26030439138412476, -0.2445012331008911, -0.12230119109153748, -0.007711853366345167, -0.6737651824951172, -0.10800183564424515, -0.33294597268104553, -0.2317110151052475, -0.004022158682346344, 0.004606116097420454, 0.19254006445407867, 0.008823391050100327, -0.10908733308315277, 0.16678869724273682, 0.3002009093761444, 0.22761407494544983, -0.14441317319869995, -0.30847829580307007, -0.15830932557582855, 0.06936925649642944, -0.6872428059577942, -0.13350810110569, -0.3824402689933777, -0.19318300485610962, -0.005582799203693867, -0.060317512601614, 0.1255616992712021, -0.0024695030879229307, -0.196039080619812, 0.11679308116436005, 0.3946702778339386, 0.23375636339187622, -0.20190756022930145, -0.37347695231437683, -0.14308170974254608, 0.01570100151002407, -0.5531243681907654, -0.11432481557130814, -0.3209897577762604, -0.17711064219474792, -0.09314308315515518, 0.09928934276103973, -0.5352045297622681, -0.0971926748752594, 0.4559859037399292, -0.3058036267757416, -1.0671230554580688, -0.7751907706260681, 0.584943413734436, 1.015892744064331, 0.508540153503418, -0.021600715816020966, 1.9096455574035645, 0.1827467679977417, 1.0147353410720825, 0.6155074238777161, 0.01219812873750925, 0.0025344383902847767, 0.2250780463218689, 0.02116336300969124, -0.22028198838233948, 0.16762802004814148, 0.4090783894062042, 0.24281910061836243, -0.08612984418869019, -0.36186909675598145, -0.2265002578496933, 0.0034545280504971743, -0.5789405107498169, -0.10642388463020325, -0.2415536344051361, -0.2441365122795105, 0.08194265514612198, -0.008733485825359821, 0.16660885512828827, 0.08257859945297241, -0.22078965604305267, 0.16629835963249207, 0.3517415225505829, 0.23065690696239471, -0.15408556163311005, -0.29143035411834717, -0.13115449249744415, 0.11122339218854904, -0.5787959098815918, 0.004624115768820047, -0.36610808968544006, -0.1939840167760849, 0.0623706690967083, -0.036843717098236084, 0.18939346075057983, 0.04985719174146652, -0.10358752310276031, 0.09333932399749756, 0.3374357521533966, 0.2163628786802292, -0.23613087832927704, -0.2858027517795563, -0.17387033998966217, 0.050595417618751526, -0.6977003812789917, -0.15645961463451385, -0.39156946539878845, -0.14197885990142822, 0.0876254141330719, -0.019354673102498055, 0.13420319557189941, 0.02201182208955288, -0.16068774461746216, 0.08117110282182693, 0.2296222299337387, 0.21488234400749207, -0.20104971528053284, -0.2686039209365845, -0.16160711646080017, 0.08129261434078217, -0.652517557144165, -0.07218564301729202, -0.5340859889984131, -0.17614369094371796, 0.061576321721076965, 0.05223134905099869, 0.14997364580631256, 0.03325966000556946, -0.18107610940933228, 0.03960691764950752, 0.3192703127861023, 0.24052873253822327, -0.12377236783504486, -0.34431472420692444, -0.20917150378227234, 0.018342090770602226, -0.6219536066055298, -0.09336281567811966, -0.28195321559906006, -0.2937276363372803, 0.0235443077981472, -0.0027033009100705385, 0.10739007592201233, 0.046614643186330795, -0.08115506172180176, 0.12110333144664764, 0.270417720079422, 0.2213689088821411, -0.1299290955066681, -0.32950854301452637, -0.1701372116804123, 0.033308230340480804, -0.5967797636985779, -0.09497448056936264, -0.35363489389419556, -0.19216452538967133, 0.007023653481155634, -0.0426291823387146, 0.2392052859067917, -0.01983903907239437, -0.0494665801525116, 0.19238698482513428, 0.26925376057624817, 0.2507275342941284, -0.1744096875190735, -0.3824688792228699, -0.20669829845428467, -0.0006225851830095053, -0.5958732962608337, -0.08511790633201599, -0.2477802038192749, -0.18481150269508362, 0.029092775657773018, 0.03650638088583946, 0.2056257575750351, 0.047016292810440063, -0.16235673427581787, 0.18911905586719513, 0.3955167829990387, 0.24760809540748596, -0.27040421962738037, -0.32577773928642273, -0.22683127224445343, -0.074549101293087, -0.6467897891998291, -0.04547766223549843, -0.27499082684516907, -0.18518760800361633, 0.07452251762151718, -0.09725073724985123, 0.1731850951910019, 0.03476252034306526, -0.10268740355968475, 0.10422280430793762, 0.39512965083122253, 0.23256869614124298, -0.17364652454853058, -0.3236819803714752, -0.23213306069374084, 0.020025130361318588, -0.6488503217697144, -0.1547556072473526, -0.2911945581436157, -0.21740314364433289, -0.00839796382933855, -0.054822441190481186, 0.18122172355651855, 0.10394743084907532, -0.15409637987613678, 0.12204014509916306, 0.42360469698905945, 0.28445619344711304, -0.13811606168746948, -0.3682508170604706, -0.13109180331230164, 0.008349400013685226, -0.579111635684967, -0.10972919315099716, -0.37809279561042786, -0.13587363064289093, 0.024417340755462646, -0.053904637694358826, 0.17819032073020935, -0.005922918673604727, -0.07920264452695847, 0.035961639136075974, 0.34739580750465393, 0.27512630820274353, -0.19676661491394043, -0.3845396935939789, -0.21306373178958893, 0.032517921179533005, -0.6843562126159668, -0.027239080518484116, -0.41532230377197266, -0.22705936431884766, 0.036036841571331024, -0.09856420010328293, 0.20691224932670593, 0.027625013142824173, -0.19760340452194214, 0.022475358098745346, 0.2766462564468384, 0.26033034920692444, -0.2493857741355896, -0.41266930103302, -0.18120557069778442, 0.12512323260307312, -0.6189551949501038, -0.02140527404844761, -0.32847896218299866, -0.160161092877388, -0.03293289244174957, -0.05748220160603523, 0.18737924098968506, 0.027656113728880882, -0.1553993821144104, 0.0651259645819664, 0.3420065641403198, 0.21906878054141998, -0.3126359283924103, -0.40415072441101074, -0.14586162567138672, 0.08910209685564041, -0.5421531796455383, -0.05471362918615341, -0.2944457232952118, -0.1539113074541092, -0.0003914197732228786, -0.12085359543561935, 0.1758560687303543, -0.05955809727311134, -0.1617961972951889, 0.1288256198167801, 0.2839222550392151, 0.2593934237957001, -0.12954336404800415, -0.326349675655365, -0.14352817833423615, 0.056585218757390976, -0.5461840629577637, -0.05198006331920624, -0.27763983607292175, -0.20251867175102234, 0.006913098506629467, -0.020215556025505066, 0.14402106404304504, -0.00010450674744788557, -0.17618674039840698, -0.01635143905878067, 0.3602007031440735, 0.19458690285682678, -0.19690772891044617, -0.2719268798828125, -0.19569428265094757, -0.01585225760936737, -0.5646367073059082, -0.03223592787981033, -0.2902885377407074, -0.21707476675510406, 0.07162570208311081, -0.070265032351017, 0.140511617064476, 0.05836416780948639, -0.17571838200092316, 0.13677570223808289, 0.47450995445251465, 0.17347103357315063, -0.22915448248386383, -0.2884179949760437, -0.17689818143844604, 0.06930269300937653, -0.6833354234695435, -0.0429840087890625, -0.34664270281791687, -0.19229166209697723, 0.05077734217047691, -0.09695692360401154, 0.12060482800006866, 0.09851415455341339, -0.13113902509212494, 0.10373708605766296, 0.3527219891548157, 0.27560579776763916, -0.24934110045433044, -0.394376665353775, -0.10441885888576508, 0.018811961635947227, -0.5903171300888062, -0.08983441442251205, -0.2702786326408386, -0.18204627931118011, 0.05828383192420006, -0.008520137518644333, 0.13446123898029327, -0.06473408639431, -0.19953985512256622, 0.1799459159374237, 0.4199599027633667, 0.24043259024620056, -0.20662808418273926, -0.2925950288772583, -0.17806476354599, -0.05371479317545891, -0.5695550441741943, -0.03569228574633598, -0.4649318754673004, -0.2149697095155716, 0.041405048221349716, -0.01189426239579916, 0.18191926181316376, -0.0032351359259337187, -0.15158766508102417, 0.11315978318452835, 0.34152552485466003, 0.23918363451957703, -0.12894007563591003, -0.39175570011138916, -0.18711744248867035, -0.09031525254249573, -0.6057837009429932, -0.018137993291020393, -0.3116905391216278, -0.15342658758163452, 0.05625687539577484, -0.018830833956599236, 0.19800905883312225, 0.015186532400548458, -0.08638332039117813, 0.08099116384983063, 0.28180810809135437, 0.3002678453922272, -0.09900912642478943, -0.3924328684806824, -0.17472583055496216, -0.023023901507258415, -0.6518613696098328, -0.05724656581878662, -0.4012053310871124, -0.2131928950548172, -0.03907683119177818, -0.06436798721551895, 0.11500822007656097, -0.02440108172595501, -0.23179525136947632, 0.1593874990940094, 0.3803786635398865, 0.1619054675102234, -0.2213856428861618, -0.37794724106788635, -0.20419147610664368, -0.06109597533941269, -0.5951129198074341, -0.07934440672397614, -0.3157074451446533, -0.1729724109172821, 0.10299380868673325, -0.12670022249221802, 0.2612418234348297, 0.09199272096157074, -0.11714641749858856, 0.21759937703609467, 0.36517465114593506, 0.2988526225090027, -0.22846496105194092, -0.28770744800567627, -0.1612529754638672, -0.011311807669699192, -0.526985228061676, -0.07479120045900345, -0.33867183327674866, -0.20917420089244843, -0.0035620061680674553, -0.009392984211444855, 0.21123218536376953, 0.04627057909965515, -0.2638286352157593, 0.14881962537765503, 0.3374137878417969, 0.19696249067783356, -0.16138021647930145, -0.31628283858299255, -0.22179795801639557, 0.13486997783184052, -0.6933135390281677, -0.14620135724544525, -0.3968234658241272, -0.1282702088356018, 0.02542870305478573, -0.05455943942070007, 0.1712038218975067, -0.0012787414016202092, -0.1803128868341446, 0.06635899841785431, 0.3903234899044037, 0.30880147218704224, -0.24429889023303986, -0.33246392011642456, -0.13023391366004944, -0.04283939674496651, -0.6579605340957642, -0.11145442724227905, -0.40366631746292114, -0.19421705603599548, -0.0025850790552794933, -0.004400149453431368, 0.17284061014652252, 0.04918332025408745, -0.16041386127471924, 0.06666990369558334, 0.35039207339286804, 0.2575288712978363, -0.22683703899383545, -0.32309553027153015, -0.132314532995224, 0.019152335822582245, -0.5471576452255249, -0.09116816520690918, -0.31308507919311523, -0.21974694728851318, 0.04595301300287247, -0.07745406776666641, 0.2300678789615631, 0.037352997809648514, -0.08618748933076859, 0.035114992409944534, 0.33024996519088745, 0.21315038204193115, -0.2181413620710373, -0.37905579805374146, -0.17567132413387299, 0.010385842062532902, -0.6348814368247986, -0.08732741326093674, -0.3297436237335205, -0.1555294245481491, -0.010892238467931747, -0.035912174731492996, 0.17060816287994385, 0.05196968466043472, -0.1489926427602768, 0.09763652831315994, 0.30537092685699463, 0.2588638663291931, -0.23763848841190338, -0.3186401128768921, -0.17480522394180298, 0.013241120614111423, -0.6773917078971863, -0.019363336265087128, -0.30449920892715454, -0.156234011054039, 0.14559084177017212, -0.03858257457613945, 0.2344818413257599, 0.022360563278198242, -0.12609535455703735, 0.11668664216995239, 0.368155300617218, 0.29069146513938904, -0.2067410796880722, -0.342378705739975, -0.14287878572940826, 0.09864089637994766, -0.6749500036239624, -0.14985784888267517, -0.36331915855407715, -0.1948137879371643, 0.04297039285302162, -0.03134571388363838, 0.23345120251178741, 0.07735444605350494, -0.1482631117105484, 0.07494241744279861, 0.33762526512145996, 0.23996345698833466, -0.20282423496246338, -0.4230578541755676, -0.19269630312919617, -0.009995801374316216, -0.6422762274742126, -0.07226621359586716, -0.30551454424858093, -0.2259073704481125, 0.012292654253542423, -0.0088596660643816, 0.16254596412181854, 0.039374690502882004, -0.11233293265104294, 0.11951824277639389, 0.3674338459968567, 0.24795658886432648, -0.15744955837726593, -0.3356478214263916, -0.19488690793514252, 0.0037077402230352163, -0.6937386393547058, -0.10762901604175568, -0.4332602918148041, -0.23330678045749664, 0.10099881142377853, -0.06378104537725449, 0.21368874609470367, 0.02837946079671383, -0.18082715570926666, 0.16533374786376953, 0.38403600454330444, 0.17018863558769226, -0.13947415351867676, -0.39816489815711975, -0.1785399615764618, 0.04465387016534805, -0.6478248834609985, -0.003604013239964843, -0.30105170607566833, -0.2781539857387543, 0.0105998320505023, -0.021852917969226837, 0.14556176960468292, 0.01773812808096409, -0.10581047087907791, 0.11425883322954178, 0.3088947534561157, 0.2665089964866638, -0.22215774655342102, -0.40518492460250854, -0.22614705562591553, 0.025595976039767265, -0.6342378854751587, -0.05927962437272072, -0.39642533659935, -0.20850668847560883, -0.017636841163039207, -0.08227629214525223, 0.166055366396904, -0.04061757028102875, -0.11233635246753693, 0.22276391088962555, 0.3778668940067291, 0.2056335061788559, -0.2051035463809967, -0.42219722270965576, -0.18852509558200836, 0.03661389648914337, -0.6033629775047302, -0.11113835126161575, -0.32415762543678284, -0.14148980379104614, 0.12197612226009369, -0.07099635154008865, 0.13110169768333435, 0.03127274662256241, -0.29220423102378845, 0.07186806946992874, 0.3413769602775574, 0.23481155931949615, -0.2707379162311554, -0.35670286417007446, -0.20847469568252563, 0.04912235215306282, -0.6787559390068054, -0.073668472468853, -0.350762277841568, -0.1709316372871399, -0.09307368844747543, -0.052987851202487946, 0.21229246258735657, 0.010467542335391045, -0.22214657068252563, 0.11474967747926712, 0.3717600703239441, 0.2442542016506195, -0.24953405559062958, -0.3814590573310852, -0.18533208966255188, -0.03252844139933586, -0.7288411259651184, -0.09801555424928665, -0.3422020971775055, -0.22466321289539337, 0.030442897230386734, -0.031773705035448074, 0.21914048492908478, 0.13874316215515137, -0.1092107743024826, 0.028203342109918594, 0.22207847237586975, 0.25305554270744324, -0.18797484040260315, -0.40251898765563965, -0.1560426652431488, -0.027098780497908592, -0.6126987338066101, -0.07582712173461914, -0.3192417621612549, -0.286469966173172, 0.012298195622861385, -0.04338640347123146, 0.10306796431541443, 0.12266886979341507, -0.17283838987350464, 0.1679183542728424, 0.321708619594574, 0.24628010392189026, -0.1323491781949997, -0.32923397421836853, -0.16004155576229095, 0.024545371532440186, -0.7011401057243347, -0.0790288895368576, -0.3869371712207794, -0.18943952023983002, 0.007021632976830006, -0.0381595604121685, 0.17044708132743835, 0.026733435690402985, -0.14753706753253937, 0.06580086797475815, 0.30222558975219727, 0.22456084191799164, -0.13796734809875488, -0.3257172107696533, -0.14029650390148163, 0.08024659752845764, -0.6918037533760071, -0.03405359387397766, -0.39286860823631287, -0.21602682769298553, 0.05843353271484375, 0.04476065933704376, 0.13520345091819763, 0.02381214126944542, -0.08798787742853165, 0.02392936311662197, 0.3730313777923584, 0.242364764213562, -0.23562902212142944, -0.3370937705039978, -0.13989797234535217, -0.03435678780078888, -0.6138327121734619, -0.05088271573185921, -0.40120556950569153, -0.12163956463336945, 0.08871667087078094, -0.07895293831825256, 0.14498016238212585, 0.0670909658074379, -0.1705721616744995, 0.0064338394440710545, 0.40736982226371765, 0.2582177221775055, -0.09619295597076416, -0.3265519440174103, -0.16009464859962463, 0.15497322380542755, -0.6414255499839783, -0.10714200139045715, -0.32765498757362366, -0.14252245426177979, -0.024034468457102776, -0.08136986196041107, 0.19680055975914001, -0.13131532073020935, -0.2004336565732956, 0.1381939798593521, 0.32558873295783997, 0.17645451426506042, -0.0952659323811531, -0.4149520695209503, -0.19581255316734314, -0.007388375233858824, -0.6286749839782715, -0.07802647352218628, -0.3431631624698639, -0.19643548130989075, 0.029887717217206955, -0.03129291161894798, 0.055365417152643204, 0.05060518905520439, -0.2160225510597229, -0.009912505745887756, 0.4239931106567383, 0.19350290298461914, -0.1825508177280426, -0.38305339217185974, -0.20356757938861847, 0.03543093800544739, -0.6852434277534485, -0.04091750085353851, -0.3200802803039551, -0.14108781516551971, 0.017410878092050552, 0.01637250930070877, 0.2603464424610138, 0.10788606852293015, -0.11649587005376816, 0.12087807059288025, 0.4973984658718109, 0.3693874478340149, -0.2298751324415207, -0.2982448637485504, -0.13954630494117737, 0.014861572533845901, -0.6251665949821472, -0.03516196087002754, -0.2922469675540924, -0.1883717179298401, 0.05121122673153877, 0.033650413155555725, 0.19019772112369537, 0.0558030791580677, -0.09628821164369583, 0.18008288741111755, 0.41627776622772217, 0.2766394019126892, -0.18115349113941193, -0.4056054949760437, -0.1550411432981491, -0.040610287338495255, -0.6840341687202454, -0.027025770395994186, -0.34414374828338623, -0.17569972574710846, 0.0028733834624290466, 0.024103596806526184, 0.18150031566619873, 0.08793836832046509, -0.11212769150733948, 0.10668221861124039, 0.33099934458732605, 0.2593540549278259, -0.15014374256134033, -0.3095424175262451, -0.22942350804805756, 0.06253230571746826, -0.6435275673866272, -0.018788030371069908, -0.3739893138408661, -0.16741743683815002, -0.005608054809272289, -0.029488762840628624, 0.15941020846366882, 0.0142733259126544, -0.17412908375263214, 0.09074081480503082, 0.2678939700126648, 0.3016086518764496, -0.14345993101596832, -0.366564005613327, -0.23863200843334198, -0.021087264642119408, -0.5854514837265015, -0.028275176882743835, -0.3541325628757477, -0.15444646775722504, -0.017852498218417168, -0.06658842414617538, 0.19804206490516663, 0.02375628799200058, -0.1752610057592392, -0.027186617255210876, 0.3884865343570709, 0.2739647626876831, -0.2786903977394104, -0.3144090175628662, -0.1703624576330185, 0.02518993802368641, -0.5855725407600403, -0.09654555469751358, -0.3034366965293884, -0.16511750221252441, 0.07229732722043991, 0.04584118351340294, 0.20724844932556152, 0.0004740685108117759, -0.05588049441576004, 0.043054141104221344, 0.3317895233631134, 0.20955517888069153, -0.19668230414390564, -0.29031434655189514, -0.1047549694776535, 0.0344240702688694, -0.6633017659187317, -0.08723224699497223, -0.36927470564842224, -0.15166817605495453, -0.11938408762216568, 0.117679663002491, -0.517853319644928, -0.07974549382925034, 0.44005724787712097, -0.2932074964046478, -1.055469274520874, -0.784866452217102, 0.5621672868728638, 1.034900426864624, 0.5094543099403381, -0.03210799768567085, 1.9144009351730347, 0.1728605479001999, 1.0248578786849976, 0.6017751097679138, -0.032120052725076675, -0.02076210454106331, 0.24170204997062683, 0.010345215909183025, -0.20922769606113434, 0.021755745634436607, 0.32089313864707947, 0.3367558419704437, -0.20193123817443848, -0.39632248878479004, -0.2001178115606308, -0.01292904932051897, -0.6195515394210815, -0.09255203604698181, -0.38361185789108276, -0.20871105790138245, 0.057655055075883865, -0.00010441572521813214, 0.20183829963207245, 0.005677107721567154, -0.23055356740951538, 0.0584939643740654, 0.3691297769546509, 0.19109690189361572, -0.17891331017017365, -0.3739241063594818, -0.19875246286392212, 0.06722354888916016, -0.6218955516815186, 0.008961592800915241, -0.38607755303382874, -0.14277315139770508, 0.07967556267976761, -0.1857067495584488, 0.1698981672525406, 0.10566317290067673, -0.10689684748649597, 0.07635371387004852, 0.3432861864566803, 0.1926262229681015, -0.1812296360731125, -0.4340839087963104, -0.2204096019268036, 0.06720642000436783, -0.6338604688644409, -0.07772251963615417, -0.3444104492664337, -0.06221550330519676, 0.08598093688488007, -0.07473867386579514, 0.25764918327331543, 0.02411220781505108, -0.10881386697292328, 0.06908825039863586, 0.30803292989730835, 0.2978743016719818, -0.22998860478401184, -0.3834912180900574, -0.12475255876779556, -0.04685525223612785, -0.7037947773933411, -0.0322984978556633, -0.43710896372795105, -0.23689648509025574, 0.01593923382461071, -0.10322195291519165, 0.11844217777252197, 0.061845600605010986, -0.13117706775665283, 0.05113419517874718, 0.3813999891281128, 0.33374258875846863, -0.2556651532649994, -0.34136253595352173, -0.07081170380115509, 0.04180619493126869, -0.5921629667282104, -0.06955885142087936, -0.3790130913257599, -0.14214162528514862, 0.06690778583288193, -0.038532763719558716, 0.15104492008686066, -0.05883430689573288, -0.13800440728664398, 0.06383413076400757, 0.3483330011367798, 0.29132744669914246, -0.15054413676261902, -0.3084752559661865, -0.056891053915023804, 0.06950412690639496, -0.6701360940933228, -0.10046631097793579, -0.2227073609828949, -0.23247651755809784, 0.03201477229595184, 0.051286350935697556, 0.26007750630378723, 0.025820352137088776, -0.13370256125926971, 0.016664303839206696, 0.3095082938671112, 0.23951570689678192, -0.19182728230953217, -0.360591322183609, -0.16202956438064575, 0.1208832785487175, -0.6096151471138, -0.044115856289863586, -0.3390235900878906, -0.2079055905342102, 0.09586876630783081, -0.08551967889070511, 0.1680327206850052, -0.02623690664768219, -0.14144323766231537, 0.1043722853064537, 0.4009510576725006, 0.31766173243522644, -0.1583910435438156, -0.26331233978271484, -0.09021911025047302, 0.027016134932637215, -0.6141648292541504, -0.156365305185318, -0.323102742433548, -0.19956478476524353, -0.050203222781419754, -0.05752590671181679, 0.22218289971351624, -0.03338676318526268, -0.08115215599536896, 0.12215973436832428, 0.3435063064098358, 0.25066229701042175, -0.1500479131937027, -0.4159950315952301, -0.06760617345571518, -0.029422776773571968, -0.6710963845252991, 0.001487193861976266, -0.29961228370666504, -0.1330406218767166, 0.08652058988809586, -0.01713680848479271, 0.11049249768257141, 0.004909958690404892, -0.13587774336338043, 0.09756064414978027, 0.2522865831851959, 0.30999892950057983, -0.10498714447021484, -0.2712205648422241, -0.2205372452735901, 0.03304653987288475, -0.7391570806503296, 0.0249496903270483, -0.4361087381839752, -0.17878477275371552, 0.09235052019357681, 0.02860948257148266, 0.19151023030281067, -0.04273054748773575, -0.15085850656032562, 0.08500384539365768, 0.37334367632865906, 0.3071659505367279, -0.15844222903251648, -0.2383444905281067, -0.13659173250198364, -0.008192676119506359, -0.5513332486152649, -0.07527761906385422, -0.3250217139720917, -0.1902713030576706, -0.08044002950191498, -0.02211560681462288, 0.1887837052345276, 0.01806670054793358, -0.15286406874656677, 0.03419959545135498, 0.35904887318611145, 0.3329249322414398, -0.2004077434539795, -0.3705315887928009, -0.20003071427345276, 0.11210259795188904, -0.7005943655967712, -0.017303727567195892, -0.3335443139076233, -0.10326538234949112, 0.09978481382131577, -0.04037250578403473, 0.09723827242851257, 0.1000339463353157, -0.0728975310921669, 0.04317466914653778, 0.3921873867511749, 0.21774034202098846, -0.23035378754138947, -0.42489099502563477, -0.12299900501966476, 0.024804620072245598, -0.6081072092056274, 0.01307253073900938, -0.3056836724281311, -0.17008206248283386, 0.07168523222208023, -0.0891851931810379, 0.17258845269680023, -0.03176112473011017, -0.19079537689685822, 0.093828946352005, 0.40802010893821716, 0.3606893718242645, -0.24580392241477966, -0.3357588052749634, -0.17318978905677795, 0.03760997951030731, -0.603711724281311, -0.04143015667796135, -0.3874737322330475, -0.17957642674446106, 0.07883787155151367, -0.06162614747881889, 0.2732199430465698, 0.01680629514157772, -0.17749272286891937, 0.11323872208595276, 0.34204623103141785, 0.18942002952098846, -0.19285210967063904, -0.2646401822566986, -0.06951765716075897, -0.017103873193264008, -0.587568461894989, -0.052924744784832, -0.32492056488990784, -0.13594163954257965, 0.07682511955499649, -0.028240583837032318, 0.127874493598938, -0.02764708735048771, -0.1490006148815155, 0.11460043489933014, 0.3800843060016632, 0.2968771457672119, -0.14256501197814941, -0.46457913517951965, -0.1244722530245781, -0.04112418740987778, -0.5771287679672241, -0.042718999087810516, -0.3075869083404541, -0.22635649144649506, 0.04835410416126251, -0.1293172389268875, 0.22907403111457825, 0.012010017409920692, -0.2752537429332733, 0.15149395167827606, 0.43876439332962036, 0.21397219598293304, -0.1309087723493576, -0.38191309571266174, -0.2829703092575073, -0.006646424066275358, -0.6165253520011902, -0.05287365987896919, -0.3339349925518036, -0.18448905646800995, 0.04504932090640068, -0.04675851762294769, 0.14424262940883636, 0.015261911787092686, -0.163383349776268, 0.08327364176511765, 0.2997780442237854, 0.2306489497423172, -0.18743295967578888, -0.3289543688297272, -0.172845721244812, 0.0020652038510888815, -0.6061096787452698, -0.018827782943844795, -0.305110901594162, -0.286805659532547, 0.011451284401118755, -0.0006738747470080853, 0.20008157193660736, 0.06254248321056366, -0.1300821453332901, 0.05574293062090874, 0.4160539507865906, 0.1897992193698883, -0.14225637912750244, -0.34524407982826233, -0.24327698349952698, -0.008559740148484707, -0.5726445913314819, -0.08897130191326141, -0.39964500069618225, -0.2137497216463089, -0.006682208739221096, -0.09848660230636597, 0.19618858397006989, 0.06846056878566742, -0.17338629066944122, 0.12460091710090637, 0.33557435870170593, 0.25279295444488525, -0.18537414073944092, -0.2697611451148987, -0.14426632225513458, -0.03870745748281479, -0.6070407629013062, -0.07061062753200531, -0.36243292689323425, -0.24129827320575714, 0.03290852531790733, -0.03896322473883629, 0.15420258045196533, -0.041005413979291916, -0.2311583310365677, 0.10481899231672287, 0.3638359606266022, 0.3669351041316986, -0.20047742128372192, -0.2933541238307953, -0.10988408327102661, 0.008194861933588982, -0.6289921402931213, -0.08104339987039566, -0.2503569424152374, -0.2747318148612976, 0.03223923221230507, 0.033249545842409134, 0.16892769932746887, 5.079049515188672e-06, -0.13415558636188507, 0.1247691735625267, 0.383297860622406, 0.27186083793640137, -0.15403971076011658, -0.2971402704715729, -0.2249884158372879, -0.007696391548961401, -0.5893476605415344, -0.03512021526694298, -0.327384889125824, -0.16568626463413239], "scores": [[-0.9927071697508774, -0.9781303331637462, -0.980726951457204, -0.9686601654444504, -0.9859965111361388, -0.9865717629616055, -0.9872280709373202, -0.9823165548094318, -0.9829072992534935, -0.9878495070628652, -0.9810894835188151, -0.9856951503570026, -0.9774452052979116, -0.9725822976425459, -0.9778063795567629, -0.9730776266008456, -0.9851865014968185, -0.9687661134241308, -0.9923840676188775, -0.9910699275930659, -0.976550872420269, -0.9848451806828293, -0.9876516408567366, -0.9898345110903177, 0.9994412463254624, -0.9770638868134249, -0.9806166377026753, -0.986895191372311, -0.9673911747362711, -0.9820734209775862, -0.9911574853118776, -0.9775900274006717, -0.9815438076069051, -0.9889390243317184, -0.9860499051870153, -0.990346344737256, -0.977599989333287, -0.9757162752791518, -0.9844969925705689, -0.9828237073298414, -0.9836747738809793, -0.9834124515102077, -0.9762736657604103, -0.9863316208064562, -0.9859690212879606, -0.9785105649864247, -0.9719845837543548, -0.9776116742536747], [-0.9924277098064016, -0.9928188631407819, -0.9912300483421472, -0.9923449212553379, -0.9865435859838525, -0.9925936264910195, -0.9919715057932824, -0.9812501550700913, -0.9917742573817456, -0.9778690656937898, -0.9823141793667827, -0.9873755735746267, -0.9731441633092237, -0.984034399574426, -0.9904464797766082, -0.9839821098809262, -0.9761437268630193, -0.9718396732600318, -0.9782555598981743, -0.9750649788330894, -0.9877874211014852, -0.9881886750379459, -0.9871399101894514, -0.9830468506826351, -0.9827622881763346, 0.9996494859150404, -0.987558404612867, -0.9869249809227716, -0.9687118370134803, -0.9871407693269674, -0.9818398625900191, -0.9761055044155703, -0.9832896629231536, -0.9804907391451436, -0.9769434411712624, -0.9704453412490723, -0.9826590242372737, -0.9812950039848866, -0.977409858711416, -0.988268575382572, -0.981042315966165, -0.9816448417852881, -0.9760361575575511, -0.990216513142874, -0.984578168970806, -0.9877420311091998, -0.9770648701114597, -0.9900559255184145]], "kept": [[0, 1, 2, 3, 13, 17, 24, 28, 44, 45, 46, 47], [0, 1, 2, 3, 17, 25, 28, 35, 44, 45, 46, 47]]}, {"name": "small_n_keydiff", "n": 20, "head_dim": 8, "n_kv_heads": 4, "budget": 5, "recent_frac": 0.3, "sink": 4, "keys": [0.06934703141450882, -0.18056292831897736, -0.071917325258255, -0.7295069694519043, 0.5098626017570496, 0.2949908673763275, -0.07241827249526978, 0.22133097052574158, 0.08255092799663544, -0.18323107063770294, -0.11445430666208267, -0.7585928440093994, 0.5683770179748535, 0.3439999520778656, -0.07863762974739075, 0.24681606888771057, 0.00474861916154623, -0.11371274292469025, -0.017960557714104652, -0.7959012985229492, 0.43983322381973267, 0.25935661792755127, -0.053093232214450836, 0.23272664844989777, 0.10920361429452896, -0.11675024777650833, -0.11026711016893387, -0.6997772455215454, 0.5178157687187195, 0.3780212104320526, -0.15165197849273682, 0.20519793033599854, 0.0674334391951561, -0.06280075013637543, -0.15901890397071838, -0.7679321765899658, 0.4981394410133362, 0.38306179642677307, -0.10691644996404648, 0.29250821471214294, -0.03833997994661331, -0.09644563496112823, -0.06905238330364227, -0.7849124073982239, 0.5339828729629517, 0.39538607001304626, -0.09077022969722748, 0.2762759327888489, 0.1740308403968811, -0.1391751766204834, -0.1348148137331009, -0.7525317668914795, 0.5587069988250732, 0.32476168870925903, -0.10410349816083908, 0.22102788090705872, 0.08777997642755508, -0.20618876814842224, -0.19728952646255493, -0.8356729745864868, 0.5862373113632202, 0.3382878601551056, -0.019660208374261856, 0.22584305703639984, 0.018178721889853477, -0.12897533178329468, -0.12462511658668518, -0.7766884565353394, 0.48205047845840454, 0.42293715476989746, -0.07744865119457245, 0.24711018800735474, 0.04145890846848488, -0.18735776841640472, -0.07621290534734726, -0.7192104458808899, 0.4883483946323395, 0.3278917670249939, -0.1404464840888977, 0.23579315841197968, -0.16588741540908813, 0.4600149393081665, 0.3621181845664978, 2.145049571990967, -1.5726239681243896, -1.0108551979064941, 0.28067320585250854, -0.6784366965293884, 0.033552248030900955, -0.18801619112491608, -0.15469001233577728, -0.7550371289253235, 0.44779980182647705, 0.32144972681999207, -0.07510826736688614, 0.27171096205711365, 0.0876416340470314, -0.030004963278770447, -0.10486161708831787, -0.7367958426475525, 0.6198855042457581, 0.28223729133605957, -0.04674229770898819, 0.1785336285829544, 0.07299207895994186, -0.2512916326522827, -0.07583200931549072, -0.7218915224075317, 0.47791972756385803, 0.41851913928985596, -0.05689844489097595, 0.2285812646150589, 0.018014324828982353, -0.1550399214029312, -0.12899214029312134, -0.6777403354644775, 0.5662075281143188, 0.3012103736400604, -0.12263470888137817, 0.19968760013580322, -0.012179368175566196, -0.182450532913208, -0.125416100025177, -0.6794315576553345, 0.5923227667808533, 0.29418089985847473, -0.06761734932661057, 0.20406801998615265, 0.15904588997364044, -0.15530796349048615, -0.09568934887647629, -0.7607872486114502, 0.4857693314552307, 0.3446829617023468, -0.11436890065670013, 0.24395529925823212, -0.022804711014032364, -0.11977154016494751, -0.16576971113681793, -0.6278579831123352, 0.5118804574012756, 0.3894614577293396, -0.16491533815860748, 0.24773818254470825, 0.010885011404752731, -0.17721624672412872, -0.11956588178873062, -0.7298328876495361, 0.5395165681838989, 0.3719809651374817, -0.06252706795930862, 0.2219167947769165, -0.012885031290352345, -0.193104088306427, -0.1305835098028183, -0.8200704455375671, 0.56390780210495, 0.3226162791252136, -0.1077321469783783, 0.2737096846103668, 0.08868201822042465, -0.1418217420578003, -0.1708623766899109, -0.6984580755233765, 0.5436126589775085, 0.2903595268726349, -0.043000586330890656, 0.251685231924057, -0.0431947223842144, -0.1467743217945099, -0.18403059244155884, -0.6571633815765381, 0.5666552186012268, 0.2794186472892761, -0.13940981030464172, 0.2313835322856903, 0.04334595426917076, -0.07661163806915283, -0.0837470069527626, -0.7281005382537842, 0.5551735758781433, 0.23298422992229462, -0.07967294752597809, 0.2696112096309662, 0.04633886739611626, -0.18838250637054443, -0.09250402450561523, -0.6160483956336975, 0.5365472435951233, 0.27856430411338806, -0.022875938564538956, 0.13236349821090698, 0.08044673502445221, -0.1819695234298706, -0.07426577806472778, -0.7564528584480286, 0.5924961566925049, 0.3496118485927582, -0.18150821328163147, 0.12760020792484283, 0.039481718093156815, -0.08816120773553848, -0.11444419622421265, -0.7106230854988098, 0.5145086050033569, 0.3714931905269623, -0.07629983872175217, 0.24030816555023193, 0.006050347350537777, -0.09347912669181824, -0.05264167860150337, -0.7790468335151672, 0.6413785219192505, 0.37072351574897766, -0.11956647783517838, 0.13107307255268097, 0.08812881261110306, -0.12499622255563736, -0.15057729184627533, -0.8035067915916443, 0.5825276374816895, 0.30518224835395813, -0.11875343322753906, 0.3685731291770935, 0.16168510913848877, -0.10518494993448257, -0.14356674253940582, -0.6815060973167419, 0.41223031282424927, 0.3399248421192169, -0.13315708935260773, 0.16201061010360718, -0.0238537285476923, -0.12661470472812653, -0.11633068323135376, -0.7059929370880127, 0.5684226155281067, 0.2922939956188202, -0.15694540739059448, 0.13747259974479675, 0.038526665419340134, -0.11486397683620453, -0.12826687097549438, -0.6819227337837219, 0.46725112199783325, 0.31114211678504944, -0.13346301019191742, 0.16387197375297546, -0.1725168079137802, 0.4599766135215759, 0.36774858832359314, 2.1417202949523926, -1.584946870803833, -1.0115666389465332, 0.3030107915401459, -0.6700875163078308, 0.03676484525203705, -0.2041129171848297, -0.14478811621665955, -0.7805370092391968, 0.44766268134117126, 0.2862911522388458, -0.12916667759418488, 0.211546391248703, 0.0587356761097908, -0.07652698457241058, -0.08018310368061066, -0.7266241312026978, 0.5625997185707092, 0.30015334486961365, -0.022741608321666718, 0.25376471877098083, -0.012505472637712955, -0.11510251462459564, -0.07999107986688614, -0.848258912563324, 0.4888990521430969, 0.3109070658683777, -0.1915917843580246, 0.1729319989681244, 0.007868585176765919, -0.1275169998407364, -0.10577253997325897, -0.5826704502105713, 0.5065908432006836, 0.39225974678993225, -0.09122385084629059, 0.2398371547460556, -0.002694996539503336, -0.24841678142547607, -0.15915310382843018, -0.6758707165718079, 0.605444073677063, 0.3647977411746979, -0.07025213539600372, 0.18435251712799072, 0.13200119137763977, -0.0856814756989479, -0.14733731746673584, -0.7162976861000061, 0.4765011966228485, 0.4079585373401642, -0.074848473072052, 0.22153763473033905, 0.07563149183988571, -0.13084067404270172, -0.14137053489685059, -0.7760574221611023, 0.462116539478302, 0.24767987430095673, -0.07210353016853333, 0.33721795678138733, -0.029666969552636147, -0.14595741033554077, -0.10024650394916534, -0.7632104158401489, 0.5365250706672668, 0.27975136041641235, -0.0351460725069046, 0.19657841324806213, 0.06031451374292374, -0.11865971237421036, -0.14228667318820953, -0.8035407066345215, 0.4997410178184509, 0.3727479577064514, -0.09939280897378922, 0.23778033256530762, 0.0012664040550589561, -0.14571064710617065, -0.08438734710216522, -0.7121567726135254, 0.6217045187950134, 0.324208527803421, -0.14723369479179382, 0.14540790021419525, 0.11288678646087646, -0.15582269430160522, -0.07021460682153702, -0.6910957098007202, 0.607558012008667, 0.39790356159210205, -0.11404390633106232, 0.23998510837554932, 0.10955509543418884, -0.21467554569244385, -0.04825089871883392, -0.653379499912262, 0.57545006275177, 0.3245515823364258, -0.08005912601947784, 0.17042890191078186, 0.05818420276045799, -0.18224076926708221, -0.11646749824285507, -0.7387104630470276, 0.5404939651489258, 0.39356479048728943, -0.16064633429050446, 0.2725604176521301, -0.04430413246154785, -0.26092344522476196, -0.12676413357257843, -0.7136426568031311, 0.5484438538551331, 0.25222891569137573, -0.09816311299800873, 0.2326170951128006, 0.11804674565792084, -0.15806376934051514, -0.09434611350297928, -0.7628504037857056, 0.5169260501861572, 0.31232142448425293, -0.060000959783792496, 0.17684990167617798, 0.013570371083915234, -0.12407238036394119, -0.0313238650560379, -0.6948440074920654, 0.5414817929267883, 0.3172910809516907, -0.04074491560459137, 0.2532867193222046, 0.14194613695144653, -0.13633620738983154, -0.16822955012321472, -0.742476224899292, 0.4769335687160492, 0.2720571458339691, -0.031980305910110474, 0.2666577100753784, -0.006645724177360535, -0.06380575895309448, -0.10856766253709793, -0.7618237733840942, 0.5456148982048035, 0.3311384916305542, -0.13379891216754913, 0.12914414703845978, 0.05712249130010605, -0.1382720023393631, -0.2497495710849762, -0.6862337589263916, 0.5571854710578918, 0.31110841035842896, -0.08956002444028854, 0.21225795149803162, 0.013431366533041, -0.17087453603744507, -0.05321550741791725, -0.7865099906921387, 0.538163423538208, 0.34380409121513367, -0.1305476874113083, 0.20427390933036804, -0.17705869674682617, 0.45020508766174316, 0.3811255395412445, 2.157050371170044, -1.577629566192627, -0.9922161102294922, 0.2706287205219269, -0.6788594722747803, 0.04375749081373215, -0.1728607714176178, -0.05574225261807442, -0.7379197478294373, 0.5930232405662537, 0.3916890025138855, -0.14744943380355835, 0.2780655324459076, 0.02276759035885334, -0.0160488523542881, -0.14008721709251404, -0.7610254883766174, 0.4307499825954437, 0.2699459493160248, -0.12419862300157547, 0.17638947069644928, -0.006147850304841995, -0.1876387745141983, -0.13112308084964752, -0.6415117979049683, 0.4411390423774719, 0.30701613426208496, -0.14286085963249207, 0.15053348243236542, -0.008020338602364063, -0.19815853238105774, -0.08494352549314499, -0.7236741781234741, 0.49940454959869385, 0.36516278982162476, -0.08112770318984985, 0.21512442827224731, 0.07450078427791595, -0.19428934156894684, -0.08942662924528122, -0.6906328797340393, 0.5056886076927185, 0.27362439036369324, -0.019162960350513458, 0.23100091516971588, 0.20784974098205566, -0.1855422407388687, -0.07815355062484741, -0.7356901168823242, 0.5389872193336487, 0.37626776099205017, -0.1351315677165985, 0.221286341547966, 0.06333357095718384, -0.14719778299331665, -0.0909731462597847, -0.6627010107040405, 0.5679852366447449, 0.3477488160133362, -0.11253482848405838, 0.16980668902397156, 0.018511570990085602, -0.1792992353439331, -0.0866599977016449, -0.7321596741676331, 0.4641868472099304, 0.36591553688049316, -0.13532871007919312, 0.1879994422197342, 0.0439729243516922, -0.1292129009962082, -0.2414548695087433, -0.7700518369674683, 0.5350832343101501, 0.31639015674591064, -0.13839340209960938, 0.19921822845935822, 0.12454919517040253, -0.1756581962108612, -0.085162453353405, -0.674385666847229, 0.5037927627563477, 0.3567701578140259, -0.11767806112766266, 0.1957699954509735, -0.0010132236639037728, -0.10912397503852844, -0.14070366322994232, -0.7415862679481506, 0.46993154287338257, 0.42504045367240906, -0.05807589739561081, 0.21965064108371735, 0.020549776032567024, -0.22936178743839264, -0.10620501637458801, -0.6995580792427063, 0.45588377118110657, 0.3926756978034973, -0.05636835843324661, 0.17818717658519745, 0.02750166691839695, -0.17423781752586365, -0.11276175081729889, -0.7212576270103455, 0.4443568289279938, 0.40830308198928833, -0.07935065776109695, 0.28932952880859375, 0.11802761256694794, -0.1632668375968933, -0.10369759052991867, -0.6771458983421326, 0.5647844672203064, 0.3264022469520569, -0.12607544660568237, 0.2280551940202713, 0.01520935632288456, -0.18066848814487457, -0.22230467200279236, -0.7097970247268677, 0.47495922446250916, 0.30680620670318604, -0.09780119359493256, 0.2419826239347458, 0.14974509179592133, -0.1426519751548767, -0.19145894050598145, -0.7074418067932129, 0.49651750922203064, 0.3545674681663513, -0.12944257259368896, 0.1908705234527588, 0.029723305255174637, -0.18424992263317108, -0.21203474700450897, -0.7473927140235901, 0.5256032347679138, 0.3946226239204407, -0.10982508212327957, 0.20770375430583954, 0.0842784121632576, -0.12590166926383972, -0.0408194325864315, -0.6847235560417175, 0.5253312587738037, 0.4015146493911743, -0.018692081794142723, 0.20252195000648499, 0.06319348514080048, -0.2369038164615631, -0.13900206983089447, -0.7581667304039001, 0.5156001448631287, 0.31051647663116455, -0.10346867144107819, 0.22113913297653198, 0.07995989918708801, -0.14132486283779144, -0.14888592064380646, -0.7784373164176941, 0.5880463719367981, 0.405463308095932, -0.1178884282708168, 0.2543615698814392, -0.1584513932466507, 0.4400157034397125, 0.3545571565628052, 2.149765968322754, -1.56759512424469, -1.0097275972366333, 0.2883966267108917, -0.6628654599189758, 0.08759760856628418, -0.12848109006881714, -0.13110923767089844, -0.7045478820800781, 0.6573908925056458, 0.27825602889060974, -0.11086318641901016, 0.21438468992710114, 0.020802440121769905, -0.21996593475341797, -0.041130684316158295, -0.7029743790626526, 0.4384041726589203, 0.29945796728134155, -0.07489614188671112, 0.2464618682861328, -0.01672986149787903, -0.15744313597679138, -0.10775161534547806, -0.69435054063797, 0.5460337996482849, 0.24344748258590698, -0.0028170377481728792, 0.23684754967689514, 0.14558598399162292, -0.009716840460896492, -0.12882356345653534, -0.7756673097610474, 0.5695103406906128, 0.32555651664733887, -0.11773666739463806, 0.17462654411792755, 0.13967572152614594, -0.17500551044940948, -0.19077493250370026, -0.7363577485084534, 0.46844086050987244, 0.3424934148788452, -0.1161344051361084, 0.17341965436935425, 0.08537480235099792, -0.18504348397254944, -0.08460763841867447, -0.6509299874305725, 0.5082061290740967, 0.3022724688053131, -0.08520137518644333, 0.2708633542060852], "scores": [[-0.9975564464117103, -0.9986921566543168, -0.9859543376514985, -0.9939001075479557, -0.992929782327592, -0.9928588310100869, -0.9919402600013656, -0.9933020173155681, -0.9953986103817825, -0.9968082074212746, 0.9996876621868835, -0.9947242876128518, -0.983888899273506, -0.9896373002833608, -0.9949639330609933, -0.9916001626100808, -0.9930081039718504, -0.9859501584404684, -0.9978722657769832, -0.9966405819089261], [-0.9942010490943345, -0.9898516243748926, -0.9924466736738322, -0.9888981549844156, -0.9900840976638492, -0.996489969256307, -0.9898599898733748, -0.9911011997851448, -0.9830094998951321, -0.9925474946692191, -0.9975211775661803, 0.9991339214376107, -0.9910461131356549, -0.9932941354799205, -0.9876273717828549, -0.9875261001896075, -0.986752519443589, -0.9879559941997538, -0.9852625380698514, -0.9939445261239175], [-0.9952231962262665, -0.9926643586088945, -0.9937184686319827, -0.9915817638097764, -0.9960746573516778, -0.9861736641084138, -0.9954020156986553, -0.9929270954414495, -0.9853682477407277, -0.9898161103649217, -0.9874138209697798, -0.9971145378690174, 0.9992159206364314, -0.9957179451724578, -0.9804138929075586, -0.9924890603993186, -0.9956476811416943, -0.9934482215336178, -0.9881305173708601, -0.9970292151347532], [-0.9949033781182776, -0.9922472211840898, -0.9958357446059833, -0.9916387372795002, -0.9922119969835057, -0.9921166884490611, -0.9951436295226932, -0.9928904655710866, -0.9936645247712148, -0.9962474316307899, -0.9895789351606823, -0.9965436340260065, -0.9987064355722328, 0.9995029352261207, -0.9878110181992688, -0.9894351216229182, -0.9856806608257324, -0.9829428712743062, -0.9935500825362488, -0.9945848242230804]], "kept": [[0, 1, 2, 3, 19], [0, 1, 2, 3, 19], [0, 1, 2, 3, 19], [0, 1, 2, 3, 19]]}, {"name": "tight_budget", "n": 40, "head_dim": 8, "n_kv_heads": 2, "budget": 4, "recent_frac": 0.3, "sink": 4, "keys": [0.47380974888801575, -0.4813322126865387, 0.11721540242433548, -0.1614769548177719, -0.12874385714530945, -0.08802423626184464, -0.5645245313644409, -0.07830052077770233, 0.541167676448822, -0.6594098806381226, 0.15381400287151337, -0.15383552014827728, -0.11346650868654251, 0.022669127210974693, -0.48451173305511475, -0.0840219110250473, 0.5079284310340881, -0.6204559206962585, 0.20268046855926514, -0.15732643008232117, -0.12685741484165192, -0.00450623594224453, -0.5560899972915649, -0.07334648072719574, 0.5611973404884338, -0.6184646487236023, 0.11050190776586533, -0.11034020036458969, -0.25608760118484497, -0.003556575160473585, -0.5597492456436157, -0.14219146966934204, 0.5308926701545715, -0.6124549508094788, 0.08368770033121109, -0.19766570627689362, -0.11337324231863022, -0.05725928023457527, -0.44148707389831543, -0.021390071138739586, 0.5267611742019653, -0.5919005274772644, 0.09564992040395737, -0.19014039635658264, -0.08547656983137131, -0.025494994595646858, -0.522508442401886, -0.09790089726448059, 0.528528094291687, -0.7721768617630005, 0.14043231308460236, -0.1192770004272461, -0.1966223418712616, -0.05155424028635025, -0.5599719882011414, -0.020899418741464615, 0.41536203026771545, -0.6932069063186646, 0.14140507578849792, -0.08602536469697952, -0.2225797474384308, -0.07952390611171722, -0.4953659772872925, -0.0892212763428688, 0.5966024398803711, -0.7070435285568237, 0.12365266680717468, -0.19626569747924805, -0.044381339102983475, -0.05570447817444801, -0.5303795337677002, -0.14466971158981323, 0.6011616587638855, -0.6098432540893555, 0.14360426366329193, -0.0869513526558876, -0.09721815586090088, -0.08658424764871597, -0.55177903175354, -0.09877046942710876, 0.5855740308761597, -0.7205012440681458, 0.07610759139060974, -0.15990760922431946, -0.1034485325217247, -0.0258544459939003, -0.5742218494415283, -0.14526113867759705, 0.5168496966362, -0.5868039727210999, 0.1437789797782898, -0.13306288421154022, -0.1305372714996338, -0.03996023163199425, -0.5239337682723999, -0.017900142818689346, 0.4773480296134949, -0.6407701969146729, 0.10038705915212631, -0.11667744815349579, -0.1034475564956665, 0.07287981361150742, -0.4368342459201813, 0.016076387837529182, 0.4150952100753784, -0.6644980311393738, 0.07549554109573364, -0.11720933765172958, -0.22863081097602844, 0.004103017970919609, -0.4584178328514099, -0.12386401742696762, 0.4681429862976074, -0.6875407695770264, 0.10809086263179779, -0.11179686337709427, -0.012285182252526283, -0.06449418514966965, -0.4733918607234955, -0.050989579409360886, 0.60506671667099, -0.6103742718696594, 0.17435358464717865, -0.1977291703224182, -0.12429151684045792, -0.09531053900718689, -0.4365196228027344, -0.025878475978970528, 0.5018131732940674, -0.6701055765151978, 0.13015931844711304, -0.1789201945066452, -0.16120892763137817, -0.030558191239833832, -0.3886103928089142, -0.07106714695692062, 0.4892771244049072, -0.7060400247573853, 0.039175521582365036, -0.11759625375270844, -0.07213933020830154, -0.05416318029165268, -0.4951381981372833, -0.05296464264392853, 0.5240031480789185, -0.7237901091575623, 0.0830201581120491, -0.13827145099639893, -0.15383782982826233, -0.07844340056180954, -0.5527229905128479, -0.07543530315160751, 0.559725821018219, -0.6678112149238586, 0.09823253750801086, -0.10315950214862823, -0.08244097232818604, 0.030138462781906128, -0.6162912249565125, -0.015917522832751274, -1.5467463731765747, 1.9284738302230835, -0.3289775550365448, 0.4299583435058594, 0.33569565415382385, 0.15685981512069702, 1.534585952758789, 0.1749061942100525, 0.5100213289260864, -0.6075146198272705, 0.07835745066404343, -0.03580010309815407, -0.06371915340423584, 0.05415685102343559, -0.5130964517593384, -0.07791490107774734, 0.5254228115081787, -0.6107533574104309, 0.07655468583106995, -0.12486033141613007, -0.1155197024345398, 0.026162171736359596, -0.5449020862579346, -0.006452382542192936, 0.48487722873687744, -0.695513904094696, 0.07041110098361969, -0.20335471630096436, -0.10736196488142014, -0.0030588761437684298, -0.5035512447357178, -0.027544215321540833, 0.5986791253089905, -0.6339808702468872, 0.11568418890237808, -0.15759849548339844, -0.1950916051864624, -0.016634969040751457, -0.599590003490448, -0.026126649230718613, 0.5163543820381165, -0.5911431312561035, 0.10253755748271942, -0.18500542640686035, -0.09678397327661514, -0.08263372629880905, -0.520814836025238, -0.05666693300008774, 0.5103428959846497, -0.6569222807884216, 0.06430382281541824, -0.15330210328102112, -0.22159650921821594, -0.06248852238059044, -0.5716698169708252, -0.002747286343947053, 0.5805667638778687, -0.7450334429740906, 0.11317190527915955, -0.15016360580921173, -0.16701526939868927, -0.02803925797343254, -0.5348514914512634, -0.14714042842388153, 0.5037365555763245, -0.6548948287963867, 0.11124762147665024, -0.2054070681333542, -0.08389543741941452, -0.017849061638116837, -0.569058358669281, -0.09170325100421906, 0.5130535364151001, -0.6757784485816956, 0.1931138038635254, -0.13342681527137756, -0.1652085930109024, -0.09402736276388168, -0.5146405100822449, 0.056028470396995544, 0.5081571936607361, -0.6411083340644836, 0.13162833452224731, -0.1458517163991928, -0.0006464411853812635, -0.08119816333055496, -0.4745568335056305, -0.05073857307434082, 0.548130214214325, -0.7063143253326416, 0.19433170557022095, -0.14992178976535797, -0.11297132074832916, -0.10137072950601578, -0.548251748085022, -0.030548499897122383, 0.5663954019546509, -0.6098194718360901, 0.1663745790719986, -0.10812097042798996, -0.11325639486312866, -0.012795616872608662, -0.48208683729171753, -0.0637892335653305, 0.5533740520477295, -0.583197295665741, 0.11765463650226593, -0.16165566444396973, -0.07874391973018646, 0.04041416943073273, -0.5222951173782349, -0.06337594985961914, 0.5102505087852478, -0.586331307888031, 0.014066156931221485, -0.12555314600467682, -0.05506628751754761, -0.09529107809066772, -0.4373181164264679, -0.0319950208067894, 0.48961859941482544, -0.6372751593589783, 0.029201462864875793, -0.1721239686012268, -0.02333429455757141, -0.09931211918592453, -0.4195384085178375, -0.06283915042877197, 0.5668035745620728, -0.6452873349189758, -0.006148007698357105, -0.17247797548770905, -0.10518680512905121, -0.11216939985752106, -0.5791375041007996, -0.04045117646455765, 0.48459964990615845, -0.732273519039154, 0.07172030955553055, -0.1008085384964943, -0.13836678862571716, -0.007883287034928799, -0.43540334701538086, -0.05713469907641411, 0.4605862498283386, -0.7125604152679443, 0.10461337864398956, -0.09979262948036194, -0.10122872143983841, -0.02725905552506447, -0.5339340567588806, -0.043739333748817444, 0.49350497126579285, -0.6637626886367798, 0.04131374508142471, -0.22245606780052185, -0.13877731561660767, -0.11000188440084457, -0.5597043037414551, -0.02520809695124626, 0.511716365814209, -0.501775860786438, 0.15224316716194153, -0.17550480365753174, -0.07577911764383316, -0.03678172081708908, -0.546051025390625, -0.00363072264008224, 0.4783039689064026, -0.8004012107849121, 0.14562930166721344, -0.17470836639404297, -0.04037760943174362, -0.08710552752017975, -0.5708346962928772, 0.01866261288523674, 0.46329548954963684, -0.6373124718666077, 0.17841242253780365, -0.13812054693698883, -0.11870872974395752, -0.04950074478983879, -0.4698143005371094, -0.09910108149051666, 0.5476024746894836, -0.6186283826828003, 0.22256167232990265, -0.16067169606685638, -0.1604209840297699, -0.017837565392255783, -0.5316248536109924, -0.04430466145277023, 0.5236992239952087, -0.5890624523162842, 0.07308419793844223, -0.14845938980579376, -0.2422851175069809, -0.09912761300802231, -0.434950053691864, -0.09283576905727386, 0.4819965660572052, -0.7083775401115417, 0.13491778075695038, -0.13494591414928436, -0.1740228235721588, -0.021249396726489067, -0.49876171350479126, -0.02105880156159401, 0.4849964380264282, -0.5605817437171936, 0.12641945481300354, -0.14968989789485931, -0.14062120020389557, -0.0668632835149765, -0.46882739663124084, -0.07220880687236786, 0.5709776282310486, -0.6184483766555786, 0.17325082421302795, -0.1031402051448822, -0.117826446890831, -0.15008890628814697, -0.4968384802341461, -0.019306669011712074, 0.5252150297164917, -0.6196606159210205, 0.08234506100416183, -0.16286928951740265, -0.16179893910884857, 0.013473103754222393, -0.5038473010063171, -0.08338477462530136, 0.4504777789115906, -0.6551475524902344, 0.17805762588977814, -0.120749332010746, -0.1509055644273758, 0.016852831467986107, -0.4403415024280548, -0.009721661917865276, 0.5565112233161926, -0.5896995663642883, 0.13308902084827423, -0.10509537905454636, -0.15368075668811798, 0.02035597525537014, -0.5002079606056213, 0.04245011880993843, 0.41495800018310547, -0.6122117042541504, 0.15499915182590485, -0.16099204123020172, -0.09205727279186249, -0.01720559597015381, -0.48289790749549866, -0.08574343472719193, 0.5099735260009766, -0.5965301990509033, 0.1697920709848404, -0.13889656960964203, -0.11082786321640015, -0.04459709674119949, -0.45988374948501587, -0.11146239191293716, 0.45061710476875305, -0.641266405582428, 0.050599630922079086, -0.17320089042186737, -0.11036469787359238, -0.030496669933199883, -0.565371572971344, -0.01868019811809063, 0.4283846616744995, -0.6202113628387451, 0.027039481326937675, -0.20759159326553345, -0.08256838470697403, -0.074940025806427, -0.5515449047088623, -0.08099930733442307, 0.5403220653533936, -0.6501845121383667, 0.10445693135261536, -0.0911065936088562, -0.15049901604652405, -0.07821860164403915, -0.5378884673118591, -0.00031436290009878576, 0.5666406750679016, -0.5831283330917358, 0.10423853993415833, -0.15168555080890656, -0.07204083353281021, -0.07434249669313431, -0.470233678817749, -0.02254583314061165, 0.48215451836586, -0.6516419053077698, 0.09996352344751358, -0.057178396731615067, 0.014711681753396988, -0.059618279337882996, -0.5110637545585632, -0.1728396862745285, 0.5276470184326172, -0.6478338241577148, 0.0522838793694973, -0.10456570237874985, -0.19296717643737793, -0.06536456197500229, -0.504824161529541, -0.12121047079563141, 0.5599074363708496, -0.5947040915489197, 0.036605026572942734, -0.21423734724521637, -0.08456651121377945, -0.0561467669904232, -0.5723371505737305, -0.07352180778980255, 0.5116945505142212, -0.6546341180801392, 0.14730049669742584, -0.12463550269603729, -0.02620629407465458, 0.014096108265221119, -0.477843314409256, -0.004906502086669207, -1.553436279296875, 1.959099292755127, -0.30766746401786804, 0.443833589553833, 0.3437568247318268, 0.15881890058517456, 1.5207619667053223, 0.17280003428459167, 0.5335524082183838, -0.5634208917617798, 0.11478518694639206, -0.15732240676879883, -0.09922489523887634, -0.07659733295440674, -0.503665030002594, -0.05913407355546951, 0.5306515097618103, -0.5513179898262024, 0.0856415405869484, -0.14199675619602203, -0.12465406954288483, -0.020991338416934013, -0.5708457231521606, -0.09339694678783417, 0.5150959491729736, -0.620174765586853, 0.1036120057106018, -0.16885973513126373, -0.15802085399627686, -0.029034284874796867, -0.439657062292099, -0.042616695165634155, 0.5264771580696106, -0.6252885460853577, 0.05879342183470726, -0.1594291627407074, -0.12807580828666687, -0.039859894663095474, -0.5133683085441589, -0.1195804700255394, 0.5733318328857422, -0.5628617405891418, 0.09276621043682098, -0.12390337139368057, -0.10132140666246414, -0.015388735570013523, -0.5857561230659485, 0.02189001441001892, 0.4936777949333191, -0.5842708945274353, 0.10995962470769882, -0.16332319378852844, -0.12170211970806122, -0.009763268753886223, -0.5549308061599731, -0.07751936465501785, 0.49296140670776367, -0.733254611492157, 0.17094732820987701, -0.11490097641944885, -0.11574523895978928, 0.003561219433322549, -0.5595884919166565, -0.06243296340107918, 0.6033405661582947, -0.5984988808631897, -0.0053464509546756744, -0.310449481010437, -0.16657467186450958, -0.001557750510983169, -0.5353893041610718, -0.06964767724275589, 0.5474839806556702, -0.6917423009872437, 0.11656996607780457, -0.10424050688743591, -0.09132882207632065, -0.03482464328408241, -0.5418692827224731, -0.033046115189790726, 0.49568891525268555, -0.582234799861908, 0.2344307005405426, -0.18084071576595306, -0.22340458631515503, -0.07589390873908997, -0.4929247498512268, -0.006253904663026333, 0.5710211992263794, -0.6866797208786011, 0.06626430153846741, -0.08048036694526672, -0.16929887235164642, -0.0001103483882616274, -0.5231114029884338, -0.020440291613340378, 0.5523786544799805, -0.6086495518684387, 0.14694739878177643, -0.19331496953964233, -0.09980343282222748, -0.05392279475927353, -0.4963465929031372, 0.002901948755607009, 0.5035150051116943, -0.6830806732177734, 0.06952029466629028, -0.15938633680343628, -0.07704895734786987, -0.09536270052194595, -0.4538041055202484, -0.0442378968000412, 0.5057648420333862, -0.7140647172927856, 0.11817272752523422, -0.10927195101976395, -0.2029677778482437, 0.004976062569767237, -0.4293796420097351, -0.04183535650372505, 0.5416594743728638, -0.5267974138259888, 0.16752059757709503, -0.1887332797050476, -0.11551174521446228, -0.06246846914291382, -0.5193108320236206, -0.1769738644361496, 0.5549206733703613, -0.6793189644813538, 0.11410864442586899, -0.15065237879753113, -0.07112471759319305, -0.06941235065460205, -0.5515384078025818, -0.03213934600353241, 0.5873146653175354, -0.6634727120399475, 0.1744234710931778, -0.13073626160621643, -0.14131595194339752, 0.02163597382605076, -0.5526983141899109, -0.008058060891926289, 0.4921610951423645, -0.6249712705612183, 0.04421865567564964, -0.18572504818439484, -0.06142687052488327, -0.04521607235074043, -0.5098523497581482, -0.08075088262557983], "scores": [[-0.9839619770618151, -0.9960142629315449, -0.9932822508957411, -0.9858342072312987, -0.9950663032535021, -0.9958637619916275, -0.9947181229770066, -0.9844001127712625, -0.9927854204393161, -0.9918085452625576, -0.9962408771658828, -0.9967041097927791, -0.9885152576299057, -0.9835143090840788, -0.9908167922396489, -0.9869273992042711, -0.9903779649845701, -0.9940814181318186, -0.997739735511437, -0.9925587033472443, 0.9998575519994184, -0.9874461140711148, -0.9947612065349204, -0.9949642105538926, -0.9939586642808791, -0.9964783278178146, -0.991719103187644, -0.9959747083271031, -0.9957478035625418, -0.9877685845711464, -0.9911513141994484, -0.9950708337123442, -0.994805562828454, -0.9930946941833565, -0.9904405896666761, -0.9871365407706127, -0.9895484398579568, -0.9908633866399978, -0.994997447706425, -0.9910984565694915], [-0.9892736539316225, -0.9849231993559078, -0.9945524620214219, -0.9939174900377179, -0.9851288279158523, -0.9942764884765576, -0.9981071672267484, -0.989613203817056, -0.9964438322713656, -0.9897888532585837, -0.9905254065068346, -0.9938789172151583, -0.9953533659744748, -0.9928279529046863, -0.9871805617731844, -0.9955903946012274, -0.994821415673201, -0.9770677089556361, -0.9914684094758843, -0.9913596704587385, -0.9909364978244775, 0.9994344206486792, -0.9973190049848656, -0.9936967090195441, -0.996745386710056, -0.9958203220985364, -0.9902925372682634, -0.9969910179140984, -0.9937343378432435, -0.9776769218179314, -0.9974026196072141, -0.9839894590127014, -0.9930578589079291, -0.9959361207876514, -0.9925630946450104, -0.9876266874962965, -0.9841656554753395, -0.9977305416928992, -0.9954561751873393, -0.9939790711693458]], "kept": [[0, 1, 2, 3], [0, 1, 2, 3]]}, {"name": "reserved_overflow", "n": 30, "head_dim": 8, "n_kv_heads": 3, "budget": 6, "recent_frac": 0.8, "sink": 8, "keys": [-0.3310459852218628, -0.0550963431596756, 0.6515293717384338, 0.3322467803955078, -0.6153426766395569, 0.023526495322585106, -0.3143995404243469, 0.16979050636291504, -0.3464188873767853, -0.012094845995306969, 0.6232651472091675, 0.2094331532716751, -0.6639891862869263, -0.03560157120227814, -0.22073426842689514, 0.051651109009981155, -0.17650803923606873, -0.15866513550281525, 0.6396062970161438, 0.20886267721652985, -0.5923813581466675, -0.10790383815765381, -0.2569284439086914, 0.06766269356012344, -0.32485002279281616, -0.01792415976524353, 0.6486960053443909, 0.3038083016872406, -0.5832157135009766, -0.05099356546998024, -0.279633492231369, 0.046987418085336685, -0.21323861181735992, -0.02463657408952713, 0.6042761206626892, 0.22310560941696167, -0.6710661053657532, -0.03121274895966053, -0.25163987278938293, 0.05056329071521759, -0.14773087203502655, -0.09250511974096298, 0.6253241300582886, 0.2763943672180176, -0.6788286566734314, -0.02043219469487667, -0.23907779157161713, 0.09586133807897568, -0.31643733382225037, 0.001388534321449697, 0.6221374869346619, 0.2619363069534302, -0.5888205170631409, 0.031038807705044746, -0.1867821216583252, 0.06581393629312515, -0.25161728262901306, -0.05136101320385933, 0.589974582195282, 0.3141748309135437, -0.6699442267417908, -0.06501302868127823, -0.13692328333854675, 0.05033179745078087, -0.3095928430557251, 0.025428546592593193, 0.6232746839523315, 0.30656924843788147, -0.6726493835449219, -0.01443983893841505, -0.32417017221450806, -0.03829292580485344, -0.2994583249092102, -0.07108689099550247, 0.6796596050262451, 0.3747765123844147, -0.6827681064605713, 0.04845117777585983, -0.26996076107025146, 0.04952757805585861, -0.3246961832046509, -0.14113423228263855, 0.6008372902870178, 0.19380994141101837, -0.6313526630401611, -0.037126559764146805, -0.17720887064933777, 0.0648050308227539, -0.24237468838691711, -0.010313178412616253, 0.6656659245491028, 0.2515988349914551, -0.635711133480072, -0.008048269897699356, -0.18590761721134186, 0.09630832076072693, -0.2323407083749771, -0.027243472635746002, 0.612533688545227, 0.2790166139602661, -0.7017916440963745, 0.06536269187927246, -0.23524130880832672, 0.06632369011640549, -0.22066740691661835, -0.06767041981220245, 0.6270549297332764, 0.31295469403266907, -0.62538743019104, -0.04779442027211189, -0.21776577830314636, -0.028101978823542595, -0.16785947978496552, -0.0757414922118187, 0.642054557800293, 0.36635398864746094, -0.5235878825187683, 0.0037894961424171925, -0.20850040018558502, 0.11166303604841232, 0.750726580619812, 0.2073666900396347, -1.911268949508667, -0.758678138256073, 1.8813899755477905, 0.0191964004188776, 0.7216143012046814, -0.1669963151216507, -0.2935836613178253, -0.19925187528133392, 0.6267542839050293, 0.32040733098983765, -0.6866352558135986, 0.05968933925032616, -0.18976640701293945, 0.054411932826042175, -0.25101718306541443, -0.06304328143596649, 0.596705973148346, 0.23934894800186157, -0.6335031390190125, 0.02859605848789215, -0.20015306770801544, 0.020017141476273537, -0.20935094356536865, -0.10607846826314926, 0.6313145160675049, 0.2719407379627228, -0.6323075890541077, -0.008348879404366016, -0.22627165913581848, 0.029514091089367867, -0.2832699120044708, -0.0773906409740448, 0.6878238916397095, 0.2483820617198944, -0.5359488725662231, 0.00871182605624199, -0.3884636461734772, 0.0351763591170311, -0.16738489270210266, -0.0668676495552063, 0.6178686618804932, 0.20978248119354248, -0.5649850368499756, 0.04814038798213005, -0.20614197850227356, 0.07855913788080215, -0.1785912960767746, -0.026124853640794754, 0.5782703161239624, 0.29188764095306396, -0.6226935982704163, -0.007516827434301376, -0.20157431066036224, 0.058573368936777115, -0.3973143398761749, 0.010684467852115631, 0.6916463375091553, 0.2473360151052475, -0.6497947573661804, 0.00933158304542303, -0.30396342277526855, 0.07247432321310043, -0.2621464431285858, -0.08483248203992844, 0.5936043858528137, 0.21953366696834564, -0.5758601427078247, 0.06475021690130234, -0.2146018147468567, 0.05487721413373947, -0.29453176259994507, -0.03129725530743599, 0.7203071713447571, 0.17274028062820435, -0.5724619626998901, -0.11028432101011276, -0.2567441761493683, 0.07638527452945709, -0.3188859522342682, -0.14303447306156158, 0.6490440368652344, 0.2973012328147888, -0.6549475789070129, 0.06519376486539841, -0.2787398099899292, 0.14487160742282867, -0.2446163445711136, -0.05541977286338806, 0.6705071330070496, 0.26951682567596436, -0.5388859510421753, 0.10832331329584122, -0.21243782341480255, 0.048868805170059204, -0.30944162607192993, -0.1049036756157875, 0.6402227282524109, 0.2483985275030136, -0.6333585977554321, -0.001748415408656001, -0.21496672928333282, -0.010380316525697708, -0.3019103705883026, -0.011088894680142403, 0.6816749572753906, 0.22667638957500458, -0.5664082169532776, -0.03582040220499039, -0.25233331322669983, -0.034119509160518646, -0.20828048884868622, -0.06762249022722244, 0.693249523639679, 0.28892871737480164, -0.6262013912200928, 0.006299509666860104, -0.17964023351669312, 0.049413442611694336, -0.257750540971756, -0.06317555904388428, 0.6806971430778503, 0.30037471652030945, -0.5344500541687012, -0.0049479347653687, -0.17532044649124146, 0.025862421840429306, -0.21888573467731476, -0.05920157581567764, 0.7283563017845154, 0.2858824133872986, -0.6515471935272217, 0.029023628681898117, -0.22025138139724731, 0.04354151710867882, -0.2630894184112549, -0.11206945776939392, 0.7799769639968872, 0.262610524892807, -0.58004230260849, -0.057850368320941925, -0.2915869653224945, 0.051960334181785583, -0.2556001842021942, -0.02457478828728199, 0.5920740365982056, 0.28296980261802673, -0.5528972744941711, -0.04518313333392143, -0.1917206048965454, 0.011658944189548492, -0.2307884842157364, -0.046475645154714584, 0.7291747331619263, 0.2214740812778473, -0.6202136278152466, 0.05272495746612549, -0.29580506682395935, 0.05923805013298988, -0.2916071116924286, -0.09975580126047134, 0.676382303237915, 0.29101404547691345, -0.4997696280479431, 0.06937193125486374, -0.17277392745018005, 0.051315855234861374, -0.26318249106407166, -0.057883478701114655, 0.5284225344657898, 0.33870387077331543, -0.6624059677124023, -0.0527585931122303, -0.26047980785369873, 0.10299821197986603, -0.18646442890167236, -0.08410150557756424, 0.6634118556976318, 0.2523844540119171, -0.6792827844619751, 0.008132793009281158, -0.23678700625896454, -0.018203649669885635, -0.1941470056772232, -0.14434285461902618, 0.6528239846229553, 0.20228974521160126, -0.6705947518348694, -0.03587159141898155, -0.20328204333782196, -0.00437941076233983, -0.2590215802192688, 0.011525641195476055, 0.6060317158699036, 0.2529059946537018, -0.5969659090042114, -0.03018580749630928, -0.2150578498840332, 0.02500271238386631, -0.21477439999580383, -0.0632137656211853, 0.593782901763916, 0.32474735379219055, -0.6351038813591003, -0.01518693845719099, -0.28518620133399963, 0.0344436950981617, -0.2546752691268921, -0.057819850742816925, 0.6954295635223389, 0.18830366432666779, -0.6479196548461914, 0.058522406965494156, -0.2299536168575287, 0.07825399190187454, -0.17030367255210876, -0.04963633418083191, 0.6637417674064636, 0.2520795166492462, -0.5421420335769653, 0.051138218492269516, -0.2751882076263428, 0.08517579734325409, -0.3074677288532257, -0.012268928810954094, 0.6738637089729309, 0.24595704674720764, -0.6508204936981201, -0.09134197980165482, -0.2011943906545639, 0.05163058266043663, -0.26592034101486206, -0.030760660767555237, 0.6067479848861694, 0.22431117296218872, -0.6657489538192749, -0.0695604607462883, -0.11920707672834396, 0.04744914919137955, -0.26816239953041077, -0.02220253273844719, 0.4822121858596802, 0.25180181860923767, -0.6307142972946167, -0.05081827566027641, -0.21708577871322632, -0.044599998742341995, 0.7538400888442993, 0.18553023040294647, -1.9132479429244995, -0.7667989134788513, 1.8814336061477661, 0.01915026642382145, 0.7163488864898682, -0.16836120188236237, -0.24374057352542877, -0.14035993814468384, 0.6285440325737, 0.18302354216575623, -0.7420589923858643, -0.13002660870552063, -0.19608289003372192, 0.04273872822523117, -0.16097642481327057, -0.14873117208480835, 0.6134938597679138, 0.2618006765842438, -0.6469722390174866, 0.014447957277297974, -0.24098360538482666, 0.017966290935873985, -0.3755180239677429, -0.12292417883872986, 0.7177350521087646, 0.28388041257858276, -0.5700055360794067, -0.03882204368710518, -0.25665003061294556, 0.006166120525449514, -0.23801690340042114, -0.07086054235696793, 0.6557806730270386, 0.3350878059864044, -0.6793904304504395, 0.028575804084539413, -0.3161465525627136, 0.1061536893248558, -0.27888697385787964, -0.07692162692546844, 0.640483021736145, 0.32971587777137756, -0.6388310790061951, -0.032973237335681915, -0.3050508201122284, 0.06487378478050232, -0.2994721233844757, -0.09844140708446503, 0.6323990821838379, 0.2809735834598541, -0.6730248928070068, 0.025441862642765045, -0.1572805941104889, -0.023427637293934822, -0.18857043981552124, 0.03446356952190399, 0.604159951210022, 0.2781519293785095, -0.7058960199356079, 0.009004934690892696, -0.26202622056007385, 0.06640840321779251, -0.30257847905158997, -0.11441409587860107, 0.6111748814582825, 0.21153706312179565, -0.6685976982116699, -0.01873929798603058, -0.30204033851623535, 0.04296685382723808, -0.2335444837808609, -0.1899738758802414, 0.642448365688324, 0.20884981751441956, -0.6946372389793396, -0.07687550038099289, -0.2900691032409668, 0.011915761046111584, -0.24146948754787445, -0.1385239064693451, 0.6836694478988647, 0.20431281626224518, -0.6051236391067505, 0.04437877610325813, -0.3087647557258606, 0.10310962796211243, -0.14843934774398804, -0.09577465802431107, 0.6630820035934448, 0.24173931777477264, -0.6318513751029968, -0.015748335048556328, -0.2687593996524811, 0.08991698175668716, -0.20961256325244904, -0.06996607780456543, 0.7130128741264343, 0.3057379424571991, -0.5979505777359009, -0.039655473083257675, -0.14617504179477692, 0.12187016010284424, -0.3593476414680481, -0.12374424934387207, 0.6148441433906555, 0.29948368668556213, -0.5697513818740845, -0.01714872568845749, -0.22804997861385345, 0.09149337559938431, -0.3148186206817627, -0.03798365965485573, 0.6241517663002014, 0.26248183846473694, -0.6337714791297913, -0.0611870214343071, -0.28960540890693665, 0.059168215841054916, -0.24420303106307983, -0.08039199560880661, 0.6928807497024536, 0.23552612960338593, -0.6920500993728638, -0.01928560435771942, -0.2485109567642212, 0.05059698596596718, -0.19246000051498413, -0.041644781827926636, 0.6924993991851807, 0.2846876084804535, -0.639369547367096, 0.049949634820222855, -0.25267815589904785, 0.06969910115003586, -0.3005616366863251, -0.061948247253894806, 0.6237358450889587, 0.27229756116867065, -0.5400779247283936, 0.11687152087688446, -0.18846097588539124, 0.05605393275618553, -0.20897206664085388, -0.05530593544244766, 0.6097902655601501, 0.25074467062950134, -0.6432484984397888, -0.031773850321769714, -0.1702663004398346, 0.030590442940592766, -0.2750305235385895, -0.03154247626662254, 0.6055077910423279, 0.19644904136657715, -0.6732489466667175, 0.02259703166782856, -0.22041143476963043, 0.07263373583555222, -0.2733134329319, -0.11229307949542999, 0.6856848001480103, 0.18175724148750305, -0.641181230545044, -0.06764116138219833, -0.2593913674354553, 0.050202544778585434, -0.1885284185409546, -0.20983080565929413, 0.7088679671287537, 0.21006044745445251, -0.6155354380607605, 0.03921877220273018, -0.27699360251426697, 0.053785063326358795, -0.22491326928138733, -0.07225742936134338, 0.6040950417518616, 0.26410427689552307, -0.6244378685951233, -0.07564543932676315, -0.17045293748378754, 0.06986087560653687, -0.370292603969574, -0.10720758885145187, 0.7273082733154297, 0.33444979786872864, -0.6160203218460083, -0.08457887172698975, -0.2499302476644516, 0.11186080425977707, -0.22917474806308746, -0.08203081786632538, 0.6047782897949219, 0.2966236472129822, -0.6398770213127136, 0.016130270436406136, -0.21315492689609528, 0.1411631852388382, -0.2394212931394577, -0.05218140408396721, 0.5524348020553589, 0.31115007400512695, -0.5846433043479919, -0.0006337009253911674, -0.35130733251571655, 0.031041521579027176, -0.24478349089622498, 0.039812881499528885, 0.6713612079620361, 0.2952270209789276, -0.5880878567695618, 0.0668412297964096, -0.2945268154144287, 0.016516245901584625, -0.30780884623527527, -0.07954327762126923, 0.6238812804222107, 0.2603961229324341, -0.6807370185852051, 0.004365235101431608, -0.2807947099208832, 0.16469913721084595, -0.2754948139190674, 0.06206158921122551, 0.7231947779655457, 0.19509993493556976, -0.5816571712493896, 0.029972167685627937, -0.31505757570266724, 0.08915407210588455, -0.25365832448005676, -0.04823727160692215, 0.5985330939292908, 0.2518428564071655, -0.6090506911277771, -0.009654578752815723, -0.2687615156173706, 0.05064573511481285, -0.2983059287071228, -0.0486634187400341, 0.5129425525665283, 0.33030396699905396, -0.6266382932662964, 0.023077456280589104, -0.195615753531456, -0.0037319327238947153, 0.7610815763473511, 0.21804533898830414, -1.9373493194580078, -0.7676986455917358, 1.900517225265503, -0.000548365933354944, 0.6865613460540771, -0.16983604431152344, -0.37301695346832275, -0.07613483816385269, 0.6177866458892822, 0.3326479196548462, -0.6442173719406128, -0.023613089695572853, -0.2767672836780548, 0.015145977959036827, -0.2775508165359497, -0.036411553621292114, 0.5744317173957825, 0.24365969002246857, -0.7038586735725403, -0.00978521816432476, -0.2310364842414856, -0.00417251605540514, -0.27891772985458374, -0.06160098686814308, 0.6786713004112244, 0.3013709485530853, -0.6365867257118225, -0.05195815488696098, -0.32109585404396057, 0.09627215564250946, -0.18068858981132507, -0.04883693903684616, 0.7182105183601379, 0.23070798814296722, -0.6508480310440063, 0.07668491452932358, -0.29038310050964355, 0.08743520826101303, -0.23131971061229706, -0.05953969061374664, 0.7265800833702087, 0.20931777358055115, -0.6527152061462402, -0.10694621503353119, -0.29240483045578003, 0.031331777572631836, -0.2701691687107086, -0.07745176553726196, 0.6547117829322815, 0.21987532079219818, -0.7306618690490723, 0.0055902935564517975, -0.21864910423755646, -0.0012396457605063915, -0.2902265787124634, 0.021215219050645828, 0.69129878282547, 0.3080936074256897, -0.59758061170578, 0.005306687671691179, -0.26544302701950073, 0.08585052937269211, -0.2606731057167053, -0.08938014507293701, 0.6391341686248779, 0.24935440719127655, -0.5865384340286255, -0.050299208611249924, -0.11450342833995819, 0.006426911801099777, -0.2721794545650482, -0.019806059077382088, 0.5976122617721558, 0.22917510569095612, -0.6298224925994873, -0.15126919746398926, -0.2653121054172516, 0.008572167716920376, -0.2631056606769562, -0.042113836854696274, 0.6719141006469727, 0.32904064655303955, -0.5585699081420898, 0.05003451555967331, -0.24169503152370453, 0.1097131222486496, -0.30067870020866394, -0.07197252660989761, 0.7123212218284607, 0.29339396953582764, -0.7014450430870056, -0.031699832528829575, -0.19399017095565796, 0.08841320127248764, -0.19609889388084412, -0.15595778822898865, 0.5762574672698975, 0.24120531976222992, -0.679896891117096, -0.0030970433726906776, -0.23857448995113373, 0.07522249966859818], "scores": [[-0.9892935256725782, -0.9922522379310489, -0.982890300588487, -0.9938808736328294, -0.994418372049177, -0.9900764674286718, -0.9945322001277987, -0.9894557092451631, -0.9876943277134009, -0.9962724361889291, -0.9888971718114964, -0.995800346874971, -0.9937726823103912, -0.9932031308133707, -0.98413399895371, 0.9996784564510381, -0.9868099050032573, -0.9975023843239723, -0.9970051172952823, -0.9835919971790351, -0.9932372626793176, -0.9944089237243015, -0.9903506146104045, -0.9964143975908272, -0.9840291696190477, -0.9909333567246519, -0.9900141937779605, -0.9955394377338832, -0.9899034680802837, -0.9954687246852747], [-0.9923996965288949, -0.9966761170015384, -0.9914067772436546, -0.9952588595936873, -0.9926388606381276, -0.984420892447216, -0.9864267456727969, -0.9949472830604766, -0.9919797537263677, -0.995036394445288, -0.9947184685643902, -0.9930090941580365, -0.9896202207403842, -0.9926642281707316, -0.9881948553255564, -0.9829841677779465, 0.9996742282031014, -0.98341440145221, -0.9924723365495485, -0.988193966918481, -0.9937447113410518, -0.9958704060855127, -0.991409124192019, -0.9873870614021487, -0.9935189367805257, -0.9885460023614695, -0.9902045705454154, -0.9932208175384234, -0.9888961130229108, -0.9887246297217169], [-0.9964448309306689, -0.9980468618357503, -0.9946411897546276, -0.9861447262929354, -0.9950795638085934, -0.9949354112423062, -0.9932366781163172, -0.9818554270820504, -0.9941996450454121, -0.9898557138604689, -0.9935745194554446, -0.9884324623803511, -0.9885589326535446, -0.9930802140138858, -0.9838876405327599, -0.9993049958383795, -0.9860393666808165, 0.9995734735315155, -0.991562665125212, -0.9925366858571721, -0.9966618487904201, -0.9899883431321648, -0.9911550568513884, -0.9936261872711777, -0.9940183127812442, -0.9883464729348481, -0.9865483305684745, -0.9915483577412078, -0.9968889376373468, -0.989368284547274]], "kept": [[0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5]]}, {"name": "full_budget", "n": 40, "head_dim": 16, "n_kv_heads": 4, "budget": 40, "recent_frac": 0.3, "sink": 4, "keys": [-0.2033780813217163, -0.3871213495731354, -0.09315542131662369, 0.08003490418195724, 0.24651910364627838, 0.054919105023145676, -0.14042742550373077, -0.22440464794635773, 0.2064642757177353, 0.447559654712677, -0.01439761184155941, -0.31918537616729736, -0.28706270456314087, 0.388774037361145, -0.014076580293476582, -0.4292137622833252, -0.2010875642299652, -0.3441769480705261, -0.11408734321594238, 0.08462567627429962, 0.22761832177639008, -0.040510255843400955, -0.12603384256362915, -0.250400573015213, 0.24449719488620758, 0.4418962597846985, -0.03213727846741676, -0.29274243116378784, -0.2931111454963684, 0.3990846574306488, 0.05257595703005791, -0.5296697616577148, -0.21086440980434418, -0.34174954891204834, -0.01359099056571722, 0.04536285623908043, 0.31908684968948364, -0.0276984553784132, -0.1538376659154892, -0.23695674538612366, 0.2584189474582672, 0.4344775378704071, 0.18934018909931183, -0.2742530405521393, -0.19577568769454956, 0.4394659399986267, 0.02006378024816513, -0.4337497055530548, -0.13167382776737213, -0.3487876057624817, -0.05225104093551636, 0.10337401926517487, 0.3126455843448639, 0.009004744701087475, -0.14489121735095978, -0.18281401693820953, 0.1911335438489914, 0.36281824111938477, 0.11254271119832993, -0.3712729215621948, -0.29808109998703003, 0.3333072066307068, 0.09874296933412552, -0.4482786953449249, -0.2477450966835022, -0.3857611417770386, -0.040634460747241974, 0.05169316381216049, 0.21858085691928864, 0.05794983357191086, -0.19710850715637207, -0.2110549956560135, 0.18564431369304657, 0.38380011916160583, 0.06504885852336884, -0.2394101917743683, -0.2638700306415558, 0.33446642756462097, -0.04144292697310448, -0.4404865801334381, -0.21680612862110138, -0.3157055079936981, -0.08490324765443802, 0.08050327003002167, 0.24611897766590118, 0.001262193312868476, -0.1292617917060852, -0.2139507234096527, 0.19100448489189148, 0.501126229763031, 0.09170946478843689, -0.3851533532142639, -0.15134884417057037, 0.4148223102092743, 0.003323692362755537, -0.38489583134651184, -0.1983104944229126, -0.35972097516059875, -0.09336646646261215, 0.05476360395550728, 0.2845907211303711, 0.08069101721048355, -0.1535257250070572, -0.17389188706874847, 0.27976468205451965, 0.3453373610973358, 0.08062967658042908, -0.3216707110404968, -0.29098811745643616, 0.34613943099975586, 0.049630098044872284, -0.40855512022972107, -0.22577276825904846, -0.32624053955078125, -0.02558058500289917, 0.11926325410604477, 0.32561826705932617, 0.050319962203502655, -0.11287283897399902, -0.10360138863325119, 0.21714098751544952, 0.4125404357910156, 0.10325074940919876, -0.3523050546646118, -0.25491055846214294, 0.4370928406715393, 0.08193369209766388, -0.3528302013874054, -0.19866962730884552, -0.40207645297050476, 0.035671330988407135, 0.15907983481884003, 0.22924789786338806, 0.09604136645793915, -0.13565298914909363, -0.285629540681839, 0.1649646908044815, 0.38062170147895813, 0.1472959667444229, -0.3459523022174835, -0.25070619583129883, 0.38359376788139343, 0.03144571930170059, -0.4759519696235657, -0.1882389336824417, -0.2751198410987854, -0.030492134392261505, 0.058060240000486374, 0.22469398379325867, 0.03319885954260826, -0.17260213196277618, -0.2264413684606552, 0.10199765861034393, 0.5035914778709412, 0.11358451098203659, -0.3550442159175873, -0.19261480867862701, 0.46455615758895874, -0.0690833106637001, -0.45769551396369934, -0.21859194338321686, -0.2965473234653473, -0.0677642896771431, 0.09291362762451172, 0.27926620841026306, 0.11991646885871887, -0.029274098575115204, -0.221174418926239, 0.13967019319534302, 0.540694534778595, 0.01876632496714592, -0.33501896262168884, -0.23619624972343445, 0.42159411311149597, 0.1018461361527443, -0.41069942712783813, -0.24281242489814758, -0.3036409318447113, -0.04919513687491417, 0.19828113913536072, 0.2814347743988037, -0.039598461240530014, -0.1895241141319275, -0.12242543697357178, 0.15897582471370697, 0.30084356665611267, 0.0387185700237751, -0.30634811520576477, -0.1785837858915329, 0.34670838713645935, 0.08372852206230164, -0.3904833197593689, -0.23416268825531006, -0.33833953738212585, 0.026781493797898293, 0.19045943021774292, 0.324960857629776, 0.043847501277923584, -0.0803576409816742, -0.20196695625782013, 0.18122774362564087, 0.36304888129234314, 0.06803172826766968, -0.3104496896266937, -0.09934373944997787, 0.38778963685035706, 0.11393525451421738, -0.3642234206199646, -0.20851720869541168, -0.27049019932746887, -0.17053170502185822, 0.10918155312538147, 0.3250478506088257, -0.09264308214187622, -0.19523167610168457, -0.14212940633296967, 0.17313185334205627, 0.35115694999694824, 0.04907377436757088, -0.33347827196121216, -0.20180295407772064, 0.4201260805130005, 0.03638293966650963, -0.46351540088653564, -0.20188705623149872, -0.26270225644111633, -0.04423971101641655, 0.13638247549533844, 0.2746506929397583, -0.03734062984585762, -0.17281529307365417, -0.16237126290798187, 0.191143199801445, 0.36087489128112793, -0.013139964081346989, -0.30081477761268616, -0.25500187277793884, 0.3799052834510803, 0.03392186015844345, -0.41605302691459656, -0.18447089195251465, -0.28085362911224365, -0.015269937925040722, 0.17103984951972961, 0.32250291109085083, 0.010469780303537846, -0.1399865746498108, -0.17022234201431274, 0.1704677790403366, 0.4335388243198395, 0.1254340261220932, -0.34641116857528687, -0.32654452323913574, 0.4571145474910736, 0.051282353699207306, -0.39481478929519653, -0.2658085525035858, -0.24231043457984924, -0.02476002275943756, 0.09813148528337479, 0.26458537578582764, 0.03294497728347778, -0.14560969173908234, -0.20886759459972382, 0.13000239431858063, 0.4692073464393616, 0.06577467173337936, -0.30261707305908203, -0.233379527926445, 0.2795209288597107, 0.07675142586231232, -0.44670602679252625, -0.21365469694137573, -0.3254706561565399, -0.11031480133533478, 0.1466042399406433, 0.24739153683185577, -0.017932720482349396, -0.1803826093673706, -0.1869954615831375, 0.22327136993408203, 0.3763142228126526, -0.00775577686727047, -0.25322431325912476, -0.29127949476242065, 0.45717331767082214, 0.047679562121629715, -0.46516576409339905, -0.23119527101516724, -0.3058667480945587, -0.01722242683172226, 0.09452668577432632, 0.3295314908027649, 0.13159909844398499, -0.14768853783607483, -0.25240543484687805, 0.13326238095760345, 0.33655011653900146, 0.03577028587460518, -0.3425378203392029, -0.30428048968315125, 0.4239410161972046, -0.0801725909113884, -0.4962882399559021, -0.09843068569898605, -0.27706992626190186, -0.03555143252015114, 0.12729814648628235, 0.3707316219806671, 0.05403166264295578, -0.1333942711353302, -0.2564410865306854, 0.18229904770851135, 0.3693726658821106, 0.0324447937309742, -0.34206080436706543, -0.24025766551494598, 0.38094624876976013, 0.09394988417625427, -0.496557354927063, 0.611250638961792, 0.9917982816696167, 0.16194097697734833, -0.31679877638816833, -0.836624801158905, -0.08202638477087021, 0.3980172574520111, 0.57875657081604, -0.5551645755767822, -1.2063647508621216, -0.209880992770195, 0.9194701910018921, 0.7040749192237854, -1.2050305604934692, -0.15706437826156616, 1.3037043809890747, -0.23358401656150818, -0.3764893710613251, -0.08875469118356705, 0.08473067730665207, 0.261779248714447, 0.024614794179797173, -0.15905015170574188, -0.17558449506759644, 0.21212594211101532, 0.4239603579044342, 0.03459838032722473, -0.3371741771697998, -0.21628211438655853, 0.38757964968681335, 0.048236772418022156, -0.5548480153083801, -0.3341984748840332, -0.31210264563560486, -0.045213084667921066, 0.17410679161548615, 0.3182429373264313, 0.03833712264895439, -0.1364549994468689, -0.26627010107040405, 0.23014119267463684, 0.37588757276535034, 0.09287778288125992, -0.2963467836380005, -0.2244204729795456, 0.442065566778183, 0.025901051238179207, -0.44961002469062805, -0.24198634922504425, -0.34781765937805176, 0.021606018766760826, 0.08626925945281982, 0.3414655923843384, 0.06855075061321259, -0.10080903023481369, -0.15478233993053436, 0.13526856899261475, 0.424739271402359, 0.1471741646528244, -0.34113866090774536, -0.2403806746006012, 0.3629738390445709, 0.0662449300289154, -0.48417478799819946, -0.11870928853750229, -0.3478018045425415, -0.13882409036159515, 0.19397473335266113, 0.33071091771125793, 0.053175754845142365, -0.11064374446868896, -0.21282298862934113, 0.17481760680675507, 0.44226783514022827, 0.11363480240106583, -0.25971823930740356, -0.22739776968955994, 0.37411239743232727, 0.04078999161720276, -0.37476828694343567, -0.12719736993312836, -0.346025675535202, -0.08555621653795242, 0.16681267321109772, 0.31836721301078796, 0.062113091349601746, -0.2028898000717163, -0.21362096071243286, 0.16023261845111847, 0.41646212339401245, -0.01649354211986065, -0.24276122450828552, -0.22301873564720154, 0.4191736578941345, -0.008812662214040756, -0.43156930804252625, -0.13246075809001923, -0.4074448347091675, -0.04640261456370354, 0.1393754482269287, 0.2475874125957489, 0.06798724085092545, -0.15377657115459442, -0.13344475626945496, 0.21235798299312592, 0.41728517413139343, 0.168199360370636, -0.2647540271282196, -0.17259353399276733, 0.3379558026790619, 0.018064003437757492, -0.38841983675956726, -0.15391267836093903, -0.33174705505371094, -0.02270469069480896, 0.051517508924007416, 0.29287654161453247, -0.030114011839032173, -0.120244599878788, -0.2192416787147522, 0.15393251180648804, 0.4088818430900574, 0.15862655639648438, -0.3569369614124298, -0.2681128680706024, 0.45081499218940735, 0.029216023162007332, -0.3417850732803345, -0.18806679546833038, -0.3244284689426422, -0.04384681209921837, 0.1914941817522049, 0.3149528503417969, 0.05308578908443451, -0.12468023598194122, -0.2730540335178375, 0.141718789935112, 0.4802340865135193, 0.108847476541996, -0.2810456156730652, -0.2061721831560135, 0.3702521324157715, 0.06729257106781006, -0.4752880036830902, -0.17962516844272614, -0.36466681957244873, -0.07238201051950455, 0.1271059364080429, 0.265845388174057, 0.059629276394844055, -0.11447121202945709, -0.17696018517017365, 0.25220295786857605, 0.33237263560295105, -0.01206055749207735, -0.27553677558898926, -0.19098562002182007, 0.49801382422447205, 0.0882072001695633, -0.3887375295162201, -0.24334728717803955, -0.29132896661758423, -0.027999071404337883, 0.05164193734526634, 0.2373688966035843, 0.021698622032999992, -0.12286043167114258, -0.3421723544597626, 0.1894976645708084, 0.4176096022129059, 0.05891174077987671, -0.29300761222839355, -0.21523001790046692, 0.34684988856315613, 0.07120336592197418, -0.4428136348724365, -0.24648721516132355, -0.3185030221939087, 0.01269119419157505, 0.03142941743135452, 0.1995292752981186, -0.05540812388062477, -0.03840753436088562, -0.2075122743844986, 0.18300805985927582, 0.36673808097839355, 0.020731167867779732, -0.4076918661594391, -0.2295050024986267, 0.4104097783565521, -0.04843234643340111, -0.48240575194358826, -0.23639896512031555, -0.33026018738746643, -0.049484558403491974, 0.16072484850883484, 0.35040757060050964, 0.08067290484905243, -0.07019669562578201, -0.2532539367675781, 0.15825779736042023, 0.4632473587989807, 0.08218660950660706, -0.30541276931762695, -0.33697474002838135, 0.30446964502334595, 0.08693882822990417, -0.4131694734096527, -0.21399615705013275, -0.3496086597442627, -0.14799614250659943, 0.12299595028162003, 0.33390581607818604, 0.10935872793197632, -0.1348162293434143, -0.20514293015003204, 0.13118253648281097, 0.46722906827926636, 0.15309633314609528, -0.3370073735713959, -0.22348739206790924, 0.40196529030799866, 0.02170298993587494, -0.5551207661628723, -0.2174932360649109, -0.2985878586769104, -0.13385611772537231, 0.06249113380908966, 0.11800674349069595, 0.09147656708955765, -0.11225930601358414, -0.1926591694355011, 0.1724587231874466, 0.3742027282714844, 0.06381324678659439, -0.32186460494995117, -0.25044316053390503, 0.3957412540912628, 0.05667690560221672, -0.35279935598373413, -0.27166488766670227, -0.31586164236068726, -0.09291905909776688, 0.04746113717556, 0.163776233792305, 0.05759478732943535, -0.18679209053516388, -0.23291738331317902, 0.20897632837295532, 0.3513932228088379, 0.0685558170080185, -0.3105928897857666, -0.19843316078186035, 0.44746384024620056, 0.006551764439791441, -0.3848893940448761, -0.20453056693077087, -0.3233201801776886, -0.07281669974327087, 0.14271217584609985, 0.22107474505901337, 0.03975439444184303, -0.07898413389921188, -0.20961658656597137, 0.2868710160255432, 0.36877158284187317, 0.07793836295604706, -0.33097928762435913, -0.26946285367012024, 0.34114325046539307, 0.022891033440828323, -0.43336012959480286, -0.23984813690185547, -0.285340279340744, 0.013825314119458199, 0.027490388602018356, 0.3234269320964813, 0.03615318611264229, -0.12560708820819855, -0.07240812480449677, 0.24569863080978394, 0.4222823679447174, 0.07640926539897919, -0.41463470458984375, -0.19940577447414398, 0.3599974513053894, 0.059180986136198044, -0.42536675930023193, -0.15117570757865906, -0.2274349182844162, -0.012510904110968113, 0.16808262467384338, 0.27045759558677673, 0.050038859248161316, -0.09881067276000977, -0.2669144868850708, 0.2880595326423645, 0.3703234791755676, 0.003876391565427184, -0.2246466428041458, -0.13210339844226837, 0.36227649450302124, 0.019728858023881912, -0.2746526300907135, -0.19356241822242737, -0.42816296219825745, -0.09230445325374603, 0.2045411318540573, 0.315140962600708, 0.06485513597726822, -0.07604525238275528, -0.16114415228366852, 0.1693221628665924, 0.3263512849807739, 0.09007209539413452, -0.3621506094932556, -0.2408372461795807, 0.4853803515434265, 0.07742687314748764, -0.427273690700531, -0.1959092766046524, -0.2924204170703888, -0.015629328787326813, 0.038594506680965424, 0.29145872592926025, 0.11303253471851349, -0.09955468773841858, -0.2331893891096115, 0.1520010083913803, 0.3029707670211792, 0.028084615245461464, -0.2260005921125412, -0.24150152504444122, 0.35468125343322754, 0.02158833108842373, -0.43578556180000305, -0.204520583152771, -0.34665873646736145, -0.10014574229717255, 0.13626369833946228, 0.359111487865448, -0.001147668226622045, -0.16433671116828918, -0.13546544313430786, 0.11279521137475967, 0.37649714946746826, 0.048848703503608704, -0.2979597747325897, -0.2517722547054291, 0.3531016409397125, 0.0659196749329567, -0.48963379859924316, -0.28631895780563354, -0.4303729236125946, -0.024256089702248573, 0.01759241707623005, 0.3045376241207123, 0.06078563630580902, -0.09692815691232681, -0.20366722345352173, 0.15857239067554474, 0.4103504419326782, 0.0380668118596077, -0.3706589639186859, -0.17525333166122437, 0.35270991921424866, 0.1401888132095337, -0.4045100510120392, -0.22192111611366272, -0.30707767605781555, -0.0001835436705732718, 0.03551173210144043, 0.2358461171388626, -0.025213195011019707, -0.129021555185318, -0.17472054064273834, 0.18743081390857697, 0.36965638399124146, 0.04133098945021629, -0.2469967007637024, -0.18217818439006805, 0.40399956703186035, -0.015136090107262135, -0.3988419473171234, -0.1848103255033493, -0.31632354855537415, 0.024810563772916794, 0.14546218514442444, 0.23346050083637238, -0.020987845957279205, -0.06825742870569229, -0.18464726209640503, 0.2303352952003479, 0.4007469117641449, 0.020279014483094215, -0.29013878107070923, -0.1570795476436615, 0.41738858819007874, -0.0028838810976594687, -0.4425412118434906, -0.22061391174793243, -0.31659287214279175, -0.006788310594856739, 0.2071496844291687, 0.29790931940078735, 0.010425851680338383, -0.12869320809841156, -0.18938978016376495, 0.16949257254600525, 0.41199707984924316, 0.07251813262701035, -0.3603748083114624, -0.20583410561084747, 0.3840966820716858, 0.006007153075188398, -0.4009491205215454, -0.21877293288707733, -0.23704729974269867, -0.03314568102359772, 0.11286703497171402, 0.24889595806598663, 0.011330622248351574, -0.09049671143293381, -0.22505739331245422, 0.2281644493341446, 0.30516868829727173, 0.08177618682384491, -0.26158374547958374, -0.21721062064170837, 0.4010688066482544, 0.10471136122941971, -0.4692128300666809, -0.24458922445774078, -0.31994935870170593, 0.002190167782828212, 0.1297602653503418, 0.27621564269065857, 0.046003762632608414, -0.11141499131917953, -0.2534559965133667, 0.04912732541561127, 0.35853636264801025, -0.003230542642995715, -0.3315998315811157, -0.294401079416275, 0.357868492603302, 0.07583323866128922, -0.4215727746486664, -0.1965247392654419, -0.24195487797260284, -0.0806174948811531, 0.08959110826253891, 0.3497423231601715, -0.024017293006181717, -0.2144019901752472, -0.1250666081905365, 0.20740626752376556, 0.44076424837112427, 0.10437370836734772, -0.2563232183456421, -0.2566963732242584, 0.39867010712623596, -0.00013521780783776194, -0.4181036353111267, -0.18232056498527527, -0.34627944231033325, -0.022862492129206657, 0.2576664090156555, 0.2302061915397644, 0.07449796795845032, -0.08422788232564926, -0.13560457527637482, 0.15190744400024414, 0.4721370339393616, 0.10682749003171921, -0.30611106753349304, -0.24853837490081787, 0.41644126176834106, 0.10397060960531235, -0.37903955578804016, -0.0775502547621727, -0.3392834961414337, -0.02319188043475151, 0.09056295454502106, 0.32899513840675354, 0.04156595095992088, -0.15939147770404816, -0.25346967577934265, 0.1553259938955307, 0.42592623829841614, 0.14910195767879486, -0.25980859994888306, -0.2900937795639038, 0.3807511627674103, 0.059597600251436234, -0.4296819865703583, -0.18968446552753448, -0.2843948006629944, -0.04599762335419655, 0.07437726110219955, 0.18995042145252228, 0.013965575024485588, -0.2179102897644043, -0.20015625655651093, 0.2041540890932083, 0.34107309579849243, 0.06196111440658569, -0.3227071762084961, -0.14272138476371765, 0.41987019777297974, -0.04464486613869667, -0.4092249572277069, -0.179878368973732, -0.3407067060470581, -0.08237920701503754, 0.1321585774421692, 0.31287726759910583, 0.046351686120033264, -0.18987946212291718, -0.2106444388628006, 0.1047094315290451, 0.3277224004268646, 0.0360138900578022, -0.36297792196273804, -0.26280584931373596, 0.5170704126358032, 0.07052148878574371, -0.4792933464050293, -0.3122662603855133, -0.3761017620563507, -0.13885916769504547, -0.011141303926706314, 0.3124369978904724, -0.0028771108482033014, -0.12145055830478668, -0.046621181070804596, 0.1933111846446991, 0.40569594502449036, 0.07896185666322708, -0.32592374086380005, -0.2856254279613495, 0.3805570900440216, 0.10130608826875687, -0.4499166011810303, -0.2737957239151001, -0.39738792181015015, -0.11917819827795029, 0.09907472133636475, 0.30047017335891724, 0.045667123049497604, -0.08321964740753174, -0.21026574075222015, 0.22295346856117249, 0.41790926456451416, 0.09897581487894058, -0.4021351933479309, -0.1790688931941986, 0.3171345293521881, -0.008666079491376877, -0.3938212990760803, -0.22077594697475433, -0.24251244962215424, -0.05533837527036667, 0.051571402698755264, 0.2886044383049011, 0.005950505845248699, -0.06077045947313309, -0.11447407305240631, 0.22198878228664398, 0.3849419355392456, 0.0712193176150322, -0.2545858919620514, -0.2787080705165863, 0.35709530115127563, 0.0923263281583786, -0.39394867420196533, -0.21129393577575684, -0.34287336468696594, -0.15974727272987366, 0.16311763226985931, 0.2758795917034149, -0.044563353061676025, -0.19402417540550232, -0.23994813859462738, 0.2708304822444916, 0.4403197467327118, 0.0006814416265115142, -0.21378464996814728, -0.1712663620710373, 0.3443697392940521, -0.08265525102615356, -0.3821733593940735, -0.22933217883110046, -0.335774302482605, -0.08793642371892929, 0.167399600148201, 0.3561336100101471, 0.06775607168674469, -0.1448049694299698, -0.1818409115076065, 0.21704146265983582, 0.32733505964279175, 0.12992262840270996, -0.3222838342189789, -0.22487179934978485, 0.40443187952041626, 0.0958101749420166, -0.37667420506477356, -0.08853328973054886, -0.34633398056030273, -0.10651861876249313, 0.11167580634355545, 0.2500934302806854, 0.07100532203912735, -0.17681923508644104, -0.18741492927074432, 0.23392361402511597, 0.4577832520008087, 0.07907512038946152, -0.23464204370975494, -0.17248116433620453, 0.3136821389198303, 0.06966018676757812, -0.4198661148548126, -0.21268360316753387, -0.3593237102031708, -0.057937122881412506, 0.12537217140197754, 0.29761749505996704, 0.0972445160150528, -0.08586741983890533, -0.22259606420993805, 0.14985840022563934, 0.4502783715724945, 0.034755971282720566, -0.296159565448761, -0.21234169602394104, 0.45082417130470276, 0.07813788205385208, -0.3560667335987091, -0.22018885612487793, -0.2934580147266388, -0.09258690476417542, 0.1157781332731247, 0.2904062867164612, -0.017145320773124695, -0.2654370367527008, -0.14507846534252167, 0.20619384944438934, 0.41517892479896545, 0.034796327352523804, -0.3209218680858612, -0.21317945420742035, 0.40050965547561646, 0.1104365661740303, -0.42233818769454956, 0.6041936278343201, 0.9834110140800476, 0.1841108649969101, -0.3086074888706207, -0.8485358953475952, -0.08689276874065399, 0.40801122784614563, 0.5904675722122192, -0.5627331137657166, -1.2111330032348633, -0.22287589311599731, 0.9105051755905151, 0.7088791728019714, -1.1872601509094238, -0.1614018976688385, 1.2843945026397705, -0.27448272705078125, -0.3009287714958191, -0.0767802819609642, 0.17524300515651703, 0.2796403765678406, 0.020103909075260162, -0.08448360115289688, -0.17975753545761108, 0.2675274610519409, 0.35457196831703186, 0.09111794084310532, -0.2709790766239166, -0.20880091190338135, 0.4565051794052124, 0.10342954099178314, -0.42674461007118225, -0.22527824342250824, -0.2779861092567444, -0.057042643427848816, 0.15474481880664825, 0.3524038791656494, 0.04017507657408714, -0.17923006415367126, -0.20436812937259674, 0.22887079417705536, 0.3978812098503113, 0.13052144646644592, -0.3002472221851349, -0.227105513215065, 0.3705945312976837, 0.04998756945133209, -0.37093454599380493, -0.21173247694969177, -0.3411175310611725, -0.09623625129461288, 0.147746741771698, 0.21028125286102295, -0.001407671021297574, -0.08789922297000885, -0.12006731331348419, 0.178500235080719, 0.457766056060791, -0.02799593098461628, -0.2651572823524475, -0.22016140818595886, 0.3394674062728882, -0.035443659871816635, -0.45242419838905334, -0.18318510055541992, -0.3075323700904846, -0.05317457392811775, 0.14593666791915894, 0.32014450430870056, -0.03900753706693649, -0.15355680882930756, -0.21766717731952667, 0.24011072516441345, 0.3842264711856842, 0.12079337239265442, -0.29900485277175903, -0.11800196021795273, 0.3977190852165222, 0.05337022244930267, -0.4330871105194092, -0.18910221755504608, -0.20481616258621216, -0.08417020738124847, 0.06352395564317703, 0.2749355435371399, 0.06343144178390503, -0.18411953747272491, -0.15263020992279053, 0.22661767899990082, 0.4002237021923065, 0.05750631168484688, -0.34343138337135315, -0.24946808815002441, 0.3596886098384857, 0.11471668630838394, -0.3467825949192047, -0.21127423644065857, -0.32893994450569153, -0.1545799821615219, 0.079970583319664, 0.3010447919368744, 0.042764853686094284, -0.12928511202335358, -0.15088313817977905, 0.19230809807777405, 0.38842788338661194, 0.1494685262441635, -0.40046876668930054, -0.2941804826259613, 0.3621293902397156, 0.015449367463588715, -0.4675217568874359, -0.20912498235702515, -0.36334145069122314, -0.06341984122991562, 0.13491728901863098, 0.2845233976840973, 0.034739360213279724, -0.18316100537776947, -0.25085559487342834, 0.1720854789018631, 0.45287463068962097, 0.08710277080535889, -0.25158828496932983, -0.18796803057193756, 0.4539403021335602, 0.010730315931141376, -0.36802101135253906, -0.1609240174293518, -0.22509601712226868, -0.03180551156401634, 0.12120821326971054, 0.2763131260871887, 0.0360245555639267, -0.21890030801296234, -0.2067749947309494, 0.18467910587787628, 0.4558931589126587, 0.0357263945043087, -0.2991633713245392, -0.14664778113365173, 0.371730238199234, 0.09288816899061203, -0.4561516344547272, -0.2035520374774933, -0.30546069145202637, -0.017230400815606117, 0.0805962011218071, 0.2320798635482788, 0.1031932458281517, -0.1273736208677292, -0.1549200415611267, 0.2319113314151764, 0.4442823529243469, 0.05544568970799446, -0.312444269657135, -0.314398854970932, 0.40699851512908936, 0.02779550850391388, -0.430728554725647, -0.1060057282447815, -0.2942725718021393, -0.05841744691133499, 0.037797961384058, 0.31887033581733704, 0.10315784811973572, -0.1772964894771576, -0.17330436408519745, 0.1610788106918335, 0.39923548698425293, 0.05035717785358429, -0.3851725161075592, -0.2626788914203644, 0.3181690573692322, 0.11930567771196365, -0.3783038258552551, -0.19643989205360413, -0.3146211802959442, -0.058254558593034744, 0.11481605470180511, 0.29243004322052, 0.016070645302534103, -0.1448669284582138, -0.2604547142982483, 0.13733065128326416, 0.3768734037876129, 0.07857494056224823, -0.32401618361473083, -0.14373111724853516, 0.388744592666626, 0.03350164368748665, -0.4083513915538788, -0.14796598255634308, -0.23729072511196136, -0.013146992772817612, 0.07626575231552124, 0.2745839059352875, 0.07958868891000748, -0.12382309883832932, -0.15486027300357819, 0.18318986892700195, 0.48547837138175964, 0.13963699340820312, -0.25019127130508423, -0.22930878400802612, 0.3195803165435791, -0.014559482224285603, -0.41312846541404724, -0.16600225865840912, -0.4702707827091217, -0.08364641666412354, 0.1620451956987381, 0.3301841616630554, 0.004258008673787117, -0.1047697439789772, -0.19207535684108734, 0.17670322954654694, 0.462240993976593, 0.12490507960319519, -0.3305232524871826, -0.2827174961566925, 0.33504414558410645, -0.017547324299812317, -0.36350324749946594, -0.10595196485519409, -0.31693556904792786, -0.008544891141355038, 0.07278812676668167, 0.2805974781513214, 0.07381589710712433, -0.12002414464950562, -0.19480624794960022, 0.10666961222887039, 0.33702754974365234, 0.049434952437877655, -0.3314598798751831, -0.23972363770008087, 0.3682199716567993, 0.04169166833162308, -0.4253652095794678, -0.17214219272136688, -0.3238458037376404, -0.08654269576072693, 0.09275387227535248, 0.32814887166023254, 0.01904619298875332, -0.1825771927833557, -0.24223528802394867, 0.20154032111167908, 0.43505150079727173, 0.1290709227323532, -0.32033148407936096, -0.16934846341609955, 0.3821328282356262, 0.040661513805389404, -0.4563368260860443, -0.17213565111160278, -0.27645257115364075, -0.061441246420145035, 0.08150717616081238, 0.20853830873966217, 0.03585664927959442, -0.15259115397930145, -0.20338453352451324, 0.15660692751407623, 0.3647880256175995, 0.04812183976173401, -0.28337135910987854, -0.19029287993907928, 0.3979960083961487, 0.06719718873500824, -0.43875032663345337, -0.20418475568294525, -0.39431506395339966, -0.017791928723454475, 0.1343982219696045, 0.3539063334465027, -0.0014776684110984206, -0.1300404965877533, -0.23873035609722137, 0.1986803114414215, 0.40111804008483887, 0.10159097611904144, -0.3123689889907837, -0.2057761400938034, 0.3746993839740753, 0.08030203729867935, -0.44383013248443604, -0.21929006278514862, -0.3236044943332672, -0.042741402983665466, 0.037216030061244965, 0.20982590317726135, 0.037534523755311966, -0.1388782262802124, -0.29228299856185913, 0.10461556911468506, 0.3700263798236847, 0.08442366868257523, -0.29527807235717773, -0.1880338042974472, 0.45539599657058716, 0.16639599204063416, -0.4988662898540497, -0.19988253712654114, -0.2760991156101227, -0.12126436084508896, 0.03740314766764641, 0.2930256128311157, 0.08163459599018097, -0.075240358710289, -0.27982398867607117, 0.12839166820049286, 0.40269896388053894, 0.043414633721113205, -0.28937602043151855, -0.2301875501871109, 0.48422735929489136, 0.04872223362326622, -0.4199996292591095, -0.19409513473510742, -0.2738778591156006, -0.11416985839605331, 0.124625563621521, 0.2462013065814972, -0.04001658782362938, -0.07845877856016159, -0.16779892146587372, 0.2555058002471924, 0.42020514607429504, 0.10110907256603241, -0.2978802025318146, -0.20843790471553802, 0.3793863356113434, 0.13547448813915253, -0.47125813364982605, -0.11390525847673416, -0.33638936281204224, -0.017082171514630318, 0.11347543448209763, 0.30629903078079224, 0.10220170766115189, -0.1278361976146698, -0.23842786252498627, 0.2002224326133728, 0.4403132498264313, -0.04943016171455383, -0.25607284903526306, -0.1820949912071228, 0.31364336609840393, 0.04550129175186157, -0.3874346613883972, -0.253156840801239, -0.34689992666244507, -0.07566612213850021, 0.13445816934108734, 0.3393654525279999, 0.02839767001569271, -0.13150469958782196, -0.14683590829372406, 0.21525044739246368, 0.4033694267272949, 0.13568075001239777, -0.3731134831905365, -0.22792205214500427, 0.3794115483760834, 0.028269445523619652, -0.417930543422699, -0.19471605122089386, -0.20561492443084717, -0.02747289650142193, 0.12428320944309235, 0.36591678857803345, 0.07971885055303574, -0.22633635997772217, -0.18641060590744019, 0.22507555782794952, 0.3787747919559479, 0.08474762737751007, -0.32642850279808044, -0.21721799671649933, 0.31955331563949585, -0.043318185955286026, -0.45511487126350403, -0.18594719469547272, -0.2613030672073364, -0.07331009954214096, 0.06952988356351852, 0.3292858898639679, 0.03089030645787716, -0.046502210199832916, -0.1974700391292572, 0.09175650775432587, 0.3685072362422943, 0.022498033940792084, -0.3733445107936859, -0.2746497690677643, 0.3649957478046417, 0.03786872327327728, -0.45583146810531616, -0.17761649191379547, -0.30716514587402344, -0.05158202722668648, 0.11554570496082306, 0.19047266244888306, -0.05382528901100159, -0.17946064472198486, -0.1810963749885559, 0.1149037703871727, 0.38649696111679077, 0.08123975992202759, -0.20701348781585693, -0.22393688559532166, 0.3721354901790619, 0.10551440715789795, -0.4190313220024109, -0.21297787129878998, -0.27296844124794006, -0.10622544586658478, 0.11470688879489899, 0.2868722677230835, 0.072612464427948, -0.19348378479480743, -0.2585776448249817, 0.16706483066082, 0.39274731278419495, 0.07003480195999146, -0.3247874081134796, -0.2165699452161789, 0.43166157603263855, 0.0614546500146389, -0.48257043957710266, -0.1421343833208084, -0.4051640033721924, -0.07213867455720901, 0.14830540120601654, 0.251274973154068, 0.09098481386899948, -0.042733993381261826, -0.11717372387647629, 0.2071380615234375, 0.454317569732666, 0.04183407127857208, -0.28693294525146484, -0.2158699631690979, 0.4016067683696747, 0.057614222168922424, -0.441005676984787, -0.2970859408378601, -0.3095528483390808, -0.04658045619726181, 0.20590046048164368, 0.31707054376602173, 0.030155586078763008, -0.0942993089556694, -0.16443833708763123, 0.1847044825553894, 0.3866993486881256, -0.005646397825330496, -0.30067867040634155, -0.15765391290187836, 0.3042490780353546, 0.0027121894527226686, -0.3930843472480774, -0.19001330435276031, -0.30849117040634155, -0.15457969903945923, 0.07470567524433136, 0.30691176652908325, 0.05614512041211128, -0.12298250198364258, -0.23923732340335846, 0.21243402361869812, 0.3462260365486145, 0.020551588386297226, -0.36473530530929565, -0.16672676801681519, 0.3764616549015045, 0.05269456282258034, -0.5401694774627686, -0.1451280415058136, -0.3063507676124573, -0.14459216594696045, 0.023944087326526642, 0.364330917596817, -0.02030963823199272, -0.04607115313410759, -0.21882817149162292, 0.06169058009982109, 0.46073395013809204, 0.09222786873579025, -0.27294033765792847, -0.2639707624912262, 0.3861652612686157, 0.0975017249584198, -0.47189706563949585, -0.2979547381401062, -0.2843427360057831, -0.0933033898472786, 0.16733062267303467, 0.25740328431129456, 0.005212489515542984, -0.15057934820652008, -0.313248872756958, 0.243914395570755, 0.3739044666290283, 0.05721741542220116, -0.23401685059070587, -0.23302710056304932, 0.4413854777812958, 0.06640651077032089, -0.39241284132003784, -0.1860785335302353, -0.2720203697681427, -0.07289796322584152, 0.12676090002059937, 0.24611368775367737, 0.08581887185573578, -0.24696072936058044, -0.18629536032676697, 0.20337769389152527, 0.4225189685821533, 0.06676401197910309, -0.24004609882831573, -0.13295388221740723, 0.3670620024204254, -0.04561002925038338, -0.4852072298526764, -0.2404124140739441, -0.3363651633262634, -0.06507286429405212, 0.15770846605300903, 0.26021960377693176, 0.024768711999058723, -0.13241645693778992, -0.2394670844078064, 0.16946563124656677, 0.543196439743042, 0.06249691918492317, -0.35511475801467896, -0.3475351333618164, 0.448637992143631, 0.03361285850405693, -0.46262094378471375, -0.07765110582113266, -0.2580461800098419, -0.14795753359794617, 0.08636405318975449, 0.33137646317481995, 0.06828777492046356, -0.19362811744213104, -0.10682837665081024, 0.2777727544307709, 0.349928617477417, -0.021262945607304573, -0.37663328647613525, -0.2054862380027771, 0.45054107904434204, 0.05876521021127701, -0.4608965218067169, -0.20475083589553833, -0.301697313785553, -0.12332485616207123, 0.018593408167362213, 0.27690407633781433, 0.08791559189558029, -0.12878108024597168, -0.18630149960517883, 0.1511702984571457, 0.45095422863960266, -0.018756501376628876, -0.3107703626155853, -0.26889175176620483, 0.38849806785583496, 0.002784817712381482, -0.3941170275211334, -0.20442074537277222, -0.36018338799476624, -0.0376681312918663, 0.10035545378923416, 0.25467950105667114, -0.0038257238920778036, -0.09996145963668823, -0.24353013932704926, 0.1731978803873062, 0.3909427225589752, 0.0816948339343071, -0.3177717924118042, -0.13196344673633575, 0.30831408500671387, 0.07820998877286911, -0.43369802832603455, -0.14559532701969147, -0.41203591227531433, -0.0027279651258140802, 0.03341447561979294, 0.31481772661209106, 0.0508992075920105, -0.13766464591026306, -0.23373334109783173, 0.14320793747901917, 0.4089975953102112, 0.028227975592017174, -0.40586718916893005, -0.33037880063056946, 0.43546608090400696, 0.08966939151287079, -0.40631213784217834, -0.1985803246498108, -0.31164708733558655, -0.03858457878232002, 0.11542900651693344, 0.24105818569660187, -0.010892611928284168, -0.19630464911460876, -0.18874862790107727, 0.13506247103214264, 0.39680635929107666, 0.1322333663702011, -0.3263871967792511, -0.19064027070999146, 0.47426411509513855, -0.04191602021455765, -0.5029984712600708, -0.17651435732841492, -0.3653380274772644, -0.03581719473004341, 0.13897904753684998, 0.25573286414146423, 0.06786730140447617, -0.05072829872369766, -0.14770899713039398, 0.1417189985513687, 0.39833584427833557, 0.040523454546928406, -0.22828173637390137, -0.2479555308818817, 0.3627106845378876, 0.052577946335077286, -0.4046248495578766, -0.2180098444223404, -0.3236071765422821, -0.06156642735004425, 0.14151428639888763, 0.33107975125312805, 0.041234441101551056, -0.16029101610183716, -0.1742352545261383, 0.14185340702533722, 0.46030426025390625, 0.08577185869216919, -0.2934767007827759, -0.35854023694992065, 0.5166410803794861, 0.12395232170820236, -0.36582455039024353, 0.5998181104660034, 0.9848651885986328, 0.17731162905693054, -0.3176513612270355, -0.856867253780365, -0.07464529573917389, 0.41521739959716797, 0.5924631953239441, -0.5362080335617065, -1.2064534425735474, -0.19904282689094543, 0.9173902273178101, 0.7362604141235352, -1.190348744392395, -0.1537628471851349, 1.303437352180481, -0.17079107463359833, -0.3755992650985718, 0.025383910164237022, 0.12138788402080536, 0.26245102286338806, -0.03901265561580658, -0.13964083790779114, -0.20115308463573456, 0.22228559851646423, 0.46235617995262146, 0.07315469533205032, -0.2794031798839569, -0.16287216544151306, 0.43227046728134155, 0.041441578418016434, -0.3953332304954529, -0.22131244838237762, -0.3023952841758728, -0.05517324432730675, 0.0534956194460392, 0.3163362443447113, 0.03541158139705658, -0.09674002975225449, -0.25736382603645325, 0.24529118835926056, 0.4675074815750122, 0.05404563248157501, -0.307471364736557, -0.23189875483512878, 0.40005308389663696, 0.09576348215341568, -0.4624016284942627, -0.2532382309436798, -0.33776089549064636, 0.03645756468176842, 0.08805475383996964, 0.23515833914279938, -0.015868160873651505, -0.12622667849063873, -0.13352340459823608, 0.15167023241519928, 0.417184054851532, 0.08077769726514816, -0.2808014452457428, -0.2285775989294052, 0.4521402418613434, 0.0868821069598198, -0.45891526341438293, -0.07583095878362656, -0.39505285024642944, -0.12555402517318726, 0.15821021795272827, 0.29348552227020264, 0.016803212463855743, -0.1252984255552292, -0.1587088704109192, 0.1667054146528244, 0.37615036964416504, 0.017125830054283142, -0.34767767786979675, -0.27632424235343933, 0.45637428760528564, 0.06597466766834259, -0.40146544575691223, -0.22579655051231384, -0.27500930428504944, -0.1382339894771576, 0.12962017953395844, 0.31835708022117615, -0.0003113810671493411, -0.18722237646579742, -0.16760021448135376, 0.1517646163702011, 0.36255311965942383, 0.06785448640584946, -0.36113467812538147, -0.2437237948179245, 0.49988678097724915, 0.02964806742966175, -0.38375821709632874, -0.25075411796569824, -0.32369348406791687, -0.07416468858718872, 0.09715337306261063, 0.32109755277633667, 0.13574187457561493, -0.14258837699890137, -0.1286976933479309, 0.2045031636953354, 0.44389164447784424, 0.045318711549043655, -0.3407101631164551, -0.24710379540920258, 0.3597487211227417, 0.05085054039955139, -0.4821251332759857, -0.19803600013256073, -0.3192283511161804, 0.030004063621163368, 0.1915350705385208, 0.30860063433647156, 0.003065337659791112, -0.1059333011507988, -0.15314070880413055, 0.17714837193489075, 0.3645983040332794, -0.020810270681977272, -0.307298868894577, -0.14094147086143494, 0.3880976140499115, 0.07067868113517761, -0.4109644293785095, -0.1739368587732315, -0.2892330288887024, -0.16087457537651062, 0.09799724072217941, 0.3021266460418701, 0.02389765903353691, -0.07533769309520721, -0.21233610808849335, 0.2333298623561859, 0.4956689476966858, 0.056483134627342224, -0.30346086621284485, -0.25114086270332336, 0.43730470538139343, -0.055005308240652084, -0.5017740726470947, -0.2121972143650055, -0.2835860848426819, -0.0303016547113657, 0.06534315645694733, 0.36442282795906067, 0.03479272127151489, -0.16652598977088928, -0.16631387174129486, 0.2533561885356903, 0.40483763813972473, -0.0060460008680820465, -0.3317396640777588, -0.23112379014492035, 0.44674152135849, 0.0024904748424887657, -0.4148085415363312, -0.2561659514904022, -0.3227948248386383, -0.12603291869163513, 0.08360908925533295, 0.20830687880516052, -0.01939675584435463, -0.2145155668258667, -0.29111218452453613, 0.24967937171459198, 0.4675038158893585, 0.09425651282072067, -0.32169800996780396, -0.1619095802307129, 0.38594216108322144, 0.07439548522233963, -0.37068021297454834, -0.18524254858493805, -0.25211861729621887, -0.06602195650339127, 0.09969332814216614, 0.2917725741863251, -0.02754661999642849, -0.18800626695156097, -0.15295590460300446, 0.10265369713306427, 0.4658210277557373, 0.08470674604177475, -0.243751659989357, -0.1964603066444397, 0.4308331310749054, 0.14105086028575897, -0.4305792450904846, -0.2008884847164154, -0.2689777910709381, -0.09121542423963547, 0.16475598514080048, 0.32024165987968445, 0.0056508989073336124, -0.19754835963249207, -0.2927328646183014, 0.2832861840724945, 0.33479782938957214, 0.010687632486224174, -0.25972676277160645, -0.21108025312423706, 0.41115760803222656, 0.0576993003487587, -0.4485335648059845, -0.24242804944515228, -0.36322110891342163, -0.029045501723885536, 0.16110824048519135, 0.34412330389022827, 0.027970021590590477, -0.2472756952047348, -0.16695183515548706, 0.18101394176483154, 0.2740170657634735, 0.1396368145942688, -0.29273849725723267, -0.2381930649280548, 0.40990149974823, 0.059067681431770325, -0.4156300127506256, -0.1906481832265854, -0.32497477531433105, -0.0301629900932312, 0.12479739636182785, 0.4062124490737915, 0.037458065897226334, -0.15963508188724518, -0.22155670821666718, 0.20877385139465332, 0.415823757648468, 0.08385773003101349, -0.3125944137573242, -0.2432725727558136, 0.43910878896713257, 0.12265326082706451, -0.4090023636817932, -0.20060265064239502, -0.38318848609924316, -0.11597209423780441, 0.11045939475297928, 0.27341699600219727, 0.025775272399187088, -0.18963384628295898, -0.19805997610092163, 0.09132017195224762, 0.3490760326385498, 0.030643928796052933, -0.2743838131427765, -0.26633742451667786, 0.35223984718322754, 0.06590498983860016, -0.4334043860435486, -0.15130163729190826, -0.31125789880752563, -0.049651138484478, 0.09611926227807999, 0.31637176871299744, -0.008549125865101814, -0.21305249631404877, -0.1489909291267395, 0.16273005306720734, 0.4088519811630249, 0.09939847886562347, -0.255053848028183, -0.38318416476249695, 0.3450883626937866, 0.1120818480849266, -0.51552414894104, -0.17538008093833923, -0.3470951020717621, -0.10629603266716003, 0.09787841141223907, 0.19735923409461975, 0.047544971108436584, -0.07844780385494232, -0.2665565609931946, 0.152296781539917, 0.4661307632923126, 0.08616528660058975, -0.32274532318115234, -0.1662660837173462, 0.3130757510662079, 0.03634442761540413, -0.4257930815219879, -0.21540382504463196, -0.2892109751701355, -0.1458984613418579, 0.1065322756767273, 0.24242936074733734, 0.08718529343605042, -0.08504100888967514, -0.2326369732618332, 0.13043329119682312, 0.40344491600990295, 0.02427947334945202, -0.3352871537208557, -0.21684005856513977, 0.37400415539741516, 0.1410892754793167, -0.4272722601890564, -0.2258094847202301, -0.3854678273200989, -0.0882495790719986, 0.15953749418258667, 0.3092785179615021, 0.09851527959108353, -0.07195556163787842, -0.16749899089336395, 0.15056414902210236, 0.3980419337749481, 0.07071996480226517, -0.38980162143707275, -0.22194258868694305, 0.3448399305343628, 0.04560184106230736, -0.4566132426261902, -0.17051059007644653, -0.19730384647846222, -0.12203644961118698, 0.10307586938142776, 0.38448289036750793, -0.01965836063027382, -0.14894504845142365, -0.20061638951301575, 0.20131570100784302, 0.32458850741386414, 0.08290289342403412, -0.2761755883693695, -0.1103191077709198, 0.393420547246933, 0.02135624550282955, -0.42363640666007996, -0.3303997814655304, -0.3120095133781433, 0.016573315486311913, 0.10277999192476273, 0.28048449754714966, 0.01511877216398716, -0.13018633425235748, -0.25721991062164307, 0.2637902498245239, 0.3832698166370392, 0.04525318741798401, -0.22276271879673004, -0.23645615577697754, 0.37689438462257385, 0.13392846286296844, -0.42028096318244934, -0.151095911860466, -0.3565910756587982, -0.14941498637199402, 0.11432085931301117, 0.35518351197242737, 0.005279575940221548, -0.11773890256881714, -0.1573372632265091, 0.2005527913570404, 0.4110836684703827, 0.10801876336336136, -0.2941443920135498, -0.2632811963558197, 0.4520716071128845, 0.09241267293691635, -0.40962105989456177, -0.1612161546945572, -0.45965954661369324, -0.04639843851327896, 0.10830893367528915, 0.18418057262897491, 0.1333393156528473, -0.11771950870752335, -0.21504426002502441, 0.1690390557050705, 0.4214667081832886, 0.06897932291030884, -0.3851994276046753, -0.2560465633869171, 0.4220035970211029, 0.0833786204457283, -0.38457196950912476, -0.2502979338169098, -0.35846439003944397, -0.040688756853342056, 0.0762258991599083, 0.29538917541503906, -0.012012055143713951, -0.10522784292697906, -0.18915122747421265, 0.2179969847202301, 0.4107581079006195, 0.03125566616654396, -0.36457258462905884, -0.2326136976480484, 0.36999133229255676, -0.021017203107476234, -0.45442941784858704, -0.15599575638771057, -0.26323962211608887, -0.019779127091169357, 0.17647898197174072, 0.26736655831336975, 0.023608900606632233, -0.18118001520633698, -0.22981126606464386, 0.27097630500793457, 0.43202850222587585, 0.10454560816287994, -0.17112071812152863, -0.13687245547771454, 0.358327180147171, 0.10399852693080902, -0.4637880325317383, -0.14499835669994354, -0.41896870732307434, -0.10439497977495193, 0.06286318600177765, 0.38752874732017517, -0.057550426572561264, -0.07681818306446075, -0.1362069696187973, 0.1965440809726715, 0.4525921046733856, 0.20227332413196564, -0.28898611664772034, -0.24732175469398499, 0.4702710807323456, 0.09448374807834625, -0.4836080074310303, -0.185119166970253, -0.31002217531204224, -0.11481142044067383, 0.06098815053701401, 0.2532883584499359, -0.02903241664171219, -0.10677707195281982, -0.17535194754600525, 0.07806041836738586, 0.38896793127059937, 0.07977231591939926, -0.37058180570602417, -0.248116597533226, 0.4160495698451996, -0.03180548921227455, -0.44203630089759827, -0.31541621685028076, -0.28760573267936707, -0.0956311747431755, 0.03402029350399971, 0.19150984287261963, -0.053681109100580215, -0.06995641440153122, -0.2353920042514801, 0.15904495120048523, 0.3879251182079315, -0.006982677616178989, -0.26614582538604736, -0.3637540936470032, 0.4439294934272766, 0.013009890913963318, -0.42713499069213867, -0.18337416648864746, -0.28572121262550354, -0.12395074218511581, 0.13601996004581451, 0.28005802631378174, 0.019764013588428497, -0.15122564136981964, -0.1379181146621704, 0.2101961225271225, 0.47093096375465393, 0.11453862488269806, -0.4068065881729126, -0.24311156570911407, 0.448782354593277, 0.004605366848409176, -0.5157157778739929, -0.3158889412879944, -0.22797204554080963, -0.10880935937166214, 0.11289295554161072, 0.2272017002105713, 0.04855785891413689, -0.0532381534576416, -0.2691318392753601, 0.18241439759731293, 0.36674049496650696, -0.02834472805261612, -0.3522498905658722, -0.2369077503681183, 0.4601310193538666, 0.1153860092163086, -0.43130627274513245, -0.2838796079158783, -0.2953682541847229, -0.06211871653795242, 0.10345693677663803, 0.2235412746667862, 0.019118869677186012, -0.037025775760412216, -0.11885110288858414, 0.25105249881744385, 0.4402233064174652, 0.047435831278562546, -0.3062537610530853, -0.23029740154743195, 0.417389452457428, 0.03258209675550461, -0.4744325280189514, -0.1419612318277359, -0.3479670584201813, -0.018573304638266563, 0.0955919548869133, 0.31695660948753357, 0.05370272323489189, -0.09926535934209824, -0.15958257019519806, 0.13769957423210144, 0.3368576467037201, 0.0655662789940834, -0.22163161635398865, -0.28039613366127014, 0.35017114877700806, 0.1669141948223114, -0.5090839862823486, -0.22827687859535217, -0.3497098386287689, -0.09620840102434158, 0.071586973965168, 0.3373768925666809, 0.03787441924214363, -0.1278783231973648, -0.2794034779071808, 0.22665125131607056, 0.41457399725914, 0.07558891922235489, -0.26414239406585693, -0.22191263735294342, 0.3961030840873718, 0.0718713030219078, -0.4661678671836853, -0.24517899751663208, -0.39780768752098083, -0.012248339131474495, 0.07133909314870834, 0.2913811206817627, 0.0840335413813591, -0.04488297551870346, -0.21298575401306152, 0.19472286105155945, 0.4112149178981781, 0.001485165674239397, -0.33043938875198364, -0.09536581486463547, 0.3945406973361969, 0.1409100592136383, -0.39100006222724915, -0.10924763977527618, -0.2610088586807251, -0.04975946620106697, 0.04173450544476509, 0.3555298149585724, 0.06368056684732437, -0.11221693456172943, -0.15036195516586304, 0.14589616656303406, 0.413007527589798, 0.06787043809890747, -0.38177812099456787, -0.2989017367362976, 0.39129021763801575, -0.062321651726961136, -0.39257413148880005, -0.1597926914691925, -0.33268705010414124, -0.04733836650848389, 0.011898837052285671, 0.2908995747566223, 0.0654301717877388, -0.11161574721336365, -0.14820200204849243, 0.18256226181983948, 0.5073899030685425, 0.0896231010556221, -0.26615655422210693, -0.1831558644771576, 0.3582175374031067, -0.007080461829900742, -0.39070209860801697, -0.26319098472595215, -0.35976290702819824, -0.01170752476900816, 0.04853270202875137, 0.23193693161010742, -0.03622785210609436, -0.1906583607196808, -0.09991233050823212, 0.21357183158397675, 0.454681396484375, -0.0313291922211647, -0.42922812700271606, -0.2969815731048584, 0.45024651288986206, 0.10972274094820023, -0.5314770340919495, -0.2887853682041168, -0.29640787839889526, -0.04967743530869484, 0.1259177029132843, 0.3150472044944763, -0.025074316188693047, -0.1318928301334381, -0.21028830111026764, 0.2475702315568924, 0.38927632570266724, -0.029606489464640617, -0.24313615262508392, -0.14714768528938293, 0.44580331444740295, 0.032448891550302505, -0.47971686720848083, -0.14920417964458466, -0.3571913242340088, -0.03641892969608307, 0.1429402083158493, 0.2739860415458679, -0.07604369521141052, -0.11320454627275467, -0.2474195808172226, 0.1049872562289238, 0.40216147899627686, 0.07313674688339233, -0.3126246929168701, -0.20739756524562836, 0.29854828119277954, 0.07836858183145523, -0.34903350472450256, -0.2049950212240219, -0.35615089535713196, -0.07211073487997055, 0.16519850492477417, 0.27105697989463806, -0.014689976349473, -0.22506926953792572, -0.1245942935347557, 0.2572370767593384, 0.405086487531662, 0.17201004922389984, -0.32776162028312683, -0.20238421857357025, 0.3294866979122162, -0.002069035079330206, -0.43179357051849365, 0.6016976833343506, 0.9842420220375061, 0.18951523303985596, -0.2935035824775696, -0.8448853492736816, -0.07756523787975311, 0.40349283814430237, 0.580612063407898, -0.5483757257461548, -1.2184139490127563, -0.20128624141216278, 0.9308857917785645, 0.7116462588310242, -1.1758184432983398, -0.15674158930778503, 1.2840869426727295, -0.27763602137565613, -0.28542226552963257, -0.11945952475070953, 0.12354537844657898, 0.29269060492515564, -0.05144540220499039, -0.13689133524894714, -0.20821478962898254, 0.15155401825904846, 0.5088478326797485, 0.07560724765062332, -0.32538673281669617, -0.14972876012325287, 0.39567920565605164, 0.04332858696579933, -0.5322296023368835, -0.16889461874961853, -0.2783319354057312, 0.03515410050749779, 0.05284569039940834, 0.3350231349468231, 0.09830668568611145, -0.10071077197790146, -0.19491121172904968, 0.1020435094833374, 0.41546449065208435, 0.039417434483766556, -0.3949292302131653, -0.20761781930923462, 0.3911170959472656, 0.038274914026260376, -0.3302488625049591, -0.22652168571949005, -0.43465593457221985, -0.17472413182258606, 0.1307458132505417, 0.27077314257621765, -0.011579778976738453, -0.24544119834899902, -0.23756760358810425, 0.128316268324852, 0.3997161090373993, 0.023323675617575645, -0.3516266345977783, -0.24130897223949432, 0.3563641905784607, 0.06573490798473358, -0.604702889919281, -0.2633722722530365, -0.268771231174469, -0.05272718146443367, 0.04788961634039879, 0.2711973488330841, -0.0035924201365560293, -0.15758371353149414, -0.12399689853191376, 0.163223996758461, 0.4150059223175049, 0.022867852821946144, -0.34575867652893066, -0.3182411789894104, 0.36239781975746155, 0.027707288041710854, -0.44772064685821533, -0.1982058882713318, -0.32590726017951965, -0.03014744445681572, 0.0918646901845932, 0.3469720184803009, 0.056308235973119736, -0.06894198805093765, -0.1991652548313141, 0.24594050645828247, 0.4093293249607086, -0.01747276820242405, -0.3089260458946228, -0.24839572608470917, 0.32221871614456177, -0.0010235850932076573, -0.46242713928222656, -0.08522891998291016, -0.3146669864654541, 0.028752854093909264, 0.15037088096141815, 0.3356929421424866, 0.06711122393608093, -0.2136775404214859, -0.13270455598831177, 0.13422222435474396, 0.4272220730781555, 0.0616479367017746, -0.2253902405500412, -0.1783800721168518, 0.48922282457351685, 0.0744004026055336, -0.4304756820201874, -0.16089428961277008, -0.20856565237045288, -0.035922929644584656, 0.10622873157262802, 0.21981751918792725, -0.010761572048068047, -0.1418769806623459, -0.16124045848846436, 0.2169945240020752, 0.40080130100250244, 0.07417958229780197, -0.22937703132629395, -0.1453515738248825, 0.4575325548648834, 0.12718364596366882, -0.367739200592041, -0.10746193677186966, -0.3922877609729767, -0.0873807743191719, 0.11630319803953171, 0.3017379641532898, 0.09461399912834167, -0.2296888530254364, -0.18742772936820984, 0.19945749640464783, 0.3722393810749054, 0.060915205627679825, -0.25584885478019714, -0.2144097089767456, 0.39137211441993713, 0.06106087937951088, -0.43614354729652405, -0.15174651145935059, -0.3336452841758728, -0.011944407597184181, 0.07693923264741898, 0.2810065448284149, -0.028455542400479317, -0.14698584377765656, -0.15215449035167694, 0.13566817343235016, 0.389078825712204, 0.07372301071882248, -0.32348185777664185, -0.19517327845096588, 0.33583495020866394, 0.09436215460300446, -0.3630167543888092, -0.11494162678718567, -0.3312731385231018, -0.1011166125535965, 0.0260999146848917, 0.317465603351593, 0.0695667564868927, -0.12161128968000412, -0.1765650361776352, 0.22085486352443695, 0.38901636004447937, -0.00520017696544528, -0.3771441578865051, -0.2765578031539917, 0.3523435890674591, -0.02852076292037964, -0.4508914053440094, -0.2502971589565277, -0.2947980761528015, -0.06424500793218613, 0.14553265273571014, 0.21658840775489807, -0.026963092386722565, -0.21647340059280396, -0.1794300079345703, 0.3439711332321167, 0.418337881565094, 0.04710027575492859, -0.2944469749927521, -0.23841041326522827, 0.4471856951713562, 0.05138498917222023, -0.473398357629776, -0.29372939467430115, -0.30968961119651794, -0.009993530809879303, 0.1131434291601181, 0.2387079894542694, 0.0066177877597510815, -0.15234027802944183, -0.16445431113243103, 0.18411670625209808, 0.4662500321865082, 0.05885253846645355, -0.33766570687294006, -0.1437128335237503, 0.3482740819454193, 0.01197112537920475, -0.3240886926651001, -0.18380744755268097, -0.23782169818878174, -0.18626336753368378, 0.16889864206314087, 0.2833963930606842, 0.03370708227157593, -0.11121556907892227, -0.3326375186443329, 0.1419636309146881, 0.4298076629638672, 0.23177777230739594, -0.3430720269680023, -0.11791174858808517, 0.4307663142681122, 0.10342731326818466, -0.4774751365184784, -0.1671520620584488, -0.37102213501930237, -0.12150812149047852, 0.17004890739917755, 0.24795660376548767, 0.002747882390394807, -0.10112576186656952, -0.234691321849823, 0.12342139333486557, 0.4186013638973236, 0.08258213847875595, -0.2286045253276825, -0.2936885356903076, 0.4345916509628296, 0.08747749775648117, -0.45132094621658325, -0.20666876435279846, -0.32690635323524475, -0.11276627331972122, 0.03863993287086487, 0.31869301199913025, -0.025732578709721565, -0.18570460379123688, -0.15425963699817657, 0.15927517414093018, 0.38624486327171326, -0.002347861882299185, -0.36821648478507996, -0.3075789511203766, 0.3281269967556, 0.11332294344902039, -0.47905123233795166, -0.31634846329689026, -0.3852788209915161, -0.026853231713175774, 0.04490399360656738, 0.2787010669708252, -0.02159462310373783, -0.2273687720298767, -0.22429083287715912, 0.1324663758277893, 0.41295742988586426, 0.10144368559122086, -0.31181812286376953, -0.10703931003808975, 0.34190401434898376, 0.11583908647298813, -0.41121336817741394], "scores": [[-0.9900531624389137, -0.9827313967889009, -0.9833436290631039, -0.9868309944389874, -0.9822332089676672, -0.9863759730940074, -0.9863742868285197, -0.9866738569138636, -0.9838407056419323, -0.9771370158293124, -0.9787839682153481, -0.9783151399116897, -0.9758214938677477, -0.9754788806258986, -0.9889305425230899, -0.9872026420211696, -0.9824592587832607, -0.9854225110600596, -0.9783685026446226, -0.9827458629412629, 0.9995756041877734, -0.9913784234040237, -0.9884143803672711, -0.9862168113279217, -0.9823312653771906, -0.9828884516321728, -0.9788213548524198, -0.9824534154294642, -0.9884392869159719, -0.9805294958035897, -0.9838130082617104, -0.9712822060818689, -0.9813203240493309, -0.9860262504834726, -0.9779580449212484, -0.9809826760380893, -0.9888629541195051, -0.9754875410457768, -0.9632892836185954, -0.981257378059502], [-0.981064245384258, -0.9874490167453631, -0.977206144650754, -0.9870825757463242, -0.9827300748254172, -0.9904349834926158, -0.9833593434079635, -0.9767579900630751, -0.9834725752632627, -0.9760186647183402, -0.9821489130047512, -0.9799255042295144, -0.9836036552266529, -0.9700090204692248, -0.9811324221271057, -0.983343750485874, -0.9643228384766567, -0.9873577588018465, -0.9804485909235797, -0.9883563027122855, -0.9868208790563022, 0.9997284045613235, -0.9842719371391817, -0.9908176099040273, -0.9769011560191202, -0.9874606411388951, -0.9821765040376712, -0.9848265519738197, -0.9892680962431732, -0.9844692043141747, -0.9873466707787226, -0.9779764694907066, -0.9925476969948329, -0.9794832866978929, -0.9778120836506345, -0.986694377881929, -0.9936471360031063, -0.9937827418554035, -0.9933507529733832, -0.9741714407609564], [-0.9838247179044004, -0.9840345465555529, -0.9783885165767597, -0.9897456428550659, -0.9749405720916309, -0.9825629019529302, -0.9828155045429215, -0.9934934234939207, -0.981764729207952, -0.9766466623489054, -0.9837609448630618, -0.97323539894024, -0.9788759105324093, -0.9767928113738867, -0.9902738091545793, -0.9693413374875643, -0.9852144978427172, -0.985621756003213, -0.9801824885776339, -0.9829645327520973, -0.9867466134891829, -0.9819918721943937, 0.9994321151563219, -0.9843988935872974, -0.992455559407602, -0.9840986328596163, -0.9815636703915287, -0.9853698486431275, -0.988694544547271, -0.9815562284521864, -0.9848026088607633, -0.9880423018334099, -0.9766081157021469, -0.9824721718542451, -0.9812044900517125, -0.9754243801588423, -0.9910638260923281, -0.9865177555276556, -0.975558939528402, -0.9808014246728878], [-0.9850488440244606, -0.986509862342904, -0.9729684442421597, -0.9759756743728931, -0.9880296617412785, -0.9743176482062564, -0.991905836922575, -0.9699835005049942, -0.9767303180192917, -0.9831926611134594, -0.9652876158079903, -0.9888206856621442, -0.9719805740000709, -0.9844014597407569, -0.974919125770806, -0.9926541333245144, -0.9744647601477228, -0.9718289451722217, -0.9816719681457671, -0.9786084680272797, -0.9817639685021871, -0.978948027525884, -0.9776183756057706, 0.9995353270873023, -0.9862930882256842, -0.9731889144070676, -0.9773390670761821, -0.9853953687566839, -0.984698584051192, -0.9687517207635873, -0.9773527031458551, -0.9819355913844239, -0.9905704945041488, -0.9793476576429958, -0.9800817789079357, -0.9791489738340817, -0.9594279531407077, -0.9831424315495938, -0.9827962561805812, -0.9729055006724165]], "kept": [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]]}, {"name": "big_dim_keydiff", "n": 40, "head_dim": 32, "n_kv_heads": 3, "budget": 10, "recent_frac": 0.3, "sink": 4, "keys": [-0.048693448305130005, 0.016022924333810806, 0.0012311515165492892, -0.20870418846607208, -0.0875617265701294, -0.14320795238018036, -0.01781250163912773, 0.2477240115404129, -0.08750692009925842, -0.11408457159996033, 0.03133044019341469, 0.07126089930534363, 0.08786512911319733, -0.2532215416431427, 0.037440355867147446, 0.13738438487052917, -0.28613853454589844, 0.013528568670153618, -0.32123008370399475, -0.3036952018737793, -0.3443737030029297, -0.015599223785102367, -0.24899446964263916, 0.08541618287563324, 0.026301074773073196, -0.0019687467720359564, -0.4037573039531708, -0.13559946417808533, 0.0009899466531351209, -0.0017492674523964524, -0.28284186124801636, -0.14965809881687164, -0.028732573613524437, 0.04665498808026314, -0.00687558576464653, -0.11106636375188828, -0.15211202204227448, -0.2271595597267151, 0.043712690472602844, 0.15368811786174774, -0.11618862301111221, -0.12213800847530365, 0.15543398261070251, 0.10192408412694931, 0.0035632853396236897, -0.19429311156272888, -0.018038548529148102, 0.2075931429862976, -0.27546626329421997, -0.10167627781629562, -0.3417136073112488, -0.24976927042007446, -0.3579637110233307, -0.10013706237077713, -0.2401314377784729, 0.0290916096419096, 0.08793333172798157, -0.0026766995433717966, -0.47689059376716614, -0.06839729845523834, -0.026160461828112602, 0.07402242720127106, -0.2894752621650696, -0.06112925708293915, -0.0643121525645256, 0.07379879057407379, -0.13622398674488068, -0.2700939178466797, -0.1011594831943512, -0.2324238121509552, 0.019570158794522285, 0.36554694175720215, -0.13461628556251526, -0.14847083389759064, 0.10285342484712601, 0.09210455417633057, 0.011103653348982334, -0.1861608326435089, 0.02959436923265457, 0.1574120670557022, -0.3057487905025482, -0.09045132249593735, -0.35757872462272644, -0.2964549660682678, -0.33510756492614746, -0.08733152598142624, -0.19095201790332794, 0.06090796738862991, 0.034092266112565994, -0.0648825392127037, -0.4816139042377472, -0.20170366764068604, -0.06573735922574997, 0.03955810144543648, -0.39563363790512085, -0.04796794801950455, -0.08707232028245926, 0.09430170804262161, -0.09408862888813019, -0.12937790155410767, -0.0793880820274353, -0.2642691731452942, 0.07382495701313019, 0.32539448142051697, -0.09632037580013275, -0.13096947968006134, 0.084589883685112, 0.018696274608373642, 0.07485329359769821, -0.20300890505313873, -0.008088299073278904, 0.09175185859203339, -0.2853686511516571, -0.1503785103559494, -0.2964895963668823, -0.25143513083457947, -0.2998034358024597, -0.043767474591732025, -0.274275541305542, 0.03493640944361687, 0.0016153891338035464, -0.034933168441057205, -0.4944467544555664, -0.11681243032217026, -0.07809571921825409, -0.018926184624433517, -0.20650240778923035, -0.12386003881692886, -0.052472181618213654, 0.07333110272884369, 0.018549829721450806, -0.24102868139743805, -0.09636173397302628, -0.21903006732463837, -0.07668345421552658, 0.29005545377731323, -0.09420232474803925, -0.11370156705379486, 0.05496765673160553, 0.09019310027360916, -0.007040911354124546, -0.18300947546958923, -0.06094181910157204, 0.07061154395341873, -0.18728841841220856, -0.11184749007225037, -0.3447590470314026, -0.24542026221752167, -0.37015676498413086, -0.06983175128698349, -0.20805123448371887, 0.03617728874087334, 0.02205475978553295, -0.034220047295093536, -0.4168580174446106, -0.06779079884290695, 0.009963029064238071, -0.006762458477169275, -0.3583037257194519, -0.04282188042998314, 0.04855486378073692, 0.04942936822772026, -0.024719586595892906, -0.12925530970096588, -0.04437641799449921, -0.14135827124118805, -0.011413362808525562, 0.3290577530860901, -0.15535977482795715, -0.0741875097155571, 0.11727983504533768, 0.1111348494887352, 0.11387436836957932, -0.10164204239845276, -0.06278762221336365, 0.04698310047388077, -0.21322055160999298, -0.1372428834438324, -0.3599633276462555, -0.20174437761306763, -0.4302893877029419, -0.14993272721767426, -0.22659040987491608, 0.053489964455366135, 0.017336787655949593, -0.0334043875336647, -0.5187091827392578, -0.17749106884002686, -0.01749972626566887, -0.027169322595000267, -0.3713793456554413, -0.0650143250823021, -0.0028374246321618557, 0.07679121196269989, -0.10127852857112885, -0.20123040676116943, -0.13588778674602509, -0.23175953328609467, 0.021137915551662445, 0.21416038274765015, -0.0752268135547638, -0.10028586536645889, 0.19384127855300903, -0.0021855621598660946, 0.06431907415390015, -0.18033871054649353, -0.006230264902114868, 0.05892348662018776, -0.2770746946334839, -0.049332395195961, -0.3634669780731201, -0.2396780252456665, -0.3626353442668915, 0.013294784352183342, -0.24062898755073547, -0.05875011160969734, -0.0049766856245696545, -0.13377095758914948, -0.6382553577423096, -0.12832210958003998, 0.05751100927591324, 0.023772107437253, -0.3478325605392456, -0.13733336329460144, 0.056763168424367905, 0.06434611976146698, -0.04941381886601448, -0.17100055515766144, -0.0840156227350235, -0.14715716242790222, 0.038995884358882904, 0.26409435272216797, -0.1451735496520996, -0.09171821922063828, 0.05837087705731392, 0.12214617431163788, -0.04362858459353447, -0.18274536728858948, -0.005896692629903555, 0.06518439948558807, -0.16796642541885376, -0.013471920043230057, -0.38252225518226624, -0.20514468848705292, -0.3291656970977783, -0.17511168122291565, -0.22703546285629272, 0.048203468322753906, 0.0337878093123436, -0.08917487412691116, -0.4891507625579834, -0.11072928458452225, 0.05023772642016411, 0.038137465715408325, -0.28948304057121277, -0.013849874958395958, -0.027529889717698097, 0.036993272602558136, -0.1426515281200409, -0.08987216651439667, -0.03771902248263359, -0.14158503711223602, 0.04481244087219238, 0.25881654024124146, -0.08225568383932114, -0.12987399101257324, 0.08240320533514023, 0.07016906887292862, 0.0955154299736023, -0.14807991683483124, -0.008451784960925579, 0.10244707763195038, -0.28581494092941284, -0.006357007659971714, -0.33400869369506836, -0.24035321176052094, -0.36540859937667847, -0.09988637268543243, -0.24289843440055847, 0.09495357424020767, 0.010000078938901424, -0.04669325426220894, -0.48673513531684875, -0.09633662551641464, -0.0888175293803215, 0.00964619405567646, -0.33192506432533264, -0.04606912285089493, -0.03829742595553398, 0.08531712740659714, 0.024408092722296715, -0.18400727212429047, -0.11601445078849792, -0.17785581946372986, 0.011266067624092102, 0.2036283016204834, -0.06998416036367416, -0.016497857868671417, 0.07967762649059296, 0.0573100820183754, -0.03232264518737793, -0.15990988910198212, -0.06787759065628052, 0.07607012987136841, -0.1900816410779953, -0.13176491856575012, -0.30527517199516296, -0.16751284897327423, -0.3351331949234009, -0.01676413230597973, -0.1419428139925003, 0.04143425077199936, -2.3348162358161062e-05, -0.10299267619848251, -0.47359806299209595, -0.02785913646221161, 0.0388127863407135, -0.025688450783491135, -0.331974059343338, -0.11550693958997726, 0.014845910482108593, 0.046199213713407516, -0.041091080754995346, -0.15349049866199493, -0.10087431967258453, -0.189436137676239, 0.021697137504816055, 0.2491106241941452, -0.06785408407449722, -0.023729871958494186, 0.12218184024095535, 0.07024441659450531, -0.06438197195529938, -0.15646645426750183, -0.10286270081996918, 0.060964975506067276, -0.21133305132389069, -0.05118050426244736, -0.3668400049209595, -0.32923129200935364, -0.3666669428348541, -0.07837061583995819, -0.207713320851326, 0.1641571968793869, 0.040473464876413345, -0.07429668307304382, -0.5342109799385071, -0.10462106019258499, -0.018006643280386925, -0.03615984693169594, -0.28338751196861267, -0.14784404635429382, 0.05583786964416504, 0.10959731787443161, 0.002423783764243126, -0.19202998280525208, -0.06020960956811905, -0.19403091073036194, -0.008073084056377411, 0.2363518327474594, -0.1580158919095993, -0.18946684896945953, 0.13229897618293762, 0.057892121374607086, 0.03074519895017147, -0.12577879428863525, -0.09218540042638779, 0.09221017360687256, -0.24529826641082764, -0.0668875202536583, -0.3781967759132385, -0.19227178394794464, -0.33757975697517395, -0.10510280728340149, -0.2860935628414154, 0.09154422581195831, 0.05281851440668106, -0.13028419017791748, -0.4082978069782257, -0.07191499322652817, 0.05800016596913338, 0.002231067279353738, -0.30399060249328613, -0.1466212272644043, 0.12707898020744324, 0.04769207537174225, 0.027562880888581276, -0.2006920874118805, -0.07774358242750168, -0.27099505066871643, -0.007775838021188974, 0.30249685049057007, -0.15561732649803162, -0.06366228312253952, 0.10944511741399765, 0.015256344340741634, -0.005155907012522221, -0.19881759583950043, -0.008004745468497276, 0.10460948199033737, -0.29542940855026245, -0.10172164440155029, -0.41068753600120544, -0.30820706486701965, -0.35050833225250244, -0.0002899917308241129, -0.31602394580841064, 0.05144607648253441, -0.0028708679601550102, -0.08418839424848557, -0.433011531829834, -0.12772482633590698, 0.06574810296297073, -0.017575856298208237, -0.26988017559051514, -0.1016625240445137, -0.03746858239173889, 0.08584854006767273, -0.05956290662288666, -0.13816693425178528, -0.08830021321773529, -0.24171824753284454, 0.006264057010412216, 0.255906879901886, -0.045106980949640274, -0.16258710622787476, 0.09061674028635025, -0.018640046939253807, 0.05249860882759094, -0.22993846237659454, -0.0958467572927475, 0.12845583260059357, -0.19878076016902924, -0.16271460056304932, -0.41374483704566956, -0.2808944880962372, -0.40457433462142944, -0.025462308898568153, -0.2799237370491028, 0.015194933861494064, 0.05879221111536026, -0.07310685515403748, -0.4540444016456604, -0.15038560330867767, -0.06977391242980957, -0.0703563541173935, -0.19614514708518982, -0.10631132870912552, 0.012429174967110157, 0.05491231009364128, -0.04381569102406502, -0.1658431440591812, 0.009475212544202805, -0.23937377333641052, -0.06650558859109879, 0.20271113514900208, -0.1597655862569809, -0.07992555946111679, 0.13360214233398438, 0.019388113170862198, -0.049597833305597305, -0.19360414147377014, 0.06402741372585297, -0.009561792016029358, -0.22773367166519165, -0.14028002321720123, -0.30732470750808716, -0.29762694239616394, -0.3623703122138977, -0.11974766105413437, -0.2884182631969452, 0.12056262791156769, 0.07065587490797043, -0.055414531379938126, -0.5191920399665833, -0.19650670886039734, -0.02884981967508793, 0.019870948046445847, -0.29340800642967224, -0.09498797357082367, -0.05585797503590584, 0.05315109342336655, -0.053747449070215225, -0.10379935801029205, 0.00740103330463171, -0.19427688419818878, -0.026947777718305588, 0.2500600218772888, -0.12341119349002838, -0.15439535677433014, 0.0896507203578949, 0.015288668684661388, 0.05022929236292839, -0.1810600906610489, 0.006971636787056923, 0.12226967513561249, -0.2904277443885803, -0.13389024138450623, -0.3712068498134613, -0.2711687684059143, -0.3364044427871704, -0.04465528950095177, -0.3076706826686859, 0.05462673678994179, -0.03751607984304428, -0.06615500897169113, -0.49040108919143677, -0.20558001101016998, -0.0045916116796433926, 0.028965279459953308, -0.2971065044403076, -0.11151424050331116, -0.018447676673531532, 0.007637812756001949, -0.06529879570007324, -0.19595804810523987, -0.0813513696193695, -0.24763424694538116, 0.023147014901041985, 0.26047006249427795, -0.10010827332735062, -0.13923390209674835, 0.12020022422075272, -0.015776194632053375, 0.042946621775627136, -0.16371244192123413, 0.00864003598690033, 0.15058231353759766, -0.2867472171783447, -0.0994652584195137, -0.3274901509284973, -0.22219878435134888, -0.33776387572288513, -0.12014971673488617, -0.2126609981060028, 0.10974422097206116, 0.08011051267385483, -0.02363722212612629, -0.5535673499107361, -0.05468910560011864, -0.016529716551303864, -0.10520987212657928, -0.27034491300582886, -0.16490696370601654, -0.06458952277898788, 0.02471684105694294, 0.011815778911113739, -0.18686972558498383, -0.07238605618476868, -0.10002904385328293, 0.09106900542974472, 0.24814143776893616, -0.1051061823964119, -0.18031719326972961, 0.057860326021909714, 0.08872180432081223, 0.03971049562096596, -0.17035239934921265, 0.04421130567789078, 0.09279827773571014, -0.256868839263916, -0.049930110573768616, -0.3301357626914978, -0.19018346071243286, -0.3282485008239746, -0.0599038302898407, -0.22144575417041779, 0.0011409108992666006, -0.05234602466225624, -0.006297535728663206, -0.47844043374061584, -0.08639586716890335, -0.09405120462179184, 0.0031604100950062275, -0.3191981315612793, -0.13350898027420044, -0.11251824349164963, 0.039721112698316574, -0.006948565132915974, -0.14927756786346436, -0.11598297208547592, -0.18817123770713806, 0.049207523465156555, 0.11528822034597397, -0.0992569550871849, -0.09011365473270416, 0.12668965756893158, 0.15249040722846985, 0.07667624205350876, -0.1602361798286438, 0.009570022113621235, 0.17076393961906433, -0.28103137016296387, -0.08850166201591492, -0.31403112411499023, -0.14585848152637482, -0.3560628890991211, -0.04685373604297638, -0.22963134944438934, 0.11843085289001465, 0.028111277148127556, 0.03813682869076729, -0.524016261100769, -0.11111816018819809, -0.019075240939855576, 0.060741398483514786, -0.23694226145744324, -0.16577059030532837, -0.045523375272750854, 0.07330740988254547, -0.08474846184253693, -0.24444885551929474, -0.03401163965463638, -0.1627289354801178, 0.03602639585733414, 0.22953683137893677, -0.04158320650458336, -0.12927165627479553, 0.1474049985408783, 0.021870631724596024, -0.022782815620303154, -0.16563116014003754, -0.04064538702368736, 0.1651892364025116, -0.24106061458587646, -0.13264082372188568, -0.3556997776031494, -0.2612946331501007, -0.3023078739643097, -0.07606135308742523, -0.26151400804519653, 0.11183251440525055, 0.14155660569667816, 0.06461931020021439, -0.4725222587585449, -0.0908736139535904, 0.06750594824552536, 0.015194371342658997, -0.33802273869514465, -0.08445459604263306, 0.0069738104939460754, -0.15671013295650482, 0.1513977348804474, 0.5019180178642273, 0.2609904408454895, 0.5592715740203857, -0.039173852652311325, -0.7434058785438538, 0.2770785689353943, 0.3421608507633209, -0.27287572622299194, -0.18400372564792633, -0.05896377936005592, 0.5399066805839539, 0.012207858264446259, -0.3991631865501404, 0.7552968263626099, 0.25804266333580017, 1.0963926315307617, 0.7196372747421265, 1.0606709718704224, 0.12321663647890091, 0.716327965259552, -0.14545409381389618, -0.08257816731929779, 0.10881367325782776, 1.4105565547943115, 0.3090713918209076, 0.017277304083108902, -0.03855924308300018, 0.8654815554618835, 0.2727823257446289, 0.03379916399717331, 0.031053666025400162, 0.044396378099918365, -0.08284792304039001, -0.057638250291347504, -0.15321294963359833, -0.08999552577733994, 0.28519460558891296, -0.10273555666208267, -0.09557837247848511, 0.12670619785785675, 0.05038930103182793, -0.0646001324057579, -0.15747058391571045, -0.04259796068072319, 0.11490697413682938, -0.28429049253463745, -0.10357096791267395, -0.4748190939426422, -0.1828833967447281, -0.3354322016239166, 0.011136259883642197, -0.1405632644891739, 0.05240032821893692, -0.06049486994743347, -0.08002546429634094, -0.5360286235809326, -0.12691007554531097, -0.005184584762901068, -0.07869324833154678, -0.27208152413368225, -0.16581620275974274, 0.015100769698619843, 0.05099022760987282, -0.06749533861875534, -0.171979621052742, -0.11292453855276108, -0.218051016330719, -0.07277029752731323, 0.25182774662971497, -0.0007534435717388988, -0.0182489063590765, 0.15867410600185394, 0.1027439683675766, -0.013900049962103367, -0.10372677445411682, -0.008335156366229057, 0.12797290086746216, -0.2686263620853424, -0.0818941667675972, -0.3811038136482239, -0.24791985750198364, -0.4023294746875763, -0.0631168857216835, -0.1255275309085846, 0.04779097065329552, 0.017693912610411644, -0.008712410926818848, -0.4402773380279541, -0.15750479698181152, -0.019524922594428062, 0.06732990592718124, -0.27571341395378113, -0.0842006728053093, 0.07793339341878891, 0.022576147690415382, -0.047676485031843185, -0.1943303942680359, -0.011420227587223053, -0.28502047061920166, -0.022139566019177437, 0.22685126960277557, -0.059853147715330124, -0.08695988357067108, 0.16234293580055237, -0.011247752234339714, 0.057447705417871475, -0.19052830338478088, -0.038810040801763535, 0.15839511156082153, -0.30007198452949524, -0.1903379261493683, -0.37785398960113525, -0.3186192810535431, -0.3805452883243561, -0.025855964049696922, -0.22395537793636322, 0.13061563670635223, 0.019644204527139664, -0.11199770122766495, -0.5134882926940918, -0.14778555929660797, -0.07005824148654938, 0.043184470385313034, -0.3214668929576874, -0.18916702270507812, 0.03507864847779274, 0.05089345946907997, -0.033971916884183884, -0.1630413830280304, -0.05435290187597275, -0.185526505112648, 0.07317764312028885, 0.27455824613571167, -0.0734267458319664, -0.09678031504154205, 0.01975134201347828, 0.05922210216522217, 0.006958131678402424, -0.16548697650432587, -0.07346348464488983, 0.2131224125623703, -0.24884557723999023, -0.1470581293106079, -0.4447874128818512, -0.257795125246048, -0.35258328914642334, -0.08034830540418625, -0.23494966328144073, 0.01921188272535801, 0.05720985308289528, -0.07155821472406387, -0.47760844230651855, -0.05287439003586769, 0.11941646039485931, -0.02896604873239994, -0.3124306797981262, -0.13229015469551086, 0.039448924362659454, -0.0009401789284311235, -0.0760316327214241, -0.1698075234889984, -0.1348695605993271, -0.2352936863899231, -0.012413624674081802, 0.1482871025800705, -0.165304034948349, -0.13792537152767181, 0.09999476373195648, 0.058165889233350754, -0.06877447664737701, -0.19905363023281097, 0.034393101930618286, 0.1592099517583847, -0.2580024003982544, -0.13086074590682983, -0.3277822434902191, -0.2726771831512451, -0.406561940908432, -0.08454301208257675, -0.16713809967041016, 0.06227967143058777, 0.08758929371833801, -0.059297941625118256, -0.42877596616744995, -0.13189154863357544, -0.01703835278749466, 0.14573362469673157, -0.25084784626960754, -0.11535689234733582, -0.004008445423096418, 0.0727759599685669, 0.00870910007506609, -0.1927841156721115, -0.17300158739089966, -0.20140953361988068, 0.012136791832745075, 0.25878602266311646, -0.027195623144507408, -0.10143930464982986, 0.1332290768623352, 0.012398205697536469, 0.06331902742385864, -0.07105332612991333, 0.03311668708920479, 0.14409446716308594, -0.24636559188365936, 0.002864853711798787, -0.405702143907547, -0.24928586184978485, -0.32508912682533264, -0.007233647629618645, -0.2614217698574066, 0.06663674116134644, 0.01573086716234684, -0.029307568445801735, -0.4822880029678345, -0.15889602899551392, -0.010222663171589375, 0.06527368724346161, -0.33755621314048767, -0.10235296189785004, 0.03347150981426239, 0.0029715311247855425, -0.04268186166882515, -0.22133395075798035, -0.02920411340892315, -0.07178636640310287, 0.1124802827835083, 0.24235008656978607, -0.05602017417550087, -0.11122377216815948, 0.09768852591514587, 0.1448417454957962, -0.04603502154350281, -0.12309958785772324, -0.007977577857673168, 0.20184370875358582, -0.24470333755016327, -0.12012585997581482, -0.345486044883728, -0.20693239569664001, -0.3463113307952881, -0.02003178931772709, -0.26563912630081177, -0.05542352423071861, 0.07462812215089798, -0.0003731379401870072, -0.4682745039463043, -0.09839581698179245, 0.04264780506491661, -0.0014392249286174774, -0.32453200221061707, -0.0997258722782135, 0.05968736484646797, -0.012890858575701714, 0.007777720224112272, -0.20029009878635406, -0.1409728080034256, -0.12442434579133987, 0.006523014511913061, 0.18829739093780518, -0.11096677929162979, -0.07072103768587112, 0.15218694508075714, 0.04609895497560501, 0.04023999348282814, -0.14016008377075195, -0.03776010498404503, 0.1491684764623642, -0.255658894777298, -0.11329347640275955, -0.38395047187805176, -0.24037902057170868, -0.3466068506240845, -0.07270820438861847, -0.2608526051044464, 0.10693753510713577, 0.04031829908490181, 0.008923593908548355, -0.4155923128128052, -0.07237423211336136, 0.10437732934951782, -0.01984908990561962, -0.24878481030464172, -0.10620557516813278, 0.09302166849374771, 0.14148679375648499, -0.14952434599399567, -0.21677076816558838, -0.05271999537944794, -0.14793407917022705, 0.048195730894804, 0.24976584315299988, -0.07027408480644226, -0.0846225917339325, 0.0886896625161171, 0.1188189908862114, -0.09305090457201004, -0.14417463541030884, -0.05723140761256218, 0.1793505698442459, -0.2654992341995239, -0.13093161582946777, -0.34064796566963196, -0.2892974317073822, -0.39373791217803955, -0.12279842048883438, -0.2408907413482666, 0.07611184567213058, 0.08077824115753174, -0.04241751879453659, -0.4232906699180603, -0.10091830044984818, -0.013831407763063908, 0.05009425804018974, -0.23628564178943634, -0.10737010091543198, -0.01195557415485382, 0.04842321574687958, -0.047675322741270065, -0.21334786713123322, -0.03453533723950386, -0.20744791626930237, 0.03448969125747681, 0.21203549206256866, -0.07508993148803711, -0.09769400954246521, 0.07154916226863861, 0.16849836707115173, 0.03847595304250717, -0.08701787143945694, 0.04242816939949989, 0.0983029380440712, -0.27317434549331665, -0.06468715518712997, -0.3562844395637512, -0.2412576824426651, -0.362409770488739, -0.13485752046108246, -0.25082698464393616, -0.05963540077209473, 0.04821791872382164, -0.07163401693105698, -0.5114542841911316, -0.11278153955936432, 0.004466633312404156, -0.05018419772386551, -0.37663534283638, -0.1436016708612442, -0.1018538698554039, 0.008122306317090988, 0.027730371803045273, -0.2211567610502243, -0.05336537957191467, -0.25600919127464294, 0.026323527097702026, 0.23732241988182068, -0.09602076560258865, -0.08883477747440338, 0.1803952306509018, 0.07718920707702637, 0.026141904294490814, -0.22453269362449646, 0.023638203740119934, 0.11911306530237198, -0.21246466040611267, -0.08867757767438889, -0.27230027318000793, -0.3428765833377838, -0.36292949318885803, -0.0003596210735850036, -0.2570899724960327, 0.011662017554044724, 0.01633291132748127, -0.10432755947113037, -0.4697357416152954, 0.020206745713949203, 0.04808459058403969, -0.03403434529900551, -0.33287233114242554, -0.11053476482629776, 0.0504533126950264, 0.015390436165034771, -0.08632534742355347, -0.12408998608589172, -0.04270301014184952, -0.2061181664466858, -0.044520650058984756, 0.17582187056541443, -0.12797963619232178, -0.22880010306835175, 0.13007421791553497, 0.035952355712652206, 0.04398864135146141, -0.0824480727314949, 0.05312100797891617, 0.07385993748903275, -0.2106025516986847, -0.028599407523870468, -0.3966608941555023, -0.2913956046104431, -0.3535841107368469, -0.12450487166643143, -0.16601860523223877, -0.06899751722812653, -0.025713428854942322, -0.04880938306450844, -0.4870377779006958, -0.09351013600826263, 0.004405444487929344, 0.010735510848462582, -0.23236097395420074, -0.19726718962192535, 0.0002242810296593234, 0.020735565572977066, -0.04518810659646988, -0.15728947520256042, -0.13152708113193512, -0.2194748818874359, 0.05099685490131378, 0.2707619369029999, -0.12704254686832428, -0.015279112383723259, 0.20804211497306824, -0.005669233854860067, 0.03501370549201965, -0.050414592027664185, 0.03366615250706673, 0.14246968924999237, -0.26446783542633057, -0.11355197429656982, -0.36996880173683167, -0.27126720547676086, -0.31085410714149475, -0.06434138864278793, -0.2616129219532013, -0.008837363682687283, 0.0271474476903677, -0.0800372064113617, -0.484720915555954, -0.049725569784641266, 0.009129260666668415, 0.046652376651763916, -0.2714007496833801, -0.08733958750963211, -0.006133904680609703, 0.0410744771361351, -0.013857929036021233, -0.22253945469856262, -0.01889950968325138, -0.18572352826595306, -0.026067500934004784, 0.2288518100976944, -0.12687964737415314, -0.1092633381485939, 0.05669107660651207, 0.12456045299768448, -0.01915799267590046, -0.28949040174484253, -0.04207870364189148, 0.030990619212388992, -0.2560584545135498, -0.03353435918688774, -0.32695508003234863, -0.31061023473739624, -0.3862122893333435, 0.04449797421693802, -0.22362978756427765, 0.051476605236530304, 0.08243007212877274, 0.0873059332370758, -0.4108547568321228, -0.09484558552503586, 0.00856742449104786, 0.05227009952068329, -0.3198402225971222, -0.14323434233665466, 0.002854960970580578, 0.009104224853217602, -0.05488689988851547, -0.16352130472660065, 0.031116226688027382, -0.22997993230819702, 0.005278072785586119, 0.24509397149085999, -0.07116460055112839, -0.06493744254112244, 0.0680590569972992, 0.026386288926005363, -0.06211157515645027, -0.2221745401620865, 0.021293263882398605, 0.13369902968406677, -0.3048367202281952, -0.105130635201931, -0.3577693998813629, -0.2185467928647995, -0.37776923179626465, -0.05695570632815361, -0.3304770886898041, -0.006298619322478771, 0.11051272600889206, -0.14473815262317657, -0.4925735592842102, -0.09085643291473389, -0.02788444608449936, -0.01899176463484764, -0.29225921630859375, -0.09054664522409439, -0.0038733636029064655, -0.03742082044482231, -0.059183962643146515, -0.21108309924602509, -0.1133224293589592, -0.1760787069797516, 0.04340395703911781, 0.298147976398468, -0.11785078048706055, -0.07107352465391159, 0.1512695997953415, 0.12428578734397888, 0.08941235393285751, -0.18314844369888306, -0.014235374517738819, 0.1726692169904709, -0.32239121198654175, -0.07602080702781677, -0.385871022939682, -0.2621701955795288, -0.43517524003982544, -0.0889609158039093, -0.24058005213737488, 0.09569332748651505, 0.07912827283143997, -0.03934744745492935, -0.4851478338241577, -0.14335030317306519, 0.010950841009616852, 0.009037211537361145, -0.2589110732078552, -0.002714035101234913, -0.00138624117244035, -0.018402937799692154, -0.09485629945993423, -0.24123483896255493, -0.14579112827777863, -0.12216389924287796, 0.02270049788057804, 0.17730039358139038, -0.060348521918058395, -0.054254915565252304, 0.07458949834108353, 0.0336543470621109, 0.003055070759728551, -0.16163398325443268, 0.026469729840755463, 0.1905275136232376, -0.19344109296798706, -0.02722206525504589, -0.2912469804286957, -0.21066135168075562, -0.4246448278427124, -0.05098191276192665, -0.22385933995246887, 0.03211435675621033, 0.0751899853348732, -0.0533418245613575, -0.5219707489013672, -0.02960139699280262, 0.02413323149085045, 0.032378800213336945, -0.24286983907222748, -0.03717166185379028, 0.017284296452999115, -0.06641828268766403, -0.08559325337409973, -0.19115641713142395, -0.13514567911624908, -0.17749357223510742, 0.07097762823104858, 0.22906383872032166, -0.1497717648744583, -0.015871919691562653, 0.07001134753227234, 0.005805643741041422, 0.03180781751871109, -0.15458019077777863, -0.040781281888484955, 0.1709393560886383, -0.2784309685230255, -0.1326398104429245, -0.3504135310649872, -0.20519229769706726, -0.37979209423065186, -0.027902424335479736, -0.24422994256019592, 0.19249235093593597, -0.005801042541861534, 0.033877454698085785, -0.4783007502555847, -0.10815444588661194, 0.026916271075606346, 0.06611564010381699, -0.22585450112819672, -0.07386578619480133, -0.02978731319308281, 0.02958248183131218, -0.026331119239330292, -0.1392824649810791, -0.016033092513680458, -0.16650591790676117, 0.06437592953443527, 0.32914766669273376, -0.08479010313749313, -0.1915515959262848, 0.03361937403678894, -0.0044362531043589115, 0.09950347244739532, -0.21823474764823914, 0.056063298135995865, 0.16070327162742615, -0.1683015525341034, -0.03633756563067436, -0.3644615709781647, -0.25373226404190063, -0.3438411355018616, -0.03567065671086311, -0.26609253883361816, 0.0496392548084259, 0.11009294539690018, -0.12043803930282593, -0.46275901794433594, -0.147163987159729, 0.00018487028137315065, 0.06340890377759933, -0.29208579659461975, -0.051649030297994614, -0.0792594626545906, 0.11181424558162689, -0.08197024464607239, -0.1372070610523224, -0.0768132358789444, -0.3167055547237396, -0.02643568627536297, 0.2643345296382904, -0.015331325121223927, -0.10291024297475815, 0.1054680272936821, -0.0031803143210709095, 0.0914812907576561, -0.08551046252250671, -0.004100026562809944, 0.12042882293462753, -0.33501750230789185, -0.04232620447874069, -0.224229097366333, -0.2078750878572464, -0.2849143445491791, -0.017329338937997818, -0.28880614042282104, 0.11810430139303207, -0.03210180252790451, -0.045864369720220566, -0.4617408215999603, -0.05767086520791054, 0.011632299050688744, 0.040757834911346436, -0.3272796869277954, -0.15476694703102112, 0.0033065401948988438, 0.020955046638846397, -0.11720306426286697, -0.2316247820854187, -0.11044206470251083, -0.2800563871860504, -0.056005917489528656, 0.17155227065086365, -0.08391715586185455, -0.09687444567680359, 0.19294027984142303, -0.007422599941492081, -0.014055891893804073, -0.1302337497472763, -0.016367468982934952, 0.1150667816400528, -0.16860651969909668, -0.10341256856918335, -0.4171527922153473, -0.30957773327827454, -0.3313148021697998, -0.029171623289585114, -0.30814048647880554, 0.0020008578430861235, 0.0563625730574131, -0.006467786151915789, -0.5078445076942444, -0.07070554047822952, 0.01942574605345726, -0.06773797422647476, -0.30480846762657166, -0.06954209506511688, -0.028421279042959213, -0.05048180744051933, -0.06596451252698898, -0.1307228058576584, -0.006915360223501921, -0.2969665825366974, 0.14203359186649323, 0.19695229828357697, -0.04518158733844757, -0.055563755333423615, 0.14173409342765808, 0.07493481785058975, -0.03814692795276642, -0.14453637599945068, -0.04129043594002724, 0.005281931720674038, -0.11262007802724838, -0.0510871559381485, -0.2684392035007477, -0.28931188583374023, -0.2747243642807007, 0.03505975380539894, -0.16182787716388702, 0.05021035298705101, -0.030496161431074142, -0.04419676586985588, -0.5858218669891357, -0.18691696226596832, -0.004570044577121735, -0.028153812512755394, -0.29742762446403503, -0.09574836492538452, 0.09676885604858398, 0.121346116065979, -0.05405740812420845, -0.10788248479366302, -0.1249212995171547, -0.2042722851037979, 0.05687873438000679, 0.1912260800600052, -0.10697916895151138, -0.1833159476518631, 0.09845685213804245, 0.14907968044281006, -0.01768088899552822, -0.16457034647464752, -0.050394874066114426, 0.0739392340183258, -0.315857470035553, -0.013754653744399548, -0.24122129380702972, -0.2195785641670227, -0.38391417264938354, -0.009189121425151825, -0.2562715411186218, 0.09134058654308319, 0.0441766232252121, -0.021345917135477066, -0.44531962275505066, -0.13079102337360382, -0.02938537485897541, -0.0646461695432663, -0.31129875779151917, -0.1614150106906891, -0.007001553196460009, 0.008583239279687405, -0.04675205796957016, -0.1456654965877533, -0.15694177150726318, -0.22809138894081116, -0.035798076540231705, 0.29063379764556885, -0.00771202752366662, -0.07469683140516281, 0.07511228322982788, 0.1407042145729065, -0.05578143149614334, -0.10089641064405441, -0.038339488208293915, -0.0075979819521307945, -0.12152477353811264, -0.0075911046005785465, -0.41093724966049194, -0.23528032004833221, -0.3588244915008545, -0.039322663098573685, -0.3137196898460388, 0.017195971682667732, 0.053025007247924805, -0.024196315556764603, -0.41572150588035583, -0.09782883524894714, 0.10829409956932068, -0.014370466582477093, -0.27623867988586426, -0.18611426651477814, -0.06242414936423302, 0.12289108335971832, -0.09895391762256622, -0.14179369807243347, -0.08902417868375778, -0.23965488374233246, -0.006044944282621145, 0.2275974303483963, -0.11739842593669891, -0.0795932486653328, 0.12520045042037964, 0.03858260437846184, -0.025431184098124504, -0.16174745559692383, -0.010529548861086369, 0.1828307956457138, -0.12130033224821091, -0.04434603825211525, -0.3610795736312866, -0.2523329555988312, -0.3909342288970947, -0.08976734429597855, -0.2897261381149292, 0.031503912061452866, 0.00826976913958788, 0.001855410635471344, -0.4956333637237549, -0.11168882995843887, -0.07008068263530731, 0.10364489257335663, -0.2638559639453888, -0.0008547168108634651, 0.04007679596543312, 0.1309465765953064, -0.06423815339803696, -0.13246941566467285, 0.04933967813849449, -0.2486041635274887, 0.0701809674501419, 0.33820632100105286, -0.07197374850511551, -0.07901304215192795, 0.15656495094299316, 0.03393913060426712, 0.04079454392194748, -0.22026456892490387, -0.0400911308825016, 0.09983853995800018, -0.2613784074783325, -0.0790034756064415, -0.33619141578674316, -0.3162761926651001, -0.24676074087619781, 0.014742745086550713, -0.20237721502780914, 0.032944150269031525, 0.025061609223484993, -0.007272803690284491, -0.4546114206314087, -0.18786947429180145, 0.028796980157494545, 0.17082849144935608, -0.38386979699134827, -0.03931449353694916, 0.01915760524570942, 0.05074402689933777, 0.0185370072722435, -0.2290072739124298, -0.07758089154958725, -0.11232970654964447, 0.0012625920353457332, 0.23184596002101898, -0.08713126927614212, -0.14011143147945404, 0.17204166948795319, 0.02086150087416172, 0.06477051973342896, -0.2401985079050064, 0.02884984202682972, 0.1260267049074173, -0.3868231177330017, -0.08679941296577454, -0.41418224573135376, -0.2665698230266571, -0.2690219283103943, -0.10186275839805603, -0.2940528690814972, 0.12531481683254242, 0.03558678925037384, 0.04390542209148407, -0.45905303955078125, -0.1471695750951767, -0.012575352564454079, 0.08462253212928772, -0.24083001911640167, -0.09025280177593231, 0.0027396902441978455, 0.0506797656416893, -0.08154714107513428, -0.06903347373008728, -0.08556598424911499, -0.24557289481163025, -0.013153358362615108, 0.2906500995159149, -0.06005215644836426, -0.12432361394166946, 0.058220382779836655, 0.06868299841880798, -0.06611042469739914, -0.2427864819765091, 0.03554120287299156, 0.10677824169397354, -0.23479081690311432, -0.023862631991505623, -0.32594743371009827, -0.24267126619815826, -0.23870041966438293, -0.010108763352036476, -0.2577783763408661, 0.08890550583600998, 0.0505942665040493, -0.07933042198419571, -0.47193264961242676, -0.1102975532412529, -0.11024460196495056, 0.01255353819578886, -0.30375805497169495, -0.11283034086227417, -0.08432233333587646, 0.038232285529375076, -0.05385388433933258, -0.16308768093585968, -0.14430361986160278, -0.15491610765457153, -0.05111604928970337, 0.24180705845355988, -0.025947192683815956, -0.15286152064800262, 0.06633314490318298, 0.12262154370546341, 0.00018828161410056055, -0.19112713634967804, 0.005839155521243811, 0.185337632894516, -0.3672963082790375, -0.1290508210659027, -0.4110887348651886, -0.29945141077041626, -0.39198869466781616, -0.08751111477613449, -0.22409455478191376, 0.005664285738021135, 0.03590615466237068, -0.014760095626115799, -0.5130860805511475, -0.09512253850698471, 0.0747089684009552, 0.0379188098013401, -0.32185426354408264, -0.09362440556287766, -0.04666459560394287, 0.07032159715890884, -0.007691322825849056, -0.2102246731519699, -0.09663283079862595, -0.13056263327598572, -0.014056766405701637, 0.26291424036026, -0.14533667266368866, -0.15911294519901276, 0.06641007214784622, 0.17626988887786865, -0.0005285507068037987, -0.20236225426197052, -0.03368845582008362, 0.12313655763864517, -0.22126850485801697, -0.07701773941516876, -0.2564523220062256, -0.23304101824760437, -0.27572929859161377, -0.030598146840929985, -0.20927628874778748, -0.020909326151013374, 0.023273969069123268, -0.03877850994467735, -0.4878312647342682, -0.2094602733850479, 0.034864239394664764, 0.0020585947204381227, -0.31110885739326477, -0.10305694490671158, -0.011171452701091766, 0.07681971043348312, -0.1153653934597969, -0.21487955749034882, -0.07140397280454636, -0.18014830350875854, -0.0074652209877967834, 0.2246134877204895, -0.1347513645887375, -0.12019141763448715, 0.15161985158920288, 0.014551352709531784, 0.09608986228704453, -0.13366524875164032, -0.028176965191960335, 0.17057041823863983, -0.3273601830005646, -0.1730710119009018, -0.4238321781158447, -0.2642180621623993, -0.3643700182437897, -0.07019082456827164, -0.23458422720432281, -0.055487025529146194, 0.05439441278576851, -0.10629615187644958, -0.5087259411811829, -0.12245886772871017, -0.06697751581668854, -0.04713387042284012, -0.33950936794281006, -0.09089455753564835, 0.029731616377830505, 0.08467654883861542, -0.10153833031654358, -0.2079063206911087, -0.13196580111980438, -0.164936825633049, -0.09050349146127701, 0.31522607803344727, -0.11727037280797958, -0.15644989907741547, 0.1095559224486351, 0.11908484250307083, 0.03705311939120293, -0.1232270896434784, 0.001189162372611463, 0.19289861619472504, -0.19180700182914734, -0.09146619588136673, -0.28738275170326233, -0.23549406230449677, -0.3437548577785492, -0.0905073881149292, -0.253994345664978, 0.09014980494976044, -0.036005403846502304, -0.0028499243780970573, -0.4582526385784149, -0.08793827891349792, -0.1194915920495987, 0.016253964975476265, -0.256792277097702, -0.12844611704349518, 0.004747841507196426, -0.06243887543678284, -0.018244128674268723, -0.20144909620285034, -0.04236779361963272, -0.27956676483154297, 0.04598088935017586, 0.24146215617656708, -0.05264993757009506, -0.16355915367603302, 0.06166037917137146, 0.17957645654678345, -0.02157365158200264, -0.2111656814813614, -0.01935788057744503, 0.08028874546289444, -0.19454216957092285, -0.00024159254098776728, -0.39454835653305054, -0.3300323486328125, -0.2895006537437439, 0.010985822416841984, -0.19847239553928375, 0.10909624397754669, -0.013749321922659874, -0.043755944818258286, -0.5448889136314392, -0.17272448539733887, 0.03988808020949364, -0.012179551646113396, -0.17522670328617096, -0.08658936619758606, -0.03178873285651207, 0.09635904431343079, 0.038249339908361435, -0.1440209150314331, -0.14679290354251862, -0.16887730360031128, 0.08014742285013199, 0.22864706814289093, -0.13171222805976868, -0.06645126640796661, 0.09958161413669586, 0.012326785363256931, 0.0009366262820549309, -0.21364744007587433, 0.009284654632210732, 0.1369009017944336, -0.1965426504611969, -0.051009006798267365, -0.43751975893974304, -0.22338122129440308, -0.3032529056072235, -0.021639740094542503, -0.18805237114429474, 0.02379182167351246, -0.017914649099111557, 0.011172376573085785, -0.5289566516876221, -0.20979322493076324, 0.04209801182150841, -0.014793135225772858, -0.30450859665870667, -0.16019734740257263, -0.06604387611150742, -0.0013646557927131653, -0.0698462724685669, -0.1498275101184845, -0.20082619786262512, -0.1498187780380249, -0.0477323904633522, 0.19999352097511292, -0.059137359261512756, -0.1210053563117981, 0.015761541202664375, 0.15968352556228638, -0.023107124492526054, -0.16349852085113525, 0.0044657387770712376, 0.2019830346107483, -0.2768484354019165, -0.036087773740291595, -0.39643752574920654, -0.18762430548667908, -0.38624873757362366, -0.06254415959119797, -0.14475703239440918, 0.06711924076080322, 0.035114891827106476, 0.0767713412642479, -0.45761558413505554, -0.14262321591377258, 0.027564415708184242, -0.037667933851480484, -0.23278945684432983, -0.029999487102031708, -0.02841545268893242, 0.025069693103432655, -0.04298856481909752, -0.1676197052001953, -0.13482001423835754, -0.15902304649353027, -0.09072626382112503, 0.23166614770889282, -0.11322717368602753, -0.13093693554401398, 0.10794442147016525, 0.006052768789231777, 0.04934971034526825, -0.18487201631069183, -0.03803958371281624, 0.06307537108659744, -0.31322047114372253, -0.06847602128982544, -0.40753552317619324, -0.24052080512046814, -0.2950003743171692, 0.007198427803814411, -0.16153551638126373, 0.03645600378513336, -0.024630511179566383, 0.015695150941610336, -0.46020323038101196, -0.04561881348490715, -0.008614180609583855, 0.08119840919971466, -0.24688591063022614, -0.054477330297231674, 0.025408649817109108, 0.07634973526000977, -0.040161989629268646, -0.15703214704990387, -0.036425910890102386, -0.21302641928195953, 0.09815177321434021, 0.247996985912323, -0.04376101866364479, -0.1217857226729393, 0.08024992793798447, 0.16906596720218658, 0.004456717055290937, -0.23972515761852264, -0.0412328876554966, 0.1155552789568901, -0.1463054120540619, -0.07615446299314499, -0.31260737776756287, -0.29381075501441956, -0.3304691016674042, -0.010859785601496696, -0.14947400987148285, 0.016457147896289825, 0.05980141460895538, -0.020201928913593292, -0.5267691612243652, -0.10278915613889694, -0.02234072983264923, -0.09485215693712234, -0.2592718303203583, -0.0632862076163292, -0.021714232861995697, -0.010463310405611992, 0.022180262953042984, -0.15502749383449554, -0.03510057553648949, -0.11691135168075562, -0.014677819795906544, 0.29391685128211975, -0.10508757829666138, -0.12040109187364578, 0.08830244839191437, 0.06738794595003128, 0.07562196999788284, -0.16967293620109558, -0.013215416111052036, 0.11758981645107269, -0.2369993031024933, -0.13123565912246704, -0.384475976228714, -0.24634477496147156, -0.3763919174671173, -0.05035750940442085, -0.27009421586990356, 0.15146856009960175, 0.09607997536659241, -0.012833517044782639, -0.47836434841156006, 0.007721861358731985, 0.006980661768466234, 0.01707383617758751, -0.27404895424842834, -0.07500121742486954, 0.07074838876724243, 0.055516552180051804, 0.05315246060490608, -0.2124285250902176, -0.05123312771320343, -0.237518772482872, 0.10222256928682327, 0.23678083717823029, -0.09593959152698517, -0.07106181234121323, 0.15535026788711548, 0.011860048398375511, 0.0026160068809986115, -0.24561047554016113, 0.0013364790938794613, 0.12634804844856262, -0.2715117037296295, -0.12071532756090164, -0.3786383867263794, -0.28105050325393677, -0.321651428937912, 0.031363800168037415, -0.3360915780067444, 0.045472294092178345, 0.008718352764844894, -0.010712695308029652, -0.5232717990875244, -0.1091381385922432, -0.06114659458398819, 0.056251004338264465, -0.28424155712127686, -0.09837210178375244, 0.018741903826594353, 0.04414062201976776, -0.12950482964515686, -0.13854093849658966, -0.07152213156223297, -0.19513142108917236, 0.06061376631259918, 0.3142338991165161, -0.1476442664861679, -0.15632747113704681, 0.16994749009609222, 0.1383964866399765, -0.01699570007622242, -0.23489981889724731, -0.03740715608000755, 0.10651085525751114, -0.33326828479766846, -0.08493389934301376, -0.4233520030975342, -0.30425024032592773, -0.30229419469833374, -0.0785931646823883, -0.2431289255619049, 0.08272548764944077, 0.07028814405202866, -0.05218213051557541, -0.4734775424003601, -0.14308664202690125, 0.07820200175046921, 0.053913600742816925, -0.34436726570129395, -0.09009547531604767, -0.012164534069597721, -0.17418316006660461, 0.1425309181213379, 0.5147976875305176, 0.25518718361854553, 0.5679089426994324, -0.04094843193888664, -0.7649387717247009, 0.29197219014167786, 0.34210404753685, -0.29576411843299866, -0.19829927384853363, -0.07118520140647888, 0.5231983065605164, 0.02570216916501522, -0.4024359881877899, 0.7709795236587524, 0.2526645064353943, 1.0929359197616577, 0.7444432973861694, 1.0395846366882324, 0.13116882741451263, 0.7103466987609863, -0.15181198716163635, -0.07738912105560303, 0.09152580797672272, 1.4362246990203857, 0.2922096848487854, 0.018903745338320732, -0.08212537318468094, 0.8886653184890747, 0.2714821398258209, 0.06574640423059464, 0.11151730269193649, -0.0400388240814209, -0.25608140230178833, -0.1382337510585785, -0.13859228789806366, 0.09702357649803162, 0.26813989877700806, -0.03066418319940567, -0.1476617455482483, 0.08311387151479721, 0.06751128286123276, 0.04107879474759102, -0.2523047924041748, 0.04150456190109253, 0.19754400849342346, -0.31150177121162415, -0.15425923466682434, -0.3418150544166565, -0.2764463424682617, -0.46664100885391235, -0.004004527349025011, -0.23177564144134521, 0.027438320219516754, 0.13529916107654572, -0.028385912999510765, -0.5133548974990845, -0.09675875306129456, -0.06117044761776924, -0.016669226810336113, -0.26988714933395386, -0.11800828576087952, -0.02231619693338871, 0.13991202414035797, -0.008831340819597244, -0.1281861662864685, -0.059494804590940475, -0.16321736574172974, 0.08419676125049591, 0.25233182311058044, -0.0002544228336773813, -0.16484373807907104, 0.07983134686946869, -0.0014759278856217861, 0.002735305344685912, -0.14994218945503235, 0.0670335665345192, 0.09825029969215393, -0.25379979610443115, -0.10445278137922287, -0.39627304673194885, -0.3031997084617615, -0.34506645798683167, -0.13103695213794708, -0.23402820527553558, 0.0523182675242424, -0.003260573372244835, -0.08587630093097687, -0.5448988080024719, -0.008249926380813122, -0.07626558840274811, 0.09930122643709183, -0.25919705629348755, -0.1084994226694107, -0.05762133374810219, 0.10858946293592453, 0.034668780863285065, -0.21502868831157684, -0.10087289661169052, -0.1348462551832199, 0.028156686574220657, 0.35330715775489807, -0.12153735011816025, -0.0901591032743454, 0.03003683127462864, 0.1722484827041626, -0.0165930837392807, -0.2862626612186432, -0.011816979385912418, 0.12444514781236649, -0.1582864373922348, -0.10343324393033981, -0.35997524857521057, -0.2140588015317917, -0.2687859833240509, -0.04759518429636955, -0.18758146464824677, 0.07717490196228027, 0.014815174974501133, -0.03145340085029602, -0.4742497205734253, -0.14761757850646973, 0.051517054438591, -0.04786490276455879, -0.23064684867858887, -0.042577408254146576, -0.0044199563562870026, 0.01819256879389286, -0.06313503533601761, -0.14468872547149658, -0.04878271743655205, -0.17308148741722107, 0.06097433716058731, 0.22543738782405853, -0.08130130916833878, -0.19569094479084015, 0.13076455891132355, 0.07511270046234131, -0.000854752492159605, -0.1177871972322464, 0.0290475282818079, -0.009403033182024956, -0.2520708441734314, -0.15924346446990967, -0.30601006746292114, -0.11927700787782669, -0.37402617931365967, -0.040704745799303055, -0.25210702419281006, 0.0688113272190094, 0.061854392290115356, 0.029028572142124176, -0.5250545144081116, -0.02323460578918457, -0.05574267730116844, 0.03527054563164711, -0.23127645254135132, -0.05463843792676926, -0.046696241945028305, 0.02136860601603985, -0.07493032515048981, -0.17088592052459717, -0.028587976470589638, -0.252018004655838, 0.034702595323324203, 0.2838077247142792, -0.06511569023132324, -0.09395834058523178, 0.1684991419315338, 0.09184430539608002, 0.06277191638946533, -0.14768698811531067, 0.08412685990333557, 0.036426443606615067, -0.1899595856666565, -0.09954379498958588, -0.3733789026737213, -0.2922045886516571, -0.44153428077697754, -0.06352844834327698, -0.2739200294017792, 0.07597936689853668, -0.043986182659864426, 0.03624953329563141, -0.4620867371559143, -0.11421148478984833, -0.04463815316557884, -0.01794424094259739, -0.3457096517086029, -0.1175164133310318, 0.0005350274732336402, 0.04759661480784416, -0.12253817170858383, -0.18161886930465698, -0.13059478998184204, -0.17943333089351654, 0.10460011661052704, 0.28456026315689087, -0.10868612676858902, -0.1985178142786026, 0.021847013384103775, -0.01688227616250515, 0.056222379207611084, -0.21411778032779694, -0.0023987656459212303, 0.10031597316265106, -0.2495051771402359, -0.01640840247273445, -0.39359036087989807, -0.2607785165309906, -0.38841670751571655, -0.0007049578707665205, -0.2908412218093872, -0.005378491710871458, -0.039756204932928085, -0.1057548001408577, -0.4521386921405792, 0.0266399048268795, -0.058935798704624176, 0.06899340450763702, -0.27664992213249207, -0.14539951086044312, -0.013602470979094505, 0.044818952679634094, -0.08634887635707855, -0.07179711759090424, -0.18124166131019592, -0.16837561130523682, 0.027943743392825127, 0.22432830929756165, -0.08555498719215393, -0.07339341193437576, 0.10021998733282089, 0.08832015842199326, 0.0545695535838604, -0.27746710181236267, 0.0683901309967041, 0.24533188343048096, -0.20502525568008423, -0.035809025168418884, -0.42975541949272156, -0.24965216219425201, -0.3887017071247101, -0.07379341870546341, -0.1912035197019577, 0.01107564102858305, 0.023121876642107964, -0.13076016306877136, -0.4836050271987915, -0.09731551259756088, -0.04440649598836899, -0.014578228816390038, -0.19488413631916046, -0.15078559517860413, -0.05229834467172623, 0.016014263033866882, 0.00812717154622078, -0.0632503405213356, -0.06932684779167175, -0.22329333424568176, -0.010019098408520222, 0.30654972791671753, -0.13858352601528168, -0.044626329094171524, 0.16341213881969452, 0.07096429169178009, 0.05223045498132706, -0.13837701082229614, 0.07218172401189804, 0.0788440927863121, -0.19500477612018585, -0.1052536591887474, -0.39773109555244446, -0.23859712481498718, -0.3588167130947113, -0.09380835294723511, -0.3194218873977661, 0.0041659255512058735, 0.1022448018193245, 0.06279735267162323, -0.44317761063575745, -0.04252007231116295, -0.03500863164663315, 0.01755852811038494, -0.2750670611858368, -0.14436285197734833, -0.04926470294594765, 0.06132365018129349, -0.06782897561788559, -0.2215052992105484, -0.06660903245210648, -0.20059140026569366, 0.06243521720170975, 0.2530500590801239, -0.10804545134305954, -0.11089000105857849, 0.08398405462503433, 0.03405158594250679, -0.00017384793318342417, -0.1383933573961258, 0.026797058060765266, 0.19572806358337402, -0.33871471881866455, -0.10614679008722305, -0.38007181882858276, -0.23514212667942047, -0.37070736289024353, -0.07009285688400269, -0.20040661096572876, 0.06017504632472992, -0.028691260144114494, 0.03139419108629227, -0.42584583163261414, -0.13051432371139526, 0.032397035509347916, -0.015342050231993198, -0.23292036354541779, -0.10118993371725082, -0.03990961238741875, 0.03780956566333771, -0.09010529518127441, -0.1587204784154892, -0.07718037813901901, -0.18563729524612427, -0.016463253647089005, 0.07025502622127533, -0.0883883461356163, -0.21208538115024567, 0.10479264706373215, 0.12399357557296753, 0.03361823782324791, -0.19533614814281464, -0.014522658661007881, 0.12476354837417603, -0.2815271019935608, -0.1156817302107811, -0.405537873506546, -0.18432264029979706, -0.3488771617412567, 0.003379866713657975, -0.25205790996551514, 0.05487905815243721, 0.021107809618115425, -0.012189322151243687, -0.4343516528606415, -0.08983111381530762, -0.04285779222846031, -0.017581580206751823, -0.2062467634677887, -0.10430026054382324, -0.014322600327432156, 0.12971515953540802, 0.002579278079792857, -0.14163556694984436, -0.10590546578168869, -0.0697675570845604, 0.031016208231449127, 0.28065839409828186, -0.10158543288707733, -0.1879890263080597, 0.025845127180218697, 0.08780279755592346, 0.039515428245067596, -0.18786422908306122, -0.06411296874284744, 0.059333786368370056, -0.27045753598213196, -0.15929865837097168, -0.31439730525016785, -0.18609750270843506, -0.20664779841899872, -0.0005178304272703826, -0.26919785141944885, 0.007687421515583992, -0.14295946061611176, -0.06507558375597, -0.48453211784362793, -0.05342598259449005, 0.02141302451491356, 0.05083359777927399, -0.27852392196655273, -0.05725287273526192, 0.07261250913143158, 0.003220941871404648, -0.07467880100011826, -0.23775404691696167, -0.00500262388959527, -0.12258398532867432, 0.04898172616958618, 0.19338154792785645, -0.08438289165496826, -0.12900562584400177, 0.09662458300590515, 0.058047983795404434, 0.04592159762978554, -0.1956063061952591, 0.03719974681735039, 0.14348642528057098, -0.26871880888938904, -0.09208492934703827, -0.3813724219799042, -0.20303654670715332, -0.3616727292537689, -0.029689183458685875, -0.2591755986213684, -0.022984422743320465, 0.07493216544389725, -0.0882374495267868, -0.44172537326812744, -0.08080870658159256, -0.09196865558624268, -0.02581796608865261, -0.31491032242774963, -0.13133427500724792, -0.05887103080749512, 0.09346956759691238, -0.06378904730081558, -0.14904293417930603, -0.10746106505393982, -0.1721610128879547, -0.026190025731921196, 0.25198328495025635, -0.06393784284591675, -0.1150200217962265, 0.06972771883010864, 0.013120565563440323, 0.028305605053901672, -0.1586170345544815, -0.018446750938892365, 0.09295066446065903, -0.22693584859371185, 0.0375942625105381, -0.3744661509990692, -0.22689376771450043, -0.32922986149787903, -0.17275694012641907, -0.23336224257946014, 0.01594473607838154, 0.09886134415864944, -0.048030417412519455, -0.5145602226257324, -0.11274131387472153, -0.0008389309514313936, -0.0065791672095656395, -0.2918384373188019, -0.15800637006759644, -0.01103016547858715, 0.09524843096733093, 0.04664483293890953, -0.13300171494483948, -0.07233894616365433, -0.12823913991451263, 0.0623229518532753, 0.33650124073028564, -0.03889733552932739, -0.12316683679819107, 0.10550560802221298, 0.07641195505857468, 0.03503091633319855, -0.1658482551574707, -0.04465021938085556, 0.026234883815050125, -0.2876489460468292, -0.18779398500919342, -0.3559451103210449, -0.24350392818450928, -0.4318782389163971, -0.010025294497609138, -0.19621245563030243, 0.05606326088309288, 0.11301025748252869, 0.05825215205550194, -0.535983145236969, -0.04998109117150307, 0.012875409796833992, 0.04233210161328316, -0.24248291552066803, -0.1262567639350891, -0.029386114329099655, 0.015131181105971336, -0.08955889940261841, -0.17325304448604584, -0.15852685272693634, -0.23971380293369293, 0.04024393483996391, 0.24976098537445068, -0.07388238608837128, -0.21173667907714844, 0.053390566259622574, 0.03968275710940361, 0.021550696343183517, -0.20517852902412415, 0.04113195464015007, 0.12850287556648254, -0.2532142996788025, -0.08581534773111343, -0.3239437937736511, -0.26731178164482117, -0.2919798195362091, -0.02143077366054058, -0.23185351490974426, 0.05335749685764313, 0.05564213916659355, 0.015438452363014221, -0.42840704321861267, -0.08302333950996399, 0.034668952226638794, 0.027334021404385567, -0.22658078372478485, -0.09162870049476624, -0.10352445393800735, 0.12335891276597977, -0.04642122611403465, -0.22688138484954834, -0.09584682434797287, -0.23161911964416504, 0.11637798696756363, 0.2781933546066284, -0.1714656949043274, -0.08996543288230896, 0.07264623045921326, 0.16329306364059448, -0.03271087259054184, -0.2882157862186432, -0.0022353194653987885, 0.11401531100273132, -0.2748942971229553, -0.19355422258377075, -0.33800259232521057, -0.1600634902715683, -0.4435519576072693, 0.01412748172879219, -0.26799121499061584, 0.16872093081474304, 0.051427338272333145, -0.04472334310412407, -0.4204143285751343, -0.07551492750644684, -0.0387897826731205, 0.05692695826292038, -0.3147400915622711, -0.17972277104854584, 0.10181211680173874, 0.03898563235998154, -0.0822644829750061, -0.15139135718345642, -0.16391722857952118, -0.2551063597202301, -0.013614815659821033, 0.2273230254650116, -0.08943566679954529, -0.06114577502012253, 0.06888188421726227, 0.0920938104391098, 0.05000120773911476, -0.23006808757781982, 0.0058940076269209385, 0.08428330719470978, -0.2008700668811798, -0.06373776495456696, -0.3585178554058075, -0.3107157349586487, -0.3422999978065491, -0.08278507739305496, -0.20425723493099213, 0.045304346829652786, -0.010990286245942116, 0.010944769717752934, -0.4396066665649414, -0.13148120045661926, 0.04893440753221512, 0.006496377754956484, -0.3046628534793854, -0.08410832285881042, -0.037399180233478546, 0.034382790327072144, -0.059837743639945984, -0.09181331098079681, -0.010953152552247047, -0.20037095248699188, 0.0918894037604332, 0.25114476680755615, -0.07400109618902206, -0.07309477031230927, 0.10718562453985214, 0.18821300566196442, 0.03209127113223076, -0.23574377596378326, 0.05974477156996727, 0.11081350594758987, -0.269279420375824, -0.23607590794563293, -0.3092937767505646, -0.20280829071998596, -0.3081485331058502, 0.004486592020839453, -0.1550053507089615, 0.048970576375722885, 0.07650776952505112, -0.07252311706542969, -0.49337348341941833, -0.04897833988070488, 0.010389207862317562, -0.045254647731781006, -0.2711388170719147, -0.0994439497590065, -0.04070080816745758, 0.07817913591861725, -0.07203906774520874, -0.24685195088386536, -0.14149340987205505, -0.09896893799304962, -0.06437892466783524, 0.26444920897483826, -0.06252733618021011, -0.08531420677900314, 0.031309645622968674, 0.051376983523368835, 0.026966292411088943, -0.1358538568019867, -0.03777682036161423, 0.07324295490980148, -0.20393630862236023, -0.03123457543551922, -0.3408394455909729, -0.2741701006889343, -0.3277614414691925, -0.03896816819906235, -0.19767393171787262, -0.011184977367520332, 0.035439979285001755, -0.04328431561589241, -0.5311676263809204, -0.091033935546875, -0.007421092130243778, 0.0008981779683381319, -0.2694038450717926, -0.16123822331428528, -0.0024495830293744802, 0.07803549617528915, -0.03761288523674011, -0.1145060732960701, -0.1459912657737732, -0.2628214657306671, 0.02332090586423874, 0.27606675028800964, 0.012961060740053654, -0.08644303679466248, 0.04888581484556198, 0.1077553778886795, -0.022357551380991936, -0.25015610456466675, 0.12492626905441284, 0.1477079540491104, -0.21831747889518738, -0.058314744383096695, -0.2687392234802246, -0.22952541708946228, -0.28721460700035095, -0.09579789638519287, -0.3091634213924408, 0.12259852141141891, 0.10336992889642715, -0.009794834069907665, -0.46438008546829224, -0.09350752085447311, -0.04357505589723587, 0.09834800660610199, -0.25626125931739807, -0.06500310450792313, 0.019038770347833633, 0.10886042565107346, -0.02656351402401924, -0.131059467792511, -0.14438217878341675, -0.1619257926940918, 0.12072759121656418, 0.22738295793533325, -0.0994715690612793, -0.07267647236585617, 0.14044839143753052, 0.09867524355649948, -0.06095605716109276, -0.15077780187129974, -0.06471250206232071, 0.09018158167600632, -0.22666232287883759, -0.10128346085548401, -0.3188619017601013, -0.0863238126039505, -0.30924052000045776, -0.008291070349514484, -0.2843417227268219, 0.01912245713174343, -0.02355615235865116, -0.061960916966199875, -0.530271053314209, -0.13480085134506226, -0.019603075459599495, -0.02861258201301098, -0.36016014218330383, -0.15837155282497406, 0.02292856201529503, 0.0594521127641201, -0.0005167631898075342, -0.11610549688339233, -0.10296758264303207, -0.15392525494098663, -0.06065850704908371, 0.3674602806568146, -0.03325991332530975, -0.12802532315254211, 0.034423258155584335, 0.08645632117986679, -0.07357972115278244, -0.2148294299840927, 0.03190910071134567, 0.13518913090229034, -0.27861323952674866, -0.03895527869462967, -0.4112747609615326, -0.22949571907520294, -0.3729937970638275, -0.08680222928524017, -0.21638557314872742, 0.001626698998734355, 0.124727264046669, -0.064566969871521, -0.3761924207210541, -0.15651488304138184, 0.02113058790564537, -0.038562674075365067, -0.35626116394996643, -0.08231310546398163, 0.03411044925451279, -0.007248361129313707, 0.007393573876470327, -0.2082074135541916, -0.07937450706958771, -0.16843849420547485, 0.0411016047000885, 0.1634185016155243, -0.012101998552680016, -0.08918400853872299, 0.07130355387926102, 0.016246195882558823, -0.0647796168923378, -0.2439354807138443, 0.02171814627945423, 0.11436301469802856, -0.310640811920166, -0.11283900588750839, -0.2811374366283417, -0.26026448607444763, -0.37732192873954773, 0.0230966005474329, -0.21646077930927277, 0.0534847117960453, -0.0007621530094183981, -0.11295166611671448, -0.5342105627059937, -0.12583942711353302, -0.020148267969489098, 0.039228226989507675, -0.2616170644760132, -0.07915230095386505, 0.011252932250499725, 0.006088551599532366, -0.15955153107643127, -0.19437125325202942, -0.13364310562610626, -0.22005878388881683, -0.0023601793218404055, 0.19112876057624817, -0.14855565130710602, -0.1222895160317421, 0.09837175905704498, 0.03465397655963898, 0.044838208705186844, -0.18683704733848572, -0.05652443319559097, 0.13345320522785187, -0.1599443256855011, -0.12023856490850449, -0.3010360598564148, -0.1748419553041458, -0.2731914222240448, -0.08556510508060455, -0.1387866586446762, 0.05708814412355423, 0.01693749614059925, -0.04684852063655853, -0.44560474157333374, -0.09925997257232666, -0.01801181770861149, 0.08559555560350418, -0.32534343004226685, -0.1424555778503418, 0.03988775983452797, 0.09164465218782425, 0.002885097172111273, -0.22193744778633118, -0.03186606615781784, -0.16332636773586273, 0.09949629008769989, 0.1908947378396988, -0.08211810886859894, -0.13115203380584717, 0.013094933703541756, 0.031709492206573486, 0.07627716660499573, -0.19931234419345856, -0.04165865480899811, 0.17482969164848328, -0.20462118089199066, -0.03715306147933006, -0.3988277316093445, -0.2466311901807785, -0.29656654596328735, -0.07723540812730789, -0.2770219147205353, 0.09268227219581604, 0.10774437338113785, -0.09293486177921295, -0.6055618524551392, -0.1743321716785431, -0.05978960543870926, 0.014532276429235935, -0.25841283798217773, -0.09762343764305115, -0.0347844697535038, 0.06413327157497406, -0.09990736842155457, -0.17067347466945648, -0.11801405996084213, -0.053410280495882034, 0.052206866443157196, 0.21155838668346405, -0.14287683367729187, -0.13918256759643555, 0.010339473374187946, 0.030929166823625565, -0.016987400129437447, -0.13166803121566772, -0.026858793571591377, 0.091889388859272, -0.3210883140563965, -0.14532418549060822, -0.36266085505485535, -0.25448721647262573, -0.2913869321346283, -0.07589023560285568, -0.18659614026546478, 0.06412698328495026, 0.02939515747129917, -0.13574634492397308, -0.54018634557724, -0.06474490463733673, 0.060724008828401566, 0.03173855319619179, -0.25107893347740173, -0.11068448424339294, -0.018335038796067238, 0.023316655308008194, -0.0030788411386311054, -0.15118759870529175, -0.09764410555362701, -0.2038470059633255, 0.0391927994787693, 0.2694648802280426, -0.13463222980499268, -0.031474366784095764, 0.10543467849493027, 0.06797784566879272, 0.09378823637962341, -0.12466346472501755, -0.009927120991051197, 0.1582159399986267, -0.3075639307498932, -0.1185344010591507, -0.47235962748527527, -0.1827123910188675, -0.4430200755596161, -0.010369941592216492, -0.2810892164707184, 0.002322097308933735, 0.01987539790570736, -0.02201051637530327, -0.4450781047344208, -0.013234762474894524, 0.008763719350099564, 0.06547588109970093, -0.31912973523139954, -0.054612308740615845, 0.007895944640040398, 0.0333055779337883, -0.05661274492740631, -0.17143021523952484, -0.0763796791434288, -0.16735787689685822, -0.014076999388635159, 0.21627044677734375, -0.055656548589468, -0.2021012157201767, 0.06167289614677429, 0.14508476853370667, -0.11543592810630798, -0.21051691472530365, 0.028857190161943436, 0.04890061914920807, -0.12415789812803268, -0.21591243147850037, -0.2875150740146637, -0.16586367785930634, -0.3022818863391876, -0.05551298335194588, -0.22169645130634308, 0.006408176384866238, 0.07009317725896835, -0.09132436662912369, -0.48937711119651794, -0.0520038828253746, -0.06488212943077087, 0.003202683525159955, -0.2809920608997345, -0.15738779306411743, 0.058451130986213684, 0.07350349426269531, -0.1275932788848877, -0.17067591845989227, -0.12470240145921707, -0.22030533850193024, 0.015204759314656258, 0.2178245633840561, -0.04480454698204994, -0.15250658988952637, 0.14166872203350067, 0.06526591628789902, 0.06271239370107651, -0.20293104648590088, -0.030792975798249245, 0.07379341125488281, -0.22516606748104095, -0.03960249572992325, -0.2533836364746094, -0.17920437455177307, -0.3356104791164398, -0.05498223006725311, -0.22012019157409668, 0.04988378658890724, 0.10280076414346695, 0.011866074055433273, -0.5175215005874634, -0.13619787991046906, 0.059197112917900085, 0.03682011738419533, -0.30079326033592224, -0.03098156675696373, -0.019147198647260666, 0.04247600957751274, -0.07487235963344574, -0.26641109585762024, -0.04272279143333435, -0.2478816658258438, -0.017067115753889084, 0.21611551940441132, -0.056680597364902496, -0.12070279568433762, 0.09335405379533768, -0.016519099473953247, 0.015552103519439697, -0.24968096613883972, 0.009831070899963379, 0.14975282549858093, -0.2549348771572113, 0.015664925798773766, -0.33295661211013794, -0.21393953263759613, -0.34098395705223083, -0.05407160520553589, -0.1912594437599182, 0.10634315013885498, -0.03543107584118843, 0.017393356189131737, -0.44999226927757263, -0.05378134176135063, 0.01414195541292429, -0.04079737514257431, -0.34785109758377075, 0.0009311511530540884, 0.01285369973629713, -0.014397988095879555, -0.034407179802656174, -0.18643364310264587, -0.17810633778572083, -0.3019368350505829, -0.06876581907272339, 0.2668987512588501, -0.09543374180793762, -0.12973162531852722, 0.10681713372468948, 0.09751041233539581, -0.08520141243934631, -0.1909218728542328, -0.04692451283335686, 0.06975693255662918, -0.2814250588417053, -0.0926358625292778, -0.37266087532043457, -0.3164641261100769, -0.3722569942474365, 0.05635794252157211, -0.2922830879688263, -0.03308397904038429, 0.04338901862502098, -0.0054997168481349945, -0.452481210231781, -0.15780222415924072, 0.04073776304721832, -0.017850497737526894, -0.34594911336898804, -0.12072224915027618, -0.005202518776059151, 0.09216935932636261, -0.020622188225388527, -0.16945242881774902, -0.03929036110639572, -0.18335296213626862, 0.056166715919971466, 0.18311567604541779, -0.17173883318901062, -0.07848763465881348, 0.023079106584191322, 0.05237793177366257, 0.015018818899989128, -0.20026186108589172, -0.02820325270295143, 0.20104800164699554, -0.38054507970809937, -0.0422985777258873, -0.24458444118499756, -0.2425515204668045, -0.3490482568740845, -0.008115163072943687, -0.14811263978481293, -0.008131776936352253, 0.03401140868663788, -0.044641949236392975, -0.44771435856819153, -0.09385701268911362, 0.02381567284464836, 0.0003344772267155349, -0.28675517439842224, -0.12381549179553986, -0.03298201784491539, 0.049387793987989426, -0.05891427397727966, -0.22695164382457733, -0.00743272015824914, -0.1941816359758377, -0.05781751871109009, 0.30838584899902344, -0.13044427335262299, -0.0918595939874649, 0.09173238277435303, 0.07998470216989517, 0.02847501076757908, -0.20462270081043243, -0.0034814642276614904, 0.23307128250598907, -0.36594530940055847, -0.08319440484046936, -0.3375939726829529, -0.2963443100452423, -0.41661691665649414, 0.06351209431886673, -0.2581305503845215, 0.15522043406963348, 0.05641510710120201, -0.03152407333254814, -0.4518848955631256, -0.07006607204675674, 0.04981577768921852, 0.04320690408349037, -0.28197476267814636, -0.08148399740457535, 0.06782817095518112, 0.09731319546699524, -0.10191486030817032, -0.20010890066623688, -0.0693981945514679, -0.21857351064682007, -0.007456831634044647, 0.205513134598732, -0.1061122715473175, -0.09609227627515793, 0.14234131574630737, 0.06791543960571289, 0.12366645783185959, -0.17968498170375824, 0.1541423797607422, 0.18149928748607635, -0.2292288839817047, -0.11144393682479858, -0.5005677342414856, -0.2514873147010803, -0.3391886055469513, 0.05168895795941353, -0.3085428476333618, 0.14656499028205872, 0.017064722254872322, 0.0008717191522009671, -0.60267573595047, -0.09486483037471771, -0.026353586465120316, 0.05553758144378662, -0.2846440374851227, -0.14741036295890808, 0.04758188873529434, 0.0911962240934372, -0.02197537012398243, -0.11788169294595718, -0.030887266620993614, -0.18851801753044128, -0.05651535466313362, 0.18406100571155548, -0.025542566552758217, -0.08175128698348999, 0.09768469631671906, 0.04572147876024246, 0.02150181122124195, -0.17462460696697235, -0.022366713732481003, 0.09646543115377426, -0.14584916830062866, -0.0750642642378807, -0.38963329792022705, -0.3088338375091553, -0.3843841850757599, -0.008447378873825073, -0.2016361802816391, 0.040352992713451385, 0.032212477177381516, -0.05058375000953674, -0.5310752391815186, -0.036732036620378494, 0.009464781731367111, 0.1452338695526123, -0.3270137906074524, -0.09717504680156708, 0.04438101872801781, 0.044210050255060196, -0.12119852751493454, -0.21577094495296478, -0.08089813590049744, -0.17025907337665558, 0.006637252867221832, 0.30348336696624756, -0.04339185357093811, -0.10113470256328583, 0.09392782300710678, 0.05480927228927612, 0.04747521132230759, -0.1298830211162567, -0.0024985377676784992, 0.153916597366333, -0.270280659198761, -0.15706974267959595, -0.40939077734947205, -0.20544244349002838, -0.32574936747550964, 0.03507697954773903, -0.23675043880939484, 0.09450935572385788, 0.09577778726816177, 0.014391383156180382, -0.5061107873916626, -0.11104463785886765, -0.0026783468201756477, -0.0278773233294487, -0.265490859746933, -0.03909895196557045, -0.023810239508748055, 0.1550924926996231, -0.06634997576475143, -0.17847536504268646, -0.09784958511590958, -0.22284124791622162, 0.03226042911410332, 0.25431039929389954, -0.009961536154150963, -0.08690816164016724, 0.11946770548820496, 0.056376997381448746, -0.050787679851055145, -0.1803394854068756, -0.048043373972177505, 0.1790088415145874, -0.2555871605873108, -0.0032051156740635633, -0.3225775361061096, -0.2019539475440979, -0.33656665682792664, -0.03400008752942085, -0.2068917155265808, 0.0974041149020195, -0.005901571363210678, -0.010717091150581837, -0.34462904930114746, -0.10630084574222565, -0.021944154053926468, -0.013207385316491127, -0.3616335093975067, -0.030854053795337677, 0.0032807777170091867, 0.07144616544246674, 0.040427934378385544, -0.0428679995238781, -0.13036292791366577, -0.1346995085477829, 0.04255979135632515, 0.2732887268066406, -0.06903119385242462, -0.118350550532341, 0.17214739322662354, 0.09828630834817886, 0.037275444716215134, -0.11339310556650162, 0.03942517191171646, 0.15455634891986847, -0.23758725821971893, 0.0014304360374808311, -0.3337279260158539, -0.24808190762996674, -0.39081692695617676, -0.061493903398513794, -0.2919003963470459, 0.05679355934262276, 0.042618781328201294, -0.13772684335708618, -0.47462761402130127, -0.06922087073326111, -0.0007025949307717383, -0.02618127316236496, -0.22563402354717255, -0.12498524785041809, -0.06105490401387215, 0.0025561146903783083, -0.052413210272789, -0.11905821412801743, -0.02753675915300846, -0.1768241971731186, -0.04820264130830765, 0.22057494521141052, -0.13781872391700745, -0.12610021233558655, 0.003964561969041824, 0.07309139519929886, 0.0522511750459671, -0.1832476556301117, 0.01754085347056389, 0.08454432338476181, -0.32674798369407654, -0.11829306930303574, -0.30144011974334717, -0.19218552112579346, -0.3708975911140442, 0.015797650441527367, -0.2557596266269684, 0.1269042193889618, 0.053794875741004944, -0.05789397656917572, -0.43588632345199585, -0.14052048325538635, -0.055929481983184814, -0.05111166462302208, -0.2762760519981384, -0.09295632690191269, -0.0036231563426554203, 0.03597217798233032, 0.020080454647541046, -0.18950775265693665, -0.12210626155138016, -0.1251218169927597, 0.08243171125650406, 0.2702520489692688, -0.18003152310848236, -0.1472592055797577, 0.12322838604450226, 0.06834006309509277, 0.09762773662805557, -0.24421003460884094, 0.028390269726514816, 0.10006501525640488, -0.1879129409790039, -0.09024810791015625, -0.30781078338623047, -0.27610719203948975, -0.4185744524002075, -0.09995285421609879, -0.26989030838012695, 2.998867057613097e-05, 0.08965000510215759, 0.03161439672112465, -0.4620310962200165, -0.15352609753608704, -0.015085521154105663, -0.012882659211754799, -0.21650156378746033, -0.07450120151042938, 0.0017500064568594098, -0.17615550756454468, 0.16611100733280182, 0.4839506149291992, 0.2580735683441162, 0.5843495726585388, -0.03792309761047363, -0.7597558498382568, 0.28581270575523376, 0.34370648860931396, -0.2775212228298187, -0.1927517205476761, -0.05782795324921608, 0.531302809715271, 0.013091991655528545, -0.39109402894973755, 0.7627526521682739, 0.27402743697166443, 1.0852731466293335, 0.7287554144859314, 1.0198304653167725, 0.14045833051204681, 0.7262187600135803, -0.1629810929298401, -0.10886450856924057, 0.09457530081272125, 1.436312198638916, 0.2948012351989746, 0.022905699908733368, -0.06204439327120781, 0.8798595666885376, 0.27071356773376465, -0.002340150997042656, 0.0063135651871562, -0.09331214427947998, -0.22082781791687012, -0.07630321383476257, -0.22310210764408112, -0.04170187562704086, 0.2652144134044647, -0.06677854061126709, -0.20531411468982697, 0.1770668774843216, 0.06906794756650925, 0.014665588736534119, -0.12252555787563324, -0.0020381591748446226, 0.10122372955083847, -0.1443362832069397, -0.1402120441198349, -0.3642241060733795, -0.20137101411819458, -0.4015817940235138, -0.049535103142261505, -0.22022710740566254, 0.1713530421257019, -0.01501291710883379, -0.03771420568227768, -0.4427480697631836, -0.09776408225297928, -0.06930934637784958, 0.041822560131549835, -0.2838277816772461, -0.06801681220531464, 0.0568058118224144, 0.10712460428476334, -0.03142566233873367, -0.1527981460094452, -0.1408592313528061, -0.1941431760787964, 0.02238359861075878, 0.23325835168361664, -0.08814969658851624, -0.15853965282440186, 0.16929954290390015, 0.17505784332752228, 0.0004466482496354729, -0.15584638714790344, -0.01723674312233925, 0.1945597380399704, -0.23231516778469086, -0.08262558281421661, -0.3705262243747711, -0.23742437362670898, -0.3981180489063263, -0.010657078586518764, -0.27525511384010315, 0.07801476120948792, 0.036108698695898056, 0.01941603235900402, -0.47653305530548096, -0.050662796944379807, -0.05528461933135986, 0.051888346672058105, -0.30041012167930603, -0.0805874764919281, 0.003819983219727874, 0.10327934473752975, -0.15279440581798553, -0.15217337012290955, -0.16066056489944458, -0.1270865947008133, -0.08726964145898819, 0.20312002301216125, -0.17672409117221832, -0.035311855375766754, 0.028133302927017212, 0.07314961403608322, -0.0068656764924526215, -0.2658524215221405, -0.022393178194761276, 0.1645534485578537, -0.26327842473983765, -0.03943825885653496, -0.3774392902851105, -0.24338781833648682, -0.36733946204185486, -0.027255287393927574, -0.14460976421833038, 0.0693826973438263, 0.03934900462627411, -0.06625185906887054, -0.5134574174880981, -0.028612881898880005, -0.06477890908718109, -0.02424098178744316, -0.2940778434276581, -0.11161389946937561, 0.09804195910692215, -0.061846811324357986, -0.033745672553777695, -0.13269193470478058, -0.14196032285690308, -0.21965628862380981, 0.032998017966747284, 0.33478477597236633, -0.13905580341815948, -0.1962347775697708, 0.14120671153068542, 0.14364105463027954, 0.0031723768915981054, -0.28655365109443665, 0.06706444174051285, 0.12149015814065933, -0.19787009060382843, -0.08866634964942932, -0.37812539935112, -0.266326367855072, -0.27442726492881775, 0.027288861572742462, -0.3464478552341461, 0.0044579217210412025, 0.11848088353872299, -0.05369579792022705, -0.5437070727348328, -0.10159306228160858, -0.014585323631763458, 0.01654885895550251, -0.31318342685699463, -0.10580222308635712, -0.02256288193166256, 0.07316011935472488, -0.09245077520608902, -0.18280968070030212, -0.09313933551311493, -0.12214039266109467, 0.030038075521588326, 0.24782410264015198, -0.0662207305431366, -0.09696774929761887, 0.14697496592998505, 0.09277955442667007, 0.10961949080228806, -0.10863346606492996, -0.007578848861157894, 0.11069119721651077, -0.2099372148513794, -0.06066080555319786, -0.3640715181827545, -0.3175783157348633, -0.43971821665763855, -0.02702077478170395, -0.2010132521390915, 0.09144244343042374, -0.03473164513707161, 0.01653606817126274, -0.5023912787437439, -0.0823952704668045, 0.011667585000395775, -0.018708210438489914, -0.3187715411186218, -0.19395101070404053, -0.04560451954603195, -0.03366231545805931, -0.029945923015475273, -0.16525661945343018, -0.04023303464055061, -0.16960032284259796, -0.0203703660517931, 0.27061915397644043, -0.11255138367414474, -0.0868205726146698, 0.10835187882184982, 0.01802522875368595, 0.020033910870552063, -0.1730976104736328, -0.02091432549059391, 0.1761620044708252, -0.1684124618768692, -0.10487894713878632, -0.3768779933452606, -0.35613390803337097, -0.3545454740524292, -0.06624283641576767, -0.3476143181324005, 0.06569535285234451, 0.05917760357260704, -0.07203646004199982, -0.494314044713974, -0.15014053881168365, 0.028960121795535088, -0.054456088691949844, -0.30486834049224854, 0.007255232892930508, 0.002811903366819024, 0.07986138015985489, -0.05914648622274399, -0.22704875469207764, -0.0875692069530487, -0.16710375249385834, -0.039764564484357834, 0.2703897953033447, -0.03078462742269039, -0.06378918886184692, 0.013445965945720673, 0.01868990994989872, -0.0607600100338459, -0.12230655550956726, 0.024987688288092613, 0.16657370328903198, -0.29798251390457153, -0.14482556283473969, -0.35549768805503845, -0.23868156969547272, -0.3908121883869171, -0.03732490912079811, -0.2073269635438919, 0.06267146021127701, -0.022342035546898842, -0.14096423983573914, -0.4886274039745331, -0.09359860420227051, -0.022171173244714737, -1.3955080930827535e-06, -0.32377755641937256, -0.05451647564768791, -0.002386227250099182, 0.06262581795454025, -0.10038995742797852, -0.1993485391139984, -0.07774574309587479, -0.23769833147525787, -0.05159743130207062, 0.2368515580892563, -0.03298158943653107, -0.1077260747551918, 0.1655632108449936, 0.060877759009599686, 0.00442068837583065, -0.24547618627548218, -0.03434000536799431, 0.2031758725643158, -0.23768749833106995, -0.04983620345592499, -0.3240881562232971, -0.24420802295207977, -0.3583398163318634, -0.03847452253103256, -0.2059902548789978, 0.07416398078203201, 0.05422555282711983, -0.027623916044831276, -0.5141801238059998, -0.11489611864089966, -0.05623963847756386, 0.07164900749921799, -0.3943074941635132, -0.10119157284498215, 0.11959308385848999, 0.039388060569763184, -0.10855300724506378, -0.16922520101070404, -0.021466821432113647, -0.19544996321201324, 0.04203495755791664, 0.2583067715167999, -0.11167377978563309, 0.040240731090307236, 0.09614076465368271, 0.04059600457549095, -0.004575210157781839, -0.15683558583259583, 0.014932091347873211, 0.17001725733280182, -0.2470078021287918, -0.15807317197322845, -0.3327469527721405, -0.25917890667915344, -0.3624045252799988, -0.11857922375202179, -0.268231600522995, 0.014928760007023811, 0.027803748846054077, -0.10800207406282425, -0.4731506109237671, -0.09559954702854156, -0.06515537947416306, -0.009816707111895084, -0.33208295702934265, -0.044917650520801544, -0.12618069350719452, 0.039298541843891144, -0.13484011590480804, -0.18991103768348694, -0.04359859600663185, -0.21895800530910492, 0.018286949023604393, 0.21107037365436554, 0.003944001160562038, -0.0716138407588005, 0.14894279837608337, 0.0010448184330016375, -0.035129986703395844, -0.12187644094228745, -0.015247187577188015, 0.1185205727815628, -0.23737584054470062, -0.14925110340118408, -0.3206997215747833, -0.2428145557641983, -0.32413750886917114, -0.0779644027352333, -0.213893860578537, 0.016026590019464493, 0.07734597474336624, 0.021151185035705566, -0.43008914589881897, -0.07599316537380219, 0.022523829713463783, -0.11388026177883148, -0.24746108055114746, -0.10400036722421646, 0.004979756195098162, 0.042279478162527084, -0.12137062847614288, -0.19525912404060364, -0.053008899092674255, -0.11655896157026291, -0.019778594374656677, 0.20811054110527039, -0.020442014560103416, -0.1614021211862564, 0.19626502692699432, 0.06998632103204727, -0.024246495217084885, -0.12664742767810822, 0.07913866639137268, 0.0596792995929718, -0.17611689865589142, -0.10056252032518387, -0.28502658009529114, -0.262753963470459, -0.42695915699005127, -0.019335348159074783, -0.19322645664215088, 0.156068816781044, 0.11580682545900345, -0.0005690217949450016, -0.48756879568099976, -0.12784892320632935, -0.023910630494356155, -0.020130500197410583, -0.30293288826942444, -0.21752475202083588, 0.00917590968310833, 0.05501803010702133, 0.0764799416065216, -0.13039040565490723, -0.10994665324687958, -0.15919071435928345, -0.049920786172151566, 0.22317451238632202, -0.1864098608493805, -0.04546068236231804, 0.12225022912025452, 0.008674676530063152, 0.016899576410651207, -0.12837590277194977, 0.019226649776101112, 0.15053053200244904, -0.2969571053981781, -0.11835451424121857, -0.3545854389667511, -0.19771796464920044, -0.3895428478717804, -0.10050370544195175, -0.18235386908054352, 0.004755054134875536, 0.08348502963781357, -0.023178381845355034, -0.43163105845451355, -0.014597336761653423, 0.025266192853450775, 0.06021770462393761, -0.3350970447063446, -0.125057652592659, 0.032429832965135574, 0.13317783176898956, 0.017791323363780975, -0.05249419063329697, -0.07039110362529755, -0.1772882491350174, 0.061677612364292145, 0.2542547881603241, -0.09615354984998703, -0.16964191198349, 0.14214423298835754, 0.013854001648724079, 0.045536961406469345, -0.2468230277299881, 0.049288053065538406, 0.07997629046440125, -0.21189089119434357, -0.16595903038978577, -0.3813338577747345, -0.1848134696483612, -0.38836678862571716, -0.041074153035879135, -0.2634769082069397, -0.06420084089040756, 0.0450298972427845, -0.013718199916183949, -0.5310720205307007, -0.07313203066587448, -0.04895173758268356, -0.02875373139977455, -0.2888626158237457, -0.12436912208795547, 0.11485418677330017, -0.021281294524669647, -0.01489177905023098, -0.11888209730386734, -0.1848991960287094, -0.27405163645744324, 0.03948706388473511, 0.2292335331439972, -0.07385198771953583, -0.06030130758881569, 0.0739758163690567, 0.014255605638027191, -0.1165480986237526, -0.16563433408737183, -0.0519595593214035, 0.14708594977855682, -0.24248388409614563, -0.15571564435958862, -0.2731492519378662, -0.18956783413887024, -0.30568593740463257, -0.10365734994411469, -0.2599268853664398, 0.023924194276332855, 0.008888917043805122, -0.0915365144610405, -0.45395106077194214, -0.09035467356443405, 0.00910162553191185, -0.0038813776336610317, -0.22465437650680542, -0.0809113010764122, -0.13665850460529327, 0.024459095671772957, -0.05466387793421745, -0.12006116658449173, -0.10097158700227737, -0.19993171095848083, -0.021773293614387512, 0.28658679127693176, -0.1154094859957695, -0.1474461704492569, 0.05547543242573738, 0.09528296440839767, -0.006724776700139046, -0.25965920090675354, 0.025604059919714928, 0.09634987264871597, -0.23621487617492676, -0.06780300289392471, -0.29916271567344666, -0.19150422513484955, -0.37685099244117737, -0.05275293067097664, -0.3040756583213806, 0.13513730466365814, 0.05170830339193344, 0.034389812499284744, -0.49163323640823364, -0.20427356660366058, 0.0674186497926712, 0.02607199177145958, -0.2825460433959961, -0.08752521872520447, 0.03246621415019035, 0.08312144875526428, -0.1313406080007553, -0.11398278921842575, -0.11238332837820053, -0.2107798457145691, 0.09773895889520645, 0.2993370592594147, -0.10993940383195877, -0.21009327471256256, 0.10066509246826172, 0.09438668191432953, 0.015897180885076523, -0.23816627264022827, -0.04506317526102066, 0.06865562498569489, -0.24329517781734467, -0.045521412044763565, -0.30807310342788696, -0.28034213185310364, -0.3363482356071472, -0.016730213537812233, -0.26511281728744507, 0.04384529963135719, 0.13582025468349457, -0.025074973702430725, -0.4315354824066162, -0.12355805933475494, -0.0070791118778288364, -0.06430033594369888, -0.2904587686061859, -0.03068447671830654, 0.10517743229866028, 0.11629683524370193, -0.017470981925725937, -0.16567924618721008, -0.1159340962767601, -0.19544100761413574, 0.042009320110082626, 0.2140519618988037, -0.0463738776743412, -0.1664794683456421, 0.003621034324169159, 0.016848938539624214, -0.06334507465362549, -0.24448110163211823, 0.051077891141176224, 0.12140908092260361, -0.2431359887123108, -0.028586579486727715, -0.2963879704475403, -0.26896053552627563, -0.28167128562927246, -0.02520381473004818, -0.23938903212547302, 0.07337561994791031, -0.04165438935160637, -0.07463618367910385, -0.4810905456542969, -0.0713309645652771, -0.027768535539507866, 0.015050173737108707, -0.2407335638999939, -0.01971329376101494], "scores": [[-0.9694600783213314, -0.9752350769014719, -0.973758606385061, -0.966433897194902, -0.9734132196262224, -0.9659014678632705, -0.9686524463000995, -0.9669882938189318, -0.9711593935656206, -0.9659494805834077, -0.9733545479202754, -0.9700626343538978, -0.9720618181756813, -0.965633132187245, -0.9632746621569854, -0.9779176000293075, -0.9768870964106867, -0.9663532918801406, -0.9568715846779827, -0.970036526517887, 0.9985416425392378, -0.953014913879445, -0.9692310226185358, -0.9731955438080574, -0.9741229112928698, -0.9640142376101226, -0.9722142829786493, -0.9622092195616353, -0.972610201227328, -0.9630844469985969, -0.9730115811721509, -0.9623584745215201, -0.9511971638060007, -0.9709920411785178, -0.9562070605689867, -0.9725104847968804, -0.9792861693355663, -0.963894374672333, -0.9600769919889249, -0.9619309527586393], [-0.9451211077581689, -0.9624914406232582, -0.928466839645338, -0.9584819294009858, -0.9431688799500514, -0.9630497131120129, -0.9449949140364544, -0.9619267458340851, -0.9684092459145898, -0.9769459078666201, -0.9668266188837207, -0.9697750298257446, -0.9633083171605605, -0.9551538623336018, -0.9680244296374251, -0.9509572002975658, -0.967667452869065, -0.9673899467178897, -0.9709678328587845, -0.9734796421599841, -0.9776098248793852, 0.9983989460907801, -0.9702580770268087, -0.9639109054908185, -0.956084754733136, -0.960224942235697, -0.9723209551288166, -0.9597877652244731, -0.9572348641178676, -0.9605301488044723, -0.9765557616484043, -0.9654048791940326, -0.9415620910109193, -0.9681822565203615, -0.969526037375213, -0.9642371850925143, -0.9803193364876406, -0.9543638803708816, -0.9732130503747424, -0.9576702818899628], [-0.9697407023258322, -0.9558892615252941, -0.9574164524036887, -0.960595090237812, -0.9697689444666063, -0.9619511064781203, -0.9624800312578798, -0.9581831284691824, -0.96537683368794, -0.9496645594351372, -0.9708727779731668, -0.9628236397298706, -0.9669042883056691, -0.9585915205987804, -0.9681327992263484, -0.9624453006115443, -0.9651770900349347, -0.9773293324508712, -0.9628865060798997, -0.9614761463664547, -0.9671587003269966, -0.9628376376333152, 0.9986904404283573, -0.9649126843306594, -0.978835332295634, -0.9616748640937511, -0.9624178719263404, -0.9691334178728641, -0.9679234094530359, -0.9737758951016443, -0.9814342315139092, -0.9644800018631186, -0.9542614346294989, -0.9543213793366342, -0.9590872175249663, -0.963177084416209, -0.952439643007878, -0.9646595539138472, -0.9664968905571389, -0.9626895310570158]], "kept": [[0, 1, 2, 3, 20, 21, 32, 37, 38, 39], [0, 1, 2, 3, 4, 21, 32, 37, 38, 39], [0, 1, 2, 3, 9, 22, 36, 37, 38, 39]]}], "rope": {"head_dim": 16, "rope_theta": 1000000.0, "abs_pos": 137, "rank": 5, "pre_rope": [0.08249430428370294, -0.46441841495421887, 0.05051506297463688, 0.6862308196812632, -1.7567905055789348, 1.6844316011395088, -0.4578428392637714, -0.5964200946055478, -1.046967562282426, 0.9317920227947954, 0.6749804835796053, 1.2444412018021018, 0.893087422223549, 0.26300494250388173, 0.3285178485491658, 0.9352436940748697], "post_rope_at_abs": [-0.9592153201489803, 0.3155531917495639, 0.6080730435707898, -0.3742057991623469, -1.862300270732207, 1.6775249421526817, -0.45926178276742863, -0.5971404364025743, -0.4276123902582183, 0.9921426414932828, -0.2973173363314788, 1.3709546536980524, 0.6447912265663527, 0.30395968014610286, 0.32653124991303284, 0.9347839299907986], "post_rope_at_rank": [-0.9805620956161258, -1.0161853371605156, -0.056394719209451605, 0.6509741554351701, -1.761233964248518, 1.6841970872033156, -0.45789477677319124, -0.5964463906787036, -0.37609089772522947, 0.2264689789630685, 0.6745146851225527, 1.2632415808460964, 0.8842923427258904, 0.26450253336168084, 0.32844545313387075, 0.9352269241218659]}} \ No newline at end of file diff --git a/experiments/niah/gen_kv_compress_fixtures.py b/experiments/niah/gen_kv_compress_fixtures.py deleted file mode 100644 index 24c2965c5..000000000 --- a/experiments/niah/gen_kv_compress_fixtures.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Generate cross-check fixtures for the C++ KeyDiff KV-compression port. - -Produces a JSON fixture file consumed by cactus-engine/tests/test_kv_compress.cpp. -Each fixture holds a random PRE-RoPE key matrix (n_kv_heads, n, head_dim) plus the -reference outputs from the Python source of truth so the C++ port can be checked -bit-for-bit: - - * keydiff_score (diagnostic_separability.py) -> per-head [n] scores - * build_geom_keepset (geom_keepset.py) -> per-head kept index lists - -A separate "rope" fixture provides a single pre-RoPE vector + the rotate_half RoPE -applied at a chosen rank position, mirroring the cache's post-RoPE storage convention -(`out[i] = x[i]*cos - x[i+half]*sin`, `out[i+half] = x[i+half]*cos + x[i]*sin`, -`angle = pos * theta^(-2i/head_dim)`), so the C++ Route-B renumber math can be checked. - -The generated fixture JSON (~360 KB) is checked into the repo, so the C++ test runs -with no Python step. Regenerating it requires the untracked MLX KeyDiff harness -(diagnostic_separability.py / geom_keepset.py) to be importable on the path. - -Run: python experiments/niah/gen_kv_compress_fixtures.py -Writes: experiments/niah/fixtures/kv_compress_fixtures.json -""" -from __future__ import annotations - -import json -from pathlib import Path - -import numpy as np - -from diagnostic_separability import keydiff_score -from geom_keepset import build_geom_keepset - -OUT = Path(__file__).parent / "fixtures" / "kv_compress_fixtures.json" - -# (name, n, head_dim, budget_frac, recent_frac, sink, seed) -# Kept small (n<=48, head_dim<=32) so the fixture covers the budget/recent/sink edge cases, -# the reserved-overflow fallback, and the planted-outlier geometry pick the tests assert on. -GRID = [ - ("base_keydiff", 48, 16, 0.25, 0.30, 4, 0), - ("small_n_keydiff", 20, 8, 0.25, 0.30, 4, 2), - ("tight_budget", 40, 8, 0.10, 0.30, 4, 3), - ("reserved_overflow", 30, 8, 0.20, 0.80, 8, 4), - ("full_budget", 40, 16, 1.00, 0.30, 4, 5), - ("big_dim_keydiff", 40, 32, 0.25, 0.30, 4, 7), -] - - -def _outlier_keys(n_kv_heads, n, d, rng): - """Tight cluster around one centroid direction, with planted mid-context outliers - so the geometry selection is exercised (not just sink/recent).""" - base = rng.normal(size=(d,)) - base /= np.linalg.norm(base) - keys = base[None, None, :] + 0.05 * rng.normal(size=(n_kv_heads, n, d)) - mid = n // 2 - for h in range(n_kv_heads): - keys[h, mid + h] = -3.0 * base + 0.01 * rng.normal(size=(d,)) - return keys.astype(np.float32) - - -def _budget(budget_frac, n): - # Mirror the live harness: B = round(budget_frac * n), matching build_geom_keepset's - # `budget` argument and keepset_for_head's B = min(budget, n). - return max(1, int(round(budget_frac * n))) - - -def make_fixture(name, n, d, bf, rf, sink, seed): - rng = np.random.default_rng(seed) - keys = _outlier_keys(2 + (seed % 3), n, d, rng) - n_kv_heads = keys.shape[0] - budget = _budget(bf, n) - - scores = [] - for h in range(n_kv_heads): - kh = np.asarray(keys[h], dtype=np.float64) - scores.append([float(x) for x in keydiff_score(kh)]) - - kept = build_geom_keepset(keys, budget=budget, signal="keydiff", sink=sink, - recent_frac=rf) - return { - "name": name, - "n": n, - "head_dim": d, - "n_kv_heads": n_kv_heads, - "budget": budget, - "recent_frac": rf, - "sink": sink, - "keys": keys.reshape(-1).astype(np.float32).tolist(), - "scores": scores, - "kept": kept, - } - - -def _rope_at(vec, pos, theta): - """rotate_half RoPE applied to a single (head_dim,) vector at position `pos`, - matching cactus-kernels/src/norms_rope.cpp.""" - d = vec.shape[0] - half = d // 2 - out = np.empty_like(vec) - for i in range(half): - inv = theta ** (-(2.0 * i) / d) - ang = pos * inv - c, s = np.cos(ang), np.sin(ang) - x1, x2 = vec[i], vec[i + half] - out[i] = x1 * c - x2 * s - out[i + half] = x2 * c + x1 * s - return out - - -def make_rope_fixture(): - rng = np.random.default_rng(99) - d, theta = 16, 1000000.0 - vec = rng.normal(size=(d,)).astype(np.float64) - abs_pos, rank = 137, 5 - post_rope_at_abs = _rope_at(vec, abs_pos, theta) - post_rope_at_rank = _rope_at(vec, rank, theta) - return { - "head_dim": d, - "rope_theta": theta, - "abs_pos": abs_pos, - "rank": rank, - "pre_rope": vec.tolist(), - "post_rope_at_abs": post_rope_at_abs.tolist(), - "post_rope_at_rank": post_rope_at_rank.tolist(), - } - - -def main(): - OUT.parent.mkdir(parents=True, exist_ok=True) - fixtures = [make_fixture(*spec) for spec in GRID] - payload = {"fixtures": fixtures, "rope": make_rope_fixture()} - OUT.write_text(json.dumps(payload)) - print(f"wrote {len(fixtures)} fixtures + rope to {OUT}") - - -if __name__ == "__main__": - main() From f603b4c54273ceea5eec187a8aa9bcc602512fa9 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 19:41:08 +0100 Subject: [PATCH 12/55] Trim excessive comments across KV-compress and rope-table changes Cut comments that restate self-evident code; keep only non-obvious WHY (banker's rounding edge case, per-layer re-rope theta source-layer subtlety, fp16 rope-angle precompute rationale). Signed-off-by: Noah Cylich --- cactus-engine/src/engine.h | 9 +- cactus-engine/src/kv_compress.cpp | 34 ++--- cactus-engine/src/kv_compress.h | 80 ++++------- cactus-engine/src/model.cpp | 50 +++---- .../tests/test_kv_compress_free_functions.cpp | 136 ++++++------------ cactus-engine/tests/test_utils.h | 3 +- python/cactus/transpile/component_plan.py | 6 +- 7 files changed, 103 insertions(+), 215 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index e8c8253a8..3932f086e 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -237,9 +237,8 @@ struct Config { bool from_json(const std::string& json_path); std::string to_json() const; - // Disable rolling unless 0 < kv_compress_target_len < kv_compress_trigger_len (when trigger > 0). + // Disable rolling unless 0 < target < trigger (when trigger > 0). void validate_kv_compress(); - // Apply CACTUS_KV_COMPRESS_AT / CACTUS_KV_COMPRESS_TO overrides (nullptr = unset). Returns true if applied. bool parse_kv_compress_override(const char* trigger_env, const char* target_env); }; @@ -634,14 +633,10 @@ class Model { void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} - // Rolling KeyDiff compaction: keep abs_budget survivors per (layer, kv-head), renumber to - // 0..B-1; refuses unless every layer is compressible. See impl for the full contract. + // Keep abs_budget survivors per (layer, kv-head) and renumber to 0..B-1. void compress_kv_cache_keydiff(const cactus::kvcompress::Params& params); - // Compact to kv_compress_target_len once the live KV seq-len reaches kv_compress_trigger_len. void maybe_roll_compact(); - // Layer indices (cache_states order) that are physically compressible. std::vector compressible_layers() const; - // Apply CACTUS_KV_COMPRESS_AT / CACTUS_KV_COMPRESS_TO env overrides onto config_. void apply_kv_compress_env_override(); void set_tool_constraints(const std::vector& tools); diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index b575a84df..b999ec51b 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -10,24 +10,22 @@ namespace kvcompress { namespace { -// Python's round()/numpy round half-to-even. round(recent_frac*B) must match the reference -// exactly, so reproduce banker's rounding rather than std::round (which rounds half away from -// zero). recent_frac arrives as float and widens to a slightly-off double (e.g. 0.30f -> -// 0.30000001...), which can flip the rounding of an exact half boundary (0.3*15 = 4.5 but -// 0.30000001*15 = 4.50000018). Snap values within a tiny epsilon of a half-integer back onto it -// before banker's rounding so the result matches the Python double reference exactly. +// Python/numpy round half-to-even (not std::round's half-away-from-zero). recent_frac arrives as +// float and widens to a slightly-off double (0.30f -> 0.30000001...) which can flip an exact half +// boundary (0.3*15 = 4.5 vs 0.30000001*15 = 4.50000018), so snap near-halves before rounding to +// match the Python double reference. long py_round(double x) { double twice = x * 2.0; double twice_rounded = std::nearbyint(twice); if (std::fabs(twice - twice_rounded) < 1e-6) x = twice_rounded * 0.5; - double r = std::nearbyint(x); // honors the default FE_TONEAREST (round-half-to-even) + double r = std::nearbyint(x); // FE_TONEAREST default == round-half-to-even return static_cast(r); } } // namespace void keydiff_score(const float* keys, size_t n, size_t head_dim, float* out) { - // mu = mean key; s_i = -cos(k_i, mu). Computed in double to match the float64 reference. + // s_i = -cos(k_i, mean(k)), in double to match the float64 reference. std::vector mu(head_dim, 0.0); for (size_t i = 0; i < n; ++i) { const float* k = keys + i * head_dim; @@ -66,8 +64,7 @@ std::vector keepset_for_head(const float* scores, size_t n, const Params& p for (long i = static_cast(n) - n_recent; i < static_cast(n); ++i) if (i >= 0) reserved.insert(i); - // Reserved may exceed B (large recent_frac + sink on a tight budget): keep sink first, - // then the most-recent tokens, until B. + // Reserved can exceed B on a tight budget: keep sink first, then most-recent, until B. if (static_cast(reserved.size()) > B) { std::vector ordered; for (long i = 0; i < sink; ++i) ordered.push_back(i); @@ -82,7 +79,6 @@ std::vector keepset_for_head(const float* scores, size_t n, const Params& p std::set keep(reserved); long remaining = B - static_cast(keep.size()); if (remaining > 0) { - // argsort(-scores, kind="stable"): descending score, ties broken by ascending index. std::vector order(n); std::iota(order.begin(), order.end(), 0L); std::stable_sort(order.begin(), order.end(), [&](long a, long b) { @@ -101,9 +97,6 @@ std::vector keepset_for_head(const float* scores, size_t n, const Params& p return result; } -// --------------------------------------------------------------------------- // -// RoPE (rotate_half) helpers // -// --------------------------------------------------------------------------- // void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos) { size_t half = head_dim / 2; for (size_t i = 0; i < half; ++i) { @@ -128,15 +121,12 @@ void unrope_head(const float* post_rope, size_t n, size_t head_dim, double rope_ namespace { -// Net renumber rotation for a survivor at absolute position `abs_pos` -> rank: rotate the -// stored post-RoPE row by (rank - abs_pos). Equivalent to un-RoPE then re-RoPE at rank. inline double renumber_delta(int abs_pos, size_t rank) { return static_cast(rank) - static_cast(abs_pos); } -// Quantize one float row into int8 + per-group scales, matching the engine's KV quant convention -// (quantize_group_fp16_to_int8 in cactus-kernels/src/quants.cpp): scale floored at 1e-10, -// round-to-nearest via roundf, clamp to [-128, 127]. groups = ceil(head_dim/group_size). +// Matches the engine's KV quant convention (quantize_group_fp16_to_int8 in +// cactus-kernels/src/quants.cpp): scale floored at 1e-10, roundf, clamp to [-128, 127]. void requant_row(const float* row, int8_t* dst, float* dsc, size_t head_dim, size_t group_size) { size_t groups = (head_dim + group_size - 1) / group_size; for (size_t g = 0; g < groups; ++g) { @@ -155,9 +145,8 @@ void requant_row(const float* row, int8_t* dst, float* dsc, size_t head_dim, siz } } -// Per-head KeyDiff keep-set pipeline shared by the fp16 and int8 entry points. `fill_post(h, post)` -// gathers head h's post-RoPE rows into `post` ([n][head_dim]); the rest (un-RoPE -> score -> keepset) -// is identical regardless of storage precision. +// `fill_post(h, post)` gathers head h's post-RoPE rows into `post` ([n][head_dim]); the rest is +// shared across storage precisions. template std::vector> keepsets_per_head(size_t n, size_t kv_heads, size_t head_dim, double rope_theta, const Params& p, FillPost fill_post) { @@ -187,7 +176,6 @@ void compact_fp16(uint16_t* key_rows_u, uint16_t* val_rows_u, size_t kv_heads, s const __fp16* ksrc = key_rows + (static_cast(abs_pos) * kv_heads + h) * head_dim; const __fp16* vsrc = val_rows + (static_cast(abs_pos) * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) { krow[d] = ksrc[d]; vrow[d] = vsrc[d]; } - // Renumber K: rotate post-RoPE row by (rank - abs_pos). V is not rotated. rope_rotate_row(krow.data(), head_dim, rope_theta, renumber_delta(abs_pos, rank)); __fp16* kdst = key_rows + (rank * kv_heads + h) * head_dim; __fp16* vdst = val_rows + (rank * kv_heads + h) * head_dim; diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index 0a596d2d2..7b244c4f9 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -15,7 +15,7 @@ struct Params { int abs_budget = 0; // keep budget B = min(max(1, abs_budget), n) per (layer, kv-head) }; -// 64-byte KV cache header at the front of each K/V buffer (mirrors cactus-graph CacheMetadata). +// 64-byte KV cache header mirroring cactus-graph CacheMetadata. struct CacheHeader { uint64_t current_seq_len; uint64_t max_seq_len; @@ -27,96 +27,72 @@ struct CacheHeader { static_assert(sizeof(CacheHeader) == 64, "CacheHeader must be 64 bytes"); // KeyDiff per-token score for one (layer, kv-head): s_i = -cos(k_i, mean(k)). -// keys: row-major [n][head_dim] (pre-RoPE). out: [n]. Mirrors keydiff_score(). +// keys: row-major [n][head_dim] (pre-RoPE). out: [n]. void keydiff_score(const float* keys, size_t n, size_t head_dim, float* out); -// KeyDiff keep-set for ONE cell. Returns EXACTLY B = min(max(1, abs_budget), n) sorted unique -// indices: sink + recent + top-score middle. Mirrors keepset_for_head() EXACTLY, -// including the |reserved|>B fallback (sink-first then most-recent). Top-B uses a stable -// descending-score sort, ties broken by ascending index, to match np.argsort(-scores). +// Keep-set for one cell: exactly B = min(max(1, abs_budget), n) sorted unique indices +// (sink + recent + top-score middle). Top-B sort is stable descending-score, ties broken +// by ascending index, to match np.argsort(-scores). std::vector keepset_for_head(const float* scores, size_t n, const Params& p); -// --------------------------------------------------------------------------- // -// Route-B RoPE un-rotation / renumber (rotate_half / NeoX convention) // -// --------------------------------------------------------------------------- // -// The cache stores POST-RoPE keys. RoPE here is the rotate_half layout used by -// cactus-kernels/src/norms_rope.cpp: for pair (i, i+half), angle = pos * theta^(-2i/d), +// The cache stores POST-RoPE keys (rotate_half layout, see cactus-kernels/src/norms_rope.cpp): +// for pair (i, i+half), angle = pos * theta^(-2i/d), // out[i] = x[i]*cos - x[i+half]*sin // out[i+half] = x[i+half]*cos + x[i]*sin -// Rotating a post-RoPE row by angle for position `delta_pos` gives the row as if RoPE'd at -// (orig_pos + delta_pos). So un-RoPE = rotate by -orig_pos; renumber = rotate by (rank-orig). - -// In-place delta rotation of one key row of length head_dim by `delta_pos`. +// Rotating a post-RoPE row by `delta_pos` yields the row as if RoPE'd at (orig_pos + delta_pos), +// so un-RoPE = rotate by -orig_pos and renumber = rotate by (rank - orig). void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos); -// INT8 analog of rope_rotate_row: dequantize one quantized key row (int8 * per-group scale), -// rotate by `delta_pos`, and re-quantize in place. `scale` is the row's [groups] scales, -// groups = ceil(head_dim/group_size). +// INT8 analog of rope_rotate_row: dequant (int8 * per-group scale), rotate, re-quantize in place. +// `scale` is the row's [groups] scales, groups = ceil(head_dim/group_size). void rotate_int8_row(int8_t* int8, float* scale, size_t head_dim, size_t group_size, double rope_theta, double delta_pos); -// Recover the pre-RoPE keys for one kv-head from post-RoPE rows. `post_rope` is -// [n][head_dim], rows at absolute positions 0..n-1. Writes pre_rope [n][head_dim]. void unrope_head(const float* post_rope, size_t n, size_t head_dim, double rope_theta, float* pre_rope); -// --------------------------------------------------------------------------- // -// Per-head physical compaction + renumber on a live cache buffer (post-RoPE). // -// Mirrors run_niah_renumber.compact_renumber_kv: gather survivors in rank order, -// renumber K to contiguous positions 0..B-1, gather V unchanged (no RoPE). -// kept_per_head[h] is the sorted survivor index list for kv-head h (length B, equal -// across heads). header.current_seq_len is set to B by the caller. -// --------------------------------------------------------------------------- // - -// FP16 cache. key_rows/val_rows point past the 64-byte header at row 0. -// Layout: [max_seq][kv_heads][head_dim] fp16. K is renumbered, V is not. +// Per-head physical compaction: gather survivors in rank order, renumber K to contiguous +// positions 0..B-1, gather V unchanged. kept_per_head[h] is the survivor index list for +// kv-head h (length B, equal across heads); the caller sets header.current_seq_len to B. +// key_rows/val_rows point past the 64-byte header; layout [max_seq][kv_heads][head_dim]. void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, const std::vector>& kept_per_head, double rope_theta); // INT8 buffer (one of K or V). int8 region [max_seq][kv_heads*head_dim]; scales -// [max_seq][kv_heads*groups] float, groups = ceil(head_dim/group_size). Dequant = -// int8 * scale[group]. When `renumber` is true (the K buffer) each gathered row is -// dequantized, renumbered (rotate by rank-abs), and re-quantized per group; when false -// (the V buffer) rows are gathered and re-quantized as-is (no rotation). +// [max_seq][kv_heads*groups], groups = ceil(head_dim/group_size). `renumber` true (K buffer) +// rotates each gathered row by (rank - abs); false (V buffer) gathers as-is. void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, const std::vector>& kept_per_head, double rope_theta, bool renumber); -// Sliding-window re-rope (single position frame). Rotate the recent K rows [lo, hi) in place by -// `delta_pos` (typically negative) so a windowed cache tracks the renumbered global frontier after -// compaction. Rows [0, lo) (the attention sink) are left fixed; V is never rotated. `rope_theta` is -// the LOCAL theta for sliding layers. No-op when delta_pos == 0 or hi <= lo. The int8 variant takes -// the K int8 region + its per-group scales (same layout as compact_int8). +// Rotate recent K rows [lo, hi) by `delta_pos` so a windowed cache tracks the renumbered global +// frontier after compaction. Rows [0, lo) (the sink) stay fixed; V is never rotated. `rope_theta` +// is the LOCAL theta for sliding layers. No-op when delta_pos == 0 or hi <= lo. void rerope_recent_fp16(uint16_t* key_rows, size_t kv_heads, size_t head_dim, size_t lo, size_t hi, double rope_theta, double delta_pos); void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos); -// Build per-head keep-sets for one layer from POST-RoPE FP16 key rows: un-RoPE -> score -> -// keepset_for_head. Returns n_kv_heads lists, each length B. +// Per-head keep-sets for one layer from POST-RoPE FP16 key rows: un-RoPE -> score -> keepset. std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, size_t kv_heads, size_t head_dim, double rope_theta, const Params& p); -// Same as keepsets_from_fp16 but for the INT8 K buffer: dequantize each row -// (int8 * scale[group]) -> un-RoPE -> score -> keepset_for_head. group_size is the KV -// quantization group size; scale_rows is [max_seq][kv_heads*groups], groups = ceil(head_dim/group_size). +// As keepsets_from_fp16 but dequantizing the INT8 K buffer first. scale_rows is +// [max_seq][kv_heads*groups], groups = ceil(head_dim/group_size). std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, size_t n, size_t kv_heads, size_t head_dim, size_t group_size, double rope_theta, const Params& p); -// True if layer `li` is a sliding-window (local) layer. Its key cache re-ropes with the LOCAL -// theta during compaction; full-attention layers -- including KV-shared global *source* layers -// that are excluded from compaction yet still ride the renumbered frame -- return false and -// re-rope with the GLOBAL theta. Out-of-range (empty layer_types -> Qwen) is full/global. +// Sliding (local) layers re-rope with the LOCAL theta during compaction; full-attention layers -- +// including KV-shared global *source* layers that are excluded from compaction yet still ride the +// renumbered frame -- return false and re-rope with the GLOBAL theta. Empty layer_types -> global. bool is_sliding_layer(const std::vector& layer_types, size_t li); -// Physically-compressible layer indices, mirroring MLX compressible_layers(physical=True). -// `layer_types` token contains "sliding" for local layers; everything else is full/global. -// Empty layer_types -> all layers full (Qwen). num_kv_shared drops the shared consumer -// range and the per-type source layers (Gemma -> {4,9}). +// Physically-compressible layer indices. Empty layer_types -> all layers (Qwen). num_kv_shared +// drops the shared consumer range and the per-type source layers (Gemma -> {4,9}). std::vector physical_compressible_layers(const std::vector& layer_types, size_t num_layers, size_t num_kv_shared); diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 043d6d1b5..a9461cc2f 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -362,9 +362,6 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, CACTUS_LOG_ERROR("model", "Failed to load config.txt from: " << bundle_dir); return false; } - // Runtime KV-compression override (bundle config unchanged). Rolling bounded compaction is the - // default (4096 -> 2048); CACTUS_KV_COMPRESS_AT (trigger) / CACTUS_KV_COMPRESS_TO (target) tune - // it, and CACTUS_KV_COMPRESS_AT=0 disables it. apply_kv_compress_env_override(); if (!load_manifest()) { CACTUS_LOG_ERROR("model", "Failed to load bundle manifest from: " << bundle_dir); @@ -1571,8 +1568,7 @@ void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; if (prepare_decode) { - // Rolling bounded compaction (self-guards on kv_compress + trigger_len). Does NOT fire - // mid-chunk: the prompt grows to its full length here, then this bounds it to target_len. + // After the prompt reaches full length here -- never mid-chunk -- bound it to target_len. maybe_roll_compact(); } } @@ -2318,8 +2314,6 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* } run_step(tokens.back(), cache_total_seq_len_ + tokens.size() - 1, /*read_logits=*/true); cache_total_seq_len_ += tokens.size(); - // Rolling bounded cache: if generation advanced the cache to trigger_len, compact to - // target_len before the next step. No-op unless kv_compress + trigger_len > 0. maybe_roll_compact(); uint32_t result = argmax_last_logits(out_entropy); record_sampled_token(result); @@ -2934,38 +2928,32 @@ std::vector Model::compressible_layers() const { config_.layer_types, config_.num_layers, shared); } -// Rolling KeyDiff compaction: keep params.abs_budget survivors per (layer, kv-head) -- attention -// sink + recent window + the most distinctive middle tokens by KeyDiff key geometry -- physically -// compact them and renumber their RoPE positions to a contiguous window 0..B-1. Refuses (no-op -// with a warning) unless EVERY layer is compressible, since the engine tracks a single global -// position counter and a subset compaction would desync the non-compacted layers. +// Keep params.abs_budget survivors per (layer, kv-head) -- sink + recent + most distinctive middle +// tokens by KeyDiff -- compact them, and renumber RoPE positions to a contiguous window 0..B-1. void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) { if (!decoder_) return; using cactus::kvcompress::CacheHeader; constexpr size_t kHeaderBytes = sizeof(CacheHeader); std::vector layers = compressible_layers(); - if (layers.empty()) return; // no global layers to anchor the compressed frame + if (layers.empty()) return; std::set compressible(layers.begin(), layers.end()); const double rope_theta = static_cast(config_.rope_theta); - // Sliding-window layers re-rope into the compressed frame with their LOCAL theta (pass 2). const double rope_local_theta = (config_.rope_local_base_freq == Config::UNSET_F32) ? rope_theta : static_cast(config_.rope_local_base_freq); - const size_t old_total = cache_total_seq_len_; // pre-compaction compressed frontier (for Δ) + const size_t old_total = cache_total_seq_len_; Component& comp = *decoder_; if (!comp.graph) return; - // Pass 1: compact + renumber the global (full-attention) layers to 0..B-1. B is identical across - // them; capture it from the keep-set, not a header we just mutated. + // B is identical across the compacted layers; capture it from the keep-set, not a mutated header. size_t new_seq_len = 0; bool have_new_seq_len = false; for (size_t li = 0; li < comp.cache_states.size(); ++li) { - if (!compressible.count(li)) continue; // global layers only + if (!compressible.count(li)) continue; const auto& cs = comp.cache_states[li]; if (cs.key_node_id < 0 || cs.value_node_id < 0) continue; - // Only attention KV caches. Hybrid models (LFM2) put conv + gated-deltanet recurrent cache - // states in the same list (different / header-less layout); KeyDiff would corrupt them and - // read out of bounds, so skip anything that is not a KV_CACHE_STATE. + // Skip non-KV caches: hybrid models (LFM2) interleave header-less conv/recurrent states + // here, which KeyDiff would corrupt and read out of bounds. if (comp.graph->get_node_op_type(static_cast(cs.key_node_id)) != OpType::KV_CACHE_STATE) continue; const auto& kdesc = comp.graph->get_output_buffer(static_cast(cs.key_node_id)); @@ -2995,7 +2983,6 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) have_new_seq_len = true; } else if (kdesc.precision == Precision::INT8) { size_t max_seq = khdr->max_seq_len; - // Dequantize + un-RoPE K to derive keep-sets (post-RoPE int8 -> pre-RoPE float). auto* k_i8 = reinterpret_cast(static_cast(kraw) + kHeaderBytes); auto* k_sc = reinterpret_cast(static_cast(kraw) + kHeaderBytes + max_seq * kv_heads * head_dim); @@ -3016,19 +3003,18 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } } - // Pass 2: re-rope the non-compacted layers' recent K rows by -Δ (Δ = old frontier − B) so they - // track the renumbered global frontier in the same (compressed) position frame. RoPE is relative, - // so a uniform shift preserves all query·key offsets. Sink rows [0, sink_size) stay fixed. Each - // layer shifts with its OWN theta: sliding layers use the local theta; full-attention layers that - // were excluded from compaction (KV-shared global *source* layers, e.g. Gemma) keep the global theta. + // Shift the non-compacted layers' recent K rows by -Δ (Δ = old frontier − B) into the same + // compressed frame; RoPE is relative, so a uniform shift preserves all query·key offsets, and + // sink rows stay fixed. Each layer shifts with its OWN theta: sliding layers use the local theta, + // while KV-shared global *source* layers (excluded from compaction, e.g. Gemma) keep the global theta. const size_t Delta = (have_new_seq_len && old_total >= new_seq_len) ? old_total - new_seq_len : 0; if (Delta > 0) { const double dpos = -static_cast(Delta); for (size_t li = 0; li < comp.cache_states.size(); ++li) { - if (compressible.count(li)) continue; // skip the layers compacted in pass 1 + if (compressible.count(li)) continue; const auto& cs = comp.cache_states[li]; if (cs.key_node_id < 0) continue; - if (comp.graph->get_node_op_type(static_cast(cs.key_node_id)) != OpType::KV_CACHE_STATE) continue; // skip conv/recurrent caches + if (comp.graph->get_node_op_type(static_cast(cs.key_node_id)) != OpType::KV_CACHE_STATE) continue; const auto& kdesc = comp.graph->get_output_buffer(static_cast(cs.key_node_id)); if (kdesc.byte_size <= kHeaderBytes) continue; void* kraw = comp.graph->get_output(static_cast(cs.key_node_id)); @@ -3055,15 +3041,11 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } } - // Global layers were compacted to B; the next decode query uses position_ids = B for ALL layers - // (global renumbered, sliding shifted into the same frame). + // All layers now share the same compressed frame, so the next decode query uses position_ids = B. if (have_new_seq_len) cache_total_seq_len_ = new_seq_len; } void Model::maybe_roll_compact() { - // Only reached from the causal-LM generation path (complete.cpp -> prefill/decode). STT models - // transcribe via transcribe.cpp with their own decode loops and never call this, and multimodal - // vision/audio encoder passes aren't on this path either -- so no model-type gate is needed. if (!config_.kv_compress || config_.kv_compress_trigger_len <= 0) return; if (cache_total_seq_len_ < static_cast(config_.kv_compress_trigger_len)) return; diff --git a/cactus-engine/tests/test_kv_compress_free_functions.cpp b/cactus-engine/tests/test_kv_compress_free_functions.cpp index d27a4199a..ad5cb428f 100644 --- a/cactus-engine/tests/test_kv_compress_free_functions.cpp +++ b/cactus-engine/tests/test_kv_compress_free_functions.cpp @@ -24,8 +24,7 @@ uint16_t f32_to_f16(float f) { __fp16 v = static_cast<__fp16>(f); uint16_t u; st using EngineTestUtils::rope_reference; -// Build an FP16 cache buffer: 64B header + [max_seq][kv_heads][head_dim] fp16. Returns the -// raw buffer; `pre_rope[h][t][d]` is the planted pre-RoPE key, stored post-RoPE at position t. +// `pre_rope[h][t][d]` is the planted pre-RoPE key, stored post-RoPE at position t. std::vector make_fp16_cache(size_t n, size_t max_seq, size_t kv_heads, size_t head_dim, double theta, std::vector>>& pre_rope, std::vector>>& values) { @@ -65,9 +64,7 @@ std::vector make_fp16_value_cache(size_t n, size_t max_seq, size_t kv_head return buf; } -// Independent oracle quantizer (mirrors quants.cpp / compact_int8's convention: scale floored -// 1e-10, roundf, clamp [-128,127]). Kept separate from production requant so the tests don't -// validate it against itself. +// Independent oracle quantizer, kept separate from production requant so tests don't self-validate. void quantize_row(const std::vector& row, int8_t* dst, float* sc, size_t head_dim, size_t group_size) { size_t groups = (head_dim + group_size - 1) / group_size; @@ -91,7 +88,6 @@ bool test_compact_fp16_cache() { auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); - // Heterogeneous per-head keep-sets (different indices, same length B). std::vector> kept = { {0, 5, 10, 20, 30, 59}, {1, 4, 12, 25, 40, 58}, {0, 2, 15, 33, 50, 55}}; size_t B = kept[0].size(); @@ -101,7 +97,6 @@ bool test_compact_fp16_cache() { for (size_t h = 0; h < kv_heads; ++h) { for (size_t rank = 0; rank < B; ++rank) { int abs_pos = kept[h][rank]; - // Expected K at slot `rank` = rope(pre_rope[h][abs_pos], rank). std::vector expK = rope_reference(pre[h][abs_pos], (double)rank, theta); const uint16_t* kdst = krows + (rank * kv_heads + h) * head_dim; const uint16_t* vdst = vrows + (rank * kv_heads + h) * head_dim; @@ -115,8 +110,7 @@ bool test_compact_fp16_cache() { } bool test_dense_check_full_budget() { - // abs_budget == n -> keep all indices in order; renumber maps rank==abs_pos so RoPE delta - // is 0 -> the K rows are byte-identical and V unchanged. + // abs_budget == n keeps every index in order, so renumber delta is 0 and the buffer is unchanged. const size_t n = 40, max_seq = 64, kv_heads = 2, head_dim = 16; const double theta = 1000000.0; std::vector>> pre, vals; @@ -128,10 +122,9 @@ bool test_dense_check_full_budget() { Params p; p.recent_frac = 0.3f; p.sink = 4; p.abs_budget = (int)n; auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); - for (auto& k : kept) if (k.size() != n) return false; // B == n + for (auto& k : kept) if (k.size() != n) return false; compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); - // Byte-identical over the live region. size_t live = n * kv_heads * head_dim * sizeof(uint16_t); if (std::memcmp(kbuf.data() + kHeaderBytes, kbuf0.data() + kHeaderBytes, live) != 0) return false; if (std::memcmp(vbuf.data() + kHeaderBytes, vbuf0.data() + kHeaderBytes, live) != 0) return false; @@ -139,7 +132,7 @@ bool test_dense_check_full_budget() { } bool test_rope_renumber_contiguous() { - // Route-B delta rotation: rope(stored_post_rope_at_abs, rank - abs) == rope(pre, rank). + // rope(stored_post_rope_at_abs, rank - abs) == rope(pre, rank). const size_t head_dim = 16; const double theta = 1000000.0; std::vector pre(head_dim); for (size_t d = 0; d < head_dim; ++d) pre[d] = std::sin(0.4 * d) + 0.1 * d; @@ -154,9 +147,7 @@ bool test_rope_renumber_contiguous() { } bool test_fp16_storage_round_trip() { - // Write pre-RoPE -> post-RoPE into an fp16 cache, compact+renumber, read back, and compare - // each survivor against the reference rope(pre, rank) within fp16 tolerance. Exercises the - // full fp16 storage path (f32->f16 store, gather, delta-rotate, f16->f16 write-back). + // Each survivor must read back as rope(pre, rank) within fp16 tolerance. const size_t n = 64, max_seq = 128, kv_heads = 2, head_dim = 16; const double theta = 1000000.0; std::vector>> pre, vals; @@ -186,38 +177,32 @@ bool test_fp16_storage_round_trip() { } bool test_gemma_layer_selection() { - // Qwen: all 28 layers (full_attention), no KV sharing. std::vector qwen(28, "full_attention"); auto q = physical_compressible_layers(qwen, 28, 0); if (q.size() != 28) return false; for (size_t i = 0; i < 28; ++i) if (q[i] != i) return false; - // Gemma4-e2b: layer_types sliding/global, num_layers=35, num_kv_shared=20 -> {4,9}. std::vector g; for (int i = 0; i < 35; ++i) g.push_back(((i + 1) % 5 == 0) ? "global" : "sliding"); auto gg = physical_compressible_layers(g, 35, 20); std::vector expect = {4, 9}; if (gg != expect) return false; - // Pass-2 re-rope theta is chosen per layer via is_sliding_layer (NOT compressibility): sliding - // layers shift with the local theta, full-attention layers with the global theta. Layer 14 is a - // global KV-shared *source* -- excluded from compaction (not in {4,9}) yet must keep global theta. - if (is_sliding_layer(g, 14)) return false; // global source -> global theta (the fix) - if (is_sliding_layer(g, 4)) return false; // compacted global -> global theta - if (!is_sliding_layer(g, 3)) return false; // sliding -> local theta - for (size_t i = 0; i < 28; ++i) if (is_sliding_layer(qwen, i)) return false; // all-global Qwen + // Pass-2 theta follows is_sliding_layer, NOT compressibility: layer 14 is a global KV-shared + // *source* excluded from compaction (not in {4,9}) yet must still re-rope with the global theta. + if (is_sliding_layer(g, 14)) return false; + if (is_sliding_layer(g, 4)) return false; + if (!is_sliding_layer(g, 3)) return false; + for (size_t i = 0; i < 28; ++i) if (is_sliding_layer(qwen, i)) return false; return true; } bool test_compact_int8_cache() { - // INT8: gather + renumber K, gather V (no rotation); dequant of compacted survivor ~ the - // expected renumbered/gathered value. const size_t n = 50, max_seq = 96, kv_heads = 2, head_dim = 32; const double theta = 1000000.0; size_t groups = (head_dim + kGroupSize - 1) / kGroupSize; size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; - // Build pre-RoPE keys + values, then store post-RoPE K quantized per group. std::vector>> pre(kv_heads, std::vector>(n)); std::vector>> vals(kv_heads, std::vector>(n)); std::vector k_i8(max_seq * int8_stride, 0), v_i8(max_seq * int8_stride, 0); @@ -260,14 +245,12 @@ bool test_compact_int8_cache() { return ok; } -// Mirror Model::maybe_roll_compact: when the live seq-len reaches trigger_len, KeyDiff-keep the -// best target_len tokens (absolute budget), renumber survivors to 0..B-1, set header len = B. -// Returns B (the new bounded length). Operates on an FP16 K/V cache pair in place. +// Mirrors Model::maybe_roll_compact on an in-place FP16 K/V pair; returns the new bounded length B. size_t roll_compact_once(uint16_t* krows, uint16_t* vrows, Header* khdr, Header* vhdr, size_t kv_heads, size_t head_dim, double theta, int target_len) { size_t n = khdr->current_seq_len; Params p; p.recent_frac = 0.30f; p.sink = 4; - p.abs_budget = target_len; // absolute budget -> keep exactly min(target_len, n) per head + p.abs_budget = target_len; auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, theta, p); compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); size_t B = kept.empty() ? 0 : kept[0].size(); @@ -277,9 +260,7 @@ size_t roll_compact_once(uint16_t* krows, uint16_t* vrows, Header* khdr, Header* } bool test_rolling_bounded_compaction() { - // Drive the compactor across repeated thresholds on a synthetic cache, exactly as the rolling - // bounded path does: grow to trigger_len -> compact to target_len -> grow again -> compact. - // Asserts the cache stays bounded <= trigger_len, positions renumber to 0..B-1 each cycle, and + // Grow to trigger_len -> compact to target_len -> repeat, asserting the cache stays bounded and // a planted distinctive mid-token survives every compaction. const int trigger_len = 4096, target_len = 2048; const size_t kv_heads = 2, head_dim = 16, max_seq = trigger_len + 8; @@ -294,12 +275,11 @@ bool test_rolling_bounded_compaction() { auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); - // A strongly distinctive pre-RoPE key direction (centroid-opposite) that KeyDiff must keep. + // A centroid-opposite direction that KeyDiff must keep. std::vector distinctive(head_dim); for (size_t d = 0; d < head_dim; ++d) distinctive[d] = (d % 2 == 0 ? -1.0f : 1.0f) * 4.0f; - // Append fresh post-RoPE rows for positions [from, to) for every head. The token at - // `plant_abs` (if in range) is the distinctive one; the rest cluster on a common centroid. + // The token at `plant_abs` is the distinctive one; the rest cluster on a common centroid. auto append = [&](size_t from, size_t to, long plant_abs) { for (size_t t = from; t < to; ++t) for (size_t h = 0; h < kv_heads; ++h) { @@ -320,14 +300,12 @@ bool test_rolling_bounded_compaction() { } }; - // Does any survivor row (un-RoPE'd at its rank) match the planted distinctive direction? auto distinctive_survives = [&](size_t B) -> bool { for (size_t h = 0; h < kv_heads; ++h) for (size_t rank = 0; rank < B; ++rank) { std::vector pre(head_dim); const uint16_t* src = krows + (rank * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) pre[d] = f16_to_f32(src[d]); - // The row is stored as rope(pre, rank); un-RoPE by -rank to recover pre. rope_rotate_row(pre.data(), head_dim, theta, -(double)rank); double dot = 0, na = 0, nb = 0; for (size_t d = 0; d < head_dim; ++d) { @@ -341,32 +319,25 @@ bool test_rolling_bounded_compaction() { bool ok = true; const int cycles = 3; for (int c = 0; c < cycles; ++c) { - size_t start = khdr->current_seq_len; // survivors carried over from last cycle - long plant_abs = (c == 0) ? (trigger_len / 2) : -1; // plant once; must persist after that + size_t start = khdr->current_seq_len; + long plant_abs = (c == 0) ? (trigger_len / 2) : -1; append(start, (size_t)trigger_len, plant_abs); khdr->current_seq_len = trigger_len; vhdr->current_seq_len = trigger_len; - // The distinctive token must be present BEFORE the first compaction renumbers it. if (c == 0 && !distinctive_survives(trigger_len)) ok = false; size_t B = roll_compact_once(krows, vrows, khdr, vhdr, kv_heads, head_dim, theta, target_len); - if (B != (size_t)target_len) ok = false; // absolute budget honored - if (khdr->current_seq_len > (size_t)trigger_len) ok = false; // bounded - if (khdr->current_seq_len != (size_t)target_len) ok = false; // header set to B - - // Renumbering: survivor K rows must read back as rope(pre, rank). We verify via the - // un-RoPE-at-rank == direction check for the planted token, and that rows are dense - // (positions 0..B-1 populated, no gaps) by re-reading the live region length. - if (!distinctive_survives(B)) ok = false; // distinctive persists each cycle + if (B != (size_t)target_len) ok = false; + if (khdr->current_seq_len > (size_t)trigger_len) ok = false; + if (khdr->current_seq_len != (size_t)target_len) ok = false; + if (!distinctive_survives(B)) ok = false; } - // After all cycles the cache is still bounded at target_len, never exceeding trigger_len. if (khdr->current_seq_len != (size_t)target_len) ok = false; return ok; } bool test_config_parse_rolling_fields() { - // Config::from_json parses a key=value file. Rolling is the default (4096 -> 2048). cactus::engine::Config def; if (!def.kv_compress) return false; if (def.kv_compress_trigger_len != 4096 || def.kv_compress_target_len != 2048) return false; @@ -377,8 +348,7 @@ bool test_config_parse_rolling_fields() { ::close(fd); { std::ofstream f(tmpl); - // model_type=qwen avoids the Gemma4-only required-field validation; we only assert the - // new kv_compress rolling fields round-trip through the key=value parser. + // model_type=qwen avoids the Gemma4-only required-field validation. f << "model_type=qwen\n" << "kv_compress=true\n" << "kv_compress_trigger_len=4096\n" @@ -392,10 +362,7 @@ bool test_config_parse_rolling_fields() { } bool test_trigger_zero_gates_rolling() { - // The rolling gate (Model::maybe_roll_compact): fire iff kv_compress && trigger_len > 0 && - // current_seq_len >= trigger_len. Drive the SAME compactor under both arms on identical caches: - // trigger_len == 0 must skip (cache byte-identical); a reached trigger must compact (len -> target, - // bytes change). The skip assertion is meaningful only against this proven mutation. + // The maybe_roll_compact gate fires iff kv_compress && trigger_len > 0 && seq_len >= trigger_len. const size_t n = 5000, kv_heads = 2, head_dim = 16, max_seq = 5008; const double theta = 1000000.0; @@ -417,7 +384,7 @@ bool test_trigger_zero_gates_rolling() { cfg.kv_compress_target_len); }; - // Gate closed (trigger_len == 0): caller skips => cache byte-identical, length unchanged. + // Gate closed: cache byte-identical, length unchanged. cactus::engine::Config off; off.kv_compress_trigger_len = 0; off.kv_compress_target_len = 0; std::vector k_off = fresh_cache(), v_off(k_off.size(), 0), k_off0 = k_off, v_off0 = v_off; roll_if_gated(off, k_off, v_off); @@ -425,7 +392,7 @@ bool test_trigger_zero_gates_rolling() { if (std::memcmp(k_off.data(), k_off0.data(), k_off.size()) != 0) return false; if (std::memcmp(v_off.data(), v_off0.data(), v_off.size()) != 0) return false; - // Gate open (trigger reached): the same compactor compacts to target_len and rewrites bytes. + // Gate open: compacts to target_len and rewrites bytes. cactus::engine::Config on; on.kv_compress_trigger_len = 4096; on.kv_compress_target_len = 2048; std::vector k_on = fresh_cache(), v_on(k_on.size(), 0), k_on0 = k_on; roll_if_gated(on, k_on, v_on); @@ -435,8 +402,7 @@ bool test_trigger_zero_gates_rolling() { } bool test_degenerate_rolling_config_disabled() { - // validate_kv_compress(): when trigger_len > 0, require 0 < target_len < trigger_len. - // Bad configs must disable rolling (trigger_len/target_len reset to 0); good configs survive. + // validate_kv_compress requires 0 < target_len < trigger_len; bad configs reset both to 0. auto disabled = [](int trig, int targ) { cactus::engine::Config c; c.kv_compress = true; @@ -445,11 +411,10 @@ bool test_degenerate_rolling_config_disabled() { c.validate_kv_compress(); return c.kv_compress_trigger_len == 0 && c.kv_compress_target_len == 0; }; - if (!disabled(4096, 0)) return false; // target_len <= 0 - if (!disabled(4096, 4096)) return false; // target_len == trigger_len - if (!disabled(2048, 4096)) return false; // target_len > trigger_len - if (!disabled(4096, -1)) return false; // negative target_len - // Valid config is untouched. + if (!disabled(4096, 0)) return false; + if (!disabled(4096, 4096)) return false; + if (!disabled(2048, 4096)) return false; + if (!disabled(4096, -1)) return false; cactus::engine::Config ok; ok.kv_compress = true; ok.kv_compress_trigger_len = 4096; @@ -459,13 +424,10 @@ bool test_degenerate_rolling_config_disabled() { } bool test_env_override_parse() { - // parse_kv_compress_override maps CACTUS_KV_COMPRESS_AT / CACTUS_KV_COMPRESS_TO onto Config. - // Unset => no override; the rolling default (4096 -> 2048) is preserved. cactus::engine::Config off; if (off.parse_kv_compress_override(nullptr, nullptr)) return false; if (!off.kv_compress || off.kv_compress_trigger_len != 4096 || off.kv_compress_target_len != 2048) return false; - // Both vars override. cactus::engine::Config both; if (!both.parse_kv_compress_override("3000", "1000")) return false; if (!both.kv_compress || both.kv_compress_trigger_len != 3000 || both.kv_compress_target_len != 1000) return false; @@ -475,12 +437,10 @@ bool test_env_override_parse() { if (!one.parse_kv_compress_override(nullptr, "1500")) return false; if (one.kv_compress_trigger_len != 4096 || one.kv_compress_target_len != 1500) return false; - // CACTUS_KV_COMPRESS_AT=0 disables. cactus::engine::Config disabled; if (!disabled.parse_kv_compress_override("0", nullptr)) return false; if (disabled.kv_compress) return false; - // Degenerate (target >= trigger) disabled by the validate guard. cactus::engine::Config bad; if (!bad.parse_kv_compress_override("2048", "4096")) return false; if (bad.kv_compress_trigger_len != 0 || bad.kv_compress_target_len != 0) return false; @@ -490,12 +450,11 @@ bool test_env_override_parse() { // --- Sliding-window re-rope (single position frame) ------------------------------------------- bool test_rerope_recent_fp16_uniform() { - // Rotate recent rows [sink, n) by a uniform (negative) delta: each recent row now reads as - // rope(pre, t+delta); sink rows [0, sink) stay byte-identical. Uses the LOCAL theta. + // After a delta shift each recent row reads as rope(pre, t+delta); sink rows stay byte-identical. const size_t n = 40, max_seq = 64, kv_heads = 2, head_dim = 16, sink = 4; const double theta = 10000.0, delta = -7.0; std::vector>> pre, vals; - auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); // header sink_size=4 + auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, theta, pre, vals); auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); std::vector sink_before(krows, krows + sink * kv_heads * head_dim); @@ -513,7 +472,6 @@ bool test_rerope_recent_fp16_uniform() { } namespace { -// Build an int8 K cache storing rope(pre, t, theta) per group for t in [0,n). void build_int8_key_cache(size_t n, size_t kv_heads, size_t head_dim, double theta, std::vector>>& pre, std::vector& k_i8, std::vector& k_sc) { @@ -560,8 +518,7 @@ bool test_rerope_recent_int8() { } bool test_rotate_int8_row_matches_inline() { - // Refactor safety: the factored rotate_int8_row must be bitwise-identical to the old inline - // dequant -> rope_rotate_row -> per-group requant. + // rotate_int8_row must be bitwise-identical to the inline dequant -> rotate -> requant sequence. const size_t head_dim = 32; const double theta = 10000.0, delta = -9.0; size_t groups = (head_dim + kGroupSize - 1) / kGroupSize; std::vector pre(head_dim); @@ -572,12 +529,10 @@ bool test_rotate_int8_row_matches_inline() { quantize_row(stored, a_i8.data(), a_sc.data(), head_dim, kGroupSize); b_i8 = a_i8; b_sc = a_sc; - // Path A: literal old inline sequence. std::vector row(head_dim); for (size_t d = 0; d < head_dim; ++d) row[d] = (float)a_i8[d] * a_sc[d / kGroupSize]; rope_rotate_row(row.data(), head_dim, theta, delta); quantize_row(row, a_i8.data(), a_sc.data(), head_dim, kGroupSize); - // Path B: factored helper. rotate_int8_row(b_i8.data(), b_sc.data(), head_dim, kGroupSize, theta, delta); return std::memcmp(a_i8.data(), b_i8.data(), head_dim) == 0 @@ -585,14 +540,11 @@ bool test_rotate_int8_row_matches_inline() { } bool test_sliding_plus_global_rolling() { - // Cross-layer single-frame invariant over 3 cycles: a GLOBAL cache compacts+renumbers to B and - // a SLIDING cache re-ropes its recent rows by -Δ (Δ = old_total - B), so a shared query at - // position B matches both. Global distinctive token survives; sliding sink stays; sliding - // recent rows track to (t - sum Δ) within fp16 tolerance across cycles. + // A global cache renumbers survivors to 0..B-1 while a sliding cache re-ropes its recent rows by + // -Δ (Δ = old_total - B), keeping both in the same compressed frame across 3 cycles. const size_t kv_heads = 2, head_dim = 16, sink = 4; const double gtheta = 1000000.0, ltheta = 10000.0; - // --- Global: one keydiff compaction, distinctive token must survive + renumber to 0..B-1. --- { const size_t n = 60, max_seq = 96; std::vector>> pre, vals; @@ -604,8 +556,8 @@ bool test_sliding_plus_global_rolling() { auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, gtheta, p); size_t B = kept[0].size(); compact_fp16(krows, vrows, kv_heads, head_dim, kept, gtheta); - for (auto& k : kept) if (k.size() != B) return false; // uniform B across heads - for (size_t h = 0; h < kv_heads; ++h) // global survivors renumber to 0..B-1 + for (auto& k : kept) if (k.size() != B) return false; + for (size_t h = 0; h < kv_heads; ++h) for (size_t rank = 0; rank < B; ++rank) { std::vector exp = rope_reference(pre[h][kept[h][rank]], (double)rank, gtheta); const uint16_t* r = krows + (rank * kv_heads + h) * head_dim; @@ -614,14 +566,13 @@ bool test_sliding_plus_global_rolling() { } } - // --- Sliding: re-rope recent rows by -Δ over 3 cycles, with Δ derived from a global cycle. --- const size_t n = 16, max_seq = 32; std::vector>> pre, vals; auto kbuf = make_fp16_cache(n, max_seq, kv_heads, head_dim, ltheta, pre, vals); auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); std::vector sink_before(krows, krows + sink * kv_heads * head_dim); - const size_t old_total = 30, B = 12; // a representative global cycle + const size_t old_total = 30, B = 12; const double Delta = (double)(old_total - B); double accum = 0.0; for (int cycle = 0; cycle < 3; ++cycle) { @@ -634,14 +585,13 @@ bool test_sliding_plus_global_rolling() { std::vector exp = rope_reference(pre[h][t], (double)t + accum, ltheta); const uint16_t* r = krows + (t * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) - if (std::abs(f16_to_f32(r[d]) - exp[d]) > 8e-2f) return false; // looser: 3-cycle drift + if (std::abs(f16_to_f32(r[d]) - exp[d]) > 8e-2f) return false; // looser for 3-cycle drift } return true; } bool test_rerope_local_theta_used() { - // The sliding re-rope must use the LOCAL theta: rotating with the wrong (global) theta does NOT - // recover the expected position. Pins the orchestration's theta choice as load-bearing. + // The sliding re-rope must use the LOCAL theta; the wrong (global) theta misses the position. const size_t n = 24, max_seq = 32, kv_heads = 1, head_dim = 16, sink = 4; const double ltheta = 10000.0, gtheta = 1000000.0, delta = -5.0; std::vector>> pre, vals; @@ -650,7 +600,7 @@ bool test_rerope_local_theta_used() { auto* a = reinterpret_cast(kbuf.data() + kHeaderBytes); auto* b = reinterpret_cast(kbuf2.data() + kHeaderBytes); - rerope_recent_fp16(a, kv_heads, head_dim, sink, n, ltheta, delta); // correct + rerope_recent_fp16(a, kv_heads, head_dim, sink, n, ltheta, delta); // correct theta rerope_recent_fp16(b, kv_heads, head_dim, sink, n, gtheta, delta); // wrong theta bool correct_ok = true, wrong_differs = false; diff --git a/cactus-engine/tests/test_utils.h b/cactus-engine/tests/test_utils.h index 717b5a101..19aba3819 100644 --- a/cactus-engine/tests/test_utils.h +++ b/cactus-engine/tests/test_utils.h @@ -31,8 +31,7 @@ class TestRunner { namespace EngineTestUtils { -// Reference rotate_half RoPE for a single vector (double-precision oracle for kv_compress tests), -// mirroring norms_rope.cpp / the Python fixture generator. +// Double-precision rotate_half RoPE oracle for kv_compress tests (mirrors norms_rope.cpp). inline std::vector rope_reference(const std::vector& v, double pos, double theta) { size_t d = v.size(), half = d / 2; std::vector o(d); diff --git a/python/cactus/transpile/component_plan.py b/python/cactus/transpile/component_plan.py index 11eb06f5f..7f3e5f52f 100644 --- a/python/cactus/transpile/component_plan.py +++ b/python/cactus/transpile/component_plan.py @@ -28,10 +28,8 @@ def _plan_from_profile( if config is not None and profile.needs_image: has_vision = _has_dict_config(config, "vision_config", "visual_config", "image_config") if not has_vision: - # Text-only causal LM (no vision). qwen3 defaults to the chunked-prefill component - # pipeline (#687: ~5x faster prefill, identical output); the chunk adapters are qwen3 - # only, so other families stay on the monolithic decoder. Key on the architecture - # ("Qwen3ForCausalLM") since the bundle config's model_type collapses to "qwen". + # qwen3 defaults to the chunked-prefill pipeline (the chunk adapters are qwen3 only). + # Key on architecture since the bundle config's model_type collapses "qwen3" to "qwen". architectures = [str(a).lower() for a in (config.get("architectures") or ())] if any("qwen3forcausallm" in a for a in architectures): return ComponentPlan( From 7574363c81a8d1507c7d412c70fbfe539c091926 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 19:42:42 +0100 Subject: [PATCH 13/55] Replace standalone rope-table test with one integrated optimize-pass test The standalone suite depended on gitignored transpiled weight bundles (skipped on CI). Replace it with a self-contained synthetic-graph test in test_optimize_gemma4_attention.py asserting precompute_rope_tables rewrites the runtime cos angle to an fp16 embedding-table lookup keyed by the position input, with positions past 2048 representable. Signed-off-by: Noah Cylich --- .../tests/test_optimize_gemma4_attention.py | 53 +++ python/tests/test_precompute_rope_tables.py | 390 ------------------ 2 files changed, 53 insertions(+), 390 deletions(-) delete mode 100644 python/tests/test_precompute_rope_tables.py diff --git a/python/tests/test_optimize_gemma4_attention.py b/python/tests/test_optimize_gemma4_attention.py index 843e3f65c..5a534f9c0 100644 --- a/python/tests/test_optimize_gemma4_attention.py +++ b/python/tests/test_optimize_gemma4_attention.py @@ -1,7 +1,11 @@ +import torch + +from cactus.transpile.canonicalize.utils import rebuild_graph from cactus.transpile.graph_ir import IRGraph from cactus.transpile.graph_ir import IRNode from cactus.transpile.graph_ir import IRValue from cactus.transpile.optimize_graph import normalize_gemma4_decoder_attention_semantics +from cactus.transpile.optimize_graph import precompute_rope_tables def test_normalize_gemma4_full_attention_uses_sequence_window_compat() -> None: @@ -222,3 +226,52 @@ def test_normalize_gemma4_attention_elides_logical_fp16_mask() -> None: assert graph.nodes["attn"].inputs == ["query", "key", "value"] assert graph.nodes["attn"].attrs["is_causal"] is True assert graph.nodes["attn"].attrs["window_size"] == 512 + + +def test_precompute_rope_tables_replaces_runtime_angle_with_fp16_lookup() -> None: + head_dim = 8 + max_seq = 4096 + theta = 10000.0 + inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim)) + + graph = IRGraph( + values={ + "positions": IRValue(id="positions", shape=(1, 4), dtype="int32"), + "angle": IRValue(id="angle", shape=(1, 4, head_dim // 2), dtype="fp32", producer="matmul"), + "emb": IRValue(id="emb", shape=(1, 4, head_dim), dtype="fp32", producer="cat"), + "cos": IRValue(id="cos", shape=(1, 4, head_dim), dtype="fp32", producer="cos_node"), + }, + nodes={ + "matmul": IRNode(id="matmul", op="matmul", inputs=["positions", "inv_freq"], outputs=["angle"]), + "cat": IRNode(id="cat", op="cat", inputs=["angle", "angle"], outputs=["emb"], attrs={"dim": -1}), + "cos_node": IRNode( + id="cos_node", op="scalar_cos", inputs=["emb"], outputs=["cos"], + meta={"component": "decoder"}, + ), + }, + order=["matmul", "cat", "cos_node"], + inputs=["positions"], + outputs=["cos"], + constants={"inv_freq": inv_freq}, + meta={"component": "decoder", "max_cache_seq_len": max_seq}, + ) + rebuild_graph(graph) + + assert precompute_rope_tables(graph) is True + + cos_node = graph.nodes["cos_node"] + assert cos_node.op == "embedding" + table_id, position_id = cos_node.inputs + assert table_id.startswith("c_rope_table_") + assert position_id == "positions" + + table = graph.constants[table_id] + assert table.dtype == torch.float16 + assert tuple(table.shape) == (max_seq, head_dim) + assert graph.values["cos"].dtype == "fp16" + + # Positions past the fp16-angle range (>2048) stay distinct/representable in the table. + positions = torch.arange(max_seq, dtype=torch.float64).reshape(max_seq, 1) + freqs = positions * inv_freq.to(torch.float64).reshape(1, -1) + expected = torch.cos(torch.cat((freqs, freqs), dim=-1)).to(torch.float16) + assert torch.equal(table[[0, 1, 2049, max_seq - 1]], expected[[0, 1, 2049, max_seq - 1]]) diff --git a/python/tests/test_precompute_rope_tables.py b/python/tests/test_precompute_rope_tables.py deleted file mode 100644 index ae8af1153..000000000 --- a/python/tests/test_precompute_rope_tables.py +++ /dev/null @@ -1,390 +0,0 @@ -import json -from pathlib import Path - -import pytest -import torch - -from cactus.transpile.canonicalize.utils import rebuild_graph -from cactus.transpile.graph_ir import IRGraph -from cactus.transpile.graph_ir import IRNode -from cactus.transpile.graph_ir import IRValue -from cactus.transpile.optimize_graph import precompute_rope_tables - - -WEIGHTS = Path(__file__).resolve().parents[2] / "weights" -GEMMA_BUNDLE = WEIGHTS / "gemma-4-e2b-it" -GEMMA_16K_BUNDLE = WEIGHTS / "gemma-4-e2b-16k" -QWEN_BUNDLE = WEIGHTS / "qwen3-0.6b" - -TEST_MAX_SEQ = 16384 - -# inv_freq = 1 / theta ** (arange(0, dim, 2) / dim), exactly the constant the -# real graph carries. The serialized optimized_ir drops multi-element tensor -# payloads, so we re-materialize the genuine per-theta inv_freq. The (theta, -# head_dim) spec is read from the bundle's config.json at test time (not -# hardcoded) so a model shipped with the wrong rope_theta / head_dim fails here -# instead of being matched against a self-referential reconstruction. -def _gemma_inv_freq_spec(bundle: Path) -> dict[str, tuple[float, int]]: - text = json.loads((bundle / "config.json").read_text())["text_config"] - rope = text["rope_parameters"] - return { - "v_module_backbone_rotary_emb_sliding_attention_inv_freq": ( - float(rope["sliding_attention"]["rope_theta"]), - int(text["head_dim"]), - ), - "v_module_backbone_rotary_emb_full_attention_inv_freq": ( - float(rope["full_attention"]["rope_theta"]), - int(text["global_head_dim"]), - ), - } - - -def _qwen_inv_freq_spec(bundle: Path) -> dict[str, tuple[float, int]]: - cfg = json.loads((bundle / "config.json").read_text()) - return { - "v_module_backbone_rotary_emb_inv_freq": ( - float(cfg["rope_theta"]), - int(cfg["head_dim"]), - ), - } - - -# The IR these tests load lives in transpiled weight bundles under weights/ -# (gitignored, ~9MB), so skip when they are absent (e.g. CI) rather than failing -# collection; run locally after transpiling the bundles. -_BUNDLES_PRESENT = (GEMMA_BUNDLE / "config.json").exists() and (QWEN_BUNDLE / "config.json").exists() -pytestmark = pytest.mark.skipif( - not _BUNDLES_PRESENT, reason="requires transpiled weight bundles under weights/ (gitignored)" -) - -GEMMA_INV_FREQ = _gemma_inv_freq_spec(GEMMA_BUNDLE) if _BUNDLES_PRESENT else {} -GEMMA_16K_INV_FREQ = _gemma_inv_freq_spec(GEMMA_16K_BUNDLE) if (GEMMA_16K_BUNDLE / "config.json").exists() else {} -QWEN_INV_FREQ = _qwen_inv_freq_spec(QWEN_BUNDLE) if _BUNDLES_PRESENT else {} - - -def _inv_freq(theta: float, head_dim: int) -> torch.Tensor: - return 1.0 / (theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim)) - - -def test_gemma_inv_freq_spec_matches_config() -> None: - """Guard the reconstruction source: the (theta, head_dim) the parity tests - feed the pass must equal the bundle's real config.json rope_parameters, so a - wrong-theta or wrong-head_dim bundle fails instead of self-referentially - passing.""" - text = json.loads((GEMMA_BUNDLE / "config.json").read_text())["text_config"] - rope = text["rope_parameters"] - assert GEMMA_INV_FREQ == { - "v_module_backbone_rotary_emb_sliding_attention_inv_freq": ( - float(rope["sliding_attention"]["rope_theta"]), - int(text["head_dim"]), - ), - "v_module_backbone_rotary_emb_full_attention_inv_freq": ( - float(rope["full_attention"]["rope_theta"]), - int(text["global_head_dim"]), - ), - } - sliding = GEMMA_INV_FREQ["v_module_backbone_rotary_emb_sliding_attention_inv_freq"] - full = GEMMA_INV_FREQ["v_module_backbone_rotary_emb_full_attention_inv_freq"] - assert sliding[0] != full[0], "sliding and full thetas must be distinct (dual-theta)" - assert sliding[1] != full[1], "sliding and full head_dims must be distinct" - - -def _load_ir(path: Path, inv_freq_spec: dict[str, tuple[float, int]], *, max_seq: int = TEST_MAX_SEQ) -> IRGraph: - payload = json.loads(path.read_text()) - graph = payload["graph"] - values = { - vid: IRValue( - id=vid, - shape=tuple(v["shape"]) if v.get("shape") is not None else None, - dtype=v.get("dtype"), - ) - for vid, v in graph["values"].items() - } - nodes: dict[str, IRNode] = {} - order: list[str] = [] - for n in graph["nodes"]: - nodes[n["id"]] = IRNode( - id=n["id"], - op=n["op"], - inputs=list(n["inputs"]), - outputs=list(n["outputs"]), - attrs=dict(n.get("attrs", {})), - meta=dict(n.get("meta", {})), - kind=n.get("kind", "generic"), - ) - order.append(n["id"]) - constants = { - cid: _inv_freq(*inv_freq_spec[cid]) - for cid in graph["constants"] - if cid in inv_freq_spec - } - meta = dict(graph["meta"]) - meta["max_cache_seq_len"] = max_seq - ir = IRGraph( - values=values, - nodes=nodes, - order=order, - inputs=list(graph["inputs"]), - outputs=list(graph["outputs"]), - constants=constants, - meta=meta, - ) - rebuild_graph(ir) - return ir - - -def _cos_sin_inputs(payload_path: Path) -> dict[str, str]: - """Map each scalar_cos/scalar_sin output id -> the int position graph input the - original angle matmul consumed, mirroring the pass's index_shape constraint - (the int value whose shape == cos/sin output shape minus its last dim).""" - graph = json.loads(payload_path.read_text())["graph"] - producer = {o: n for n in graph["nodes"] for o in n["outputs"]} - values = graph["values"] - - def find_position(value_id: str, index_shape: tuple[int, ...]) -> str | None: - seen: set[str] = set() - stack = [value_id] - while stack: - cur = stack.pop() - if cur in seen: - continue - seen.add(cur) - val = values.get(cur, {}) - dtype = str(val.get("dtype")) - shape = val.get("shape") - if ( - dtype in {"int32", "int64", "i32", "i64"} - and shape is not None - and tuple(int(d) for d in shape) == index_shape - ): - return cur - node = producer.get(cur) - if node is None: - continue - stack.extend(node["inputs"]) - return None - - result: dict[str, str] = {} - for n in graph["nodes"]: - if n["op"] in {"scalar_cos", "scalar_sin"}: - out_shape = values[n["outputs"][0]]["shape"] - index_shape = tuple(int(d) for d in out_shape[:-1]) - result[n["outputs"][0]] = find_position(n["inputs"][0], index_shape) - return result - - -def _rope_nodes(ir: IRGraph) -> tuple[list[IRNode], list[str]]: - embeddings = [ - n - for n in ir.nodes.values() - if n.op == "embedding" and n.inputs and n.inputs[0].startswith("c_rope_table_") - ] - tables = [c for c in ir.constants if c.startswith("c_rope_table_")] - return embeddings, tables - - -# --------------------------------------------------------------------------- # -# fires -# --------------------------------------------------------------------------- # - - -def test_gemma_decoder_step_rewrites_four_cos_sin() -> None: - ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_step.json", GEMMA_INV_FREQ) - assert precompute_rope_tables(ir) is True - embeddings, tables = _rope_nodes(ir) - assert len(embeddings) == 4 - assert len(tables) == 4 - assert all(n.op != "scalar_cos" and n.op != "scalar_sin" for n in ir.nodes.values()) - - -def test_gemma_prefill_chunk_rewrites_eight_cos_sin() -> None: - ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_prefill_chunk.json", GEMMA_INV_FREQ) - assert precompute_rope_tables(ir) is True - embeddings, tables = _rope_nodes(ir) - assert len(embeddings) == 8 - assert len(tables) == 8 - - -def test_gemma_media_step_rewrites_its_rope_tables() -> None: - path = GEMMA_16K_BUNDLE / "components" / "decoder_media_step" / "optimized_ir.json" - if not path.exists(): - path = GEMMA_16K_BUNDLE / "optimized_ir_decoder_media_step.json" - if not path.exists(): - pytest.skip( - "gemma4 component pipeline emits no decoder_media_step component " - "(only lm_encoder_media_step, which carries no rope angle path); " - "this guards future models that do emit a decoder media step" - ) - ir = _load_ir(path, GEMMA_16K_INV_FREQ) - assert precompute_rope_tables(ir) is True - embeddings, tables = _rope_nodes(ir) - assert len(embeddings) >= 4 - assert len(tables) >= 4 - widths = {int(ir.constants[t].shape[-1]) for t in tables} - assert 256 in widths and 512 in widths - - -# --------------------------------------------------------------------------- # -# two thetas: both widths exist and each table is the correct fp64 cos/sin -# --------------------------------------------------------------------------- # - - -def _table_by_width(ir: IRGraph) -> dict[int, dict[str, torch.Tensor]]: - out: dict[int, dict[str, torch.Tensor]] = {} - for node in ir.nodes.values(): - if node.op != "embedding" or not node.inputs[0].startswith("c_rope_table_"): - continue - table = ir.constants[node.inputs[0]] - width = int(table.shape[-1]) - kind = "cos" if "cos" in node.inputs[0] else "sin" - out.setdefault(width, {})[kind] = table - return out - - -def test_gemma_two_thetas_tables_match_fresh_fp64_cos_sin() -> None: - ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_step.json", GEMMA_INV_FREQ) - precompute_rope_tables(ir) - by_width = _table_by_width(ir) - assert set(by_width.keys()) == {256, 512} - - expected = { - head_dim: _inv_freq(theta, head_dim).to(torch.float64) - for theta, head_dim in GEMMA_INV_FREQ.values() - } - assert set(expected.keys()) == {256, 512} - probe_positions = [0, 1, 2000, 2049, TEST_MAX_SEQ - 1] - for width, inv in expected.items(): - positions = torch.tensor(probe_positions, dtype=torch.float64).reshape(-1, 1) - freqs = positions * inv.reshape(1, -1) - emb = torch.cat((freqs, freqs), dim=-1) - exp_cos = torch.cos(emb).to(torch.float16) - exp_sin = torch.sin(emb).to(torch.float16) - got_cos = by_width[width]["cos"][probe_positions] - got_sin = by_width[width]["sin"][probe_positions] - assert torch.equal(got_cos, exp_cos), f"cos width {width}" - assert torch.equal(got_sin, exp_sin), f"sin width {width}" - - -def test_gemma_width256_uses_sliding_not_global_inv_freq() -> None: - """Cross-check the per-theta binding: the width-256 table must NOT match the - global theta and vice versa.""" - ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_step.json", GEMMA_INV_FREQ) - precompute_rope_tables(ir) - by_width = _table_by_width(ir) - - wrong = _inv_freq(1000000.0, 256).to(torch.float64) # global theta on sliding width - positions = torch.tensor([2049.0], dtype=torch.float64).reshape(-1, 1) - freqs = positions * wrong.reshape(1, -1) - emb = torch.cat((freqs, freqs), dim=-1) - wrong_cos = torch.cos(emb).to(torch.float16) - assert not torch.equal(by_width[256]["cos"][[2049]], wrong_cos) - - -# --------------------------------------------------------------------------- # -# layout -# --------------------------------------------------------------------------- # - - -def test_gemma_table_width_matches_output_and_halves_are_byte_equal() -> None: - payload = json.loads((GEMMA_BUNDLE / "optimized_ir_decoder_step.json").read_text()) - out_width = { - n["outputs"][0]: int(payload["graph"]["values"][n["outputs"][0]]["shape"][-1]) - for n in payload["graph"]["nodes"] - if n["op"] in {"scalar_cos", "scalar_sin"} - } - ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_decoder_step.json", GEMMA_INV_FREQ) - precompute_rope_tables(ir) - for node in ir.nodes.values(): - if node.op != "embedding" or not node.inputs[0].startswith("c_rope_table_"): - continue - out_id = node.outputs[0] - table = ir.constants[node.inputs[0]] - width = int(table.shape[-1]) - assert width == out_width[out_id] - half = width // 2 - assert torch.equal(table[:, :half], table[:, half:]) - - -# --------------------------------------------------------------------------- # -# position binding -# --------------------------------------------------------------------------- # - - -def test_gemma_embedding_index_is_original_position_input() -> None: - path = GEMMA_BUNDLE / "optimized_ir_decoder_step.json" - pos_for_output = _cos_sin_inputs(path) - ir = _load_ir(path, GEMMA_INV_FREQ) - precompute_rope_tables(ir) - for node in ir.nodes.values(): - if node.op != "embedding" or not node.inputs[0].startswith("c_rope_table_"): - continue - out_id = node.outputs[0] - assert node.inputs[1] == pos_for_output[out_id] - assert set(pos_for_output.values()) == {"v_args_2"} - - -# --------------------------------------------------------------------------- # -# Qwen regression: single width table -# --------------------------------------------------------------------------- # - - -def test_qwen_single_width_table_still_correct() -> None: - path = QWEN_BUNDLE / "optimized_ir_decoder_step.json" - pos_for_output = _cos_sin_inputs(path) - ir = _load_ir(path, QWEN_INV_FREQ) - assert precompute_rope_tables(ir) is True - embeddings, tables = _rope_nodes(ir) - assert len(embeddings) == 2 - assert len(tables) == 2 - - widths = {int(ir.constants[t].shape[-1]) for t in tables} - qwen_theta, qwen_head_dim = QWEN_INV_FREQ["v_module_backbone_rotary_emb_inv_freq"] - assert widths == {qwen_head_dim} - - inv = _inv_freq(qwen_theta, qwen_head_dim).to(torch.float64) - positions = torch.tensor([0.0, 1.0, 2049.0, TEST_MAX_SEQ - 1], dtype=torch.float64).reshape(-1, 1) - freqs = positions * inv.reshape(1, -1) - emb = torch.cat((freqs, freqs), dim=-1) - for node in ir.nodes.values(): - if node.op != "embedding" or not node.inputs[0].startswith("c_rope_table_"): - continue - table = ir.constants[node.inputs[0]] - ref = (torch.cos(emb) if "cos" in node.inputs[0] else torch.sin(emb)).to(torch.float16) - idx = [0, 1, 2049, TEST_MAX_SEQ - 1] - assert torch.equal(table[idx], ref) - assert node.inputs[1] == pos_for_output[node.outputs[0]] - - -# --------------------------------------------------------------------------- # -# inert: lm_encoder has no rope angle path -# --------------------------------------------------------------------------- # - - -def test_lm_encoder_is_inert() -> None: - ir = _load_ir(GEMMA_BUNDLE / "optimized_ir_lm_encoder_step.json", GEMMA_INV_FREQ) - assert precompute_rope_tables(ir) is False - embeddings, tables = _rope_nodes(ir) - assert tables == [] - - -def test_vision_encoder_is_inert() -> None: - """The vision tower's axial rope has the same cat(halves) <- matmul(inv_freq, - position) shape as decoder rope, but its position is a mid-graph ``select`` - (not a graph input). The pass must NOT fire, or it corrupts vision RoPE. The - inv_freq constant is materialized so the pass is rejected by the position - graph-input check, not by a missing constant.""" - path = GEMMA_BUNDLE / "optimized_ir_vision_encoder.json" - if not path.exists(): - pytest.skip("no vision encoder component for this bundle") - graph = json.loads(path.read_text())["graph"] - spec = { - cid: (10000.0, int(graph["values"][cid]["shape"][0]) * 2) - for cid in graph["constants"] - if "inv_freq" in cid - } - assert spec, "expected a vision rope inv_freq constant" - assert any(n["op"] in {"scalar_cos", "scalar_sin"} for n in graph["nodes"]), "vision encoder should have rope trig" - ir = _load_ir(path, spec) - assert precompute_rope_tables(ir) is False - _, tables = _rope_nodes(ir) - assert tables == [] From 58842d26e66542382702da34fb644345fb573a2e Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 19:47:21 +0100 Subject: [PATCH 14/55] Drop redundant keydiff_score formula comment The s_i formula is already on the header declaration; keep only the double-accumulation rationale. Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index b999ec51b..c187a6e0c 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -25,7 +25,7 @@ long py_round(double x) { } // namespace void keydiff_score(const float* keys, size_t n, size_t head_dim, float* out) { - // s_i = -cos(k_i, mean(k)), in double to match the float64 reference. + // double accumulation matches the float64 reference. std::vector mu(head_dim, 0.0); for (size_t i = 0; i < n; ++i) { const float* k = keys + i * head_dim; From 021141363ea52c6045cfeb522777e0626ccb3c29 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 19:59:32 +0100 Subject: [PATCH 15/55] Rename test_kv_compress_free_functions.cpp to test_kv_compress.cpp The old test_kv_compress.cpp (the MLX-fixture consumer) was removed, freeing the canonical name for the self-contained unit tests. Signed-off-by: Noah Cylich --- .../{test_kv_compress_free_functions.cpp => test_kv_compress.cpp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cactus-engine/tests/{test_kv_compress_free_functions.cpp => test_kv_compress.cpp} (100%) diff --git a/cactus-engine/tests/test_kv_compress_free_functions.cpp b/cactus-engine/tests/test_kv_compress.cpp similarity index 100% rename from cactus-engine/tests/test_kv_compress_free_functions.cpp rename to cactus-engine/tests/test_kv_compress.cpp From 28ab27e56d4331d692535a690d6a214fdeedd017 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 19:59:32 +0100 Subject: [PATCH 16/55] Shorten KV-compress comments Trim multi-line comments to two focused lines and drop low-value restatements. Signed-off-by: Noah Cylich --- cactus-engine/src/engine.h | 1 - cactus-engine/src/kv_compress.cpp | 9 ++--- cactus-engine/src/kv_compress.h | 40 ++++++++--------------- cactus-engine/src/model.cpp | 6 ++-- python/cactus/transpile/optimize_graph.py | 5 ++- 5 files changed, 21 insertions(+), 40 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 3932f086e..6cc03a717 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -633,7 +633,6 @@ class Model { void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} - // Keep abs_budget survivors per (layer, kv-head) and renumber to 0..B-1. void compress_kv_cache_keydiff(const cactus::kvcompress::Params& params); void maybe_roll_compact(); std::vector compressible_layers() const; diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index c187a6e0c..be09ab35b 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -10,10 +10,8 @@ namespace kvcompress { namespace { -// Python/numpy round half-to-even (not std::round's half-away-from-zero). recent_frac arrives as -// float and widens to a slightly-off double (0.30f -> 0.30000001...) which can flip an exact half -// boundary (0.3*15 = 4.5 vs 0.30000001*15 = 4.50000018), so snap near-halves before rounding to -// match the Python double reference. +// Round half-to-even like numpy. recent_frac widens from float to a slightly-off double, so snap +// near-halves first to avoid flipping an exact .5 boundary the Python reference keeps. long py_round(double x) { double twice = x * 2.0; double twice_rounded = std::nearbyint(twice); @@ -145,8 +143,7 @@ void requant_row(const float* row, int8_t* dst, float* dsc, size_t head_dim, siz } } -// `fill_post(h, post)` gathers head h's post-RoPE rows into `post` ([n][head_dim]); the rest is -// shared across storage precisions. +// fill_post(h, post) gathers head h's post-RoPE rows; scoring is shared across fp16/int8. template std::vector> keepsets_per_head(size_t n, size_t kv_heads, size_t head_dim, double rope_theta, const Params& p, FillPost fill_post) { diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index 7b244c4f9..ef9cf231e 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -30,45 +30,35 @@ static_assert(sizeof(CacheHeader) == 64, "CacheHeader must be 64 bytes"); // keys: row-major [n][head_dim] (pre-RoPE). out: [n]. void keydiff_score(const float* keys, size_t n, size_t head_dim, float* out); -// Keep-set for one cell: exactly B = min(max(1, abs_budget), n) sorted unique indices -// (sink + recent + top-score middle). Top-B sort is stable descending-score, ties broken -// by ascending index, to match np.argsort(-scores). +// Keep-set for one cell: B = min(max(1, abs_budget), n) sorted unique indices (sink + recent + +// top-score middle). Top-B is stable by descending score, ties by ascending index (np.argsort(-s)). std::vector keepset_for_head(const float* scores, size_t n, const Params& p); -// The cache stores POST-RoPE keys (rotate_half layout, see cactus-kernels/src/norms_rope.cpp): -// for pair (i, i+half), angle = pos * theta^(-2i/d), -// out[i] = x[i]*cos - x[i+half]*sin -// out[i+half] = x[i+half]*cos + x[i]*sin -// Rotating a post-RoPE row by `delta_pos` yields the row as if RoPE'd at (orig_pos + delta_pos), -// so un-RoPE = rotate by -orig_pos and renumber = rotate by (rank - orig). +// Keys are stored POST-RoPE (rotate_half, see cactus-kernels/src/norms_rope.cpp). Rotating a row by +// delta_pos re-RoPEs it to (orig + delta_pos): un-RoPE = -orig, renumber = (rank - orig). void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos); -// INT8 analog of rope_rotate_row: dequant (int8 * per-group scale), rotate, re-quantize in place. -// `scale` is the row's [groups] scales, groups = ceil(head_dim/group_size). +// INT8 analog of rope_rotate_row: dequant, rotate, re-quantize in place. scale is [groups] per row. void rotate_int8_row(int8_t* int8, float* scale, size_t head_dim, size_t group_size, double rope_theta, double delta_pos); void unrope_head(const float* post_rope, size_t n, size_t head_dim, double rope_theta, float* pre_rope); -// Per-head physical compaction: gather survivors in rank order, renumber K to contiguous -// positions 0..B-1, gather V unchanged. kept_per_head[h] is the survivor index list for -// kv-head h (length B, equal across heads); the caller sets header.current_seq_len to B. -// key_rows/val_rows point past the 64-byte header; layout [max_seq][kv_heads][head_dim]. +// Per-head compaction: gather survivors in rank order, renumber K to 0..B-1, gather V unchanged. +// kept_per_head[h] is head h's survivor indices (length B); rows point past the header, [max_seq][kv_heads][head_dim]. void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, const std::vector>& kept_per_head, double rope_theta); -// INT8 buffer (one of K or V). int8 region [max_seq][kv_heads*head_dim]; scales -// [max_seq][kv_heads*groups], groups = ceil(head_dim/group_size). `renumber` true (K buffer) -// rotates each gathered row by (rank - abs); false (V buffer) gathers as-is. +// INT8 analog of compact_fp16 (K or V). renumber=true (K) rotates each gathered row by (rank - abs); +// false (V) gathers as-is. scales are [max_seq][kv_heads*groups], groups = ceil(head_dim/group_size). void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, const std::vector>& kept_per_head, double rope_theta, bool renumber); -// Rotate recent K rows [lo, hi) by `delta_pos` so a windowed cache tracks the renumbered global -// frontier after compaction. Rows [0, lo) (the sink) stay fixed; V is never rotated. `rope_theta` -// is the LOCAL theta for sliding layers. No-op when delta_pos == 0 or hi <= lo. +// Rotate recent K rows [lo, hi) by delta_pos so a windowed cache tracks the renumbered frontier; +// sink rows [0, lo) and V stay fixed. No-op when delta_pos == 0 or hi <= lo. void rerope_recent_fp16(uint16_t* key_rows, size_t kv_heads, size_t head_dim, size_t lo, size_t hi, double rope_theta, double delta_pos); void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, @@ -79,16 +69,14 @@ std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_ size_t kv_heads, size_t head_dim, double rope_theta, const Params& p); -// As keepsets_from_fp16 but dequantizing the INT8 K buffer first. scale_rows is -// [max_seq][kv_heads*groups], groups = ceil(head_dim/group_size). +// As keepsets_from_fp16 but dequantizing the INT8 K buffer first; scale_rows [max_seq][kv_heads*groups]. std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, size_t n, size_t kv_heads, size_t head_dim, size_t group_size, double rope_theta, const Params& p); -// Sliding (local) layers re-rope with the LOCAL theta during compaction; full-attention layers -- -// including KV-shared global *source* layers that are excluded from compaction yet still ride the -// renumbered frame -- return false and re-rope with the GLOBAL theta. Empty layer_types -> global. +// True for sliding (local) layers. False for full-attention layers -- including KV-shared global +// *source* layers excluded from compaction -- so the re-rope picks local vs global theta correctly. bool is_sliding_layer(const std::vector& layer_types, size_t li); // Physically-compressible layer indices. Empty layer_types -> all layers (Qwen). num_kv_shared diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index a9461cc2f..e928ecaa9 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -3003,10 +3003,8 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } } - // Shift the non-compacted layers' recent K rows by -Δ (Δ = old frontier − B) into the same - // compressed frame; RoPE is relative, so a uniform shift preserves all query·key offsets, and - // sink rows stay fixed. Each layer shifts with its OWN theta: sliding layers use the local theta, - // while KV-shared global *source* layers (excluded from compaction, e.g. Gemma) keep the global theta. + // Shift non-compacted layers' recent K rows by -Δ into the compressed frame (RoPE is relative, so a + // uniform shift preserves q·k offsets); each uses its own theta -- local for sliding, global for shared sources. const size_t Delta = (have_new_seq_len && old_total >= new_seq_len) ? old_total - new_seq_len : 0; if (Delta > 0) { const double dpos = -static_cast(Delta); diff --git a/python/cactus/transpile/optimize_graph.py b/python/cactus/transpile/optimize_graph.py index 9a2a495eb..95630442c 100644 --- a/python/cactus/transpile/optimize_graph.py +++ b/python/cactus/transpile/optimize_graph.py @@ -367,9 +367,8 @@ def fuse_rope(graph: IRGraph) -> bool: def precompute_rope_tables(graph: IRGraph) -> bool: - """Replace the runtime ``cos/sin(position * inv_freq)`` rope angle with an - fp64-precomputed fp16 table gathered by position id (the fp16 angle matmul - cannot represent positions > 2048, randomising cos/sin past long context).""" + """Replace the runtime cos/sin(position * inv_freq) rope angle with an fp64-precomputed fp16 table + gathered by position id, fixing fp16's inability to represent angles past ~2048 in long context.""" changed = False for node_id in list(graph.order): From 10a923b1218696967052f0f4efd770c0e8df8df7 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 20:06:41 +0100 Subject: [PATCH 17/55] Hoist RoPE rotation table out of compaction row loops rope_rotate_row recomputed pow(theta, -2i/d) and cos/sin for every row. Factor a RopeRot (inv_freq via pow once, then cos/sin) so compaction and re-rope build it once: compact/un-rope reuse one inv_freq across rows, and the uniform-delta re-rope paths build the full cos/sin table a single time. Public API and per-row math are unchanged (bit-identical; 17/17 tests green). Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.cpp | 75 +++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index be09ab35b..aa4b0fc9f 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -95,25 +95,54 @@ std::vector keepset_for_head(const float* scores, size_t n, const Params& p return result; } -void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos) { - size_t half = head_dim / 2; +namespace { + +// Per-dim-pair cos/sin for a fixed delta_pos. inv_freq (the pow) and the trig are built once here +// so a row loop over a uniform delta -- or many rows sharing one inv_freq -- avoids recomputing them. +struct RopeRot { std::vector cos, sin; }; + +std::vector rope_inv_freq(size_t head_dim, double rope_theta) { + std::vector inv(head_dim / 2); + for (size_t i = 0; i < inv.size(); ++i) + inv[i] = std::pow(rope_theta, -(2.0 * static_cast(i)) / static_cast(head_dim)); + return inv; +} + +RopeRot rope_rot(const std::vector& inv_freq, double delta_pos) { + RopeRot r; + r.cos.resize(inv_freq.size()); + r.sin.resize(inv_freq.size()); + for (size_t i = 0; i < inv_freq.size(); ++i) { + double a = delta_pos * inv_freq[i]; + r.cos[i] = std::cos(a); + r.sin[i] = std::sin(a); + } + return r; +} + +void rope_apply(float* row, const RopeRot& r) { + size_t half = r.cos.size(); for (size_t i = 0; i < half; ++i) { - double inv = std::pow(rope_theta, -(2.0 * static_cast(i)) / static_cast(head_dim)); - double ang = delta_pos * inv; - double c = std::cos(ang), s = std::sin(ang); double x1 = row[i], x2 = row[i + half]; - row[i] = static_cast(x1 * c - x2 * s); - row[i + half] = static_cast(x2 * c + x1 * s); + row[i] = static_cast(x1 * r.cos[i] - x2 * r.sin[i]); + row[i + half] = static_cast(x2 * r.cos[i] + x1 * r.sin[i]); } } +} // namespace + +void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos) { + rope_apply(row, rope_rot(rope_inv_freq(head_dim, rope_theta), delta_pos)); +} + void unrope_head(const float* post_rope, size_t n, size_t head_dim, double rope_theta, float* pre_rope) { + auto inv = rope_inv_freq(head_dim, rope_theta); for (size_t t = 0; t < n; ++t) { const float* src = post_rope + t * head_dim; float* dst = pre_rope + t * head_dim; for (size_t d = 0; d < head_dim; ++d) dst[d] = src[d]; - rope_rotate_row(dst, head_dim, rope_theta, -static_cast(t)); + rope_apply(dst, rope_rot(inv, -static_cast(t))); } } @@ -143,6 +172,14 @@ void requant_row(const float* row, int8_t* dst, float* dsc, size_t head_dim, siz } } +void rotate_int8_row_rot(int8_t* int8, float* scale, size_t head_dim, size_t group_size, + const RopeRot& rot) { + std::vector row(head_dim); + for (size_t d = 0; d < head_dim; ++d) row[d] = static_cast(int8[d]) * scale[d / group_size]; + rope_apply(row.data(), rot); + requant_row(row.data(), int8, scale, head_dim, group_size); +} + // fill_post(h, post) gathers head h's post-RoPE rows; scoring is shared across fp16/int8. template std::vector> keepsets_per_head(size_t n, size_t kv_heads, size_t head_dim, @@ -165,6 +202,7 @@ void compact_fp16(uint16_t* key_rows_u, uint16_t* val_rows_u, size_t kv_heads, s const std::vector>& kept_per_head, double rope_theta) { __fp16* key_rows = reinterpret_cast<__fp16*>(key_rows_u); __fp16* val_rows = reinterpret_cast<__fp16*>(val_rows_u); + auto inv = rope_inv_freq(head_dim, rope_theta); std::vector krow(head_dim), vrow(head_dim); for (size_t h = 0; h < kv_heads; ++h) { const std::vector& kept = kept_per_head[h]; @@ -173,7 +211,7 @@ void compact_fp16(uint16_t* key_rows_u, uint16_t* val_rows_u, size_t kv_heads, s const __fp16* ksrc = key_rows + (static_cast(abs_pos) * kv_heads + h) * head_dim; const __fp16* vsrc = val_rows + (static_cast(abs_pos) * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) { krow[d] = ksrc[d]; vrow[d] = vsrc[d]; } - rope_rotate_row(krow.data(), head_dim, rope_theta, renumber_delta(abs_pos, rank)); + rope_apply(krow.data(), rope_rot(inv, renumber_delta(abs_pos, rank))); __fp16* kdst = key_rows + (rank * kv_heads + h) * head_dim; __fp16* vdst = val_rows + (rank * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) { @@ -191,6 +229,7 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t groups = (head_dim + group_size - 1) / group_size; size_t int8_stride = kv_heads * head_dim; size_t scale_stride = kv_heads * groups; + auto inv = rope_inv_freq(head_dim, rope_theta); std::vector row(head_dim); for (size_t h = 0; h < kv_heads; ++h) { const std::vector& kept = kept_per_head[h]; @@ -201,7 +240,7 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, const float* ssc = scale_rows + src_t * scale_stride + h * groups; for (size_t d = 0; d < head_dim; ++d) row[d] = static_cast(src[d]) * ssc[d / group_size]; if (renumber) - rope_rotate_row(row.data(), head_dim, rope_theta, renumber_delta(abs_pos, rank)); + rope_apply(row.data(), rope_rot(inv, renumber_delta(abs_pos, rank))); int8_t* dst = int8_rows + rank * int8_stride + h * head_dim; float* dsc = scale_rows + rank * scale_stride + h * groups; requant_row(row.data(), dst, dsc, head_dim, group_size); @@ -211,22 +250,21 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, void rotate_int8_row(int8_t* int8, float* scale, size_t head_dim, size_t group_size, double rope_theta, double delta_pos) { - std::vector row(head_dim); - for (size_t d = 0; d < head_dim; ++d) row[d] = static_cast(int8[d]) * scale[d / group_size]; - rope_rotate_row(row.data(), head_dim, rope_theta, delta_pos); - requant_row(row.data(), int8, scale, head_dim, group_size); + rotate_int8_row_rot(int8, scale, head_dim, group_size, + rope_rot(rope_inv_freq(head_dim, rope_theta), delta_pos)); } void rerope_recent_fp16(uint16_t* key_rows_u, size_t kv_heads, size_t head_dim, size_t lo, size_t hi, double rope_theta, double delta_pos) { if (delta_pos == 0.0 || hi <= lo) return; __fp16* key_rows = reinterpret_cast<__fp16*>(key_rows_u); + RopeRot rot = rope_rot(rope_inv_freq(head_dim, rope_theta), delta_pos); std::vector row(head_dim); for (size_t t = lo; t < hi; ++t) for (size_t h = 0; h < kv_heads; ++h) { __fp16* r = key_rows + (t * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) row[d] = r[d]; - rope_rotate_row(row.data(), head_dim, rope_theta, delta_pos); + rope_apply(row.data(), rot); for (size_t d = 0; d < head_dim; ++d) r[d] = static_cast<__fp16>(row[d]); } } @@ -236,11 +274,12 @@ void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, s if (delta_pos == 0.0 || hi <= lo) return; size_t groups = (head_dim + group_size - 1) / group_size; size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; + RopeRot rot = rope_rot(rope_inv_freq(head_dim, rope_theta), delta_pos); for (size_t t = lo; t < hi; ++t) for (size_t h = 0; h < kv_heads; ++h) - rotate_int8_row(int8_rows + t * int8_stride + h * head_dim, - scale_rows + t * scale_stride + h * groups, - head_dim, group_size, rope_theta, delta_pos); + rotate_int8_row_rot(int8_rows + t * int8_stride + h * head_dim, + scale_rows + t * scale_stride + h * groups, + head_dim, group_size, rot); } std::vector> keepsets_from_fp16(const uint16_t* key_rows_u, size_t n, From db17e67f4cfdd209b719ebbf46d0564bf9945bee Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 20:14:23 +0100 Subject: [PATCH 18/55] Shorten RopeRot comment Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index aa4b0fc9f..8319b3182 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -97,8 +97,7 @@ std::vector keepset_for_head(const float* scores, size_t n, const Params& p namespace { -// Per-dim-pair cos/sin for a fixed delta_pos. inv_freq (the pow) and the trig are built once here -// so a row loop over a uniform delta -- or many rows sharing one inv_freq -- avoids recomputing them. +// Precomputed cos/sin per dim-pair, built once so row loops don't recompute pow/trig. struct RopeRot { std::vector cos, sin; }; std::vector rope_inv_freq(size_t head_dim, double rope_theta) { From 9f384c0d82e40aa1441ab51a321b694a45a00783 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 20:29:18 +0100 Subject: [PATCH 19/55] Share one un-rope table across a compaction's keep-set scoring Keep-set scoring un-ropes every key row, but the un-rope rotation depends only on (position, head_dim, theta) -- identical across all compressible layers (all global, same theta) and all kv-heads. It was rebuilt per (layer, head). Build the per-position cos/sin table once (unrope_table) and thread it through keepsets_from_fp16/int8; compress_kv_cache_keydiff builds it on the first compressible layer and reuses it. Keep-sets are bit-identical (17/17 tests). A full 28-layer 4096->2048 compaction drops from ~650 ms to ~335 ms. Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.cpp | 62 +++++++++++++++++++------------ cactus-engine/src/kv_compress.h | 20 ++++++++-- cactus-engine/src/model.cpp | 8 +++- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index 8319b3182..2335ae086 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -97,9 +97,6 @@ std::vector keepset_for_head(const float* scores, size_t n, const Params& p namespace { -// Precomputed cos/sin per dim-pair, built once so row loops don't recompute pow/trig. -struct RopeRot { std::vector cos, sin; }; - std::vector rope_inv_freq(size_t head_dim, double rope_theta) { std::vector inv(head_dim / 2); for (size_t i = 0; i < inv.size(); ++i) @@ -107,8 +104,8 @@ std::vector rope_inv_freq(size_t head_dim, double rope_theta) { return inv; } -RopeRot rope_rot(const std::vector& inv_freq, double delta_pos) { - RopeRot r; +RopeRotation rope_rot(const std::vector& inv_freq, double delta_pos) { + RopeRotation r; r.cos.resize(inv_freq.size()); r.sin.resize(inv_freq.size()); for (size_t i = 0; i < inv_freq.size(); ++i) { @@ -119,7 +116,7 @@ RopeRot rope_rot(const std::vector& inv_freq, double delta_pos) { return r; } -void rope_apply(float* row, const RopeRot& r) { +void rope_apply(float* row, const RopeRotation& r) { size_t half = r.cos.size(); for (size_t i = 0; i < half; ++i) { double x1 = row[i], x2 = row[i + half]; @@ -134,15 +131,11 @@ void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delt rope_apply(row, rope_rot(rope_inv_freq(head_dim, rope_theta), delta_pos)); } -void unrope_head(const float* post_rope, size_t n, size_t head_dim, double rope_theta, - float* pre_rope) { +std::vector unrope_table(size_t n, size_t head_dim, double rope_theta) { auto inv = rope_inv_freq(head_dim, rope_theta); - for (size_t t = 0; t < n; ++t) { - const float* src = post_rope + t * head_dim; - float* dst = pre_rope + t * head_dim; - for (size_t d = 0; d < head_dim; ++d) dst[d] = src[d]; - rope_apply(dst, rope_rot(inv, -static_cast(t))); - } + std::vector table(n); + for (size_t t = 0; t < n; ++t) table[t] = rope_rot(inv, -static_cast(t)); + return table; } namespace { @@ -172,7 +165,7 @@ void requant_row(const float* row, int8_t* dst, float* dsc, size_t head_dim, siz } void rotate_int8_row_rot(int8_t* int8, float* scale, size_t head_dim, size_t group_size, - const RopeRot& rot) { + const RopeRotation& rot) { std::vector row(head_dim); for (size_t d = 0; d < head_dim; ++d) row[d] = static_cast(int8[d]) * scale[d / group_size]; rope_apply(row.data(), rot); @@ -182,13 +175,19 @@ void rotate_int8_row_rot(int8_t* int8, float* scale, size_t head_dim, size_t gro // fill_post(h, post) gathers head h's post-RoPE rows; scoring is shared across fp16/int8. template std::vector> keepsets_per_head(size_t n, size_t kv_heads, size_t head_dim, - double rope_theta, const Params& p, FillPost fill_post) { + const std::vector& unrope, + const Params& p, FillPost fill_post) { std::vector post(n * head_dim), pre(n * head_dim), scores(n); std::vector> out; out.reserve(kv_heads); for (size_t h = 0; h < kv_heads; ++h) { fill_post(h, post.data()); - unrope_head(post.data(), n, head_dim, rope_theta, pre.data()); + for (size_t t = 0; t < n; ++t) { + const float* src = post.data() + t * head_dim; + float* dst = pre.data() + t * head_dim; + for (size_t d = 0; d < head_dim; ++d) dst[d] = src[d]; + rope_apply(dst, unrope[t]); + } keydiff_score(pre.data(), n, head_dim, scores.data()); out.push_back(keepset_for_head(scores.data(), n, p)); } @@ -257,7 +256,7 @@ void rerope_recent_fp16(uint16_t* key_rows_u, size_t kv_heads, size_t head_dim, size_t lo, size_t hi, double rope_theta, double delta_pos) { if (delta_pos == 0.0 || hi <= lo) return; __fp16* key_rows = reinterpret_cast<__fp16*>(key_rows_u); - RopeRot rot = rope_rot(rope_inv_freq(head_dim, rope_theta), delta_pos); + RopeRotation rot = rope_rot(rope_inv_freq(head_dim, rope_theta), delta_pos); std::vector row(head_dim); for (size_t t = lo; t < hi; ++t) for (size_t h = 0; h < kv_heads; ++h) { @@ -273,7 +272,7 @@ void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, s if (delta_pos == 0.0 || hi <= lo) return; size_t groups = (head_dim + group_size - 1) / group_size; size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; - RopeRot rot = rope_rot(rope_inv_freq(head_dim, rope_theta), delta_pos); + RopeRotation rot = rope_rot(rope_inv_freq(head_dim, rope_theta), delta_pos); for (size_t t = lo; t < hi; ++t) for (size_t h = 0; h < kv_heads; ++h) rotate_int8_row_rot(int8_rows + t * int8_stride + h * head_dim, @@ -283,9 +282,10 @@ void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, s std::vector> keepsets_from_fp16(const uint16_t* key_rows_u, size_t n, size_t kv_heads, size_t head_dim, - double rope_theta, const Params& p) { + const std::vector& unrope, + const Params& p) { const __fp16* key_rows = reinterpret_cast(key_rows_u); - return keepsets_per_head(n, kv_heads, head_dim, rope_theta, p, [&](size_t h, float* post) { + return keepsets_per_head(n, kv_heads, head_dim, unrope, p, [&](size_t h, float* post) { for (size_t t = 0; t < n; ++t) { const __fp16* src = key_rows + (t * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) post[t * head_dim + d] = src[d]; @@ -293,14 +293,22 @@ std::vector> keepsets_from_fp16(const uint16_t* key_rows_u, siz }); } +std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, + size_t kv_heads, size_t head_dim, + double rope_theta, const Params& p) { + return keepsets_from_fp16(key_rows, n, kv_heads, head_dim, + unrope_table(n, head_dim, rope_theta), p); +} + std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, size_t n, size_t kv_heads, size_t head_dim, - size_t group_size, double rope_theta, + size_t group_size, + const std::vector& unrope, const Params& p) { size_t groups = (head_dim + group_size - 1) / group_size; size_t int8_stride = kv_heads * head_dim; size_t scale_stride = kv_heads * groups; - return keepsets_per_head(n, kv_heads, head_dim, rope_theta, p, [&](size_t h, float* post) { + return keepsets_per_head(n, kv_heads, head_dim, unrope, p, [&](size_t h, float* post) { for (size_t t = 0; t < n; ++t) { const int8_t* src = int8_rows + t * int8_stride + h * head_dim; const float* ssc = scale_rows + t * scale_stride + h * groups; @@ -310,6 +318,14 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const }); } +std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, + size_t n, size_t kv_heads, size_t head_dim, + size_t group_size, double rope_theta, + const Params& p) { + return keepsets_from_int8(int8_rows, scale_rows, n, kv_heads, head_dim, group_size, + unrope_table(n, head_dim, rope_theta), p); +} + bool is_sliding_layer(const std::vector& layer_types, size_t li) { return li < layer_types.size() && layer_types[li].find("sliding") != std::string::npos; } diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index ef9cf231e..82295d5f8 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -42,8 +42,12 @@ void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delt void rotate_int8_row(int8_t* int8, float* scale, size_t head_dim, size_t group_size, double rope_theta, double delta_pos); -void unrope_head(const float* post_rope, size_t n, size_t head_dim, double rope_theta, - float* pre_rope); +// Precomputed cos/sin per dim-pair for a fixed RoPE delta. +struct RopeRotation { std::vector cos, sin; }; + +// Un-rope rotations for positions [0, n) at (head_dim, theta): row t rotates by -t. Built once and +// shared across a compaction's keep-set scoring -- all compressible (global) layers share theta. +std::vector unrope_table(size_t n, size_t head_dim, double rope_theta); // Per-head compaction: gather survivors in rank order, renumber K to 0..B-1, gather V unchanged. // kept_per_head[h] is head h's survivor indices (length B); rows point past the header, [max_seq][kv_heads][head_dim]. @@ -64,12 +68,22 @@ void rerope_recent_fp16(uint16_t* key_rows, size_t kv_heads, size_t head_dim, void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos); -// Per-head keep-sets for one layer from POST-RoPE FP16 key rows: un-RoPE -> score -> keepset. +// Per-head keep-sets for one layer from POST-RoPE FP16 key rows: un-RoPE -> score -> keepset. The +// theta overload builds the un-rope table itself; pass a shared table to reuse it across layers. +std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, + size_t kv_heads, size_t head_dim, + const std::vector& unrope, + const Params& p); std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, size_t kv_heads, size_t head_dim, double rope_theta, const Params& p); // As keepsets_from_fp16 but dequantizing the INT8 K buffer first; scale_rows [max_seq][kv_heads*groups]. +std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, + size_t n, size_t kv_heads, size_t head_dim, + size_t group_size, + const std::vector& unrope, + const Params& p); std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, size_t n, size_t kv_heads, size_t head_dim, size_t group_size, double rope_theta, diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index e928ecaa9..e8d05189b 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2948,6 +2948,9 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) // B is identical across the compacted layers; capture it from the keep-set, not a mutated header. size_t new_seq_len = 0; bool have_new_seq_len = false; + // The un-rope table is identical across compressible layers (all global, same theta/dims), so + // build it once on the first such layer and reuse it for every layer's keep-set scoring. + std::vector unrope; for (size_t li = 0; li < comp.cache_states.size(); ++li) { if (!compressible.count(li)) continue; const auto& cs = comp.cache_states[li]; @@ -2969,12 +2972,13 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) size_t kv_heads = khdr->num_kv_heads; size_t head_dim = khdr->head_dim; if (kv_heads == 0 || head_dim == 0) continue; + if (unrope.empty()) unrope = cactus::kvcompress::unrope_table(n, head_dim, rope_theta); if (kdesc.precision == Precision::FP16) { auto* kbase = reinterpret_cast(static_cast(kraw) + kHeaderBytes); auto* vbase = reinterpret_cast(static_cast(vraw) + kHeaderBytes); auto kept = cactus::kvcompress::keepsets_from_fp16( - kbase, n, kv_heads, head_dim, rope_theta, params); + kbase, n, kv_heads, head_dim, unrope, params); cactus::kvcompress::compact_fp16(kbase, vbase, kv_heads, head_dim, kept, rope_theta); size_t B = kept.empty() ? 0 : kept[0].size(); khdr->current_seq_len = B; @@ -2990,7 +2994,7 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) auto* v_sc = reinterpret_cast(static_cast(vraw) + kHeaderBytes + max_seq * kv_heads * head_dim); auto kept = cactus::kvcompress::keepsets_from_int8( - k_i8, k_sc, n, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, rope_theta, params); + k_i8, k_sc, n, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, unrope, params); cactus::kvcompress::compact_int8(k_i8, k_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, kept, rope_theta, /*renumber=*/true); cactus::kvcompress::compact_int8(v_i8, v_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, From 8a20978cd2dd997574a3e95444bc86ade94e7df7 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 20:49:06 +0100 Subject: [PATCH 20/55] Vectorize int8 dequant in KV compaction Extract the int8 dequant (int8 * per-group scale) that was duplicated across compact_int8, rotate_int8_row, and the int8 keep-set fill into one dequant_row helper, and add a float32x4 NEON path (widen int8 -> float, multiply by the group scale). Bit-identical to scalar; ~1.5x on int8 keep-set scoring. A g_use_simd toggle (kv_set_simd) selects the scalar fallback so the new dequant_simd_matches_scalar test can compare both on a NEON build; non-ARM builds compile only the scalar path. fp16 rope/keydiff were left scalar -- clang -O3 already auto-vectorizes those double loops. Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.cpp | 45 +++++++++++++++++++++--- cactus-engine/src/kv_compress.h | 3 ++ cactus-engine/tests/test_kv_compress.cpp | 38 ++++++++++++++++++++ 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index 2335ae086..ee000e30a 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -2,12 +2,27 @@ #include #include +#include #include #include +#if defined(__ARM_NEON) +#include +#define CACTUS_KV_NEON 1 +#else +#define CACTUS_KV_NEON 0 +#endif + namespace cactus { namespace kvcompress { +namespace { +// Toggled off by tests to exercise/compare the scalar fallback on a NEON build. +bool g_use_simd = true; +} + +void kv_set_simd(bool on) { g_use_simd = on; } + namespace { // Round half-to-even like numpy. recent_frac widens from float to a slightly-off double, so snap @@ -164,10 +179,33 @@ void requant_row(const float* row, int8_t* dst, float* dsc, size_t head_dim, siz } } +void dequant_row(const int8_t* src, const float* scale, size_t head_dim, size_t group_size, + float* out) { +#if CACTUS_KV_NEON + if (g_use_simd) { + size_t groups = (head_dim + group_size - 1) / group_size; + for (size_t g = 0; g < groups; ++g) { + size_t lo = g * group_size, hi = std::min(head_dim, lo + group_size); + float32x4_t scv = vdupq_n_f32(scale[g]); + size_t d = lo; + for (; d + 4 <= hi; d += 4) { + int32_t four; + std::memcpy(&four, src + d, 4); + int16x8_t i16 = vmovl_s8(vreinterpret_s8_s32(vdup_n_s32(four))); + vst1q_f32(out + d, vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(i16))), scv)); + } + for (; d < hi; ++d) out[d] = static_cast(src[d]) * scale[g]; + } + return; + } +#endif + for (size_t d = 0; d < head_dim; ++d) out[d] = static_cast(src[d]) * scale[d / group_size]; +} + void rotate_int8_row_rot(int8_t* int8, float* scale, size_t head_dim, size_t group_size, const RopeRotation& rot) { std::vector row(head_dim); - for (size_t d = 0; d < head_dim; ++d) row[d] = static_cast(int8[d]) * scale[d / group_size]; + dequant_row(int8, scale, head_dim, group_size, row.data()); rope_apply(row.data(), rot); requant_row(row.data(), int8, scale, head_dim, group_size); } @@ -236,7 +274,7 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t src_t = static_cast(abs_pos); const int8_t* src = int8_rows + src_t * int8_stride + h * head_dim; const float* ssc = scale_rows + src_t * scale_stride + h * groups; - for (size_t d = 0; d < head_dim; ++d) row[d] = static_cast(src[d]) * ssc[d / group_size]; + dequant_row(src, ssc, head_dim, group_size, row.data()); if (renumber) rope_apply(row.data(), rope_rot(inv, renumber_delta(abs_pos, rank))); int8_t* dst = int8_rows + rank * int8_stride + h * head_dim; @@ -312,8 +350,7 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const for (size_t t = 0; t < n; ++t) { const int8_t* src = int8_rows + t * int8_stride + h * head_dim; const float* ssc = scale_rows + t * scale_stride + h * groups; - for (size_t d = 0; d < head_dim; ++d) - post[t * head_dim + d] = static_cast(src[d]) * ssc[d / group_size]; + dequant_row(src, ssc, head_dim, group_size, post + t * head_dim); } }); } diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index 82295d5f8..0d9dd5508 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -26,6 +26,9 @@ struct CacheHeader { }; static_assert(sizeof(CacheHeader) == 64, "CacheHeader must be 64 bytes"); +// Test hook: force the scalar fallback to compare against the NEON path on a NEON build. +void kv_set_simd(bool on); + // KeyDiff per-token score for one (layer, kv-head): s_i = -cos(k_i, mean(k)). // keys: row-major [n][head_dim] (pre-RoPE). out: [n]. void keydiff_score(const float* keys, size_t n, size_t head_dim, float* out); diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index ad5cb428f..bdd64660d 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -628,6 +628,43 @@ bool test_rerope_zero_delta_noop() { return kbuf == before; } +// dequant_row's NEON path must match the scalar fallback, via int8 compact and the int8 keep-set fill. +bool test_dequant_simd_matches_scalar() { + const size_t n = 64, max_seq = 96, kv_heads = 4, head_dim = 128; + const double theta = 1000000.0; + size_t groups = (head_dim + kGroupSize - 1) / kGroupSize; + size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; + Params p; p.recent_frac = 0.30f; p.sink = 4; p.abs_budget = 32; + + std::vector k8(max_seq * int8_stride, 0), v8(max_seq * int8_stride, 0); + std::vector ks(max_seq * scale_stride, 0.f), vs(max_seq * scale_stride, 0.f); + for (size_t h = 0; h < kv_heads; ++h) + for (size_t t = 0; t < n; ++t) { + std::vector k(head_dim), v(head_dim); + for (size_t d = 0; d < head_dim; ++d) { k[d] = std::sin(0.1 * (h + 1) * (t + 1) + 0.2 * d); v[d] = std::cos(0.05 * (t + 1) + 0.1 * d); } + std::vector kr = rope_reference(k, (double)t, theta); + quantize_row(kr, k8.data() + t * int8_stride + h * head_dim, ks.data() + t * scale_stride + h * groups, head_dim, kGroupSize); + quantize_row(v, v8.data() + t * int8_stride + h * head_dim, vs.data() + t * scale_stride + h * groups, head_dim, kGroupSize); + } + std::vector> kept = {{0, 5, 11, 22}, {1, 4, 13, 26}, {2, 8, 17, 30}, {3, 9, 19, 40}}; + + auto compact = [&](bool simd) { + kv_set_simd(simd); + std::vector a = k8; + std::vector b = ks; + compact_int8(a.data(), b.data(), kv_heads, head_dim, kGroupSize, kept, theta, true); + a.insert(a.end(), reinterpret_cast(b.data()), reinterpret_cast(b.data() + b.size())); + return a; + }; + auto keepsets = [&](bool simd) { + kv_set_simd(simd); + return keepsets_from_int8(k8.data(), ks.data(), n, kv_heads, head_dim, kGroupSize, theta, p); + }; + bool ok = compact(true) == compact(false) && keepsets(true) == keepsets(false); + kv_set_simd(true); + return ok; +} + int main() { TestUtils::TestRunner runner("KV Compress Free-Function Tests"); runner.run_test("compact_fp16_cache", test_compact_fp16_cache()); @@ -647,6 +684,7 @@ int main() { runner.run_test("trigger_zero_gates_rolling", test_trigger_zero_gates_rolling()); runner.run_test("degenerate_rolling_config_disabled", test_degenerate_rolling_config_disabled()); runner.run_test("env_override_parse", test_env_override_parse()); + runner.run_test("dequant_simd_matches_scalar", test_dequant_simd_matches_scalar()); runner.print_summary(); return runner.all_passed() ? 0 : 1; } From 304b865366766083989eaef231419fd4cac710bc Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Thu, 4 Jun 2026 20:59:33 +0100 Subject: [PATCH 21/55] Renumber compaction via table composition (drop per-survivor trig) Compaction's renumber rotated each survivor by (rank - abs_pos), computing cos/sin per survivor -- the dominant cost of compact. Since re-rope by +rank is the conjugate of the shared un-rope table (same cos, negated sin), renumber becomes: un-rope by -abs_pos (table[abs_pos]) then re-rope by +rank (conjugate of table[rank]) -- two table lookups, no per-row trig. compact_fp16 and compact_int8 take the shared un-rope table model.cpp already builds; the theta overloads build it from the kept indices for callers/tests. Two rotations vs one differ by ~2 ULP double, well below fp16/int8 resolution, so the stored cache is unchanged. A 28-layer 4096->2048 fp16 compaction drops from ~153 ms to ~32 ms (full compaction ~327 -> ~203 ms). 18/18 tests pass. Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.cpp | 54 ++++++++++++++++++++++++------- cactus-engine/src/kv_compress.h | 11 ++++++- cactus-engine/src/model.cpp | 6 ++-- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index ee000e30a..23e1eb50d 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -140,6 +140,22 @@ void rope_apply(float* row, const RopeRotation& r) { } } +// Apply the inverse rotation (negate the table angle): re-rope by +t given the un-rope table[t]. +void rope_apply_conj(float* row, const RopeRotation& r) { + size_t half = r.cos.size(); + for (size_t i = 0; i < half; ++i) { + double x1 = row[i], x2 = row[i + half]; + row[i] = static_cast(x1 * r.cos[i] + x2 * r.sin[i]); + row[i + half] = static_cast(x2 * r.cos[i] - x1 * r.sin[i]); + } +} + +size_t max_kept_index(const std::vector>& kept) { + size_t m = 0; + for (const auto& kh : kept) for (int idx : kh) m = std::max(m, static_cast(idx)); + return m; +} + } // namespace void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos) { @@ -155,10 +171,6 @@ std::vector unrope_table(size_t n, size_t head_dim, double rope_th namespace { -inline double renumber_delta(int abs_pos, size_t rank) { - return static_cast(rank) - static_cast(abs_pos); -} - // Matches the engine's KV quant convention (quantize_group_fp16_to_int8 in // cactus-kernels/src/quants.cpp): scale floored at 1e-10, roundf, clamp to [-128, 127]. void requant_row(const float* row, int8_t* dst, float* dsc, size_t head_dim, size_t group_size) { @@ -235,10 +247,10 @@ std::vector> keepsets_per_head(size_t n, size_t kv_heads, size_ } // namespace void compact_fp16(uint16_t* key_rows_u, uint16_t* val_rows_u, size_t kv_heads, size_t head_dim, - const std::vector>& kept_per_head, double rope_theta) { + const std::vector>& kept_per_head, + const std::vector& unrope) { __fp16* key_rows = reinterpret_cast<__fp16*>(key_rows_u); __fp16* val_rows = reinterpret_cast<__fp16*>(val_rows_u); - auto inv = rope_inv_freq(head_dim, rope_theta); std::vector krow(head_dim), vrow(head_dim); for (size_t h = 0; h < kv_heads; ++h) { const std::vector& kept = kept_per_head[h]; @@ -247,7 +259,9 @@ void compact_fp16(uint16_t* key_rows_u, uint16_t* val_rows_u, size_t kv_heads, s const __fp16* ksrc = key_rows + (static_cast(abs_pos) * kv_heads + h) * head_dim; const __fp16* vsrc = val_rows + (static_cast(abs_pos) * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) { krow[d] = ksrc[d]; vrow[d] = vsrc[d]; } - rope_apply(krow.data(), rope_rot(inv, renumber_delta(abs_pos, rank))); + // renumber abs_pos -> rank: un-rope by -abs_pos then re-rope by +rank (no per-row trig) + rope_apply(krow.data(), unrope[static_cast(abs_pos)]); + rope_apply_conj(krow.data(), unrope[rank]); __fp16* kdst = key_rows + (rank * kv_heads + h) * head_dim; __fp16* vdst = val_rows + (rank * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) { @@ -258,14 +272,19 @@ void compact_fp16(uint16_t* key_rows_u, uint16_t* val_rows_u, size_t kv_heads, s } } +void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, + const std::vector>& kept_per_head, double rope_theta) { + compact_fp16(key_rows, val_rows, kv_heads, head_dim, kept_per_head, + unrope_table(max_kept_index(kept_per_head) + 1, head_dim, rope_theta)); +} + void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, - const std::vector>& kept_per_head, double rope_theta, - bool renumber) { + const std::vector>& kept_per_head, + const std::vector& unrope, bool renumber) { size_t groups = (head_dim + group_size - 1) / group_size; size_t int8_stride = kv_heads * head_dim; size_t scale_stride = kv_heads * groups; - auto inv = rope_inv_freq(head_dim, rope_theta); std::vector row(head_dim); for (size_t h = 0; h < kv_heads; ++h) { const std::vector& kept = kept_per_head[h]; @@ -275,8 +294,10 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, const int8_t* src = int8_rows + src_t * int8_stride + h * head_dim; const float* ssc = scale_rows + src_t * scale_stride + h * groups; dequant_row(src, ssc, head_dim, group_size, row.data()); - if (renumber) - rope_apply(row.data(), rope_rot(inv, renumber_delta(abs_pos, rank))); + if (renumber) { // un-rope by -abs_pos then re-rope by +rank (no per-row trig) + rope_apply(row.data(), unrope[static_cast(abs_pos)]); + rope_apply_conj(row.data(), unrope[rank]); + } int8_t* dst = int8_rows + rank * int8_stride + h * head_dim; float* dsc = scale_rows + rank * scale_stride + h * groups; requant_row(row.data(), dst, dsc, head_dim, group_size); @@ -284,6 +305,15 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, } } +void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, + size_t head_dim, size_t group_size, + const std::vector>& kept_per_head, double rope_theta, + bool renumber) { + std::vector unrope; + if (renumber) unrope = unrope_table(max_kept_index(kept_per_head) + 1, head_dim, rope_theta); + compact_int8(int8_rows, scale_rows, kv_heads, head_dim, group_size, kept_per_head, unrope, renumber); +} + void rotate_int8_row(int8_t* int8, float* scale, size_t head_dim, size_t group_size, double rope_theta, double delta_pos) { rotate_int8_row_rot(int8, scale, head_dim, group_size, diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index 0d9dd5508..0bfe69945 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -54,11 +54,20 @@ std::vector unrope_table(size_t n, size_t head_dim, double rope_th // Per-head compaction: gather survivors in rank order, renumber K to 0..B-1, gather V unchanged. // kept_per_head[h] is head h's survivor indices (length B); rows point past the header, [max_seq][kv_heads][head_dim]. +// Renumber re-ropes via the un-rope table (un-rope by -abs_pos then re-rope by +rank), so pass the +// shared table to avoid per-survivor cos/sin; the theta overload builds it from the kept indices. +void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, + const std::vector>& kept_per_head, + const std::vector& unrope); void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, const std::vector>& kept_per_head, double rope_theta); -// INT8 analog of compact_fp16 (K or V). renumber=true (K) rotates each gathered row by (rank - abs); +// INT8 analog of compact_fp16 (K or V). renumber=true (K) re-ropes each gathered row to its new rank; // false (V) gathers as-is. scales are [max_seq][kv_heads*groups], groups = ceil(head_dim/group_size). +void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, + size_t head_dim, size_t group_size, + const std::vector>& kept_per_head, + const std::vector& unrope, bool renumber); void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, const std::vector>& kept_per_head, double rope_theta, diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index e8d05189b..5eaf52052 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2979,7 +2979,7 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) auto* vbase = reinterpret_cast(static_cast(vraw) + kHeaderBytes); auto kept = cactus::kvcompress::keepsets_from_fp16( kbase, n, kv_heads, head_dim, unrope, params); - cactus::kvcompress::compact_fp16(kbase, vbase, kv_heads, head_dim, kept, rope_theta); + cactus::kvcompress::compact_fp16(kbase, vbase, kv_heads, head_dim, kept, unrope); size_t B = kept.empty() ? 0 : kept[0].size(); khdr->current_seq_len = B; vhdr->current_seq_len = B; @@ -2996,9 +2996,9 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) auto kept = cactus::kvcompress::keepsets_from_int8( k_i8, k_sc, n, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, unrope, params); cactus::kvcompress::compact_int8(k_i8, k_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, - kept, rope_theta, /*renumber=*/true); + kept, unrope, /*renumber=*/true); cactus::kvcompress::compact_int8(v_i8, v_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, - kept, rope_theta, /*renumber=*/false); + kept, unrope, /*renumber=*/false); size_t B = kept.empty() ? 0 : kept[0].size(); khdr->current_seq_len = B; vhdr->current_seq_len = B; From d16cf73222e6c2cf1f659e74ad20a0e86e1952a3 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 00:13:31 +0100 Subject: [PATCH 22/55] Skip thinking-token cache strip on a compacted cache remove_thinking_tokens uses absolute token positions as cache row indices, but a rolling compaction renumbers rows to 0..B-1 and shrinks the cache. After a compaction the absolute ranges no longer map, so the strip decremented cache_total_seq_len_ while skipping the per-row memmove, desyncing the cache. Track a cache_renumbered_ flag (set in compress_kv_cache_keydiff, cleared in reset_cache) and early-return from remove_thinking_tokens when set; the caller still erases processed_tokens, and the bounded thinking rows linger until the next compaction evicts them. Only affects GEMMA4 + thinking after a compaction. Signed-off-by: Noah Cylich --- cactus-engine/src/engine.h | 1 + cactus-engine/src/model.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 6cc03a717..5726b2d43 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -765,6 +765,7 @@ class Model { std::unique_ptr tokenizer_; bool initialized_ = false; size_t cache_total_seq_len_ = 0; + bool cache_renumbered_ = false; size_t cache_max_seq_len_ = 4096; size_t last_logit_position_ = 0; double last_prefill_cache_copy_ms_ = 0.0; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 5eaf52052..3ddc1b887 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2814,6 +2814,7 @@ std::vector Model::get_audio_embeddings(const std::vector& mel_bin void Model::reset_cache() { cache_total_seq_len_ = 0; + cache_renumbered_ = false; last_logit_position_ = 0; context_tokens_.clear(); token_history_.clear(); @@ -2830,6 +2831,10 @@ void Model::reset_cache() { void Model::set_cache_window(size_t /*window_size*/, size_t /*sink_size*/) {} void Model::remove_thinking_tokens(const std::vector>& ranges) { + // Ranges are absolute token positions; a renumbered (compacted) cache no longer maps them, so + // skip the cache edit -- the caller still erases processed_tokens. + if (cache_renumbered_) return; + size_t total_removed = 0; for (const auto& r : ranges) total_removed += r.second; if (cache_total_seq_len_ >= total_removed) @@ -3044,7 +3049,7 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } // All layers now share the same compressed frame, so the next decode query uses position_ids = B. - if (have_new_seq_len) cache_total_seq_len_ = new_seq_len; + if (have_new_seq_len) { cache_total_seq_len_ = new_seq_len; cache_renumbered_ = true; } } void Model::maybe_roll_compact() { From 2a6103545f5029d379e349243fd6ba80545db213 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 10:29:47 +0100 Subject: [PATCH 23/55] Preserve special tokens across KV-cache compaction KeyDiff rolling compaction could evict structural special tokens (BOS, end-of-turn, think delimiters) when they were not recent or distinctive, degrading multi-turn conversations. Force-keep them: a per-cache-row token map (cache_token_ids_) tracks the text decode path and is gathered through each compaction with the first compressed layer's head-0 keep-set (canonical view -- exact in head 0, best-effort elsewhere). compress_kv_cache_keydiff unions the positions whose token is special into Params::protect, which the keep-set reserves just after the sink. Special ids come from the new Tokenizer::special_token_ids() (captures turn/think tokens the config parser misses). Guarded by cache_token_ids_ sync + no media; gated by kv_compress_preserve_special (default on) / CACTUS_KV_PRESERVE_SPECIAL. Signed-off-by: Noah Cylich --- cactus-engine/src/engine.h | 14 ++++ cactus-engine/src/kv_compress.cpp | 5 +- cactus-engine/src/kv_compress.h | 1 + cactus-engine/src/model.cpp | 41 ++++++++++- cactus-engine/tests/test_kv_compress.cpp | 91 +++++++++++++++++++++++- 5 files changed, 146 insertions(+), 6 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 5726b2d43..7e5504a2a 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -180,6 +180,7 @@ struct Config { uint32_t kv_compress_sink = 4; int32_t kv_compress_trigger_len = 4096; int32_t kv_compress_target_len = 2048; + bool kv_compress_preserve_special = true; uint32_t altup_num_inputs = 4; uint32_t laurel_rank = 64; @@ -328,6 +329,7 @@ class Tokenizer { virtual uint32_t get_unk_token() const = 0; virtual uint32_t get_bos_token() const = 0; virtual uint32_t get_eos_token() const = 0; + virtual std::unordered_set special_token_ids() const { return {}; } virtual bool has_chat_template() const { return has_chat_template_; } std::string get_default_stop_sequence() const; @@ -382,6 +384,11 @@ class BPETokenizer : public Tokenizer { uint32_t get_unk_token() const override { return unk_token_id_; } uint32_t get_bos_token() const override { return bos_token_id_; } uint32_t get_eos_token() const override { return eos_token_id_; } + std::unordered_set special_token_ids() const override { + std::unordered_set ids; + for (const auto& kv : special_tokens_) ids.insert(kv.second); + return ids; + } private: std::unordered_map token_to_id_; @@ -434,6 +441,11 @@ class SPTokenizer : public Tokenizer { uint32_t get_unk_token() const override { return unk_token_id_; } uint32_t get_bos_token() const override { return bos_token_id_; } uint32_t get_eos_token() const override { return eos_token_id_; } + std::unordered_set special_token_ids() const override { + std::unordered_set ids; + for (const auto& kv : special_tokens_) ids.insert(kv.second); + return ids; + } private: struct TrieNode { @@ -766,6 +778,8 @@ class Model { bool initialized_ = false; size_t cache_total_seq_len_ = 0; bool cache_renumbered_ = false; + std::vector cache_token_ids_; // token id per cache row (canonical head-0 view) + std::unordered_set special_ids_; // special-token ids force-kept during compaction size_t cache_max_seq_len_ = 4096; size_t last_logit_position_ = 0; double last_prefill_cache_copy_ms_ = 0.0; diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index 23e1eb50d..e4d24dffa 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -72,15 +72,18 @@ std::vector keepset_for_head(const float* scores, size_t n, const Params& p long n_recent = std::min(py_round(static_cast(p.recent_frac) * static_cast(B)), static_cast(n)); + auto in_range = [&](int idx) { return idx >= 0 && idx < static_cast(n); }; std::set reserved; for (long i = 0; i < sink; ++i) reserved.insert(i); + for (int idx : p.protect) if (in_range(idx)) reserved.insert(static_cast(idx)); for (long i = static_cast(n) - n_recent; i < static_cast(n); ++i) if (i >= 0) reserved.insert(i); - // Reserved can exceed B on a tight budget: keep sink first, then most-recent, until B. + // Reserved may exceed B: keep sink, protected, then recent, until B. if (static_cast(reserved.size()) > B) { std::vector ordered; for (long i = 0; i < sink; ++i) ordered.push_back(i); + for (int idx : p.protect) if (in_range(idx)) ordered.push_back(static_cast(idx)); for (long i = static_cast(n) - 1; i >= sink; --i) ordered.push_back(i); reserved.clear(); for (long i : ordered) { diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index 0bfe69945..0a0c7b8d8 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -13,6 +13,7 @@ struct Params { float recent_frac = 0.30f; size_t sink = 4; int abs_budget = 0; // keep budget B = min(max(1, abs_budget), n) per (layer, kv-head) + std::vector protect; // positions always kept (special tokens) }; // 64-byte KV cache header mirroring cactus-graph CacheMetadata. diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 3ddc1b887..5195ff89e 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -1498,6 +1498,7 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, if (tokens.empty() || !decoder_ || cache_total_seq_len_ != 0) { return false; } + cache_token_ids_ = tokens; if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); run_full_context_text(); @@ -1557,6 +1558,7 @@ void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); if (!context_tokens_.empty()) run_full_context_text(); cache_total_seq_len_ = context_tokens_.size(); + cache_token_ids_ = context_tokens_; return; } ChunkedPrefillResult chunked = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), prepare_decode); @@ -1565,6 +1567,7 @@ void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false); ++cache_total_seq_len_; } + cache_token_ids_.insert(cache_token_ids_.end(), tokens.begin(), tokens.end()); last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; if (prepare_decode) { @@ -2294,6 +2297,7 @@ void Model::prefill_with_media(const std::vector& tokens, run_step(t, pos, false); } cache_total_seq_len_ += tokens.size(); + cache_token_ids_.insert(cache_token_ids_.end(), tokens.begin(), tokens.end()); (void)profile_file; } @@ -2305,6 +2309,7 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* context_tokens_.insert(context_tokens_.end(), tokens.begin(), tokens.end()); run_full_context_text(); cache_total_seq_len_ = context_tokens_.size(); + cache_token_ids_ = context_tokens_; uint32_t result = argmax_last_logits(out_entropy); record_sampled_token(result); return result; @@ -2314,6 +2319,7 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* } run_step(tokens.back(), cache_total_seq_len_ + tokens.size() - 1, /*read_logits=*/true); cache_total_seq_len_ += tokens.size(); + cache_token_ids_.insert(cache_token_ids_.end(), tokens.begin(), tokens.end()); maybe_roll_compact(); uint32_t result = argmax_last_logits(out_entropy); record_sampled_token(result); @@ -2817,6 +2823,7 @@ void Model::reset_cache() { cache_renumbered_ = false; last_logit_position_ = 0; context_tokens_.clear(); + cache_token_ids_.clear(); token_history_.clear(); media_features_.clear(); media_feature_shapes_.clear(); @@ -2948,11 +2955,24 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) ? rope_theta : static_cast(config_.rope_local_base_freq); const size_t old_total = cache_total_seq_len_; + // Force-keep special-token rows. + cactus::kvcompress::Params params_local = params; + bool preserve = config_.kv_compress_preserve_special; + if (const char* e = std::getenv("CACTUS_KV_PRESERVE_SPECIAL")) preserve = (std::atoi(e) != 0); + const bool map_valid = cache_token_ids_.size() == old_total && media_features_.empty(); + if (preserve && map_valid) { + if (special_ids_.empty() && tokenizer_) special_ids_ = tokenizer_->special_token_ids(); + for (size_t pos = 0; pos < cache_token_ids_.size(); ++pos) + if (special_ids_.count(cache_token_ids_[pos])) params_local.protect.push_back(static_cast(pos)); + } + Component& comp = *decoder_; if (!comp.graph) return; // B is identical across the compacted layers; capture it from the keep-set, not a mutated header. size_t new_seq_len = 0; bool have_new_seq_len = false; + std::vector canonical_keep; + bool canonical_captured = false; // The un-rope table is identical across compressible layers (all global, same theta/dims), so // build it once on the first such layer and reuse it for every layer's keep-set scoring. std::vector unrope; @@ -2983,7 +3003,8 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) auto* kbase = reinterpret_cast(static_cast(kraw) + kHeaderBytes); auto* vbase = reinterpret_cast(static_cast(vraw) + kHeaderBytes); auto kept = cactus::kvcompress::keepsets_from_fp16( - kbase, n, kv_heads, head_dim, unrope, params); + kbase, n, kv_heads, head_dim, unrope, params_local); + if (!canonical_captured) { canonical_keep = kept.empty() ? std::vector{} : kept[0]; canonical_captured = true; } cactus::kvcompress::compact_fp16(kbase, vbase, kv_heads, head_dim, kept, unrope); size_t B = kept.empty() ? 0 : kept[0].size(); khdr->current_seq_len = B; @@ -2999,7 +3020,8 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) auto* v_sc = reinterpret_cast(static_cast(vraw) + kHeaderBytes + max_seq * kv_heads * head_dim); auto kept = cactus::kvcompress::keepsets_from_int8( - k_i8, k_sc, n, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, unrope, params); + k_i8, k_sc, n, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, unrope, params_local); + if (!canonical_captured) { canonical_keep = kept.empty() ? std::vector{} : kept[0]; canonical_captured = true; } cactus::kvcompress::compact_int8(k_i8, k_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, kept, unrope, /*renumber=*/true); cactus::kvcompress::compact_int8(v_i8, v_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, @@ -3049,7 +3071,19 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } // All layers now share the same compressed frame, so the next decode query uses position_ids = B. - if (have_new_seq_len) { cache_total_seq_len_ = new_seq_len; cache_renumbered_ = true; } + if (have_new_seq_len) { + cache_total_seq_len_ = new_seq_len; + cache_renumbered_ = true; + if (map_valid && canonical_captured) { + std::vector compacted; + compacted.reserve(canonical_keep.size()); + for (int idx : canonical_keep) + if (idx >= 0 && idx < static_cast(cache_token_ids_.size())) compacted.push_back(cache_token_ids_[idx]); + cache_token_ids_ = std::move(compacted); + } else { + cache_token_ids_.clear(); + } + } } void Model::maybe_roll_compact() { @@ -3254,6 +3288,7 @@ bool Config::from_json(const std::string& config_path) { else if (key == "kv_compress_sink") kv_compress_sink = static_cast(std::stoul(value)); else if (key == "kv_compress_trigger_len") kv_compress_trigger_len = static_cast(std::stol(value)); else if (key == "kv_compress_target_len") kv_compress_target_len = static_cast(std::stol(value)); + else if (key == "kv_compress_preserve_special") kv_compress_preserve_special = (value == "true" || value == "1"); else if (key == "layer_types") { layer_types.clear(); std::string sanitized; diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index bdd64660d..f55309ea9 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -2,6 +2,7 @@ #include "../src/kv_compress.h" #include "../src/engine.h" +#include #include #include #include @@ -447,8 +448,6 @@ bool test_env_override_parse() { return true; } -// --- Sliding-window re-rope (single position frame) ------------------------------------------- - bool test_rerope_recent_fp16_uniform() { // After a delta shift each recent row reads as rope(pre, t+delta); sink rows stay byte-identical. const size_t n = 40, max_seq = 64, kv_heads = 2, head_dim = 16, sink = 4; @@ -665,6 +664,92 @@ bool test_dequant_simd_matches_scalar() { return ok; } +bool test_keepset_protect_union() { + // A low-score position KeyDiff would evict is force-kept via Params::protect. + const size_t n = 64; + std::vector scores(n); + for (size_t i = 0; i < n; ++i) scores[i] = 0.01f * static_cast(i); + Params p; p.sink = 4; p.recent_frac = 0.25f; p.abs_budget = 24; + const int special = 9; + + auto base = keepset_for_head(scores.data(), n, p); + if (std::find(base.begin(), base.end(), special) != base.end()) return false; + + p.protect = {special}; + auto prot = keepset_for_head(scores.data(), n, p); + if (std::find(prot.begin(), prot.end(), special) == prot.end()) return false; + if (prot.size() != 24) return false; + for (int i = 0; i < 4; ++i) + if (std::find(prot.begin(), prot.end(), i) == prot.end()) return false; + return true; +} + +bool test_rolling_protect_survives() { + // A token KeyDiff would evict survives each compaction cycle when re-protected. + const int trigger_len = 256, target_len = 128; + const size_t kv_heads = 2, head_dim = 16, max_seq = trigger_len + 8; + const double theta = 1000000.0; + + std::vector kbuf(kHeaderBytes + max_seq * kv_heads * head_dim * sizeof(uint16_t), 0); + std::vector vbuf(kbuf.size(), 0); + auto* khdr = reinterpret_cast(kbuf.data()); + auto* vhdr = reinterpret_cast(vbuf.data()); + *khdr = Header{0, max_seq, kv_heads, head_dim, 4, {0, 0, 0}}; + *vhdr = *khdr; + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); + + std::vector special_v(head_dim); + for (size_t d = 0; d < head_dim; ++d) special_v[d] = (d % 2 ? -3.0f : 3.0f); + + auto append = [&](size_t from, size_t to, long special_abs) { + for (size_t t = from; t < to; ++t) + for (size_t h = 0; h < kv_heads; ++h) { + std::vector k(head_dim); + for (size_t d = 0; d < head_dim; ++d) k[d] = std::sin(0.05 * (h + 1) + 0.3 * d); + std::vector kr = rope_reference(k, (double)t, theta); + std::vector vr(head_dim); + if ((long)t == special_abs) vr = special_v; + else for (size_t d = 0; d < head_dim; ++d) vr[d] = std::cos(0.07 * (t + 1) + 0.2 * d); + for (size_t d = 0; d < head_dim; ++d) { + krows[(t * kv_heads + h) * head_dim + d] = f32_to_f16(kr[d]); + vrows[(t * kv_heads + h) * head_dim + d] = f32_to_f16(vr[d]); + } + } + }; + + auto special_rank = [&]() -> long { + for (size_t r = 0; r < khdr->current_seq_len; ++r) { + const uint16_t* v = vrows + (r * kv_heads + 0) * head_dim; + double dot = 0, na = 0, nb = 0; + for (size_t d = 0; d < head_dim; ++d) { + float x = f16_to_f32(v[d]); dot += x * special_v[d]; na += x * x; nb += special_v[d] * special_v[d]; + } + if (dot / (std::sqrt(na) * std::sqrt(nb) + 1e-9) > 0.99) return (long)r; + } + return -1; + }; + + long special_pos = trigger_len / 2; + for (int c = 0; c < 3; ++c) { + size_t start = khdr->current_seq_len; + append(start, (size_t)trigger_len, c == 0 ? special_pos : -1); + khdr->current_seq_len = trigger_len; + vhdr->current_seq_len = trigger_len; + + Params p; p.sink = 4; p.recent_frac = 0.30f; p.abs_budget = target_len; + p.protect = {(int)special_pos}; + auto kept = keepsets_from_fp16(krows, (size_t)trigger_len, kv_heads, head_dim, theta, p); + compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); + khdr->current_seq_len = kept[0].size(); + vhdr->current_seq_len = kept[0].size(); + + special_pos = special_rank(); + if (special_pos < 0) return false; + } + return true; +} + int main() { TestUtils::TestRunner runner("KV Compress Free-Function Tests"); runner.run_test("compact_fp16_cache", test_compact_fp16_cache()); @@ -685,6 +770,8 @@ int main() { runner.run_test("degenerate_rolling_config_disabled", test_degenerate_rolling_config_disabled()); runner.run_test("env_override_parse", test_env_override_parse()); runner.run_test("dequant_simd_matches_scalar", test_dequant_simd_matches_scalar()); + runner.run_test("keepset_protect_union", test_keepset_protect_union()); + runner.run_test("rolling_protect_survives", test_rolling_protect_survives()); runner.print_summary(); return runner.all_passed() ? 0 : 1; } From 4c8127d1d099148a6f1958cc5993bfb48ed6ae74 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 10:59:25 +0100 Subject: [PATCH 24/55] Tighten PR comments to non-obvious behavior only Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.h | 43 +++++++++-------------- python/cactus/transpile/component_plan.py | 2 -- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index 0a0c7b8d8..7cc82aaaf 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -12,11 +12,11 @@ namespace kvcompress { struct Params { float recent_frac = 0.30f; size_t sink = 4; - int abs_budget = 0; // keep budget B = min(max(1, abs_budget), n) per (layer, kv-head) + int abs_budget = 0; // per (layer, kv-head) keep budget, clamped to [1, n] std::vector protect; // positions always kept (special tokens) }; -// 64-byte KV cache header mirroring cactus-graph CacheMetadata. +// KV cache header mirroring cactus-graph CacheMetadata. struct CacheHeader { uint64_t current_seq_len; uint64_t max_seq_len; @@ -27,44 +27,37 @@ struct CacheHeader { }; static_assert(sizeof(CacheHeader) == 64, "CacheHeader must be 64 bytes"); -// Test hook: force the scalar fallback to compare against the NEON path on a NEON build. +// Test hook: force the scalar path on a NEON build. void kv_set_simd(bool on); -// KeyDiff per-token score for one (layer, kv-head): s_i = -cos(k_i, mean(k)). -// keys: row-major [n][head_dim] (pre-RoPE). out: [n]. +// KeyDiff score s_i = -cos(k_i, mean(k)); keys [n][head_dim] pre-RoPE. void keydiff_score(const float* keys, size_t n, size_t head_dim, float* out); -// Keep-set for one cell: B = min(max(1, abs_budget), n) sorted unique indices (sink + recent + -// top-score middle). Top-B is stable by descending score, ties by ascending index (np.argsort(-s)). +// Sorted keep-set for one cell: sink + recent + top-score middle; ties by ascending index (np.argsort(-s)). std::vector keepset_for_head(const float* scores, size_t n, const Params& p); -// Keys are stored POST-RoPE (rotate_half, see cactus-kernels/src/norms_rope.cpp). Rotating a row by -// delta_pos re-RoPEs it to (orig + delta_pos): un-RoPE = -orig, renumber = (rank - orig). +// Keys are stored POST-RoPE; rotating by delta_pos re-RoPEs to (orig + delta_pos): un-RoPE = -orig, renumber = rank - orig. void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos); -// INT8 analog of rope_rotate_row: dequant, rotate, re-quantize in place. scale is [groups] per row. +// INT8 analog of rope_rotate_row; scale is [groups] per row. void rotate_int8_row(int8_t* int8, float* scale, size_t head_dim, size_t group_size, double rope_theta, double delta_pos); // Precomputed cos/sin per dim-pair for a fixed RoPE delta. struct RopeRotation { std::vector cos, sin; }; -// Un-rope rotations for positions [0, n) at (head_dim, theta): row t rotates by -t. Built once and -// shared across a compaction's keep-set scoring -- all compressible (global) layers share theta. +// Un-rope rotations for [0, n): row t rotates by -t. Shared across a compaction's layers (one theta). std::vector unrope_table(size_t n, size_t head_dim, double rope_theta); -// Per-head compaction: gather survivors in rank order, renumber K to 0..B-1, gather V unchanged. -// kept_per_head[h] is head h's survivor indices (length B); rows point past the header, [max_seq][kv_heads][head_dim]. -// Renumber re-ropes via the un-rope table (un-rope by -abs_pos then re-rope by +rank), so pass the -// shared table to avoid per-survivor cos/sin; the theta overload builds it from the kept indices. +// Per-head compaction: gather survivors in rank order, re-rope K to 0..B-1, gather V unchanged. +// kept_per_head[h] is head h's survivor indices; the theta overload builds the un-rope table itself. void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, const std::vector>& kept_per_head, const std::vector& unrope); void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, const std::vector>& kept_per_head, double rope_theta); -// INT8 analog of compact_fp16 (K or V). renumber=true (K) re-ropes each gathered row to its new rank; -// false (V) gathers as-is. scales are [max_seq][kv_heads*groups], groups = ceil(head_dim/group_size). +// INT8 analog of compact_fp16. renumber=true (K) re-ropes each row to its new rank; false (V) gathers as-is. void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, const std::vector>& kept_per_head, @@ -74,15 +67,13 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, const std::vector>& kept_per_head, double rope_theta, bool renumber); -// Rotate recent K rows [lo, hi) by delta_pos so a windowed cache tracks the renumbered frontier; -// sink rows [0, lo) and V stay fixed. No-op when delta_pos == 0 or hi <= lo. +// Rotate recent K rows [lo, hi) by delta_pos so a windowed cache tracks the renumbered frontier; sink and V stay fixed. void rerope_recent_fp16(uint16_t* key_rows, size_t kv_heads, size_t head_dim, size_t lo, size_t hi, double rope_theta, double delta_pos); void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos); -// Per-head keep-sets for one layer from POST-RoPE FP16 key rows: un-RoPE -> score -> keepset. The -// theta overload builds the un-rope table itself; pass a shared table to reuse it across layers. +// Per-head keep-sets for one layer from POST-RoPE keys: un-RoPE -> score -> keepset. theta overload builds the table itself. std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, size_t kv_heads, size_t head_dim, const std::vector& unrope, @@ -91,7 +82,7 @@ std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_ size_t kv_heads, size_t head_dim, double rope_theta, const Params& p); -// As keepsets_from_fp16 but dequantizing the INT8 K buffer first; scale_rows [max_seq][kv_heads*groups]. +// As keepsets_from_fp16 but dequantizes the INT8 K buffer first. std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, size_t n, size_t kv_heads, size_t head_dim, size_t group_size, @@ -102,12 +93,10 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const size_t group_size, double rope_theta, const Params& p); -// True for sliding (local) layers. False for full-attention layers -- including KV-shared global -// *source* layers excluded from compaction -- so the re-rope picks local vs global theta correctly. +// True for sliding (local) layers, false for full-attention (incl. KV-shared global sources); selects local vs global theta. bool is_sliding_layer(const std::vector& layer_types, size_t li); -// Physically-compressible layer indices. Empty layer_types -> all layers (Qwen). num_kv_shared -// drops the shared consumer range and the per-type source layers (Gemma -> {4,9}). +// Compressible layer indices. Empty layer_types -> all (Qwen); num_kv_shared drops shared consumers + sources (Gemma -> {4,9}). std::vector physical_compressible_layers(const std::vector& layer_types, size_t num_layers, size_t num_kv_shared); diff --git a/python/cactus/transpile/component_plan.py b/python/cactus/transpile/component_plan.py index 7f3e5f52f..558a2756f 100644 --- a/python/cactus/transpile/component_plan.py +++ b/python/cactus/transpile/component_plan.py @@ -28,8 +28,6 @@ def _plan_from_profile( if config is not None and profile.needs_image: has_vision = _has_dict_config(config, "vision_config", "visual_config", "image_config") if not has_vision: - # qwen3 defaults to the chunked-prefill pipeline (the chunk adapters are qwen3 only). - # Key on architecture since the bundle config's model_type collapses "qwen3" to "qwen". architectures = [str(a).lower() for a in (config.get("architectures") or ())] if any("qwen3forcausallm" in a for a in architectures): return ComponentPlan( From 4390d586964d1b151ff1631da1a7c2e1b6760729 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 20:23:53 -0700 Subject: [PATCH 25/55] Grow the KV cache on demand and move it across the prefill handoff Transpiled decoders bake max_cache_seq_len (e.g. 262144 for qwen3-vl) and the runtime allocated the full buffer up front -- ~32 GB for a tiny prompt, almost all of it zeroed and untouched. Make the KV_CACHE_STATE buffer growable: it starts at 256 entries and doubles (relocating the int8 scales region, preserving stored rows) up to the baked ceiling, so RAM tracks actual occupancy. Sliding-window layers keep a fixed window-sized buffer; only global layers grow. The prefill->decode handoff now transfers buffer ownership (a cross-graph std::move via steal_cache_buffer) instead of copying into a second allocation, so both caches are never resident at once -- halving peak. The padded-prefill path truncates the moved cache and re-runs the last token instead of re-copying. copy_cache_states/ensure_cache_capacity are removed as dead. For qwen3-vl this takes a short prompt from ~32 GB to ~150 MB with byte-identical output, and a 5k-token needle prompt peaks ~1.1 GB. Signed-off-by: Noah Cylich --- cactus-engine/src/engine.h | 3 +- cactus-engine/src/model.cpp | 172 ++++------------------- cactus-engine/tests/test_kv_compress.cpp | 59 ++++++++ cactus-graph/cactus_graph.h | 1 + cactus-graph/src/ops_cache.cpp | 76 +++++++++- 5 files changed, 161 insertions(+), 150 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 7e5504a2a..518445552 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -714,7 +714,8 @@ class Model { void copy_component_outputs_to_chunk_inputs(const Component& source, Component& target, size_t token_index); void copy_component_outputs_to_chunk_inputs_range(const Component& source, Component& target, size_t token_offset); bool cache_states_compatible(const Component& source, const Component& target) const; - void copy_cache_states(const Component& source, Component& target, size_t logical_current = std::numeric_limits::max()); + void move_cache_states(Component& source, Component& target, size_t logical_current = std::numeric_limits::max()); + void set_cache_current_len(Component& comp, size_t len); void reset_component_cache_states(Component& comp); size_t component_chunk_tokens(const Component& comp, const std::string& input_name) const; size_t component_output_tokens(const Component& comp, const std::string& output_name) const; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 5195ff89e..b314548d7 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -821,7 +821,7 @@ bool Model::cache_states_compatible(const Component& source, const Component& ta return true; } -void Model::copy_cache_states(const Component& source, Component& target, size_t logical_current) { +void Model::move_cache_states(Component& source, Component& target, size_t logical_current) { if (source.cache_states.empty() || source.cache_states.size() != target.cache_states.size()) { throw std::runtime_error("prefill and step cache states are not compatible"); } @@ -832,149 +832,31 @@ void Model::copy_cache_states(const Component& source, Component& target, size_t throw std::runtime_error("prefill and step cache layer mismatch: " + src.layer_key + " != " + dst.layer_key); } for (auto [src_node, dst_node] : {std::pair{src.key_node_id, dst.key_node_id}, std::pair{src.value_node_id, dst.value_node_id}}) { - const auto& src_desc = source.graph->get_output_buffer(static_cast(src_node)); - const auto& dst_desc = target.graph->get_output_buffer(static_cast(dst_node)); - if (src_desc.precision != dst_desc.precision) { - std::ostringstream oss; - oss << "prefill and step cache precision mismatch at layer " << src.layer_key - << ": " << static_cast(src_desc.precision) - << " vs " << static_cast(dst_desc.precision); - throw std::runtime_error(oss.str()); - } - void* src_ptr = source.graph->get_output(static_cast(src_node)); - void* dst_ptr = target.graph->get_output(static_cast(dst_node)); - - const OpType src_op = source.graph->get_node_op_type(static_cast(src_node)); - const OpType dst_op = target.graph->get_node_op_type(static_cast(dst_node)); - if (src_op != dst_op) { - throw std::runtime_error( - "cache state op_type mismatch between prefill and step at layer " - + src.layer_key); - } - if (src_op == OpType::RECURRENT_CACHE_STATE) { - if (src_desc.byte_size != dst_desc.byte_size) { - throw std::runtime_error( - "recurrent cache buffer shape mismatch between prefill and step at layer " - + src.layer_key); - } - std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); - continue; - } - if (src_op == OpType::CONV_CACHE_STATE) { - if (src_desc.byte_size != dst_desc.byte_size) { - throw std::runtime_error( - "conv cache buffer shape mismatch between prefill and step at layer " - + src.layer_key); - } - std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); - continue; - } - if (src_desc.byte_size == dst_desc.byte_size && logical_current == std::numeric_limits::max()) { - std::memcpy(dst_ptr, src_ptr, src_desc.byte_size); - continue; - } - if (src_desc.precision == Precision::INT8) { - auto* src_meta = static_cast(src_ptr); - auto* dst_meta = static_cast(dst_ptr); - const size_t src_current = static_cast(src_meta[0]); - const size_t effective_current = std::min(src_current, logical_current); - const size_t src_max = static_cast(src_meta[1]); - const size_t kv_heads = static_cast(src_meta[2]); - const size_t head_dim = static_cast(src_meta[3]); - const size_t sink = static_cast(src_meta[4]); - if (src_max == 0 || kv_heads == 0 || head_dim == 0) { - throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); - } - const size_t groups = (head_dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - const size_t int8_stride = kv_heads * head_dim; - const size_t scale_stride = kv_heads * groups; - const size_t row_bytes = int8_stride + scale_stride * sizeof(float); - const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; - if (dst_max == 0) { - throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); + if (src_node < 0 || dst_node < 0) continue; + target.graph->steal_cache_buffer(static_cast(dst_node), *source.graph, static_cast(src_node)); + // KV caches carry a CacheMetadata header whose current_seq_len may need truncating to + // the logical handoff length (e.g. dropping padded tail rows). + if (target.graph->get_node_op_type(static_cast(dst_node)) == OpType::KV_CACHE_STATE && + logical_current != std::numeric_limits::max()) { + auto* meta = static_cast(target.graph->get_output(static_cast(dst_node))); + if (meta && logical_current < meta[0]) { + meta[4] = std::min(meta[4], logical_current); + meta[0] = logical_current; } - const size_t dst_current = std::min(effective_current, dst_max); - dst_meta[0] = dst_current; - dst_meta[1] = dst_max; - dst_meta[2] = kv_heads; - dst_meta[3] = head_dim; - dst_meta[4] = std::min(sink, dst_current); - std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); - - const auto* src_i8 = static_cast(src_ptr) + 64; - const auto* src_scales = reinterpret_cast( - static_cast(src_ptr) + 64 + src_max * int8_stride); - auto* dst_i8 = static_cast(dst_ptr) + 64; - auto* dst_scales = reinterpret_cast( - static_cast(dst_ptr) + 64 + dst_max * int8_stride); - auto copy_rows = [&](size_t dst_row, size_t src_row, size_t rows) { - if (rows == 0) return; - std::memcpy( - dst_i8 + dst_row * int8_stride, - src_i8 + src_row * int8_stride, - rows * int8_stride); - std::memcpy( - dst_scales + dst_row * scale_stride, - src_scales + src_row * scale_stride, - rows * scale_stride * sizeof(float)); - }; - if (effective_current <= dst_max) { - copy_rows(0, 0, effective_current); - } else { - const size_t copied_sink = std::min(sink, dst_max); - const size_t tail_rows = dst_max - copied_sink; - copy_rows(0, 0, copied_sink); - if (tail_rows > 0) copy_rows(copied_sink, effective_current - tail_rows, tail_rows); - } - continue; - } - if (PrecisionTraits::is_cq(src_desc.precision) || src_desc.byte_size < 64 || dst_desc.byte_size < 64) { - std::ostringstream oss; - oss << "prefill and step cache buffer mismatch at layer " << src.layer_key - << ": " << src_desc.byte_size << " bytes vs " << dst_desc.byte_size << " bytes"; - throw std::runtime_error(oss.str()); } + } + } +} - auto* src_meta = static_cast(src_ptr); - auto* dst_meta = static_cast(dst_ptr); - const size_t src_current = static_cast(src_meta[0]); - const size_t effective_current = std::min(src_current, logical_current); - const size_t src_max = static_cast(src_meta[1]); - const size_t kv_heads = static_cast(src_meta[2]); - const size_t head_dim = static_cast(src_meta[3]); - const size_t sink = static_cast(src_meta[4]); - if (src_max == 0 || kv_heads == 0 || head_dim == 0) { - throw std::runtime_error("prefill cache metadata is not initialized for layer " + src.layer_key); - } - const size_t row_bytes = kv_heads * head_dim * PrecisionTraits::size_of(src_desc.precision); - const size_t dst_max = (dst_desc.byte_size - 64) / row_bytes; - if (dst_max == 0) { - throw std::runtime_error("step cache capacity is zero for layer " + src.layer_key); - } - const size_t dst_current = std::min(effective_current, dst_max); - dst_meta[0] = dst_current; - dst_meta[1] = dst_max; - dst_meta[2] = kv_heads; - dst_meta[3] = head_dim; - dst_meta[4] = std::min(sink, dst_current); - std::memset(static_cast(dst_ptr) + 64, 0, dst_desc.byte_size - 64); - const auto* src_rows = static_cast(src_ptr) + 64; - auto* dst_rows = static_cast(dst_ptr) + 64; - if (effective_current <= dst_max) { - std::memcpy(dst_rows, src_rows, effective_current * row_bytes); - } else { - const size_t copied_sink = std::min(sink, dst_max); - const size_t tail_rows = dst_max - copied_sink; - if (copied_sink > 0) { - std::memcpy(dst_rows, src_rows, copied_sink * row_bytes); - } - if (tail_rows > 0) { - std::memcpy( - dst_rows + copied_sink * row_bytes, - src_rows + (effective_current - tail_rows) * row_bytes, - tail_rows * row_bytes); - } - } +void Model::set_cache_current_len(Component& comp, size_t len) { + for (const auto& state : comp.cache_states) { + for (int node_id : {state.key_node_id, state.value_node_id}) { + if (node_id < 0) continue; + if (comp.graph->get_node_op_type(static_cast(node_id)) != OpType::KV_CACHE_STATE) continue; + auto* meta = static_cast(comp.graph->get_output(static_cast(node_id))); + if (!meta || len >= meta[0]) continue; + meta[4] = std::min(meta[4], len); + meta[0] = len; } } } @@ -1129,7 +1011,7 @@ Model::ChunkedPrefillResult Model::run_chunked_prefill(const std::vectorinput_buffers[i].begin(), decoder_->input_buffers[i].end(), 0); } auto copy_start = std::chrono::high_resolution_clock::now(); - copy_cache_states(*decoder_prefill_, *decoder_, start_position + result.logical_tokens); + move_cache_states(*decoder_prefill_, *decoder_, start_position + result.logical_tokens); auto copy_end = std::chrono::high_resolution_clock::now(); last_prefill_cache_copy_ms_ = std::chrono::duration_cast(copy_end - copy_start).count() / 1000.0; } @@ -1523,7 +1405,9 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, if (decoder_prefill_) { chunked = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), true); if (chunked.logical_tokens == tokens.size() && chunked.padding_tokens > 0 && tokens.size() > 0) { - copy_cache_states(*decoder_prefill_, *decoder_, tokens.size() - 1); + // run_chunked_prefill already moved the cache into the step component; drop the padded + // last row and re-run the true last token on the step decoder for correct logits. + set_cache_current_len(*decoder_, tokens.size() - 1); cache_total_seq_len_ = tokens.size() - 1; run_step(tokens.back(), cache_total_seq_len_, true); ++cache_total_seq_len_; @@ -2151,7 +2035,7 @@ bool Model::run_chunk_prefill_path(const std::vector& tokens, decoder_prefill_chunk_->graph->execute(); } if (whole_chunks_end > 0 && decoder_ != nullptr) { - copy_cache_states(*decoder_prefill_chunk_, *decoder_); + move_cache_states(*decoder_prefill_chunk_, *decoder_); decoder_prefill_chunk_->graph->release_runtime_buffers(); unload_component_graph(*decoder_prefill_chunk_); } diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index f55309ea9..f5c256b17 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -1,6 +1,7 @@ #include "test_utils.h" #include "../src/kv_compress.h" #include "../src/engine.h" +#include "cactus_graph.h" #include #include @@ -750,8 +751,66 @@ bool test_rolling_protect_survives() { return true; } +// A global KV layer starts small (not the baked ceiling) and doubles as tokens append; every +// stored row must survive the scales-region relocation (the growable cache's key invariant). +bool test_cache_starts_small_and_grows() { + CactusGraph gb; + const size_t kv_heads = 2, head_dim = 64, ceiling = 100000, chunk = 100; + const size_t groups = (head_dim + kGroupSize - 1) / kGroupSize; + const size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; + size_t new_kv = gb.input({chunk, kv_heads, head_dim}, Precision::FP16); + size_t state = gb.kv_cache_state(ceiling, kv_heads, head_dim, /*window*/0, /*sink*/4); + size_t append = gb.kv_cache_append(new_kv, state, /*window*/0, /*sink*/4); + gb.retain_outputs({static_cast(state), static_cast(append)}); + + bool ok = true; + std::vector expect; // value per (global row, channel) + auto val = [](size_t row, size_t c) { return std::sin(0.013 * row + 0.07 * c); }; + for (size_t step = 0; step < 3; ++step) { + std::vector in(chunk * int8_stride); + for (size_t r = 0; r < chunk; ++r) + for (size_t c = 0; c < int8_stride; ++c) { + float v = static_cast(val(step * chunk + r, c)); + in[r * int8_stride + c] = f32_to_f16(v); + expect.push_back(v); + } + gb.set_input(new_kv, in.data(), Precision::FP16); + gb.execute(); + if (step == 0 && reinterpret_cast(gb.get_output(state))->max_seq_len != 256) ok = false; + } + + const auto* hdr = reinterpret_cast(gb.get_output(state)); + if (hdr->max_seq_len != 512) ok = false; // 300 rows forced 256 -> 512 + if (hdr->current_seq_len != 300) ok = false; + + const auto* base = static_cast(gb.get_output(state)); + const int8_t* i8 = reinterpret_cast(base + kHeaderBytes); + const float* sc = reinterpret_cast(base + kHeaderBytes + 512 * int8_stride); + for (size_t r = 0; r < 300 && ok; ++r) + for (size_t h = 0; h < kv_heads; ++h) + for (size_t d = 0; d < head_dim; ++d) { + float dq = static_cast(i8[r * int8_stride + h * head_dim + d]) * + sc[r * scale_stride + h * groups + d / kGroupSize]; + if (std::abs(dq - expect[r * int8_stride + h * head_dim + d]) > 0.05f) ok = false; + } + return ok; +} + +// A sliding-window layer gets a fixed window-sized buffer, never the baked ceiling. +bool test_sliding_layer_fixed_capacity() { + CactusGraph gb; + const size_t kv_heads = 2, head_dim = 64, ceiling = 100000, window = 512, sink = 4; + size_t state = gb.kv_cache_state(ceiling, kv_heads, head_dim, window, sink); + gb.retain_outputs({static_cast(state)}); + gb.execute(); + auto* hdr = reinterpret_cast(gb.get_output(state)); + return hdr->max_seq_len == window + sink + 1; +} + int main() { TestUtils::TestRunner runner("KV Compress Free-Function Tests"); + runner.run_test("cache_starts_small_and_grows", test_cache_starts_small_and_grows()); + runner.run_test("sliding_layer_fixed_capacity", test_sliding_layer_fixed_capacity()); runner.run_test("compact_fp16_cache", test_compact_fp16_cache()); runner.run_test("dense_check_full_budget", test_dense_check_full_budget()); runner.run_test("rope_renumber_contiguous", test_rope_renumber_contiguous()); diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index f4f808555..fce075efb 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -708,6 +708,7 @@ class CactusGraph { const BufferDesc& get_output_buffer(size_t node_id) const; OpType get_node_op_type(size_t node_id) const; size_t get_node_window_size(size_t node_id) const; + void steal_cache_buffer(size_t dst_node, CactusGraph& src, size_t src_node); void allocate_buffers(); size_t get_node_count() const; diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 7df2a588a..ce497caa5 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace { @@ -87,6 +88,47 @@ inline bool use_fp16_kv_cache() { return cached; } +// Initial KV-cache capacity; the buffer doubles up to the baked ceiling as tokens accrue. +constexpr size_t kInitialCacheEntries = 256; + +// Double a KV cache buffer to hold `needed` rows (capped at `ceiling`), relocating the int8 +// scales region and preserving stored rows. Returns true if reallocated. +inline bool grow_cache_buffer(BufferDesc& buf, size_t needed, size_t ceiling) { + auto* meta = get_meta(buf); + size_t cur = meta->max_seq_len; + if (needed <= cur || cur >= ceiling) return false; + size_t new_max = cur; + while (new_max < needed) new_max <<= 1; + if (new_max > ceiling) new_max = ceiling; + if (new_max <= cur) return false; + + const size_t kv_heads = meta->num_kv_heads; + const size_t hdim = meta->head_dim; + const size_t current_seq = meta->current_seq_len; + const bool fp16_cache = buf.precision == Precision::FP16; + + size_t total = fp16_cache ? fp16_cache_elements(new_max, kv_heads, hdim) + : cache_buffer_size(new_max, kv_heads, hdim); + BufferDesc grown({total}, fp16_cache ? Precision::FP16 : Precision::INT8); + grown.allocate(); + std::memset(grown.get_data(), 0, grown.byte_size); + + std::memcpy(grown.get_data(), buf.get_data(), sizeof(CacheMetadata)); + if (fp16_cache) { + std::memcpy(get_fp16_data(grown), get_fp16_data(buf), + current_seq * kv_heads * hdim * sizeof(__fp16)); + } else { + std::memcpy(get_int8_data(grown), get_int8_data(buf), current_seq * kv_heads * hdim); + const size_t groups = (hdim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; + std::memcpy(get_scales(grown, new_max, kv_heads, hdim), + get_scales(buf, cur, kv_heads, hdim), + current_seq * kv_heads * groups * sizeof(float)); + } + get_meta(grown)->max_seq_len = new_max; + buf = std::move(grown); + return true; +} + } // namespace void compute_kv_cache_state_node( @@ -96,7 +138,12 @@ void compute_kv_cache_state_node( if (node.output_buffer.get_data()) return; - size_t max_seq = node.params.max_cache_seq_len; + size_t ceiling = node.params.max_cache_seq_len; + size_t window = node.params.window_size; + bool sliding = window > 0 && window < ceiling; + // Sliding layers need a fixed window-sized buffer; global layers start small and grow. + size_t max_seq = sliding ? std::min(ceiling, window + node.params.cache_sink_size + 1) + : std::min(ceiling, kInitialCacheEntries); size_t kv_heads = node.params.num_kv_heads; size_t hdim = node.params.head_dim; const bool fp16_cache = use_fp16_kv_cache(); @@ -136,12 +183,22 @@ void compute_kv_cache_append_node( size_t int8_stride = kv_heads * hdim; size_t scale_stride = kv_heads * num_groups; + // Global layers (no effective sliding window) grow the buffer to fit; capped at the baked + // ceiling, beyond which the eviction path below takes over. Sliding layers stay fixed. + size_t ceiling = nodes[node_index_map.at(node.input_ids[1])]->params.max_cache_seq_len; + bool sliding = node.params.window_size > 0 && node.params.window_size < ceiling; + if (!sliding && current_len + new_seq_len > max_len) { + if (grow_cache_buffer(cache_buf, current_len + new_seq_len, ceiling)) { + meta = get_meta(cache_buf); + max_len = meta->max_seq_len; + } + } + if (cache_buf.precision == Precision::FP16) { size_t stride = kv_heads * hdim; __fp16* fp16_base = get_fp16_data(cache_buf); const __fp16* source = new_kv.data_as<__fp16>(); - size_t window = node.params.window_size; - if (window == 0) window = max_len; + size_t window = sliding ? node.params.window_size : max_len; size_t new_total = current_len + new_seq_len; bool needs_eviction = new_total > window; @@ -185,8 +242,7 @@ void compute_kv_cache_append_node( int8_t* int8_base = get_int8_data(cache_buf); float* scale_base = get_scales(cache_buf, max_len, kv_heads, hdim); - size_t window = node.params.window_size; - if (window == 0) window = max_len; + size_t window = sliding ? node.params.window_size : max_len; size_t new_total = current_len + new_seq_len; bool needs_eviction = new_total > window; @@ -467,3 +523,13 @@ void compute_recurrent_cache_write_node( auto& cache_buf = nodes[node_index_map.at(node.input_ids[1])]->output_buffer; std::memcpy(cache_buf.get_data(), src.get_data(), src.byte_size); } + +// Move a cache buffer from another graph's node into this one (no copy); the source node is left +// empty and reallocates small on its next execute. +void CactusGraph::steal_cache_buffer(size_t dst_node, CactusGraph& src, size_t src_node) { + auto& dst = nodes_[node_index_map_.at(dst_node)]; + auto& s = src.nodes_[src.node_index_map_.at(src_node)]; + assert(dst->op_type == s->op_type && dst->output_buffer.precision == s->output_buffer.precision); + dst->output_buffer = std::move(s->output_buffer); +} + From 659c22f4416ebf58538e539b4b3a6476cb3b3541 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 20:24:13 -0700 Subject: [PATCH 26/55] Use a media-specific default auto-prompt in the chat test tool When an image/audio is attached without --prompt, the tool auto-sent the generic "Describe the attached input.", which some VLMs (e.g. qwen3-vl) degenerate into a repetition loop on. Pick the prompt by attached media ("Describe this image." / "Transcribe or respond to this audio."). Signed-off-by: Noah Cylich --- cactus-engine/tests/chat.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index b578d5b16..75b2ae10a 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -517,7 +517,9 @@ int main(int argc, char** argv) { std::string input; if (auto_send) { auto_send = false; - input = initial_prompt.empty() ? "Describe the attached input." : initial_prompt; + if (!initial_prompt.empty()) input = initial_prompt; + else if (!current_image.empty()) input = "Describe this image."; + else input = "Transcribe or respond to this audio."; std::cout << "You: " << input << "\n"; } else { std::cout << "You: " << std::flush; From 862781396b07ca2716fe8d65b82ee1e7c863b2e2 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 27 May 2026 23:30:50 -0700 Subject: [PATCH 27/55] Support full-context cache transpilation Add explicit cache context sizing for cached component transpilation so decoder graphs can reserve the model context length instead of deriving capacity from the capture prompt. Gemma4, Qwen, and LFM cached decode builders now share the same config-driven resolver while preserving their existing capture-derived fallback sizes when no model context field is available. For Gemma4, sliding-attention layers keep the configured sliding-window cache size while full-attention layers receive the requested full cache length. Add a retranspile path that reuses existing converted weights, wire the cache context option through the CLI and transpile entrypoint, and make lowering honor per-attention cache length metadata. Add token-file input support for exact-token live generation and a context-scaling benchmark tool that records CSV/JSONL summaries and plots. Cover the new CLI plumbing, retranspile behavior, context parsing, and per-layer cache metadata with focused tests. Signed-off-by: Noah Cylich --- cactus-engine/tests/chat.cpp | 36 +- python/cactus/cli/__init__.py | 6 + python/cactus/cli/convert.py | 2 + python/cactus/cli/model.py | 7 +- python/cactus/cli/run.py | 1 + python/cactus/transpile/hf_model.py | 6 + python/cactus/transpile/lower.py | 3 +- python/cactus/transpile/model_adapters.py | 124 ++++- python/cactus/transpile/optimize_graph.py | 29 ++ python/tests/test_bindings.py | 5 + python/tests/test_cli_run.py | 4 + python/tests/test_cli_transpile_defaults.py | 106 ++++- .../tests/test_optimize_gemma4_attention.py | 51 ++ .../tools/benchmark_gemma4_context_scaling.py | 445 ++++++++++++++++++ 14 files changed, 802 insertions(+), 23 deletions(-) create mode 100755 python/tests/transpile/tools/benchmark_gemma4_context_scaling.py diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index 75b2ae10a..a5074e39b 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -301,7 +302,11 @@ bool file_exists(const std::string& path) { std::vector parse_token_ids(const std::string& text) { std::vector tokens; - std::stringstream ss(text); + std::string normalized = text; + for (char& ch : normalized) { + if (ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n') ch = ','; + } + std::stringstream ss(normalized); std::string part; while (std::getline(ss, part, ',')) { size_t start = part.find_first_not_of(" \t\r\n"); @@ -318,6 +323,16 @@ std::vector parse_token_ids(const std::string& text) { return tokens; } +bool read_text_file(const std::string& path, std::string& text) { + std::ifstream in(path); + if (!in) return false; + std::ostringstream buffer; + buffer << in.rdbuf(); + if (!in.good() && !in.eof()) return false; + text = buffer.str(); + return true; +} + bool write_text_file(const std::string& path, const std::string& text) { std::ofstream out(path); if (!out) return false; @@ -395,7 +410,7 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 << " [--system ] [--image ] [--audio ]" - << " [--prompt ] [--input-ids ] [--max-new-tokens ]" + << " [--prompt ] [--input-ids ] [--input-ids-file ] [--max-new-tokens ]" << " [--result-json ] [--thinking] [--no-cloud-handoff]" << " [--confidence-threshold ] [--cloud-timeout-ms ]\n"; } @@ -417,6 +432,7 @@ int main(int argc, char** argv) { std::string current_audio; std::string initial_prompt; std::string input_ids; + std::string input_ids_file; std::string result_json; int max_new_tokens = kMaxTokens; bool thinking = false; @@ -436,6 +452,8 @@ int main(int argc, char** argv) { initial_prompt = argv[++i]; } else if (arg == "--input-ids" && i + 1 < argc) { input_ids = argv[++i]; + } else if (arg == "--input-ids-file" && i + 1 < argc) { + input_ids_file = expand_tilde(argv[++i]); } else if (arg == "--max-new-tokens" && i + 1 < argc) { max_new_tokens = std::max(0, std::atoi(argv[++i])); } else if (arg == "--result-json" && i + 1 < argc) { @@ -459,6 +477,20 @@ int main(int argc, char** argv) { std::cerr << "Audio file not found: " << current_audio << "\n"; return 1; } + if (!input_ids.empty() && !input_ids_file.empty()) { + std::cerr << "Use either --input-ids or --input-ids-file, not both\n"; + return 1; + } + if (!input_ids_file.empty()) { + if (!file_exists(input_ids_file)) { + std::cerr << "Input ids file not found: " << input_ids_file << "\n"; + return 1; + } + if (!read_text_file(input_ids_file, input_ids)) { + std::cerr << "Failed to read input ids file: " << input_ids_file << "\n"; + return 1; + } + } std::cout << "Loading model from " << model_path << "...\n"; cactus_model_t model = cactus_init(model_path.c_str(), nullptr, false); diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 48108abe5..4c15fe11a 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -187,6 +187,8 @@ def create_parser(): help="Initial prompt to send immediately") run_parser.add_argument("--input-ids", default=None, help="Comma-separated token ids for transpiled causal-LM bundles") + run_parser.add_argument("--input-ids-file", default=None, + help="File containing token ids for transpiled causal-LM bundles") run_parser.add_argument("--max-new-tokens", type=int, default=None, help="Maximum tokens to generate for transpiled causal-LM bundles") run_parser.add_argument("--result-json", default=None, @@ -283,6 +285,8 @@ def create_parser(): help="Allow HF remote code during the transpile phase") convert_parser.add_argument("--local-files-only", action="store_true", help="Require HF model/processor files to already be local during transpile") + convert_parser.add_argument("--cache-context-length", default=None, + help="KV cache context length for cached decode graphs (default: model config)") convert_parser.add_argument("--reconvert", action="store_true", help="Force conversion from source") convert_parser.add_argument("--npu", action="store_true", @@ -293,6 +297,8 @@ def create_parser(): help="Audio encoder weight quant (0=fp16, 4=int4, 8=int8). Default int8.") convert_parser.add_argument("--npu-vision-quantize", type=int, choices=[0, 4, 8], default=None, help="Vision encoder weight quant (0=fp16, 4=int4, 8=int8). Default fp16 — int4 is known to degrade Gemma 4 vision output.") + convert_parser.add_argument("--retranspile", action="store_true", + help="Force transpilation while reusing existing converted weights") return parser diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index 2eaef3c40..5e2d906e9 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -14,6 +14,7 @@ def cmd_convert(args): bits=args.bits or 4, token=args.token, reconvert=args.reconvert, + retranspile=args.retranspile, output_dir=output_dir, transpile=TranspileOptions( task=args.task or "auto", @@ -30,6 +31,7 @@ def cmd_convert(args): npu_quantize=getattr(args, "npu_quantize", None), npu_audio_quantize=getattr(args, "npu_audio_quantize", None), npu_vision_quantize=getattr(args, "npu_vision_quantize", None), + cache_context_length=args.cache_context_length, ), ) return 0 diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index decdb5c6c..ec5f8d345 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -204,10 +204,11 @@ class TranspileOptions: npu_quantize: int | None = None npu_audio_quantize: int | None = None npu_vision_quantize: int | None = None + cache_context_length: str | int | None = None def ensure_bundle(model_id, *, bits=4, token=None, - reconvert=False, output_dir=None, transpile=None): + reconvert=False, retranspile=False, output_dir=None, transpile=None): """Return path to transpiled bundle, creating it if needed. """ from .download import get_weights_dir @@ -228,7 +229,7 @@ def ensure_bundle(model_id, *, bits=4, token=None, ) # Step 2: skip if already transpiled - if _has_transpiled_bundle(output_dir): + if _has_transpiled_bundle(output_dir) and not retranspile: return output_dir # Step 3: infer transpile spec from converted output @@ -324,6 +325,8 @@ def ensure_bundle(model_id, *, bits=4, token=None, extra_args.extend(["--npu-audio-quantize", str(int(opts.npu_audio_quantize))]) if opts.npu_vision_quantize is not None: extra_args.extend(["--npu-vision-quantize", str(int(opts.npu_vision_quantize))]) + if opts.cache_context_length is not None: + extra_args.extend(["--cache-context-length", str(opts.cache_context_length)]) rc = run_transpile(model_id, extra_args=extra_args) if rc != 0: diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index bcc049aaf..aaf8dddda 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -44,6 +44,7 @@ def cmd_run(args): ("--image", args.image), ("--audio", args.audio), ("--input-ids", args.input_ids), + ("--input-ids-file", args.input_ids_file), ("--max-new-tokens", args.max_new_tokens), ("--result-json", args.result_json), ("--confidence-threshold", args.confidence_threshold), diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index a21e73d2b..c1359fa03 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -2962,6 +2962,11 @@ def main() -> int: "so run-transpiled can do greedy autoregressive decoding." ), ) + parser.add_argument( + "--cache-context-length", + default="auto", + help="KV cache context length for cached decode graphs. Use auto to read the model config.", + ) parser.add_argument( "--audio-file", default="", @@ -3267,6 +3272,7 @@ def main() -> int: weights_dir=weights_dir, inputs_metadata=prepared.metadata, components=requested_components, + cache_context_length=args.cache_context_length, ) use_component_pipeline = False if args.component_pipeline == "on": diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index 5f2c3d1e2..e16af4f9b 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -238,7 +238,8 @@ def _lower_attention_with_internal_kv_cache( ) if layer_key not in cache_states: default_cache_len = max(512, int(key_shape[1])) - max_cache_seq_len = int(ir.meta.get("max_cache_seq_len", default_cache_len) or default_cache_len) + requested_cache_len = node.meta.get("max_cache_seq_len", ir.meta.get("max_cache_seq_len", default_cache_len)) + max_cache_seq_len = max(default_cache_len, int(requested_cache_len or default_cache_len)) sink_size = int(ir.meta.get("cache_sink_size", 4) or 4) window_size = int(node.attrs.get("window_size", 0) or 0) k_cache = g.kv_cache_state( diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index c33e4d0be..ac1d12c1d 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -290,6 +290,97 @@ def _gemma4_text_config(multimodal_backbone: torch.nn.Module) -> object: return config +def _parse_cache_context_length(value: str | int | None) -> int | None: + if value is None: + return None + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"", "auto"}: + return None + parsed = int(normalized) + else: + parsed = int(value) + if parsed <= 0: + raise ValueError("--cache-context-length must be a positive integer or auto") + return parsed + + +def _config_int_value(config: object, key: str) -> int | None: + if config is None: + return None + if isinstance(config, dict): + raw_value = config.get(key) + else: + raw_value = getattr(config, key, None) + if raw_value is None: + return None + parsed = int(raw_value) + return parsed if parsed > 0 else None + + +def _config_candidates_for_context_length(model: torch.nn.Module) -> tuple[object, ...]: + candidates: list[object] = [] + seen: set[int] = set() + + def _add(candidate: object) -> None: + if candidate is None: + return + candidate_id = id(candidate) + if candidate_id in seen: + return + seen.add(candidate_id) + candidates.append(candidate) + + for owner in ( + model, + getattr(model, "model", None), + getattr(getattr(model, "model", None), "language_model", None), + ): + config = getattr(owner, "config", None) + _add(config) + get_text_config = getattr(config, "get_text_config", None) + if callable(get_text_config): + _add(get_text_config()) + for attr in ("text_config", "llm_config", "language_config", "decoder_config"): + _add(getattr(config, attr, None)) + + return tuple(candidates) + + +def _cache_context_length( + model: torch.nn.Module, + *, + input_seq_len: int, + cache_context_length: str | int | None, + fallback_extra_tokens: int, +) -> int: + explicit_length = _parse_cache_context_length(cache_context_length) + if explicit_length is not None: + return explicit_length + + for candidate in _config_candidates_for_context_length(model): + for key in ("max_position_embeddings", "context_length", "model_max_length", "n_positions"): + value = _config_int_value(candidate, key) + if value is not None: + return value + return input_seq_len + int(fallback_extra_tokens) + + +def _max_cache_seq_len( + model: torch.nn.Module, + input_ids: torch.Tensor, + cache_context_length: str | int | None, + *, + fallback_extra_tokens: int, +) -> int: + return max(1024, _cache_context_length( + model, + input_seq_len=int(input_ids.shape[1]), + cache_context_length=cache_context_length, + fallback_extra_tokens=fallback_extra_tokens, + )) + + def _gemma4_special_token_ids(multimodal_backbone: torch.nn.Module) -> tuple[int, int, int]: config = getattr(multimodal_backbone, "config", None) text_config = _gemma4_text_config(multimodal_backbone) @@ -407,9 +498,9 @@ def _gemma4_text_attention_forward( query_states = apply_rotary_pos_emb(query_states, cos, sin, unsqueeze_dim=2) query_states = query_states.transpose(1, 2) - kv_shared_layer_index = getattr(attn, "kv_shared_layer_index", None) - if bool(getattr(attn, "is_kv_shared_layer", False)) and kv_shared_layer_index in shared_kv_states: - key_states, value_states = shared_kv_states[int(kv_shared_layer_index)] + kv_shared_key = getattr(attn, "layer_type", getattr(attn, "kv_shared_layer_index", None)) + if bool(getattr(attn, "is_kv_shared_layer", False)) and kv_shared_key in shared_kv_states: + key_states, value_states = shared_kv_states[kv_shared_key] key_states = key_states.to(query_states.device) value_states = value_states.to(query_states.device) else: @@ -424,10 +515,9 @@ def _gemma4_text_attention_forward( value_states = value_states.transpose(1, 2) layer_idx = getattr(attn, "layer_idx", None) - if layer_idx is not None and ( - bool(getattr(attn, "store_full_length_kv", False)) - or not bool(getattr(attn, "is_kv_shared_layer", False)) - ): + if bool(getattr(attn, "store_full_length_kv", False)): + shared_kv_states[kv_shared_key] = (key_states, value_states) + elif layer_idx is not None and not bool(getattr(attn, "is_kv_shared_layer", False)): shared_kv_states[int(layer_idx)] = (key_states, value_states) attention_interface = eager_attention_forward @@ -3249,6 +3339,7 @@ def _build_gemma4_multimodal_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec]: pixel_values = named_tensors["pixel_values"] pixel_position_ids = named_tensors.get("pixel_position_ids") @@ -3467,7 +3558,7 @@ def _require(component: str) -> None: graph_meta={**common_graph_meta, "component": "lm_encoder"}, metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"}, )) - cache_seq_len = max(1024, int(input_ids.shape[1]) + 256) + cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=256) prefill_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "128") or "128")) prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) if "decoder" in expanded_components: @@ -4269,6 +4360,7 @@ def _build_qwen_causal_lm_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None = None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: input_ids = named_tensors.get("input_ids") if input_ids is None: @@ -4313,7 +4405,7 @@ def _build_qwen_causal_lm_component_specs( if "decoder_step" in requested_set: step_input_ids = input_ids[:, :1] step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_step, @@ -4419,6 +4511,7 @@ def _build_qwen3_5_multimodal_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: input_ids = named_tensors.get("input_ids") attention_mask = named_tensors.get("attention_mask") @@ -4503,6 +4596,7 @@ def _require(component: str) -> None: "task": "multimodal_causal_lm_logits", "adapter_family": "qwen3_5", } + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs: list[ComponentModuleSpec] = [] if "vision_encoder" in expanded_components: specs.append(ComponentModuleSpec( @@ -4563,7 +4657,6 @@ def _require(component: str) -> None: if "decoder_prefill_chunk" in expanded_components: if prefill_decoder_inputs is None: raise RuntimeError("Qwen3.5 decoder_prefill_chunk spec requires precomputed text chunk decoder inputs") - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", module=decoder_prefill_chunk, @@ -4587,7 +4680,6 @@ def _require(component: str) -> None: raise RuntimeError("Qwen3.5 decoder_media_step spec requires precomputed decoder inputs") step_inputs_embeds = decoder_inputs[0][:, :1, :] step_position_ids = decoder_inputs[2][:, :, :1] - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_media_step", module=decoder_media_step, @@ -4608,7 +4700,6 @@ def _require(component: str) -> None: if "decoder_step" in expanded_components: step_input_ids = input_ids[:, :1] step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_step, @@ -5240,6 +5331,7 @@ def _build_lfm2_vl_multimodal_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec]: input_ids = named_tensors["input_ids"] attention_mask = named_tensors["attention_mask"] @@ -5341,6 +5433,7 @@ def _require(component: str) -> None: "task": "multimodal_causal_lm_logits", "adapter_family": "lfm2_vl", } + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs: list[ComponentModuleSpec] = [] if "vision_encoder" in expanded_components: specs.append(ComponentModuleSpec( @@ -5433,7 +5526,6 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values, spatial_shapes, pixel_attention_mask) decoder_inputs = lm_encoder(input_ids, attention_mask, image_features) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_prefill_chunk", module=decoder_last_token, @@ -5455,7 +5547,6 @@ def _require(component: str) -> None: if image_features is None: image_features = vision_encoder(pixel_values, spatial_shapes, pixel_attention_mask) decoder_inputs = lm_encoder(input_ids, attention_mask, image_features) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_last_token, @@ -5483,6 +5574,7 @@ def build_component_module_specs( weights_dir: str | None = None, inputs_metadata: dict[str, object] | None = None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: family = _family_key(model) if family == "qwen3_5" and task == "multimodal_causal_lm_logits": @@ -5491,6 +5583,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family in {"qwen3", "qwen3_5"} and task == "causal_lm_logits": return _build_qwen_causal_lm_component_specs( @@ -5498,6 +5591,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "gemma4" and task == "multimodal_causal_lm_logits": return _build_gemma4_multimodal_component_specs( @@ -5505,6 +5599,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "lfm2_vl" and task == "multimodal_causal_lm_logits": return _build_lfm2_vl_multimodal_component_specs( @@ -5512,6 +5607,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "lfm2" and task == "causal_lm_logits": return _build_lfm2_causal_lm_component_specs( diff --git a/python/cactus/transpile/optimize_graph.py b/python/cactus/transpile/optimize_graph.py index 95630442c..308e8935f 100644 --- a/python/cactus/transpile/optimize_graph.py +++ b/python/cactus/transpile/optimize_graph.py @@ -822,6 +822,9 @@ def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bo if len(attention_nodes) != len(layer_types): return False + use_internal_cache = bool(graph.meta.get("use_internal_kv_cache", False)) + sliding_window = _graph_sliding_window(graph) + graph_cache_seq_len = _graph_max_cache_seq_len(graph) changed = False for layer_index, (node, layer_type) in enumerate(zip(attention_nodes, layer_types, strict=True)): if "attention_layer_type" not in node.meta: @@ -830,6 +833,16 @@ def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bo if "attention_layer_index" not in node.meta: node.meta["attention_layer_index"] = int(layer_index) changed = True + if use_internal_cache: + normalized_layer_type = str(layer_type).strip().lower() + node_cache_seq_len = None + if normalized_layer_type in {"sliding", "sliding_attention"} and sliding_window is not None and sliding_window > 0: + node_cache_seq_len = int(sliding_window) + elif normalized_layer_type in {"global", "full", "full_attention"} and graph_cache_seq_len is not None: + node_cache_seq_len = int(graph_cache_seq_len) + if node_cache_seq_len is not None and node.meta.get("max_cache_seq_len") != node_cache_seq_len: + node.meta["max_cache_seq_len"] = node_cache_seq_len + changed = True return changed @@ -1570,6 +1583,22 @@ def _graph_sliding_window(graph: IRGraph) -> int | None: return None +def _graph_max_cache_seq_len(graph: IRGraph) -> int | None: + max_cache_seq_len = _coerce_optional_int(graph.meta.get("max_cache_seq_len")) + if max_cache_seq_len is not None: + return max_cache_seq_len + + providers = graph.meta.get("transpile_metadata_providers") + if isinstance(providers, dict): + for provider_meta in providers.values(): + if not isinstance(provider_meta, dict): + continue + max_cache_seq_len = _coerce_optional_int(provider_meta.get("max_cache_seq_len")) + if max_cache_seq_len is not None: + return max_cache_seq_len + return None + + def _coerce_string_tuple(value: object) -> tuple[str, ...]: if not isinstance(value, (tuple, list)): return () diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py index 7ab6412b7..2adf8c513 100644 --- a/python/tests/test_bindings.py +++ b/python/tests/test_bindings.py @@ -244,6 +244,7 @@ def test_defaults(self): assert opts.system_prompt is None assert opts.trust_remote_code is False assert opts.local_files_only is False + assert opts.cache_context_length is None def test_custom_values(self): from cactus.cli.model import TranspileOptions @@ -252,11 +253,13 @@ def test_custom_values(self): prompt="Hello", max_new_tokens=256, trust_remote_code=True, + cache_context_length="131072", ) assert opts.task == "causal_lm_logits" assert opts.prompt == "Hello" assert opts.max_new_tokens == 256 assert opts.trust_remote_code is True + assert opts.cache_context_length == "131072" def test_frozen(self): from cactus.cli.model import TranspileOptions @@ -297,6 +300,7 @@ def test_run_command_chunked_bundle_flags(self): "--audio", "audio.wav", "--image", "image.png", "--input-ids", "1,2,3", + "--input-ids-file", "tokens.txt", "--max-new-tokens", "4", "--result-json", "result.json", ]) @@ -304,6 +308,7 @@ def test_run_command_chunked_bundle_flags(self): assert args.audio == "audio.wav" assert args.image == "image.png" assert args.input_ids == "1,2,3" + assert args.input_ids_file == "tokens.txt" assert args.max_new_tokens == 4 assert args.result_json == "result.json" diff --git a/python/tests/test_cli_run.py b/python/tests/test_cli_run.py index 4c2b82c7c..ae29a74e6 100644 --- a/python/tests/test_cli_run.py +++ b/python/tests/test_cli_run.py @@ -19,8 +19,10 @@ def test_cmd_run_forwards_chunked_bundle_flags(monkeypatch, tmp_path: Path) -> N image_file = tmp_path / "image.png" audio_file = tmp_path / "audio.wav" result_json = tmp_path / "result.json" + input_ids_file = tmp_path / "tokens.txt" image_file.write_bytes(b"image") audio_file.write_bytes(b"audio") + input_ids_file.write_text("1 2 3", encoding="utf-8") calls = [] @@ -40,6 +42,7 @@ def fake_run(cmd): image=str(image_file), audio=str(audio_file), input_ids="1,2,3", + input_ids_file=str(input_ids_file), max_new_tokens=4, result_json=str(result_json), thinking=False, @@ -53,5 +56,6 @@ def fake_run(cmd): assert cmd[cmd.index("--image") + 1] == str(image_file) assert cmd[cmd.index("--audio") + 1] == str(audio_file) assert cmd[cmd.index("--input-ids") + 1] == "1,2,3" + assert cmd[cmd.index("--input-ids-file") + 1] == str(input_ids_file) assert cmd[cmd.index("--max-new-tokens") + 1] == "4" assert cmd[cmd.index("--result-json") + 1] == str(result_json) diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py index 62ea0c99a..e12685f7c 100644 --- a/python/tests/test_cli_transpile_defaults.py +++ b/python/tests/test_cli_transpile_defaults.py @@ -1,14 +1,77 @@ from __future__ import annotations from pathlib import Path +from types import SimpleNamespace +import pytest from cactus import cli from cactus.cli import convert as convert_cli import cactus.cli.model as model_mod +from cactus.transpile.model_adapters import _cache_context_length + + +class _ConfigWithText: + def get_text_config(self): + return SimpleNamespace(max_position_embeddings=128000) + + +def test_cache_context_length_uses_explicit_value() -> None: + model = SimpleNamespace(config=SimpleNamespace(max_position_embeddings=40960)) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length="32768", + fallback_extra_tokens=512, + ) == 32768 + + +def test_cache_context_length_reads_top_level_config() -> None: + model = SimpleNamespace(config=SimpleNamespace(max_position_embeddings=40960)) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length=None, + fallback_extra_tokens=512, + ) == 40960 + + +def test_cache_context_length_reads_text_config() -> None: + model = SimpleNamespace(config=_ConfigWithText()) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length="auto", + fallback_extra_tokens=512, + ) == 128000 + + +def test_cache_context_length_falls_back_to_capture_size() -> None: + model = SimpleNamespace(config=SimpleNamespace()) + + assert _cache_context_length( + model, + input_seq_len=2048, + cache_context_length=None, + fallback_extra_tokens=512, + ) == 2560 + + +def test_cache_context_length_rejects_non_positive_explicit_value() -> None: + with pytest.raises(ValueError): + _cache_context_length( + SimpleNamespace(config=SimpleNamespace()), + input_seq_len=2048, + cache_context_length="0", + fallback_extra_tokens=512, + ) def _write_gemma4_multimodal_config(output_dir: Path) -> None: output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "config.txt").write_text("model_type=gemma4\n", encoding="utf-8") (output_dir / "hf_config.json").write_text( ( '{"model_type":"gemma4",' @@ -21,7 +84,7 @@ def _write_gemma4_multimodal_config(output_dir: Path) -> None: def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[str]: - assets_dir = Path(model_mod.__file__).resolve().parent.parent / "assets" + default_images, default_audio = model_mod._default_multimodal_assets() return [ "--weights-dir", str(model_dir), @@ -38,9 +101,9 @@ def _gemma4_multimodal_extra_args(model_dir: Path, artifact_dir: Path) -> list[s "--components", "vision_encoder,audio_encoder,lm_encoder,decoder", "--image-file", - str(assets_dir / "test_monkey.png"), + default_images[0], "--audio-file", - str(assets_dir / "test.wav"), + str(default_audio), "--trust-remote-code", ] @@ -138,6 +201,41 @@ def _fake_cq_main(command): assert transpile_calls[0]["extra_args"] == _gemma4_multimodal_extra_args(output_dir, output_dir) +def test_cmd_convert_retranspile_reuses_weights_and_forwards_cache_context(monkeypatch, tmp_path: Path) -> None: + parser = cli.create_parser() + output_dir = tmp_path / "gemma4" + _write_gemma4_multimodal_config(output_dir) + (output_dir / "components").mkdir() + (output_dir / "components" / "manifest.json").write_text("{}", encoding="utf-8") + (output_dir / "raw_ir_decoder_step.json").write_text("{}", encoding="utf-8") + args = parser.parse_args([ + "convert", + "google/gemma-4-E2B-it", + str(output_dir), + "--retranspile", + "--cache-context-length", + "131072", + ]) + + def _unexpected_cq_main(command): + raise AssertionError(f"unexpected CQ conversion: {command}") + + import cactus.convert.cli as cq_cli + monkeypatch.setattr(cq_cli, "main", _unexpected_cq_main) + + transpile_calls: list[dict] = [] + _patch_transpile(monkeypatch, transpile_calls) + + rc = convert_cli.cmd_convert(args) + + assert rc == 0 + assert not (output_dir / "components").exists() + assert not (output_dir / "raw_ir_decoder_step.json").exists() + assert len(transpile_calls) == 1 + extra_args = transpile_calls[0]["extra_args"] + assert extra_args[extra_args.index("--cache-context-length") + 1] == "131072" + + def test_cmd_convert_supplies_default_audio_for_parakeet(monkeypatch, tmp_path: Path) -> None: parser = cli.create_parser() output_dir = tmp_path / "parakeet" @@ -163,7 +261,7 @@ def _fake_cq_main(command): extra_args = transpile_calls[0]["extra_args"] assert extra_args[extra_args.index("--task") + 1] == "tdt_transcription" assert "--audio-file" in extra_args - assert extra_args[extra_args.index("--audio-file") + 1].endswith("python/cactus/assets/test.wav") + assert Path(extra_args[extra_args.index("--audio-file") + 1]).name == "test.wav" def test_cmd_convert_supplies_default_audio_for_whisper(monkeypatch, tmp_path: Path) -> None: diff --git a/python/tests/test_optimize_gemma4_attention.py b/python/tests/test_optimize_gemma4_attention.py index 5a534f9c0..36fddc19d 100644 --- a/python/tests/test_optimize_gemma4_attention.py +++ b/python/tests/test_optimize_gemma4_attention.py @@ -182,6 +182,57 @@ def test_normalize_gemma4_attention_recovers_layer_hints_from_graph_meta() -> No assert graph.nodes["attn_1"].attrs["window_size"] == 800 +def test_normalize_gemma4_cached_attention_sets_per_layer_cache_lengths() -> None: + graph = IRGraph( + values={ + "query_0": IRValue(id="query_0", shape=(1, 8, 1, 256), dtype="fp16"), + "key_0": IRValue(id="key_0", shape=(1, 8, 1, 256), dtype="fp16"), + "value_0": IRValue(id="value_0", shape=(1, 8, 1, 256), dtype="fp16"), + "out_0": IRValue(id="out_0", shape=(1, 8, 1, 256), dtype="fp16", producer="attn_0"), + "query_1": IRValue(id="query_1", shape=(1, 8, 1, 256), dtype="fp16"), + "key_1": IRValue(id="key_1", shape=(1, 8, 1, 256), dtype="fp16"), + "value_1": IRValue(id="value_1", shape=(1, 8, 1, 256), dtype="fp16"), + "out_1": IRValue(id="out_1", shape=(1, 8, 1, 256), dtype="fp16", producer="attn_1"), + }, + nodes={ + "attn_0": IRNode( + id="attn_0", + op="attention", + inputs=["query_0", "key_0", "value_0"], + outputs=["out_0"], + attrs={"is_causal": False, "window_size": 0}, + ), + "attn_1": IRNode( + id="attn_1", + op="attention", + inputs=["query_1", "key_1", "value_1"], + outputs=["out_1"], + attrs={"is_causal": False, "window_size": 0}, + ), + }, + order=["attn_0", "attn_1"], + inputs=["query_0", "key_0", "value_0", "query_1", "key_1", "value_1"], + outputs=["out_1"], + constants={}, + meta={ + "adapter_family": "gemma4", + "component": "decoder_step", + "use_internal_kv_cache": True, + "sliding_window": 512, + "max_cache_seq_len": 131072, + "layer_types": ("sliding_attention", "full_attention"), + }, + ) + + changed = normalize_gemma4_decoder_attention_semantics(graph) + + assert changed is True + assert graph.nodes["attn_0"].attrs["window_size"] == 512 + assert graph.nodes["attn_0"].meta["max_cache_seq_len"] == 512 + assert graph.nodes["attn_1"].attrs["window_size"] == 0 + assert graph.nodes["attn_1"].meta["max_cache_seq_len"] == 131072 + + def test_normalize_gemma4_attention_elides_logical_fp16_mask() -> None: graph = IRGraph( values={ diff --git a/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py b/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py new file mode 100755 index 000000000..9a4a90c53 --- /dev/null +++ b/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json +import shutil +import statistics +import subprocess +from datetime import datetime +from pathlib import Path +from typing import Any + + +LENGTHS = (64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768) +SWEEPS = ( + ("warmup", (64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768)), + ("measured_1", (2048, 4096, 1024, 8192, 512, 16384, 256, 32768, 128, 64)), + ("measured_2", (64, 128, 32768, 256, 16384, 512, 8192, 1024, 4096, 2048)), + ("measured_3", (4096, 1024, 8192, 512, 16384, 256, 32768, 128, 64, 2048)), +) +JSON_FIELDS = ( + "success", + "time_to_first_token_ms", + "total_time_ms", + "prefill_tps", + "prefill_compute_tps", + "prefill_prepare_tps", + "ttft_prompt_tps", + "cache_prime_ms", + "cache_prime_compute_ms", + "cache_state_copy_ms", + "cache_prime_tokens", + "first_decode_ms", + "first_token_from_prefill", + "prefill_padding_tokens", + "prefill_scalar_tail_tokens", + "decode_tps", + "prompt_tokens", + "completion_tokens", + "peak_ram_usage_mb", + "ram_usage_mb", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run Gemma4 E4B context-scaling decode benchmarks through cactus run.", + ) + parser.add_argument( + "--model", + default="weights/gemma-4-e4b-it", + help="Prepared Cactus bundle to benchmark.", + ) + parser.add_argument( + "--out-dir", + type=Path, + default=None, + help="Directory for token files, raw results, logs, JSONL, CSV, and summary.", + ) + parser.add_argument( + "--cactus-bin", + default="cactus", + help="Cactus CLI executable to invoke.", + ) + parser.add_argument( + "--token-id", + type=int, + default=106, + help="Token id repeated to build exact-length prompts.", + ) + parser.add_argument( + "--max-new-tokens", + type=int, + default=32, + help="Number of tokens to generate after each prefill.", + ) + parser.add_argument( + "--skip-build", + action="store_true", + help="Skip the required cactus build step when the caller already built this checkout.", + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + help="Record failed runs and continue instead of stopping at the first failure.", + ) + parser.add_argument( + "--no-plots", + action="store_true", + help="Do not generate PNG plots after the benchmark completes.", + ) + return parser.parse_args() + + +def default_out_dir() -> Path: + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + return Path("tmp") / "gemma4-context-scaling" / timestamp + + +def write_token_file(path: Path, *, token_id: int, count: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + chunk_size = 4096 + remaining = count + first = True + with path.open("w", encoding="utf-8") as handle: + while remaining > 0: + current = min(chunk_size, remaining) + text = ",".join([str(token_id)] * current) + if not first: + handle.write(",") + handle.write(text) + first = False + remaining -= current + + +def ensure_token_files(out_dir: Path, *, token_id: int) -> dict[int, Path]: + token_dir = out_dir / "tokens" + files: dict[int, Path] = {} + for length in LENGTHS: + path = token_dir / f"tokens_{length}.txt" + write_token_file(path, token_id=token_id, count=length) + files[length] = path + return files + + +def run_command(command: list[str], *, stdout_path: Path, stderr_path: Path) -> int: + with stdout_path.open("w", encoding="utf-8") as stdout, stderr_path.open("w", encoding="utf-8") as stderr: + proc = subprocess.run(command, stdout=stdout, stderr=stderr, text=True) + return int(proc.returncode) + + +def read_result_json(result_path: Path, stdout_path: Path) -> dict[str, Any]: + if result_path.exists(): + return json.loads(result_path.read_text(encoding="utf-8")) + for line in stdout_path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped.startswith("{") and stripped.endswith("}"): + return json.loads(stripped) + raise RuntimeError(f"No result JSON found in {result_path} or {stdout_path}") + + +def decode_ms_per_token(payload: dict[str, Any]) -> float | None: + decode_tps = float(payload.get("decode_tps") or 0.0) + if decode_tps <= 0.0: + return None + return 1000.0 / decode_tps + + +def append_jsonl(path: Path, record: dict[str, Any]) -> None: + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + + +def write_csv(path: Path, records: list[dict[str, Any]]) -> None: + fieldnames = [ + "phase", + "sweep_index", + "sweep_position", + "context_tokens", + "max_new_tokens", + "returncode", + "decode_ms_per_token", + *JSON_FIELDS, + "result_json", + "stdout_log", + "stderr_log", + ] + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + for record in records: + writer.writerow({name: record.get(name) for name in fieldnames}) + + +def write_summary(path: Path, records: list[dict[str, Any]]) -> None: + measured = [record for record in records if str(record.get("phase", "")).startswith("measured")] + fieldnames = [ + "context_tokens", + "runs", + "decode_ms_per_token_mean", + "decode_ms_per_token_stdev", + "decode_tps_mean", + "decode_tps_stdev", + "total_time_ms_mean", + "total_time_ms_stdev", + "peak_ram_usage_mb_mean", + "peak_ram_usage_mb_stdev", + ] + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + for length in LENGTHS: + rows = [record for record in measured if int(record.get("context_tokens", -1)) == length] + ms_values = [value for row in rows if (value := _optional_float(row.get("decode_ms_per_token"))) is not None] + tps_values = [value for row in rows if (value := _optional_float(row.get("decode_tps"))) is not None] + total_values = [value for row in rows if (value := _optional_float(row.get("total_time_ms"))) is not None] + ram_values = [value for row in rows if (value := _optional_float(row.get("peak_ram_usage_mb"))) is not None] + writer.writerow({ + "context_tokens": length, + "runs": len(rows), + "decode_ms_per_token_mean": statistics.mean(ms_values) if ms_values else None, + "decode_ms_per_token_stdev": statistics.stdev(ms_values) if len(ms_values) > 1 else 0.0 if ms_values else None, + "decode_tps_mean": statistics.mean(tps_values) if tps_values else None, + "decode_tps_stdev": statistics.stdev(tps_values) if len(tps_values) > 1 else 0.0 if tps_values else None, + "total_time_ms_mean": statistics.mean(total_values) if total_values else None, + "total_time_ms_stdev": statistics.stdev(total_values) if len(total_values) > 1 else 0.0 if total_values else None, + "peak_ram_usage_mb_mean": statistics.mean(ram_values) if ram_values else None, + "peak_ram_usage_mb_stdev": statistics.stdev(ram_values) if len(ram_values) > 1 else 0.0 if ram_values else None, + }) + + +def _float_column(rows: list[dict[str, str]], key: str, *, default: float = 0.0) -> list[float]: + values: list[float] = [] + for row in rows: + raw = row.get(key) + values.append(float(raw) if raw not in {None, ""} else default) + return values + + +def _optional_float(value: object) -> float | None: + if value in {None, ""}: + return None + return float(value) + + +def write_plots(summary_path: Path, out_dir: Path) -> list[Path]: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + with summary_path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + contexts = [int(row["context_tokens"]) for row in rows] + ms_mean = _float_column(rows, "decode_ms_per_token_mean") + ms_std = _float_column(rows, "decode_ms_per_token_stdev") + tps_mean = _float_column(rows, "decode_tps_mean") + tps_std = _float_column(rows, "decode_tps_stdev") + ram_mean = _float_column(rows, "peak_ram_usage_mb_mean") + ram_std = _float_column(rows, "peak_ram_usage_mb_stdev") + total_mean = [value / 1000.0 for value in _float_column(rows, "total_time_ms_mean")] + total_std = [value / 1000.0 for value in _float_column(rows, "total_time_ms_stdev")] + + plt.style.use("seaborn-v0_8-whitegrid") + error_style = { + "ecolor": "#1f1f1f", + "elinewidth": 1.8, + "capthick": 1.8, + "capsize": 6, + "markeredgecolor": "#1f1f1f", + "markeredgewidth": 0.9, + } + + def configure_x_axis(ax) -> None: + ax.set_xscale("log", base=2) + ax.set_xticks(contexts) + ax.set_xticklabels([str(context) for context in contexts], rotation=35, ha="right") + ax.grid(True, which="both", axis="y", alpha=0.35) + ax.grid(True, which="major", axis="x", alpha=0.18) + + def finish(fig, path: Path) -> Path: + fig.tight_layout() + fig.savefig(path, dpi=180) + plt.close(fig) + return path + + paths: list[Path] = [] + + fig, ax = plt.subplots(figsize=(9.5, 5.5)) + ax.errorbar(contexts, ms_mean, yerr=ms_std, marker="o", linewidth=2, color="#2f6f9f", **error_style) + configure_x_axis(ax) + ax.set_xlabel("Prefill context tokens") + ax.set_ylabel("Decode latency (ms/token)") + ax.set_title("Gemma 4 E4B IT decode latency vs context length") + paths.append(finish(fig, out_dir / "decode_ms_per_token.png")) + + fig, ax = plt.subplots(figsize=(9.5, 5.5)) + ax.errorbar(contexts, tps_mean, yerr=tps_std, marker="o", linewidth=2, color="#3d7d54", **error_style) + configure_x_axis(ax) + ax.set_xlabel("Prefill context tokens") + ax.set_ylabel("Decode throughput (tokens/sec)") + ax.set_title("Gemma 4 E4B IT decode throughput vs context length") + paths.append(finish(fig, out_dir / "decode_tps.png")) + + fig, ax1 = plt.subplots(figsize=(9.5, 5.5)) + line1 = ax1.errorbar( + contexts, + ms_mean, + yerr=ms_std, + marker="o", + linewidth=2, + color="#2f6f9f", + label="Decode ms/token", + **error_style, + ) + configure_x_axis(ax1) + ax1.set_xlabel("Prefill context tokens") + ax1.set_ylabel("Decode latency (ms/token)", color="#2f6f9f") + ax1.tick_params(axis="y", labelcolor="#2f6f9f") + ax2 = ax1.twinx() + line2 = ax2.errorbar( + contexts, + ram_mean, + yerr=ram_std, + marker="s", + linewidth=2, + color="#8a5a2b", + label="Peak RAM MB", + **error_style, + ) + ax2.set_ylabel("Peak RAM (MB)", color="#8a5a2b") + ax2.tick_params(axis="y", labelcolor="#8a5a2b") + ax1.set_title("Gemma 4 E4B IT decode latency and RAM vs context length") + ax1.legend([line1, line2], ["Decode ms/token", "Peak RAM MB"], loc="upper left") + paths.append(finish(fig, out_dir / "decode_latency_and_ram.png")) + + fig, ax = plt.subplots(figsize=(9.5, 5.5)) + ax.errorbar(contexts, total_mean, yerr=total_std, marker="o", linewidth=2, color="#7b4f9f", **error_style) + configure_x_axis(ax) + ax.set_yscale("log") + ax.set_xlabel("Prefill context tokens") + ax.set_ylabel("Mean total run time (seconds, log scale)") + ax.set_title("Gemma 4 E4B IT total run time vs context length") + paths.append(finish(fig, out_dir / "total_time_seconds.png")) + + baseline = ms_mean[0] + baseline_std = ms_std[0] + slowdown = [value / baseline for value in ms_mean] + slowdown_std = [] + for value, std in zip(ms_mean, ms_std): + if value <= 0.0 or baseline <= 0.0: + slowdown_std.append(0.0) + else: + relative_std = ((std / value) ** 2 + (baseline_std / baseline) ** 2) ** 0.5 + slowdown_std.append((value / baseline) * relative_std) + fig, ax = plt.subplots(figsize=(9.5, 5.5)) + ax.errorbar(contexts, slowdown, yerr=slowdown_std, marker="o", linewidth=2, color="#b24c4c", **error_style) + ax.axhline(1.0, color="#444444", linewidth=1, linestyle="--", alpha=0.7) + configure_x_axis(ax) + ax.set_xlabel("Prefill context tokens") + ax.set_ylabel("Decode slowdown vs 64-token baseline") + ax.set_title("Gemma 4 E4B IT decode slowdown vs initial speed") + for context, value in zip(contexts, slowdown): + ax.annotate(f"{value:.2f}x", (context, value), textcoords="offset points", xytext=(0, 8), ha="center", fontsize=8) + paths.append(finish(fig, out_dir / "decode_slowdown_vs_64.png")) + + return paths + + +def validate_environment(cactus_bin: str) -> None: + if shutil.which(cactus_bin) is None: + raise RuntimeError(f"Could not find Cactus CLI executable: {cactus_bin}") + + +def main() -> int: + args = parse_args() + validate_environment(args.cactus_bin) + + out_dir = args.out_dir or default_out_dir() + out_dir.mkdir(parents=True, exist_ok=True) + token_files = ensure_token_files(out_dir, token_id=args.token_id) + + metadata = { + "model": args.model, + "token_id": args.token_id, + "max_new_tokens": args.max_new_tokens, + "sweeps": [{"phase": phase, "lengths": list(lengths)} for phase, lengths in SWEEPS], + } + (out_dir / "metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + if not args.skip_build: + build_rc = run_command( + [args.cactus_bin, "build"], + stdout_path=out_dir / "build.stdout.log", + stderr_path=out_dir / "build.stderr.log", + ) + if build_rc != 0: + raise RuntimeError(f"cactus build failed; see {out_dir / 'build.stderr.log'}") + + records: list[dict[str, Any]] = [] + jsonl_path = out_dir / "runs.jsonl" + csv_path = out_dir / "runs.csv" + summary_path = out_dir / "summary.csv" + + for sweep_index, (phase, lengths) in enumerate(SWEEPS): + for sweep_position, length in enumerate(lengths): + prefix = f"{sweep_index:02d}_{phase}_{sweep_position:02d}_{length}" + result_path = out_dir / "results" / f"{prefix}.json" + stdout_path = out_dir / "logs" / f"{prefix}.stdout.log" + stderr_path = out_dir / "logs" / f"{prefix}.stderr.log" + result_path.parent.mkdir(parents=True, exist_ok=True) + stdout_path.parent.mkdir(parents=True, exist_ok=True) + + command = [ + args.cactus_bin, + "run", + args.model, + "--input-ids-file", + str(token_files[length]), + "--max-new-tokens", + str(args.max_new_tokens), + "--result-json", + str(result_path), + ] + print(f"[{phase} {sweep_position + 1}/{len(lengths)}] context={length}", flush=True) + returncode = run_command(command, stdout_path=stdout_path, stderr_path=stderr_path) + + if returncode == 0: + payload = read_result_json(result_path, stdout_path) + else: + payload = {"success": False} + + record = { + "phase": phase, + "sweep_index": sweep_index, + "sweep_position": sweep_position, + "context_tokens": length, + "max_new_tokens": args.max_new_tokens, + "returncode": returncode, + "decode_ms_per_token": decode_ms_per_token(payload), + "result_json": str(result_path), + "stdout_log": str(stdout_path), + "stderr_log": str(stderr_path), + } + for field in JSON_FIELDS: + record[field] = payload.get(field) + records.append(record) + append_jsonl(jsonl_path, record) + write_csv(csv_path, records) + write_summary(summary_path, records) + + if returncode != 0 and not args.continue_on_error: + raise RuntimeError(f"Benchmark run failed for context={length}; see {stderr_path}") + + print(f"wrote {jsonl_path}") + print(f"wrote {csv_path}") + print(f"wrote {summary_path}") + if not args.no_plots: + for path in write_plots(summary_path, out_dir): + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 85da4e53b87011ccc166394137587a802bc109d8 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 20:33:52 -0700 Subject: [PATCH 28/55] Size Qwen chunked-prefill caches from the full context too Full-context transpilation wired the cache-length resolver into the Qwen decoder_step path but left the chunked-prefill block (decoder_prefill_chunk / decoder_media_step) on the old max(1024, prompt+512) formula. So a re-transpile lifted decoder_step's ceiling but the prefill components stayed at 1024, and prompts past 1024 tokens still overflowed the baked RoPE table during prefill. Resolve those components through _max_cache_seq_len as well. Signed-off-by: Noah Cylich --- python/cactus/transpile/model_adapters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index ac1d12c1d..2c3944ea8 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -4429,7 +4429,7 @@ def _build_qwen_causal_lm_component_specs( lm_encoder_text_chunk = Qwen3LMEncoderTextChunkAdapter(model).eval() decoder_media_step = Qwen3EmbedsCausalLMStepAdapter(model).eval() decoder_prefill_chunk = Qwen3EmbedsCausalLMPrefillChunkAdapter(model).eval() - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) prefill_chunk_size = max(1, int(os.environ.get("CACTUS_QWEN_PREFILL_CHUNK", "128") or "128")) prefill_chunk_size = min(prefill_chunk_size, int(input_ids.shape[1])) chunk_input_ids = input_ids[:, :prefill_chunk_size].contiguous() From 7f0e854a1854a81725284fea55d2b02ad8b6c78e Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 20:42:00 -0700 Subject: [PATCH 29/55] Supply cloud-handoff args in the chunked-bundle-flags run test cmd_run forwards --confidence-threshold/--cloud-timeout-ms/--no-cloud-handoff (features on this branch), but the full-context test's args Namespace omitted them, raising AttributeError after the two were merged. Add the attributes. Signed-off-by: Noah Cylich --- python/tests/test_cli_run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/tests/test_cli_run.py b/python/tests/test_cli_run.py index ae29a74e6..2943457e5 100644 --- a/python/tests/test_cli_run.py +++ b/python/tests/test_cli_run.py @@ -46,6 +46,9 @@ def fake_run(cmd): max_new_tokens=4, result_json=str(result_json), thinking=False, + no_cloud_handoff=False, + confidence_threshold=None, + cloud_timeout_ms=None, ) assert run_mod.cmd_run(args) == 0 From aed4967067a70658f1378095d10696a7a32af763 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 20:44:20 -0700 Subject: [PATCH 30/55] Size LFM2 decoder_step cache from the full context too The LFM2 decoder_step path was the last component still deriving max_cache_seq_len from the capture prompt (max(1024, prompt+512)) rather than the model context; its chunked sibling already used the resolver. Route it through _max_cache_seq_len so LFM2 reserves the full context like the other families. Signed-off-by: Noah Cylich --- python/cactus/transpile/model_adapters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 2c3944ea8..dc7ffe48c 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -5304,7 +5304,7 @@ def _build_lfm2_causal_lm_component_specs( if "decoder_step" in requested_set: step_input_ids = input_ids[:, :1] step_position_ids = torch.zeros_like(step_input_ids, dtype=torch.int64) - max_cache_seq_len = max(1024, int(input_ids.shape[1]) + 512) + max_cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) specs.append(ComponentModuleSpec( component="decoder_step", module=decoder_step, From 7309a1f621d49b15d1bf07a3f4639a2192c479df Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 20:50:59 -0700 Subject: [PATCH 31/55] Carry max_position_embeddings in common graph meta for rope precompute Full-context rope-table precompute raises when a component has neither max_cache_seq_len nor max_position_embeddings in its meta. Non-cached components (e.g. LFM2's full-context decoder) set neither, so re-transpiling multimodal bundles failed at precompute. Seed common graph meta with the model's max_position_embeddings as the rope-table fallback. Signed-off-by: Noah Cylich --- python/cactus/transpile/model_adapters.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index dc7ffe48c..1f0b1e6c6 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -63,12 +63,20 @@ def _model_name_or_path(model: torch.nn.Module) -> str: def _transpile_graph_meta(model: torch.nn.Module, *, adapter_family: str, adapter_type: str, input_names: tuple[str, ...]) -> dict[str, object]: - return { + meta: dict[str, object] = { "adapter_family": adapter_family, "adapter_type": adapter_type, "model_name_or_path": _model_name_or_path(model), "input_names": input_names, } + # Rope-table precompute falls back to this on components that don't reserve a KV cache. + for candidate in _config_candidates_for_context_length(model): + for key in ("max_position_embeddings", "context_length", "model_max_length", "n_positions"): + value = _config_int_value(candidate, key) + if value is not None: + meta["max_position_embeddings"] = value + return meta + return meta def _extract_tensor_output(output: object, *, preferred_field: str | None = None) -> torch.Tensor: From 6ef5056f7a197ec99037527150a988a643757c6a Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 21:50:38 -0700 Subject: [PATCH 32/55] Don't assert precision in the cache move handoff steal_cache_buffer asserted the destination and source buffers share a precision, but the destination is the step node's not-yet-executed baked buffer (INT8) while the source is the runtime prefill buffer, which is FP16 under CACTUS_KV_CACHE_FP16. The move replaces the destination buffer wholesale, so its precision follows the source; only op_type is invariant. This aborted every fp16-cache run at the handoff. Signed-off-by: Noah Cylich --- cactus-graph/src/ops_cache.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index ce497caa5..6b18a060b 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -529,7 +529,9 @@ void compute_recurrent_cache_write_node( void CactusGraph::steal_cache_buffer(size_t dst_node, CactusGraph& src, size_t src_node) { auto& dst = nodes_[node_index_map_.at(dst_node)]; auto& s = src.nodes_[src.node_index_map_.at(src_node)]; - assert(dst->op_type == s->op_type && dst->output_buffer.precision == s->output_buffer.precision); + // Only op_type is invariant here; the moved buffer carries its own (runtime) precision, which + // may differ from dst's not-yet-executed baked precision (e.g. CACTUS_KV_CACHE_FP16). + assert(dst->op_type == s->op_type); dst->output_buffer = std::move(s->output_buffer); } From eaabcf82449747265953c34e598edd0d190fe79a Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 21:56:44 -0700 Subject: [PATCH 33/55] Drop comments the code already expresses Keep only the non-obvious rationale (the fp16 precision invariant in the cache move, the padded-prefill re-run, the rope-table meta fallback). Signed-off-by: Noah Cylich --- cactus-engine/src/model.cpp | 5 +---- cactus-engine/tests/test_kv_compress.cpp | 7 ++----- cactus-graph/src/ops_cache.cpp | 11 +---------- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index b314548d7..6ea964858 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -834,8 +834,6 @@ void Model::move_cache_states(Component& source, Component& target, size_t logic for (auto [src_node, dst_node] : {std::pair{src.key_node_id, dst.key_node_id}, std::pair{src.value_node_id, dst.value_node_id}}) { if (src_node < 0 || dst_node < 0) continue; target.graph->steal_cache_buffer(static_cast(dst_node), *source.graph, static_cast(src_node)); - // KV caches carry a CacheMetadata header whose current_seq_len may need truncating to - // the logical handoff length (e.g. dropping padded tail rows). if (target.graph->get_node_op_type(static_cast(dst_node)) == OpType::KV_CACHE_STATE && logical_current != std::numeric_limits::max()) { auto* meta = static_cast(target.graph->get_output(static_cast(dst_node))); @@ -1405,8 +1403,7 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, if (decoder_prefill_) { chunked = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), true); if (chunked.logical_tokens == tokens.size() && chunked.padding_tokens > 0 && tokens.size() > 0) { - // run_chunked_prefill already moved the cache into the step component; drop the padded - // last row and re-run the true last token on the step decoder for correct logits. + // Cache already moved into the step component; drop the padded last row and re-run it for logits. set_cache_current_len(*decoder_, tokens.size() - 1); cache_total_seq_len_ = tokens.size() - 1; run_step(tokens.back(), cache_total_seq_len_, true); diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index f5c256b17..2e6a97e6b 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -751,8 +751,6 @@ bool test_rolling_protect_survives() { return true; } -// A global KV layer starts small (not the baked ceiling) and doubles as tokens append; every -// stored row must survive the scales-region relocation (the growable cache's key invariant). bool test_cache_starts_small_and_grows() { CactusGraph gb; const size_t kv_heads = 2, head_dim = 64, ceiling = 100000, chunk = 100; @@ -764,7 +762,7 @@ bool test_cache_starts_small_and_grows() { gb.retain_outputs({static_cast(state), static_cast(append)}); bool ok = true; - std::vector expect; // value per (global row, channel) + std::vector expect; auto val = [](size_t row, size_t c) { return std::sin(0.013 * row + 0.07 * c); }; for (size_t step = 0; step < 3; ++step) { std::vector in(chunk * int8_stride); @@ -780,7 +778,7 @@ bool test_cache_starts_small_and_grows() { } const auto* hdr = reinterpret_cast(gb.get_output(state)); - if (hdr->max_seq_len != 512) ok = false; // 300 rows forced 256 -> 512 + if (hdr->max_seq_len != 512) ok = false; if (hdr->current_seq_len != 300) ok = false; const auto* base = static_cast(gb.get_output(state)); @@ -796,7 +794,6 @@ bool test_cache_starts_small_and_grows() { return ok; } -// A sliding-window layer gets a fixed window-sized buffer, never the baked ceiling. bool test_sliding_layer_fixed_capacity() { CactusGraph gb; const size_t kv_heads = 2, head_dim = 64, ceiling = 100000, window = 512, sink = 4; diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 6b18a060b..8bd30aaf1 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -88,11 +88,8 @@ inline bool use_fp16_kv_cache() { return cached; } -// Initial KV-cache capacity; the buffer doubles up to the baked ceiling as tokens accrue. constexpr size_t kInitialCacheEntries = 256; -// Double a KV cache buffer to hold `needed` rows (capped at `ceiling`), relocating the int8 -// scales region and preserving stored rows. Returns true if reallocated. inline bool grow_cache_buffer(BufferDesc& buf, size_t needed, size_t ceiling) { auto* meta = get_meta(buf); size_t cur = meta->max_seq_len; @@ -141,7 +138,6 @@ void compute_kv_cache_state_node( size_t ceiling = node.params.max_cache_seq_len; size_t window = node.params.window_size; bool sliding = window > 0 && window < ceiling; - // Sliding layers need a fixed window-sized buffer; global layers start small and grow. size_t max_seq = sliding ? std::min(ceiling, window + node.params.cache_sink_size + 1) : std::min(ceiling, kInitialCacheEntries); size_t kv_heads = node.params.num_kv_heads; @@ -183,8 +179,6 @@ void compute_kv_cache_append_node( size_t int8_stride = kv_heads * hdim; size_t scale_stride = kv_heads * num_groups; - // Global layers (no effective sliding window) grow the buffer to fit; capped at the baked - // ceiling, beyond which the eviction path below takes over. Sliding layers stay fixed. size_t ceiling = nodes[node_index_map.at(node.input_ids[1])]->params.max_cache_seq_len; bool sliding = node.params.window_size > 0 && node.params.window_size < ceiling; if (!sliding && current_len + new_seq_len > max_len) { @@ -524,13 +518,10 @@ void compute_recurrent_cache_write_node( std::memcpy(cache_buf.get_data(), src.get_data(), src.byte_size); } -// Move a cache buffer from another graph's node into this one (no copy); the source node is left -// empty and reallocates small on its next execute. void CactusGraph::steal_cache_buffer(size_t dst_node, CactusGraph& src, size_t src_node) { auto& dst = nodes_[node_index_map_.at(dst_node)]; auto& s = src.nodes_[src.node_index_map_.at(src_node)]; - // Only op_type is invariant here; the moved buffer carries its own (runtime) precision, which - // may differ from dst's not-yet-executed baked precision (e.g. CACTUS_KV_CACHE_FP16). + // Buffer carries its own runtime precision (may be fp16); only op_type is invariant pre-move. assert(dst->op_type == s->op_type); dst->output_buffer = std::move(s->output_buffer); } From 553db06aa04f4d5d13f11db0e50b87b559889521 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 5 Jun 2026 22:04:28 -0700 Subject: [PATCH 34/55] Trim kv_compress.h comments to the load-bearing math Drop the declaration comments that restate the signature and tighten the rest to one line, keeping only what the code can't express: the KeyDiff score, the POST-RoPE rotation convention, the K/V renumber semantics, and the per-family compressible-layer selection. Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.h | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index 7cc82aaaf..b12ba504c 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -16,7 +16,7 @@ struct Params { std::vector protect; // positions always kept (special tokens) }; -// KV cache header mirroring cactus-graph CacheMetadata. +// Mirrors cactus-graph CacheMetadata. struct CacheHeader { uint64_t current_seq_len; uint64_t max_seq_len; @@ -27,37 +27,33 @@ struct CacheHeader { }; static_assert(sizeof(CacheHeader) == 64, "CacheHeader must be 64 bytes"); -// Test hook: force the scalar path on a NEON build. void kv_set_simd(bool on); -// KeyDiff score s_i = -cos(k_i, mean(k)); keys [n][head_dim] pre-RoPE. +// score s_i = -cos(k_i, mean(k)); keys pre-RoPE. void keydiff_score(const float* keys, size_t n, size_t head_dim, float* out); -// Sorted keep-set for one cell: sink + recent + top-score middle; ties by ascending index (np.argsort(-s)). +// Keep-set: sink + recent + top-score middle; ties by ascending index. std::vector keepset_for_head(const float* scores, size_t n, const Params& p); -// Keys are stored POST-RoPE; rotating by delta_pos re-RoPEs to (orig + delta_pos): un-RoPE = -orig, renumber = rank - orig. +// Keys stored POST-RoPE; rotating by delta_pos re-RoPEs to orig+delta_pos. void rope_rotate_row(float* row, size_t head_dim, double rope_theta, double delta_pos); -// INT8 analog of rope_rotate_row; scale is [groups] per row. void rotate_int8_row(int8_t* int8, float* scale, size_t head_dim, size_t group_size, double rope_theta, double delta_pos); -// Precomputed cos/sin per dim-pair for a fixed RoPE delta. struct RopeRotation { std::vector cos, sin; }; -// Un-rope rotations for [0, n): row t rotates by -t. Shared across a compaction's layers (one theta). +// Un-rope rotations for [0, n): row t rotates by -t. std::vector unrope_table(size_t n, size_t head_dim, double rope_theta); -// Per-head compaction: gather survivors in rank order, re-rope K to 0..B-1, gather V unchanged. -// kept_per_head[h] is head h's survivor indices; the theta overload builds the un-rope table itself. +// Gather survivors in rank order; re-rope K to 0..B-1, gather V unchanged. void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, const std::vector>& kept_per_head, const std::vector& unrope); void compact_fp16(uint16_t* key_rows, uint16_t* val_rows, size_t kv_heads, size_t head_dim, const std::vector>& kept_per_head, double rope_theta); -// INT8 analog of compact_fp16. renumber=true (K) re-ropes each row to its new rank; false (V) gathers as-is. +// renumber=true (K) re-ropes each row to its new rank; false (V) gathers as-is. void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, const std::vector>& kept_per_head, @@ -67,13 +63,13 @@ void compact_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, const std::vector>& kept_per_head, double rope_theta, bool renumber); -// Rotate recent K rows [lo, hi) by delta_pos so a windowed cache tracks the renumbered frontier; sink and V stay fixed. +// Rotate recent K rows [lo, hi) by delta_pos; sink and V stay fixed. void rerope_recent_fp16(uint16_t* key_rows, size_t kv_heads, size_t head_dim, size_t lo, size_t hi, double rope_theta, double delta_pos); void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos); -// Per-head keep-sets for one layer from POST-RoPE keys: un-RoPE -> score -> keepset. theta overload builds the table itself. +// Per-head keep-sets from POST-RoPE keys: un-RoPE -> score -> keepset. std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, size_t kv_heads, size_t head_dim, const std::vector& unrope, @@ -82,7 +78,6 @@ std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_ size_t kv_heads, size_t head_dim, double rope_theta, const Params& p); -// As keepsets_from_fp16 but dequantizes the INT8 K buffer first. std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, size_t n, size_t kv_heads, size_t head_dim, size_t group_size, @@ -93,10 +88,10 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const size_t group_size, double rope_theta, const Params& p); -// True for sliding (local) layers, false for full-attention (incl. KV-shared global sources); selects local vs global theta. +// Sliding (local) vs full-attention; selects local vs global theta. bool is_sliding_layer(const std::vector& layer_types, size_t li); -// Compressible layer indices. Empty layer_types -> all (Qwen); num_kv_shared drops shared consumers + sources (Gemma -> {4,9}). +// Compressible layers: empty -> all (Qwen); num_kv_shared drops shared consumers+sources (Gemma -> {4,9}). std::vector physical_compressible_layers(const std::vector& layer_types, size_t num_layers, size_t num_kv_shared); From 37e5af460c3f58639a2ebb3d7f2635288f50790f Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 07:37:39 -0700 Subject: [PATCH 35/55] Bake rope tables only into the lowered graph, not the saved IR optimize_graph baked the precomputed rope tables into the IR it returns, so the component pipeline's saved optimized_ir.json carried them too -- which the rope-table tests can't read back (the values live in graph.cactus, not the JSON). Gate precompute behind a flag and run it on the deepcopy the component pipeline lowers, leaving the saved optimized IR un-baked. graph.cactus is still baked; runtime is unchanged. Other transpile paths keep baking via the default. Signed-off-by: Noah Cylich --- python/cactus/transpile/component_pipeline.py | 9 +++++++-- python/cactus/transpile/optimize_graph.py | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/python/cactus/transpile/component_pipeline.py b/python/cactus/transpile/component_pipeline.py index 60394ebc6..332096a31 100644 --- a/python/cactus/transpile/component_pipeline.py +++ b/python/cactus/transpile/component_pipeline.py @@ -16,6 +16,7 @@ from cactus.transpile.lower import transpile_preoptimized_ir from cactus.transpile.optimize_graph import FusionConfig from cactus.transpile.optimize_graph import optimize_graph +from cactus.transpile.optimize_graph import precompute_rope_tables @dataclass @@ -96,8 +97,12 @@ def capture_component_spec( raw_ir_graph = copy.deepcopy(captured.ir_graph) optimized_ir_graph = copy.deepcopy(captured.ir_graph) canonicalize_exported_graph(optimized_ir_graph) - optimize_graph(optimized_ir_graph, config=fusion_config) - transpiled_graph = transpile_preoptimized_ir(copy.deepcopy(optimized_ir_graph)) + optimize_graph(optimized_ir_graph, config=fusion_config, precompute_rope=False) + # Bake rope tables only into the lowered graph.cactus; the saved optimized IR stays un-baked. + lowered_ir = copy.deepcopy(optimized_ir_graph) + if precompute_rope_tables(lowered_ir): + canonicalize_exported_graph(lowered_ir) + transpiled_graph = transpile_preoptimized_ir(lowered_ir) if len(spec.output_keys) != len(transpiled_graph.outputs): raise ValueError( f"component {spec.component} declared {len(spec.output_keys)} output keys but lowered " diff --git a/python/cactus/transpile/optimize_graph.py b/python/cactus/transpile/optimize_graph.py index 308e8935f..c700c7128 100644 --- a/python/cactus/transpile/optimize_graph.py +++ b/python/cactus/transpile/optimize_graph.py @@ -57,7 +57,8 @@ class FusionConfig: enable_dense_mlp_tq_fused: bool = True -def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None) -> IRGraph: +def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig | None = None, + precompute_rope: bool = True) -> IRGraph: config = config or FusionConfig() verify_ir(graph) canonicalize_exported_graph(graph) @@ -108,7 +109,7 @@ def optimize_graph(graph: IRGraph, *, max_passes: int = 8, config: FusionConfig break canonicalize_exported_graph(graph) - if precompute_rope_tables(graph): + if precompute_rope and precompute_rope_tables(graph): canonicalize_exported_graph(graph) annotate_gold_patterns(graph) From 88cf4e7ca991cb8b6750f13ba2a3c38fb773166b Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 09:54:04 -0700 Subject: [PATCH 36/55] Fix conv/recurrent cache handoff and guard MLA-dim compaction The prefill->decode cache move iterated key then value node ids; conv and recurrent caches serialize one node as both, so the second steal_cache_buffer moved the already-emptied source over the destination and blanked the buffer. Dedup the move so a shared node is transferred once. Also preflight-skip KeyDiff compaction when any compressible layer's V head dim differs from its K head dim (MLA-style): compaction strides both by one head_dim and would otherwise corrupt the value rows. Signed-off-by: Noah Cylich --- cactus-engine/src/model.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 6ea964858..81f89926f 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -831,8 +831,14 @@ void Model::move_cache_states(Component& source, Component& target, size_t logic if (src.layer_key != dst.layer_key) { throw std::runtime_error("prefill and step cache layer mismatch: " + src.layer_key + " != " + dst.layer_key); } + int moved_src = -1, moved_dst = -1; for (auto [src_node, dst_node] : {std::pair{src.key_node_id, dst.key_node_id}, std::pair{src.value_node_id, dst.value_node_id}}) { if (src_node < 0 || dst_node < 0) continue; + // Conv/recurrent caches serialize one node as both key and value; moving it twice would + // steal the already-emptied source buffer and leave the destination blank. + if (src_node == moved_src && dst_node == moved_dst) continue; + moved_src = src_node; + moved_dst = dst_node; target.graph->steal_cache_buffer(static_cast(dst_node), *source.graph, static_cast(src_node)); if (target.graph->get_node_op_type(static_cast(dst_node)) == OpType::KV_CACHE_STATE && logical_current != std::numeric_limits::max()) { @@ -2836,7 +2842,8 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) ? rope_theta : static_cast(config_.rope_local_base_freq); const size_t old_total = cache_total_seq_len_; - // Force-keep special-token rows. + // Force-keep special-token rows. Best-effort: protect is anchored to head 0's keep set, so + // per-head KeyDiff divergence can still drop a mid-context special from other heads across cycles. cactus::kvcompress::Params params_local = params; bool preserve = config_.kv_compress_preserve_special; if (const char* e = std::getenv("CACTUS_KV_PRESERVE_SPECIAL")) preserve = (std::atoi(e) != 0); @@ -2849,6 +2856,18 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) Component& comp = *decoder_; if (!comp.graph) return; + // Compaction strides K and V by one head_dim; if any compressible layer has a different V dim + // (MLA-style), skip the whole pass so we never shrink some layers while leaving others stale. + for (size_t li = 0; li < comp.cache_states.size(); ++li) { + if (!compressible.count(li)) continue; + const auto& cs = comp.cache_states[li]; + if (cs.key_node_id < 0 || cs.value_node_id < 0) continue; + if (comp.graph->get_node_op_type(static_cast(cs.key_node_id)) != OpType::KV_CACHE_STATE) continue; + void* kraw = comp.graph->get_output(static_cast(cs.key_node_id)); + void* vraw = comp.graph->get_output(static_cast(cs.value_node_id)); + if (!kraw || !vraw) continue; + if (static_cast(vraw)->head_dim != static_cast(kraw)->head_dim) return; + } // B is identical across the compacted layers; capture it from the keep-set, not a mutated header. size_t new_seq_len = 0; bool have_new_seq_len = false; From 40a554728f957e1817af0e67c8329e15021a2f29 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 12:15:17 -0700 Subject: [PATCH 37/55] Preserve special tokens per-head across KV-cache compaction Special-token protect was built from the head-0-anchored cache_token_ids_ and applied as global positions, so after the first rolling compaction (which keeps per-head-divergent rows) non-head-0 heads protected the wrong rows and could drop a mid-context special across cycles. Track special rows per (compressible layer, head) in a SpecialRowTracker, rebuilt lazily from the still-accurate appended region of cache_token_ids_ and remapped through each head's keep set after compaction. Feed a per-head protect into keepsets so every head reserves its own specials; KeyDiff scoring stays per-head. Add a specials-fit preflight that skips the pass rather than letting a special be truncated, and disable tracking once an untracked compaction (e.g. media) diverges the heads. Signed-off-by: Noah Cylich --- cactus-engine/src/engine.h | 1 + cactus-engine/src/kv_compress.cpp | 68 +++++++++++-- cactus-engine/src/kv_compress.h | 45 ++++++++- cactus-engine/src/model.cpp | 36 +++++-- cactus-engine/tests/test_kv_compress.cpp | 119 +++++++++++++++++++++++ 5 files changed, 252 insertions(+), 17 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 518445552..4f5d518ac 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -781,6 +781,7 @@ class Model { bool cache_renumbered_ = false; std::vector cache_token_ids_; // token id per cache row (canonical head-0 view) std::unordered_set special_ids_; // special-token ids force-kept during compaction + cactus::kvcompress::SpecialRowTracker special_rows_; // per-(layer,head) special rows for compaction protect size_t cache_max_seq_len_ = 4096; size_t last_logit_position_ = 0; double last_prefill_cache_copy_ms_ = 0.0; diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index e4d24dffa..55a73939c 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -229,10 +229,13 @@ void rotate_int8_row_rot(int8_t* int8, float* scale, size_t head_dim, size_t gro template std::vector> keepsets_per_head(size_t n, size_t kv_heads, size_t head_dim, const std::vector& unrope, - const Params& p, FillPost fill_post) { + const Params& p, FillPost fill_post, + const std::vector>& protect_per_head) { + const bool per_head_protect = protect_per_head.size() == kv_heads; std::vector post(n * head_dim), pre(n * head_dim), scores(n); std::vector> out; out.reserve(kv_heads); + Params ph = p; for (size_t h = 0; h < kv_heads; ++h) { fill_post(h, post.data()); for (size_t t = 0; t < n; ++t) { @@ -242,7 +245,8 @@ std::vector> keepsets_per_head(size_t n, size_t kv_heads, size_ rope_apply(dst, unrope[t]); } keydiff_score(pre.data(), n, head_dim, scores.data()); - out.push_back(keepset_for_head(scores.data(), n, p)); + if (per_head_protect) ph.protect = protect_per_head[h]; + out.push_back(keepset_for_head(scores.data(), n, per_head_protect ? ph : p)); } return out; } @@ -354,14 +358,15 @@ void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, s std::vector> keepsets_from_fp16(const uint16_t* key_rows_u, size_t n, size_t kv_heads, size_t head_dim, const std::vector& unrope, - const Params& p) { + const Params& p, + const std::vector>& protect_per_head) { const __fp16* key_rows = reinterpret_cast(key_rows_u); return keepsets_per_head(n, kv_heads, head_dim, unrope, p, [&](size_t h, float* post) { for (size_t t = 0; t < n; ++t) { const __fp16* src = key_rows + (t * kv_heads + h) * head_dim; for (size_t d = 0; d < head_dim; ++d) post[t * head_dim + d] = src[d]; } - }); + }, protect_per_head); } std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, @@ -375,7 +380,8 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const size_t n, size_t kv_heads, size_t head_dim, size_t group_size, const std::vector& unrope, - const Params& p) { + const Params& p, + const std::vector>& protect_per_head) { size_t groups = (head_dim + group_size - 1) / group_size; size_t int8_stride = kv_heads * head_dim; size_t scale_stride = kv_heads * groups; @@ -385,7 +391,7 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const const float* ssc = scale_rows + t * scale_stride + h * groups; dequant_row(src, ssc, head_dim, group_size, post + t * head_dim); } - }); + }, protect_per_head); } std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, @@ -396,6 +402,56 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const unrope_table(n, head_dim, rope_theta), p); } +std::vector remap_rows_through_kept(const std::vector& rows, const std::vector& kept) { + std::vector out; + size_t i = 0; + for (size_t rank = 0; rank < kept.size(); ++rank) { + while (i < rows.size() && rows[i] < kept[rank]) ++i; + if (i < rows.size() && rows[i] == kept[rank]) out.push_back(static_cast(rank)); + } + return out; +} + +void SpecialRowTracker::add_appended(size_t layer, size_t kv_heads, + const std::vector& appended_rows) { + if (layer_rows_.size() <= layer) layer_rows_.resize(layer + 1); + auto& heads = layer_rows_[layer]; + if (heads.empty()) heads.resize(kv_heads); + for (auto& rows : heads) + rows.insert(rows.end(), appended_rows.begin(), appended_rows.end()); // appended >= existing, stays sorted +} + +const std::vector>& SpecialRowTracker::protect(size_t layer) const { + static const std::vector> kEmpty; + return layer < layer_rows_.size() ? layer_rows_[layer] : kEmpty; +} + +size_t SpecialRowTracker::max_reserved(size_t layer, size_t sink, + const std::vector& appended) const { + const auto& heads = protect(layer); + if (heads.empty()) { + // No tracked rows yet: only sink and the appended specials beyond sink count. + size_t extra = 0; + for (int r : appended) if (static_cast(r) >= sink) ++extra; + return sink + extra; + } + size_t worst = 0; + for (const auto& rows : heads) { + size_t count = sink; + for (int r : rows) if (static_cast(r) >= sink) ++count; + for (int r : appended) if (static_cast(r) >= sink) ++count; + worst = std::max(worst, count); + } + return worst; +} + +void SpecialRowTracker::remap(size_t layer, const std::vector>& kept_per_head) { + if (layer >= layer_rows_.size()) return; + auto& heads = layer_rows_[layer]; + for (size_t h = 0; h < heads.size() && h < kept_per_head.size(); ++h) + heads[h] = remap_rows_through_kept(heads[h], kept_per_head[h]); +} + bool is_sliding_layer(const std::vector& layer_types, size_t li) { return li < layer_types.size() && layer_types[li].find("sliding") != std::string::npos; } diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index b12ba504c..accc0aac0 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace cactus { @@ -70,10 +71,13 @@ void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, s size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos); // Per-head keep-sets from POST-RoPE keys: un-RoPE -> score -> keepset. +// protect_per_head (when sized to kv_heads) overrides Params::protect for each head, so every head +// reserves its own special rows; empty -> use the shared Params::protect for all heads. std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, size_t kv_heads, size_t head_dim, const std::vector& unrope, - const Params& p); + const Params& p, + const std::vector>& protect_per_head = {}); std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, size_t kv_heads, size_t head_dim, double rope_theta, const Params& p); @@ -82,12 +86,49 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const size_t n, size_t kv_heads, size_t head_dim, size_t group_size, const std::vector& unrope, - const Params& p); + const Params& p, + const std::vector>& protect_per_head = {}); std::vector> keepsets_from_int8(const int8_t* int8_rows, const float* scale_rows, size_t n, size_t kv_heads, size_t head_dim, size_t group_size, double rope_theta, const Params& p); +// New ranks of `rows` after a keep-set gather: {r : kept[r] in rows}, ascending. Both inputs sorted. +std::vector remap_rows_through_kept(const std::vector& rows, const std::vector& kept); + +// Tracks, per compressible KV layer and head, the cache rows holding special tokens, so rolling +// compaction can re-protect every head's own specials across cycles (per-head keep sets diverge, +// so a single head-0 view is not enough). Pure bookkeeping over row indices; no engine deps. +class SpecialRowTracker { +public: + void clear() { tracked_len_ = 0; layer_rows_.clear(); valid_ = true; } + size_t tracked_len() const { return tracked_len_; } + void set_tracked_len(size_t len) { tracked_len_ = len; } + + // A compaction that ran without per-head tracking diverged the heads; the head-0 token view can + // no longer rebuild per-head truth, so disable tracking until the next clear(). + bool valid() const { return valid_; } + void invalidate() { valid_ = false; } + + // Append head-aligned special rows (rows >= tracked_len, identical across heads) to `layer`, + // sizing it to kv_heads on first touch. Does not advance tracked_len. + void add_appended(size_t layer, size_t kv_heads, const std::vector& appended_rows); + + // Per-head protected rows for a layer (empty if untracked). + const std::vector>& protect(size_t layer) const; + + // max over heads of |[0,sink) U rows[h] U appended| -- the rows a compaction must reserve. + size_t max_reserved(size_t layer, size_t sink, const std::vector& appended) const; + + // After compacting `layer`, remap each head's special rows through its keep set. + void remap(size_t layer, const std::vector>& kept_per_head); + +private: + size_t tracked_len_ = 0; + bool valid_ = true; + std::vector>> layer_rows_; // [layer][head] -> sorted rows +}; + // Sliding (local) vs full-attention; selects local vs global theta. bool is_sliding_layer(const std::vector& layer_types, size_t li); diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 81f89926f..0305281ad 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2711,6 +2711,7 @@ void Model::reset_cache() { last_logit_position_ = 0; context_tokens_.clear(); cache_token_ids_.clear(); + special_rows_.clear(); token_history_.clear(); media_features_.clear(); media_feature_shapes_.clear(); @@ -2842,22 +2843,28 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) ? rope_theta : static_cast(config_.rope_local_base_freq); const size_t old_total = cache_total_seq_len_; - // Force-keep special-token rows. Best-effort: protect is anchored to head 0's keep set, so - // per-head KeyDiff divergence can still drop a mid-context special from other heads across cycles. + // Force-keep special-token rows. Per-head: each head re-protects its own special rows so KeyDiff + // divergence can't drop a mid-context special from some heads across cycles. Disabled (best-effort + // off) when the token map is unusable (media) or a prior untracked compaction diverged the heads. cactus::kvcompress::Params params_local = params; bool preserve = config_.kv_compress_preserve_special; if (const char* e = std::getenv("CACTUS_KV_PRESERVE_SPECIAL")) preserve = (std::atoi(e) != 0); const bool map_valid = cache_token_ids_.size() == old_total && media_features_.empty(); - if (preserve && map_valid) { + const bool per_head_protect = preserve && map_valid && special_rows_.valid(); + // Head-aligned special rows appended since the last compaction (still accurate in cache_token_ids_). + std::vector appended_special; + if (per_head_protect) { if (special_ids_.empty() && tokenizer_) special_ids_ = tokenizer_->special_token_ids(); - for (size_t pos = 0; pos < cache_token_ids_.size(); ++pos) - if (special_ids_.count(cache_token_ids_[pos])) params_local.protect.push_back(static_cast(pos)); + for (size_t r = special_rows_.tracked_len(); r < old_total && r < cache_token_ids_.size(); ++r) + if (special_ids_.count(cache_token_ids_[r])) appended_special.push_back(static_cast(r)); } Component& comp = *decoder_; if (!comp.graph) return; - // Compaction strides K and V by one head_dim; if any compressible layer has a different V dim - // (MLA-style), skip the whole pass so we never shrink some layers while leaving others stale. + // Preflight before mutating anything: skip the whole pass if (a) any compressible layer's V dim + // differs from its K dim (MLA -- would corrupt value rows), or (b) a head can't fit sink + all its + // specials in the budget (we never silently drop a special to make room). + const size_t protect_budget = params.abs_budget > 0 ? static_cast(params.abs_budget) : 0; for (size_t li = 0; li < comp.cache_states.size(); ++li) { if (!compressible.count(li)) continue; const auto& cs = comp.cache_states[li]; @@ -2867,6 +2874,8 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) void* vraw = comp.graph->get_output(static_cast(cs.value_node_id)); if (!kraw || !vraw) continue; if (static_cast(vraw)->head_dim != static_cast(kraw)->head_dim) return; + if (per_head_protect && protect_budget > 0 && + special_rows_.max_reserved(li, params.sink, appended_special) > protect_budget) return; } // B is identical across the compacted layers; capture it from the keep-set, not a mutated header. size_t new_seq_len = 0; @@ -2899,13 +2908,17 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) if (kv_heads == 0 || head_dim == 0) continue; if (unrope.empty()) unrope = cactus::kvcompress::unrope_table(n, head_dim, rope_theta); + if (per_head_protect) special_rows_.add_appended(li, kv_heads, appended_special); + const std::vector>& pph = special_rows_.protect(li); + if (kdesc.precision == Precision::FP16) { auto* kbase = reinterpret_cast(static_cast(kraw) + kHeaderBytes); auto* vbase = reinterpret_cast(static_cast(vraw) + kHeaderBytes); auto kept = cactus::kvcompress::keepsets_from_fp16( - kbase, n, kv_heads, head_dim, unrope, params_local); + kbase, n, kv_heads, head_dim, unrope, params_local, pph); if (!canonical_captured) { canonical_keep = kept.empty() ? std::vector{} : kept[0]; canonical_captured = true; } cactus::kvcompress::compact_fp16(kbase, vbase, kv_heads, head_dim, kept, unrope); + if (per_head_protect) special_rows_.remap(li, kept); size_t B = kept.empty() ? 0 : kept[0].size(); khdr->current_seq_len = B; vhdr->current_seq_len = B; @@ -2920,12 +2933,13 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) auto* v_sc = reinterpret_cast(static_cast(vraw) + kHeaderBytes + max_seq * kv_heads * head_dim); auto kept = cactus::kvcompress::keepsets_from_int8( - k_i8, k_sc, n, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, unrope, params_local); + k_i8, k_sc, n, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, unrope, params_local, pph); if (!canonical_captured) { canonical_keep = kept.empty() ? std::vector{} : kept[0]; canonical_captured = true; } cactus::kvcompress::compact_int8(k_i8, k_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, kept, unrope, /*renumber=*/true); cactus::kvcompress::compact_int8(v_i8, v_sc, kv_heads, head_dim, KV_QUANT_GROUP_SIZE, kept, unrope, /*renumber=*/false); + if (per_head_protect) special_rows_.remap(li, kept); size_t B = kept.empty() ? 0 : kept[0].size(); khdr->current_seq_len = B; vhdr->current_seq_len = B; @@ -2974,6 +2988,10 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) if (have_new_seq_len) { cache_total_seq_len_ = new_seq_len; cache_renumbered_ = true; + // The special-row sets were remapped per layer above; advance to the compacted length. A + // compaction that ran without per-head tracking diverged the heads, so disable it afterward. + if (per_head_protect) special_rows_.set_tracked_len(new_seq_len); + else special_rows_.invalidate(); if (map_valid && canonical_captured) { std::vector compacted; compacted.reserve(canonical_keep.size()); diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index 2e6a97e6b..8ab6785e4 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -804,6 +804,121 @@ bool test_sliding_layer_fixed_capacity() { return hdr->max_seq_len == window + sink + 1; } +bool test_special_rows_remap_through_kept() { + // New ranks are the positions in `kept` whose value is a tracked row. + std::vector rows = {3, 7, 10}; + std::vector kept = {0, 3, 5, 7, 9, 10, 12}; // 3->rank1, 7->rank3, 10->rank5 + return remap_rows_through_kept(rows, kept) == std::vector{1, 3, 5}; +} + +bool test_per_head_protect_keeps_specials() { + // Each head reserves its OWN protected rows, even mid-context ones KeyDiff would evict. + const double theta = 1000000.0; + const size_t n = 64, kv_heads = 3, head_dim = 16; + std::vector>> pre, val; + auto kbuf = make_fp16_cache(n, n, kv_heads, head_dim, theta, pre, val); + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + Params p; p.sink = 4; p.recent_frac = 0.30f; p.abs_budget = 24; + auto unrope = unrope_table(n, head_dim, theta); + std::vector> protect(kv_heads); + for (size_t h = 0; h < kv_heads; ++h) protect[h] = {static_cast(20 + h)}; // distinct middle row per head + auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, unrope, p, protect); + if (kept.size() != kv_heads) return false; + for (size_t h = 0; h < kv_heads; ++h) + if (std::find(kept[h].begin(), kept[h].end(), static_cast(20 + h)) == kept[h].end()) return false; + return true; +} + +bool test_preflight_specials_exceed_budget() { + SpecialRowTracker tracked; + std::vector rows; + for (int r = 0; r < 50; ++r) rows.push_back(r); + tracked.add_appended(/*layer=*/0, /*kv_heads=*/2, rows); // 50 specials in each head + SpecialRowTracker untracked; + // sink=4: |[0,4) U rows| = 50 either way; exceeds a budget of 40. + return tracked.max_reserved(0, 4, {}) == 50 && tracked.max_reserved(0, 4, {}) > 40 && + untracked.max_reserved(0, 4, rows) == 50 && untracked.max_reserved(0, 4, rows) > 40; +} + +bool test_all_heads_keep_special_across_cycles() { + // Per-head KeyDiff keeps different middle rows per head; a tracked mid-context special must + // survive in EVERY head across multiple compactions (head-0-anchored protect would drop it). + const int trigger_len = 256, target_len = 128; + const size_t kv_heads = 4, head_dim = 16, max_seq = trigger_len + 8; + const double theta = 1000000.0; + std::vector kbuf(kHeaderBytes + max_seq * kv_heads * head_dim * sizeof(uint16_t), 0); + std::vector vbuf(kbuf.size(), 0); + auto* khdr = reinterpret_cast(kbuf.data()); + auto* vhdr = reinterpret_cast(vbuf.data()); + *khdr = Header{0, max_seq, kv_heads, head_dim, 4, {0, 0, 0}}; + *vhdr = *khdr; + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + auto* vrows = reinterpret_cast(vbuf.data() + kHeaderBytes); + + auto special_val = [&](size_t h) { + std::vector v(head_dim); + for (size_t d = 0; d < head_dim; ++d) v[d] = (d % 2 ? -2.0f : 2.0f) * static_cast(h + 1); + return v; + }; + const long special_abs = trigger_len / 2; // a MIDDLE position (not sink, not recent) + auto append = [&](size_t from, size_t to) { + for (size_t t = from; t < to; ++t) + for (size_t h = 0; h < kv_heads; ++h) { + std::vector k(head_dim); + // Special key is non-distinctive (KeyDiff would evict it); other rows differ per (h,t). + for (size_t d = 0; d < head_dim; ++d) + k[d] = (static_cast(t) == special_abs) ? 0.01f + : std::sin(0.11 * (h + 1) * (t + 1) + 0.3 * d) + 0.05 * d; + std::vector kr = rope_reference(k, static_cast(t), theta); + std::vector vr(head_dim); + if (static_cast(t) == special_abs) vr = special_val(h); + else for (size_t d = 0; d < head_dim; ++d) vr[d] = std::cos(0.07 * (t + 1) + 0.2 * d); + for (size_t d = 0; d < head_dim; ++d) { + krows[(t * kv_heads + h) * head_dim + d] = f32_to_f16(kr[d]); + vrows[(t * kv_heads + h) * head_dim + d] = f32_to_f16(vr[d]); + } + } + }; + auto special_rank = [&](size_t h) -> long { + std::vector sv = special_val(h); + for (size_t r = 0; r < khdr->current_seq_len; ++r) { + const uint16_t* v = vrows + (r * kv_heads + h) * head_dim; + double dot = 0, na = 0, nb = 0; + for (size_t d = 0; d < head_dim; ++d) { float x = f16_to_f32(v[d]); dot += x * sv[d]; na += x * x; nb += sv[d] * sv[d]; } + if (dot / (std::sqrt(na) * std::sqrt(nb) + 1e-9) > 0.99) return static_cast(r); + } + return -1; + }; + + SpecialRowTracker tracker; + Params p; p.sink = 4; p.recent_frac = 0.30f; p.abs_budget = target_len; + auto unrope = unrope_table(trigger_len, head_dim, theta); + for (int c = 0; c < 3; ++c) { + size_t start = khdr->current_seq_len; + append(start, static_cast(trigger_len)); + khdr->current_seq_len = trigger_len; vhdr->current_seq_len = trigger_len; + std::vector appended; + if (c == 0) appended.push_back(static_cast(special_abs)); // head-aligned special in the appended region + tracker.add_appended(0, kv_heads, appended); + auto kept = keepsets_from_fp16(krows, static_cast(trigger_len), kv_heads, head_dim, + unrope, p, tracker.protect(0)); + compact_fp16(krows, vrows, kv_heads, head_dim, kept, theta); + tracker.remap(0, kept); + size_t B = kept.empty() ? 0 : kept[0].size(); + khdr->current_seq_len = B; vhdr->current_seq_len = B; + tracker.set_tracked_len(B); + } + long r0 = special_rank(0); + if (r0 < 0) return false; + bool diverged = false; + for (size_t h = 0; h < kv_heads; ++h) { + long r = special_rank(h); + if (r < 0) return false; // special must survive in every head + if (r != r0) diverged = true; // and heads keep it at different ranks (a real per-head scenario) + } + return diverged; +} + int main() { TestUtils::TestRunner runner("KV Compress Free-Function Tests"); runner.run_test("cache_starts_small_and_grows", test_cache_starts_small_and_grows()); @@ -828,6 +943,10 @@ int main() { runner.run_test("dequant_simd_matches_scalar", test_dequant_simd_matches_scalar()); runner.run_test("keepset_protect_union", test_keepset_protect_union()); runner.run_test("rolling_protect_survives", test_rolling_protect_survives()); + runner.run_test("special_rows_remap_through_kept", test_special_rows_remap_through_kept()); + runner.run_test("per_head_protect_keeps_specials", test_per_head_protect_keeps_specials()); + runner.run_test("preflight_specials_exceed_budget", test_preflight_specials_exceed_budget()); + runner.run_test("all_heads_keep_special_across_cycles", test_all_heads_keep_special_across_cycles()); runner.print_summary(); return runner.all_passed() ? 0 : 1; } From f8a94469143f7e51b1ddab18a16c68a318f8fd14 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 12:18:47 -0700 Subject: [PATCH 38/55] Don't pass stale per-head protect when tracking is disabled keepsets received special_rows_.protect(li) unconditionally, so a compaction with per-head tracking off (media, preserve off, or post-invalidate) would still override Params::protect with stale tracker rows. Pass empty protect unless per_head_protect, clear the rows on invalidate(), and test the Params::protect fallback. Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.h | 4 ++-- cactus-engine/src/model.cpp | 3 ++- cactus-engine/tests/test_kv_compress.cpp | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index accc0aac0..7e56b4e79 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -106,9 +106,9 @@ class SpecialRowTracker { void set_tracked_len(size_t len) { tracked_len_ = len; } // A compaction that ran without per-head tracking diverged the heads; the head-0 token view can - // no longer rebuild per-head truth, so disable tracking until the next clear(). + // no longer rebuild per-head truth, so disable tracking and drop the now-stale rows until clear(). bool valid() const { return valid_; } - void invalidate() { valid_ = false; } + void invalidate() { valid_ = false; layer_rows_.clear(); } // Append head-aligned special rows (rows >= tracked_len, identical across heads) to `layer`, // sizing it to kv_heads on first touch. Does not advance tracked_len. diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 0305281ad..52f57409c 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2908,8 +2908,9 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) if (kv_heads == 0 || head_dim == 0) continue; if (unrope.empty()) unrope = cactus::kvcompress::unrope_table(n, head_dim, rope_theta); + static const std::vector> kNoProtect; if (per_head_protect) special_rows_.add_appended(li, kv_heads, appended_special); - const std::vector>& pph = special_rows_.protect(li); + const std::vector>& pph = per_head_protect ? special_rows_.protect(li) : kNoProtect; if (kdesc.precision == Precision::FP16) { auto* kbase = reinterpret_cast(static_cast(kraw) + kHeaderBytes); diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index 8ab6785e4..e3e5f6866 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -840,6 +840,23 @@ bool test_preflight_specials_exceed_budget() { untracked.max_reserved(0, 4, rows) == 50 && untracked.max_reserved(0, 4, rows) > 40; } +bool test_empty_protect_per_head_uses_params_fallback() { + // When protect_per_head is empty, every head must fall back to Params::protect (so a disabled + // tracker can't leak stale per-head rows into the keep set). + const double theta = 1000000.0; + const size_t n = 64, kv_heads = 3, head_dim = 16; + std::vector>> pre, val; + auto kbuf = make_fp16_cache(n, n, kv_heads, head_dim, theta, pre, val); + auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); + Params p; p.sink = 4; p.recent_frac = 0.30f; p.abs_budget = 24; p.protect = {25}; // global middle protect + auto unrope = unrope_table(n, head_dim, theta); + auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, unrope, p, /*protect_per_head=*/{}); + if (kept.size() != kv_heads) return false; + for (size_t h = 0; h < kv_heads; ++h) + if (std::find(kept[h].begin(), kept[h].end(), 25) == kept[h].end()) return false; // global protect honored in every head + return true; +} + bool test_all_heads_keep_special_across_cycles() { // Per-head KeyDiff keeps different middle rows per head; a tracked mid-context special must // survive in EVERY head across multiple compactions (head-0-anchored protect would drop it). @@ -946,6 +963,7 @@ int main() { runner.run_test("special_rows_remap_through_kept", test_special_rows_remap_through_kept()); runner.run_test("per_head_protect_keeps_specials", test_per_head_protect_keeps_specials()); runner.run_test("preflight_specials_exceed_budget", test_preflight_specials_exceed_budget()); + runner.run_test("empty_protect_per_head_uses_params_fallback", test_empty_protect_per_head_uses_params_fallback()); runner.run_test("all_heads_keep_special_across_cycles", test_all_heads_keep_special_across_cycles()); runner.print_summary(); return runner.all_passed() ? 0 : 1; From 6cd45c89ee6eb06ec023739394c6cf8ecbe57c3d Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 12:24:54 -0700 Subject: [PATCH 39/55] Trim per-head-protect comments to one line each Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.h | 18 ++++++------------ cactus-engine/src/model.cpp | 17 ++++++----------- cactus-engine/tests/test_kv_compress.cpp | 6 ++---- 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index 7e56b4e79..0f039ec6c 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -71,8 +71,7 @@ void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, s size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos); // Per-head keep-sets from POST-RoPE keys: un-RoPE -> score -> keepset. -// protect_per_head (when sized to kv_heads) overrides Params::protect for each head, so every head -// reserves its own special rows; empty -> use the shared Params::protect for all heads. +// protect_per_head (sized to kv_heads) overrides Params::protect per head; empty -> shared protect. std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, size_t kv_heads, size_t head_dim, const std::vector& unrope, @@ -96,31 +95,26 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const // New ranks of `rows` after a keep-set gather: {r : kept[r] in rows}, ascending. Both inputs sorted. std::vector remap_rows_through_kept(const std::vector& rows, const std::vector& kept); -// Tracks, per compressible KV layer and head, the cache rows holding special tokens, so rolling -// compaction can re-protect every head's own specials across cycles (per-head keep sets diverge, -// so a single head-0 view is not enough). Pure bookkeeping over row indices; no engine deps. +// Per-(compressible layer, head) special-token rows so compaction re-protects each head's own +// specials across cycles (keep sets diverge per head). Pure row-index bookkeeping. class SpecialRowTracker { public: void clear() { tracked_len_ = 0; layer_rows_.clear(); valid_ = true; } size_t tracked_len() const { return tracked_len_; } void set_tracked_len(size_t len) { tracked_len_ = len; } - // A compaction that ran without per-head tracking diverged the heads; the head-0 token view can - // no longer rebuild per-head truth, so disable tracking and drop the now-stale rows until clear(). + // An untracked compaction diverged the heads; disable and drop the now-stale rows until clear(). bool valid() const { return valid_; } void invalidate() { valid_ = false; layer_rows_.clear(); } - // Append head-aligned special rows (rows >= tracked_len, identical across heads) to `layer`, - // sizing it to kv_heads on first touch. Does not advance tracked_len. + // Append head-aligned special rows (>= tracked_len) to `layer`; does not advance tracked_len. void add_appended(size_t layer, size_t kv_heads, const std::vector& appended_rows); - // Per-head protected rows for a layer (empty if untracked). const std::vector>& protect(size_t layer) const; - // max over heads of |[0,sink) U rows[h] U appended| -- the rows a compaction must reserve. + // max over heads of |[0,sink) U rows[h] U appended| -- rows a compaction must reserve. size_t max_reserved(size_t layer, size_t sink, const std::vector& appended) const; - // After compacting `layer`, remap each head's special rows through its keep set. void remap(size_t layer, const std::vector>& kept_per_head); private: diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 52f57409c..5dc191aa3 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -834,8 +834,7 @@ void Model::move_cache_states(Component& source, Component& target, size_t logic int moved_src = -1, moved_dst = -1; for (auto [src_node, dst_node] : {std::pair{src.key_node_id, dst.key_node_id}, std::pair{src.value_node_id, dst.value_node_id}}) { if (src_node < 0 || dst_node < 0) continue; - // Conv/recurrent caches serialize one node as both key and value; moving it twice would - // steal the already-emptied source buffer and leave the destination blank. + // Conv/recurrent caches share one node for key and value; move it only once. if (src_node == moved_src && dst_node == moved_dst) continue; moved_src = src_node; moved_dst = dst_node; @@ -2843,15 +2842,13 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) ? rope_theta : static_cast(config_.rope_local_base_freq); const size_t old_total = cache_total_seq_len_; - // Force-keep special-token rows. Per-head: each head re-protects its own special rows so KeyDiff - // divergence can't drop a mid-context special from some heads across cycles. Disabled (best-effort - // off) when the token map is unusable (media) or a prior untracked compaction diverged the heads. + // Per-head protect: each head re-keeps its own special rows (a single head-0 view drifts across cycles). cactus::kvcompress::Params params_local = params; bool preserve = config_.kv_compress_preserve_special; if (const char* e = std::getenv("CACTUS_KV_PRESERVE_SPECIAL")) preserve = (std::atoi(e) != 0); const bool map_valid = cache_token_ids_.size() == old_total && media_features_.empty(); const bool per_head_protect = preserve && map_valid && special_rows_.valid(); - // Head-aligned special rows appended since the last compaction (still accurate in cache_token_ids_). + // Special rows appended since the last compaction (still head-aligned in cache_token_ids_). std::vector appended_special; if (per_head_protect) { if (special_ids_.empty() && tokenizer_) special_ids_ = tokenizer_->special_token_ids(); @@ -2861,9 +2858,8 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) Component& comp = *decoder_; if (!comp.graph) return; - // Preflight before mutating anything: skip the whole pass if (a) any compressible layer's V dim - // differs from its K dim (MLA -- would corrupt value rows), or (b) a head can't fit sink + all its - // specials in the budget (we never silently drop a special to make room). + // Preflight before mutating: skip the pass if any layer's V dim differs from K (MLA), or a head + // can't fit sink + all its specials in the budget. const size_t protect_budget = params.abs_budget > 0 ? static_cast(params.abs_budget) : 0; for (size_t li = 0; li < comp.cache_states.size(); ++li) { if (!compressible.count(li)) continue; @@ -2989,8 +2985,7 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) if (have_new_seq_len) { cache_total_seq_len_ = new_seq_len; cache_renumbered_ = true; - // The special-row sets were remapped per layer above; advance to the compacted length. A - // compaction that ran without per-head tracking diverged the heads, so disable it afterward. + // An untracked compaction diverged the heads; disable per-head protect afterward. if (per_head_protect) special_rows_.set_tracked_len(new_seq_len); else special_rows_.invalidate(); if (map_valid && canonical_captured) { diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index e3e5f6866..f4faac5bd 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -841,8 +841,7 @@ bool test_preflight_specials_exceed_budget() { } bool test_empty_protect_per_head_uses_params_fallback() { - // When protect_per_head is empty, every head must fall back to Params::protect (so a disabled - // tracker can't leak stale per-head rows into the keep set). + // Empty protect_per_head must fall back to Params::protect (no stale per-head rows leak in). const double theta = 1000000.0; const size_t n = 64, kv_heads = 3, head_dim = 16; std::vector>> pre, val; @@ -858,8 +857,7 @@ bool test_empty_protect_per_head_uses_params_fallback() { } bool test_all_heads_keep_special_across_cycles() { - // Per-head KeyDiff keeps different middle rows per head; a tracked mid-context special must - // survive in EVERY head across multiple compactions (head-0-anchored protect would drop it). + // A tracked mid-context special must survive in EVERY head across compactions, despite per-head divergence. const int trigger_len = 256, target_len = 128; const size_t kv_heads = 4, head_dim = 16, max_seq = trigger_len + 8; const double theta = 1000000.0; From f849932d43830348443fb38cbb3f8049c907537e Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 12:29:31 -0700 Subject: [PATCH 40/55] Drop per-head-protect comments that restate the code Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.cpp | 1 - cactus-engine/src/kv_compress.h | 16 ++++------------ cactus-engine/src/model.cpp | 6 ++---- cactus-engine/tests/test_kv_compress.cpp | 21 ++++++++------------- 4 files changed, 14 insertions(+), 30 deletions(-) diff --git a/cactus-engine/src/kv_compress.cpp b/cactus-engine/src/kv_compress.cpp index 55a73939c..eff9c4eb6 100644 --- a/cactus-engine/src/kv_compress.cpp +++ b/cactus-engine/src/kv_compress.cpp @@ -430,7 +430,6 @@ size_t SpecialRowTracker::max_reserved(size_t layer, size_t sink, const std::vector& appended) const { const auto& heads = protect(layer); if (heads.empty()) { - // No tracked rows yet: only sink and the appended specials beyond sink count. size_t extra = 0; for (int r : appended) if (static_cast(r) >= sink) ++extra; return sink + extra; diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index 0f039ec6c..d5d118eff 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -70,8 +70,8 @@ void rerope_recent_fp16(uint16_t* key_rows, size_t kv_heads, size_t head_dim, void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos); -// Per-head keep-sets from POST-RoPE keys: un-RoPE -> score -> keepset. -// protect_per_head (sized to kv_heads) overrides Params::protect per head; empty -> shared protect. +// Per-head keep-sets from POST-RoPE keys. protect_per_head (sized to kv_heads) overrides +// Params::protect per head; empty -> shared protect. std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, size_t kv_heads, size_t head_dim, const std::vector& unrope, @@ -92,29 +92,21 @@ std::vector> keepsets_from_int8(const int8_t* int8_rows, const size_t group_size, double rope_theta, const Params& p); -// New ranks of `rows` after a keep-set gather: {r : kept[r] in rows}, ascending. Both inputs sorted. std::vector remap_rows_through_kept(const std::vector& rows, const std::vector& kept); -// Per-(compressible layer, head) special-token rows so compaction re-protects each head's own -// specials across cycles (keep sets diverge per head). Pure row-index bookkeeping. +// Per-(compressible layer, head) special rows; lets compaction re-protect each head's own specials. class SpecialRowTracker { public: void clear() { tracked_len_ = 0; layer_rows_.clear(); valid_ = true; } size_t tracked_len() const { return tracked_len_; } void set_tracked_len(size_t len) { tracked_len_ = len; } - - // An untracked compaction diverged the heads; disable and drop the now-stale rows until clear(). bool valid() const { return valid_; } void invalidate() { valid_ = false; layer_rows_.clear(); } - // Append head-aligned special rows (>= tracked_len) to `layer`; does not advance tracked_len. + // appended_rows must be >= tracked_len (the still head-aligned region). void add_appended(size_t layer, size_t kv_heads, const std::vector& appended_rows); - const std::vector>& protect(size_t layer) const; - - // max over heads of |[0,sink) U rows[h] U appended| -- rows a compaction must reserve. size_t max_reserved(size_t layer, size_t sink, const std::vector& appended) const; - void remap(size_t layer, const std::vector>& kept_per_head); private: diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 5dc191aa3..95e3ac2ad 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2842,13 +2842,12 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) ? rope_theta : static_cast(config_.rope_local_base_freq); const size_t old_total = cache_total_seq_len_; - // Per-head protect: each head re-keeps its own special rows (a single head-0 view drifts across cycles). cactus::kvcompress::Params params_local = params; bool preserve = config_.kv_compress_preserve_special; if (const char* e = std::getenv("CACTUS_KV_PRESERVE_SPECIAL")) preserve = (std::atoi(e) != 0); const bool map_valid = cache_token_ids_.size() == old_total && media_features_.empty(); const bool per_head_protect = preserve && map_valid && special_rows_.valid(); - // Special rows appended since the last compaction (still head-aligned in cache_token_ids_). + // cache_token_ids_ past tracked_len is still head-aligned, so specials there apply to every head. std::vector appended_special; if (per_head_protect) { if (special_ids_.empty() && tokenizer_) special_ids_ = tokenizer_->special_token_ids(); @@ -2858,7 +2857,7 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) Component& comp = *decoder_; if (!comp.graph) return; - // Preflight before mutating: skip the pass if any layer's V dim differs from K (MLA), or a head + // Skip the whole pass (before mutating) if any layer's V dim differs from K (MLA), or a head // can't fit sink + all its specials in the budget. const size_t protect_budget = params.abs_budget > 0 ? static_cast(params.abs_budget) : 0; for (size_t li = 0; li < comp.cache_states.size(); ++li) { @@ -2985,7 +2984,6 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) if (have_new_seq_len) { cache_total_seq_len_ = new_seq_len; cache_renumbered_ = true; - // An untracked compaction diverged the heads; disable per-head protect afterward. if (per_head_protect) special_rows_.set_tracked_len(new_seq_len); else special_rows_.invalidate(); if (map_valid && canonical_captured) { diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index f4faac5bd..f1193791d 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -805,14 +805,12 @@ bool test_sliding_layer_fixed_capacity() { } bool test_special_rows_remap_through_kept() { - // New ranks are the positions in `kept` whose value is a tracked row. std::vector rows = {3, 7, 10}; - std::vector kept = {0, 3, 5, 7, 9, 10, 12}; // 3->rank1, 7->rank3, 10->rank5 + std::vector kept = {0, 3, 5, 7, 9, 10, 12}; return remap_rows_through_kept(rows, kept) == std::vector{1, 3, 5}; } bool test_per_head_protect_keeps_specials() { - // Each head reserves its OWN protected rows, even mid-context ones KeyDiff would evict. const double theta = 1000000.0; const size_t n = 64, kv_heads = 3, head_dim = 16; std::vector>> pre, val; @@ -821,7 +819,7 @@ bool test_per_head_protect_keeps_specials() { Params p; p.sink = 4; p.recent_frac = 0.30f; p.abs_budget = 24; auto unrope = unrope_table(n, head_dim, theta); std::vector> protect(kv_heads); - for (size_t h = 0; h < kv_heads; ++h) protect[h] = {static_cast(20 + h)}; // distinct middle row per head + for (size_t h = 0; h < kv_heads; ++h) protect[h] = {static_cast(20 + h)}; auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, unrope, p, protect); if (kept.size() != kv_heads) return false; for (size_t h = 0; h < kv_heads; ++h) @@ -833,31 +831,28 @@ bool test_preflight_specials_exceed_budget() { SpecialRowTracker tracked; std::vector rows; for (int r = 0; r < 50; ++r) rows.push_back(r); - tracked.add_appended(/*layer=*/0, /*kv_heads=*/2, rows); // 50 specials in each head + tracked.add_appended(0, 2, rows); SpecialRowTracker untracked; - // sink=4: |[0,4) U rows| = 50 either way; exceeds a budget of 40. return tracked.max_reserved(0, 4, {}) == 50 && tracked.max_reserved(0, 4, {}) > 40 && untracked.max_reserved(0, 4, rows) == 50 && untracked.max_reserved(0, 4, rows) > 40; } bool test_empty_protect_per_head_uses_params_fallback() { - // Empty protect_per_head must fall back to Params::protect (no stale per-head rows leak in). const double theta = 1000000.0; const size_t n = 64, kv_heads = 3, head_dim = 16; std::vector>> pre, val; auto kbuf = make_fp16_cache(n, n, kv_heads, head_dim, theta, pre, val); auto* krows = reinterpret_cast(kbuf.data() + kHeaderBytes); - Params p; p.sink = 4; p.recent_frac = 0.30f; p.abs_budget = 24; p.protect = {25}; // global middle protect + Params p; p.sink = 4; p.recent_frac = 0.30f; p.abs_budget = 24; p.protect = {25}; auto unrope = unrope_table(n, head_dim, theta); auto kept = keepsets_from_fp16(krows, n, kv_heads, head_dim, unrope, p, /*protect_per_head=*/{}); if (kept.size() != kv_heads) return false; for (size_t h = 0; h < kv_heads; ++h) - if (std::find(kept[h].begin(), kept[h].end(), 25) == kept[h].end()) return false; // global protect honored in every head + if (std::find(kept[h].begin(), kept[h].end(), 25) == kept[h].end()) return false; return true; } bool test_all_heads_keep_special_across_cycles() { - // A tracked mid-context special must survive in EVERY head across compactions, despite per-head divergence. const int trigger_len = 256, target_len = 128; const size_t kv_heads = 4, head_dim = 16, max_seq = trigger_len + 8; const double theta = 1000000.0; @@ -913,7 +908,7 @@ bool test_all_heads_keep_special_across_cycles() { append(start, static_cast(trigger_len)); khdr->current_seq_len = trigger_len; vhdr->current_seq_len = trigger_len; std::vector appended; - if (c == 0) appended.push_back(static_cast(special_abs)); // head-aligned special in the appended region + if (c == 0) appended.push_back(static_cast(special_abs)); tracker.add_appended(0, kv_heads, appended); auto kept = keepsets_from_fp16(krows, static_cast(trigger_len), kv_heads, head_dim, unrope, p, tracker.protect(0)); @@ -928,8 +923,8 @@ bool test_all_heads_keep_special_across_cycles() { bool diverged = false; for (size_t h = 0; h < kv_heads; ++h) { long r = special_rank(h); - if (r < 0) return false; // special must survive in every head - if (r != r0) diverged = true; // and heads keep it at different ranks (a real per-head scenario) + if (r < 0) return false; + if (r != r0) diverged = true; // ranks differ across heads -> a real per-head scenario, not trivially aligned } return diverged; } From 19a657d76d66bbb1968fe314cdda87f1ff9e46b5 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 12:37:52 -0700 Subject: [PATCH 41/55] Disable KV compaction for Gemma4 thinking mode Compaction renumbers the cache, which the Gemma4 thinking-token strip can't follow, so previous-turn thoughts leak once a compaction has run. Until the strip is compaction-aware, suppress compaction for thinking requests (set in do_prefill, checked in maybe_roll_compact); non-thinking and other models are unaffected. Gemma4 is the only model that strips prior thoughts. Signed-off-by: Noah Cylich --- cactus-engine/src/complete.cpp | 5 +++++ cactus-engine/src/engine.h | 2 ++ cactus-engine/src/model.cpp | 1 + 3 files changed, 8 insertions(+) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index a6c20e65b..ab1aadecd 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -525,6 +525,11 @@ PrefillResult do_prefill( const PreparedPrompt& prompt, const std::vector& target_tokens ) { + // Compaction renumbers the cache, which the Gemma4 thinking-token strip can't follow; disable it + // for thinking requests until the strip is compaction-aware. + handle->model->set_compaction_suppressed( + prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported); + PrefillResult result = {}; bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 4f5d518ac..7629c4309 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -644,6 +644,7 @@ class Model { void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} + void set_compaction_suppressed(bool v) { compaction_suppressed_ = v; } void compress_kv_cache_keydiff(const cactus::kvcompress::Params& params); void maybe_roll_compact(); @@ -779,6 +780,7 @@ class Model { bool initialized_ = false; size_t cache_total_seq_len_ = 0; bool cache_renumbered_ = false; + bool compaction_suppressed_ = false; // Gemma4 thinking strips prior thoughts, which renumbering breaks std::vector cache_token_ids_; // token id per cache row (canonical head-0 view) std::unordered_set special_ids_; // special-token ids force-kept during compaction cactus::kvcompress::SpecialRowTracker special_rows_; // per-(layer,head) special rows for compaction protect diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 95e3ac2ad..d19eb53d8 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2999,6 +2999,7 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } void Model::maybe_roll_compact() { + if (compaction_suppressed_) return; if (!config_.kv_compress || config_.kv_compress_trigger_len <= 0) return; if (cache_total_seq_len_ < static_cast(config_.kv_compress_trigger_len)) return; From 5f864bbc43f99f8f639859a2a86a9a3653659c17 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 12:53:55 -0700 Subject: [PATCH 42/55] Suppress compaction before the first-token fast path too The fast prefill_and_sample_first_token path (text-only, empty cache) bypasses do_prefill, where compaction_suppressed_ was set, so Gemma4 thinking requests on that path still compacted. Set the flag before the fast path as well. Signed-off-by: Noah Cylich --- cactus-engine/src/complete.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index ab1aadecd..497c4d626 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -804,6 +804,10 @@ int cactus_complete( } } + // Set before the fast path, which bypasses do_prefill. See do_prefill for the rationale. + handle->model->set_compaction_suppressed( + prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported); + bool first_token_from_prefill = false; if (!has_images && !has_audio && handle->processed_tokens.empty()) { reset_cache(handle); From 1f98ccfa6d227bf4165036c5f029c987221f6494 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 12:58:28 -0700 Subject: [PATCH 43/55] cleaned comment Signed-off-by: Noah Cylich --- cactus-engine/src/complete.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 497c4d626..f3913fff5 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -804,7 +804,6 @@ int cactus_complete( } } - // Set before the fast path, which bypasses do_prefill. See do_prefill for the rationale. handle->model->set_compaction_suppressed( prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported); From c391c53bbd3f5edeb0e4f34bc3110fb0f14e933f Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 13:22:26 -0700 Subject: [PATCH 44/55] Shrink the KV cache after compaction to reclaim long-prefill capacity The growable cache only ever grew to peak occupancy and was never reclaimed, so a long prefill (moved into decode) left the buffer over-allocated at prompt length even after compaction dropped occupancy to target. After compacting each layer, shrink its buffer to the smallest power of two >= trigger, which holds the decode occupancy oscillation [target, trigger] without re-growing. Factor a shared resize_cache_buffer used by grow and the new shrink_cache_buffer. Signed-off-by: Noah Cylich --- cactus-engine/src/model.cpp | 6 +++ cactus-engine/tests/test_kv_compress.cpp | 44 +++++++++++++++++++++ cactus-graph/cactus_graph.h | 1 + cactus-graph/src/ops_cache.cpp | 49 ++++++++++++++++-------- 4 files changed, 84 insertions(+), 16 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index d19eb53d8..3e8880d53 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2880,6 +2880,10 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) // The un-rope table is identical across compressible layers (all global, same theta/dims), so // build it once on the first such layer and reuse it for every layer's keep-set scoring. std::vector unrope; + // Reclaim capacity a long prefill grew into: shrink each compacted layer to the smallest power of + // two >= trigger, which holds the decode occupancy oscillation [target, trigger] without re-growing. + size_t shrink_cap = 1; + while (shrink_cap < static_cast(config_.kv_compress_trigger_len)) shrink_cap <<= 1; for (size_t li = 0; li < comp.cache_states.size(); ++li) { if (!compressible.count(li)) continue; const auto& cs = comp.cache_states[li]; @@ -2942,6 +2946,8 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) new_seq_len = B; have_new_seq_len = true; } + comp.graph->shrink_cache_buffer(static_cast(cs.key_node_id), shrink_cap); + comp.graph->shrink_cache_buffer(static_cast(cs.value_node_id), shrink_cap); } // Shift non-compacted layers' recent K rows by -Δ into the compressed frame (RoPE is relative, so a diff --git a/cactus-engine/tests/test_kv_compress.cpp b/cactus-engine/tests/test_kv_compress.cpp index f1193791d..20f46afa1 100644 --- a/cactus-engine/tests/test_kv_compress.cpp +++ b/cactus-engine/tests/test_kv_compress.cpp @@ -929,6 +929,49 @@ bool test_all_heads_keep_special_across_cycles() { return diverged; } +bool test_shrink_cache_buffer_preserves_rows() { + CactusGraph gb; + const size_t kv_heads = 2, head_dim = 64, ceiling = 100000, chunk = 300; + const size_t groups = (head_dim + kGroupSize - 1) / kGroupSize; + const size_t int8_stride = kv_heads * head_dim, scale_stride = kv_heads * groups; + size_t new_kv = gb.input({chunk, kv_heads, head_dim}, Precision::FP16); + size_t state = gb.kv_cache_state(ceiling, kv_heads, head_dim, /*window*/0, /*sink*/4); + size_t append = gb.kv_cache_append(new_kv, state, /*window*/0, /*sink*/4); + gb.retain_outputs({static_cast(state), static_cast(append)}); + + std::vector expect; + auto val = [](size_t row, size_t c) { return std::sin(0.013 * row + 0.07 * c); }; + std::vector in(chunk * int8_stride); + for (size_t r = 0; r < chunk; ++r) + for (size_t c = 0; c < int8_stride; ++c) { + float v = static_cast(val(r, c)); + in[r * int8_stride + c] = f32_to_f16(v); + expect.push_back(v); + } + gb.set_input(new_kv, in.data(), Precision::FP16); + gb.execute(); // appends 300 -> grows 256->512, occupancy 300 + + auto* hdr = reinterpret_cast(gb.get_output(state)); + if (hdr->max_seq_len != 512 || hdr->current_seq_len != 300) return false; + + gb.shrink_cache_buffer(state, 256); // clamped up to occupancy 300 + hdr = reinterpret_cast(gb.get_output(state)); + if (hdr->max_seq_len != 300 || hdr->current_seq_len != 300) return false; + + const auto* base = static_cast(gb.get_output(state)); + const int8_t* i8 = reinterpret_cast(base + kHeaderBytes); + const float* sc = reinterpret_cast(base + kHeaderBytes + 300 * int8_stride); + bool ok = true; + for (size_t r = 0; r < 300 && ok; ++r) + for (size_t h = 0; h < kv_heads; ++h) + for (size_t d = 0; d < head_dim; ++d) { + float dq = static_cast(i8[r * int8_stride + h * head_dim + d]) * + sc[r * scale_stride + h * groups + d / kGroupSize]; + if (std::abs(dq - expect[r * int8_stride + h * head_dim + d]) > 0.05f) ok = false; + } + return ok; +} + int main() { TestUtils::TestRunner runner("KV Compress Free-Function Tests"); runner.run_test("cache_starts_small_and_grows", test_cache_starts_small_and_grows()); @@ -958,6 +1001,7 @@ int main() { runner.run_test("preflight_specials_exceed_budget", test_preflight_specials_exceed_budget()); runner.run_test("empty_protect_per_head_uses_params_fallback", test_empty_protect_per_head_uses_params_fallback()); runner.run_test("all_heads_keep_special_across_cycles", test_all_heads_keep_special_across_cycles()); + runner.run_test("shrink_cache_buffer_preserves_rows", test_shrink_cache_buffer_preserves_rows()); runner.print_summary(); return runner.all_passed() ? 0 : 1; } diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index fce075efb..7e1d49752 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -709,6 +709,7 @@ class CactusGraph { OpType get_node_op_type(size_t node_id) const; size_t get_node_window_size(size_t node_id) const; void steal_cache_buffer(size_t dst_node, CactusGraph& src, size_t src_node); + void shrink_cache_buffer(size_t node_id, size_t new_capacity); void allocate_buffers(); size_t get_node_count() const; diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 8bd30aaf1..5c59fa3ca 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -90,42 +90,50 @@ inline bool use_fp16_kv_cache() { constexpr size_t kInitialCacheEntries = 256; -inline bool grow_cache_buffer(BufferDesc& buf, size_t needed, size_t ceiling) { +// Reallocate the cache to capacity new_max (grow or shrink), preserving the occupied rows and +// relocating the int8 scale region (whose offset depends on capacity). new_max must hold current_seq. +inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { auto* meta = get_meta(buf); size_t cur = meta->max_seq_len; - if (needed <= cur || cur >= ceiling) return false; - size_t new_max = cur; - while (new_max < needed) new_max <<= 1; - if (new_max > ceiling) new_max = ceiling; - if (new_max <= cur) return false; + const size_t current_seq = meta->current_seq_len; + if (new_max == cur || new_max < current_seq) return false; const size_t kv_heads = meta->num_kv_heads; const size_t hdim = meta->head_dim; - const size_t current_seq = meta->current_seq_len; const bool fp16_cache = buf.precision == Precision::FP16; size_t total = fp16_cache ? fp16_cache_elements(new_max, kv_heads, hdim) : cache_buffer_size(new_max, kv_heads, hdim); - BufferDesc grown({total}, fp16_cache ? Precision::FP16 : Precision::INT8); - grown.allocate(); - std::memset(grown.get_data(), 0, grown.byte_size); + BufferDesc resized({total}, fp16_cache ? Precision::FP16 : Precision::INT8); + resized.allocate(); + std::memset(resized.get_data(), 0, resized.byte_size); - std::memcpy(grown.get_data(), buf.get_data(), sizeof(CacheMetadata)); + std::memcpy(resized.get_data(), buf.get_data(), sizeof(CacheMetadata)); if (fp16_cache) { - std::memcpy(get_fp16_data(grown), get_fp16_data(buf), + std::memcpy(get_fp16_data(resized), get_fp16_data(buf), current_seq * kv_heads * hdim * sizeof(__fp16)); } else { - std::memcpy(get_int8_data(grown), get_int8_data(buf), current_seq * kv_heads * hdim); + std::memcpy(get_int8_data(resized), get_int8_data(buf), current_seq * kv_heads * hdim); const size_t groups = (hdim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - std::memcpy(get_scales(grown, new_max, kv_heads, hdim), + std::memcpy(get_scales(resized, new_max, kv_heads, hdim), get_scales(buf, cur, kv_heads, hdim), current_seq * kv_heads * groups * sizeof(float)); } - get_meta(grown)->max_seq_len = new_max; - buf = std::move(grown); + get_meta(resized)->max_seq_len = new_max; + buf = std::move(resized); return true; } +inline bool grow_cache_buffer(BufferDesc& buf, size_t needed, size_t ceiling) { + size_t cur = get_meta(buf)->max_seq_len; + if (needed <= cur || cur >= ceiling) return false; + size_t new_max = cur; + while (new_max < needed) new_max <<= 1; + if (new_max > ceiling) new_max = ceiling; + if (new_max <= cur) return false; + return resize_cache_buffer(buf, new_max); +} + } // namespace void compute_kv_cache_state_node( @@ -526,3 +534,12 @@ void CactusGraph::steal_cache_buffer(size_t dst_node, CactusGraph& src, size_t s dst->output_buffer = std::move(s->output_buffer); } +void CactusGraph::shrink_cache_buffer(size_t node_id, size_t new_capacity) { + auto& buf = nodes_[node_index_map_.at(node_id)]->output_buffer; + if (!buf.get_data()) return; + auto* meta = get_meta(buf); + size_t target = std::max(new_capacity, meta->current_seq_len); + if (target >= meta->max_seq_len) return; + resize_cache_buffer(buf, target); +} + From 59e8742b0b6ff80efb3fb1570dfbb4892e1904f1 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 13:26:27 -0700 Subject: [PATCH 45/55] Trim shrink/suppression comments to the load-bearing ones Signed-off-by: Noah Cylich --- cactus-engine/src/complete.cpp | 2 -- cactus-engine/src/model.cpp | 3 +-- cactus-graph/src/ops_cache.cpp | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index f3913fff5..f27eedd26 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -525,8 +525,6 @@ PrefillResult do_prefill( const PreparedPrompt& prompt, const std::vector& target_tokens ) { - // Compaction renumbers the cache, which the Gemma4 thinking-token strip can't follow; disable it - // for thinking requests until the strip is compaction-aware. handle->model->set_compaction_suppressed( prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported); diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 3e8880d53..3b65d2480 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2880,8 +2880,7 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) // The un-rope table is identical across compressible layers (all global, same theta/dims), so // build it once on the first such layer and reuse it for every layer's keep-set scoring. std::vector unrope; - // Reclaim capacity a long prefill grew into: shrink each compacted layer to the smallest power of - // two >= trigger, which holds the decode occupancy oscillation [target, trigger] without re-growing. + // Reclaim capacity a long prefill grew into; next_pow2(trigger) still fits the decode oscillation. size_t shrink_cap = 1; while (shrink_cap < static_cast(config_.kv_compress_trigger_len)) shrink_cap <<= 1; for (size_t li = 0; li < comp.cache_states.size(); ++li) { diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 5c59fa3ca..2ed462a23 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -90,8 +90,6 @@ inline bool use_fp16_kv_cache() { constexpr size_t kInitialCacheEntries = 256; -// Reallocate the cache to capacity new_max (grow or shrink), preserving the occupied rows and -// relocating the int8 scale region (whose offset depends on capacity). new_max must hold current_seq. inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { auto* meta = get_meta(buf); size_t cur = meta->max_seq_len; From 0bfe84dade31051624211f3280300985b527cd4d Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 20:33:27 -0700 Subject: [PATCH 46/55] Keep Gemma-4 thinking in the KV cache like LiteRT Stop stripping generated thinking for Gemma-4 so it persists across turns as ordinary context (Qwen already does this). - Remove the in-cache strip path: strip_thinking_from_cache, Model::remove_thinking_tokens, find_channel_token_ranges, and the now-unread cache_renumbered_ member/writes. - Remove the compaction-suppression stopgap (compaction_suppressed_, set_compaction_suppressed, set sites, and the maybe_roll_compact guard) so compaction runs normally for thinking context. - format_gemma4_style keeps assistant content verbatim (drop strip_channel) so prior-turn thinking stays in the rendered prompt. - Rename strip_thinking_block -> partition_thinking_response: it now only splits generated text into (thinking, content) for the API output. - Emit a non-user-facing context_response (raw assistant text before partitioning) in the result JSON for conversation-history persistence. Signed-off-by: Noah Cylich --- cactus-engine/src/complete.cpp | 31 +---------- cactus-engine/src/engine.h | 4 -- cactus-engine/src/model.cpp | 94 --------------------------------- cactus-engine/src/tokenizer.cpp | 23 +------- cactus-engine/src/utils.h | 32 +++-------- 5 files changed, 9 insertions(+), 175 deletions(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index f27eedd26..c3a7a7b0a 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -77,23 +77,6 @@ std::vector build_tool_constraint_specs(const std::vector& generated_tokens, - size_t prompt_len) { - const auto& cfg = handle->model->get_config(); - uint32_t open_id = cfg.channel_open_token_id; - uint32_t close_id = cfg.channel_close_token_id; - auto ranges = find_channel_token_ranges(generated_tokens, prompt_len, - open_id, close_id); - if (ranges.empty()) return; - - handle->model->remove_thinking_tokens(ranges); - for (auto it = ranges.rbegin(); it != ranges.rend(); ++it) { - auto start = handle->processed_tokens.begin() + it->first; - handle->processed_tokens.erase(start, start + it->second); - } -} - void setup_tool_constraints(CactusModelHandle* handle, const std::vector& tools, bool force_tools, float& temperature) { if (!force_tools || tools.empty()) return; @@ -525,9 +508,6 @@ PrefillResult do_prefill( const PreparedPrompt& prompt, const std::vector& target_tokens ) { - handle->model->set_compaction_suppressed( - prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported); - PrefillResult result = {}; bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); @@ -802,9 +782,6 @@ int cactus_complete( } } - handle->model->set_compaction_suppressed( - prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported); - bool first_token_from_prefill = false; if (!has_images && !has_audio && handle->processed_tokens.empty()) { reset_cache(handle); @@ -924,10 +901,6 @@ int cactus_complete( handle->model->clear_tool_constraints(); } - if (prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported && !generated_tokens.empty()) { - strip_thinking_from_cache(handle, generated_tokens, prompt.tokens.size()); - } - auto end_time = std::chrono::high_resolution_clock::now(); double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; @@ -945,7 +918,7 @@ int cactus_complete( std::string thinking_text; if (prompt.model_type == Config::ModelType::GEMMA4 || prompt.options.enable_thinking_if_supported) { std::string stripped_content; - strip_thinking_block(regular_response, thinking_text, stripped_content); + partition_thinking_response(regular_response, thinking_text, stripped_content); regular_response = stripped_content; if (!prompt.options.enable_thinking_if_supported) { thinking_text.clear(); @@ -988,7 +961,7 @@ int cactus_complete( std::string result = construct_response_json(primary_response, primary_function_calls, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_tokens, completion_tokens, confidence, handoff_succeeded, - thinking_text); + thinking_text, {}, response_text); if (result.length() >= buffer_size) { handle_error_response("Response buffer too small", response_buffer, buffer_size); diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 7629c4309..83817048e 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -642,9 +642,7 @@ class Model { bool load_npu_vision_encoder(const std::string& model_path); bool has_npu_vision_encoder() const { return npu_vision_encoder_ != nullptr; } - void remove_thinking_tokens(const std::vector>& ranges); void compact_kv_cache() {} - void set_compaction_suppressed(bool v) { compaction_suppressed_ = v; } void compress_kv_cache_keydiff(const cactus::kvcompress::Params& params); void maybe_roll_compact(); @@ -779,8 +777,6 @@ class Model { std::unique_ptr tokenizer_; bool initialized_ = false; size_t cache_total_seq_len_ = 0; - bool cache_renumbered_ = false; - bool compaction_suppressed_ = false; // Gemma4 thinking strips prior thoughts, which renumbering breaks std::vector cache_token_ids_; // token id per cache row (canonical head-0 view) std::unordered_set special_ids_; // special-token ids force-kept during compaction cactus::kvcompress::SpecialRowTracker special_rows_; // per-(layer,head) special rows for compaction protect diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 3b65d2480..f5aceb911 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2706,7 +2706,6 @@ std::vector Model::get_audio_embeddings(const std::vector& mel_bin void Model::reset_cache() { cache_total_seq_len_ = 0; - cache_renumbered_ = false; last_logit_position_ = 0; context_tokens_.clear(); cache_token_ids_.clear(); @@ -2724,97 +2723,6 @@ void Model::reset_cache() { void Model::set_cache_window(size_t /*window_size*/, size_t /*sink_size*/) {} -void Model::remove_thinking_tokens(const std::vector>& ranges) { - // Ranges are absolute token positions; a renumbered (compacted) cache no longer maps them, so - // skip the cache edit -- the caller still erases processed_tokens. - if (cache_renumbered_) return; - - size_t total_removed = 0; - for (const auto& r : ranges) total_removed += r.second; - if (cache_total_seq_len_ >= total_removed) - cache_total_seq_len_ -= total_removed; - else - cache_total_seq_len_ = 0; - - struct CacheHeader { - uint64_t current_seq_len; - uint64_t max_seq_len; - uint64_t num_kv_heads; - uint64_t head_dim; - uint64_t sink_size; - uint64_t reserved[3]; - }; - constexpr size_t kHeaderBytes = 64; - static_assert(sizeof(CacheHeader) == kHeaderBytes, "CacheHeader layout mismatch"); - - auto sorted_ranges = ranges; - std::sort(sorted_ranges.begin(), sorted_ranges.end(), - [](const auto& a, const auto& b) { return a.first < b.first; }); - - for (auto& kv : components_) { - Component& comp = kv.second; - if (!comp.graph) continue; - for (const auto& cs : comp.cache_states) { - for (int node_id : {cs.key_node_id, cs.value_node_id}) { - if (node_id < 0) continue; - const auto& desc = comp.graph->get_output_buffer(static_cast(node_id)); - if (desc.byte_size <= kHeaderBytes || !desc.get_data()) continue; - void* raw = comp.graph->get_output(static_cast(node_id)); - if (!raw) continue; - auto* hdr = static_cast(raw); - size_t cur = hdr->current_seq_len; - if (cur == 0) continue; - size_t kv_heads = hdr->num_kv_heads; - size_t hdim = hdr->head_dim; - if (kv_heads == 0 || hdim == 0) continue; - size_t token_elems = kv_heads * hdim; - size_t num_groups = (hdim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - size_t token_scales = kv_heads * num_groups; - size_t max_seq = hdr->max_seq_len; - - size_t new_len = cur; - if (desc.precision == Precision::FP16) { - auto* base = reinterpret_cast<__fp16*>(static_cast(raw) + kHeaderBytes); - for (auto it = sorted_ranges.rbegin(); it != sorted_ranges.rend(); ++it) { - size_t start = it->first; - if (start >= new_len) continue; - size_t count = std::min(it->second, new_len - start); - size_t tail_start = start + count; - size_t tail_count = new_len - tail_start; - if (tail_count > 0) { - std::memmove(base + start * token_elems, - base + tail_start * token_elems, - tail_count * token_elems * sizeof(__fp16)); - } - new_len -= count; - } - } else { - auto* int8_base = reinterpret_cast(static_cast(raw) + kHeaderBytes); - auto* scale_base = reinterpret_cast(static_cast(raw) + kHeaderBytes + - max_seq * kv_heads * hdim); - for (auto it = sorted_ranges.rbegin(); it != sorted_ranges.rend(); ++it) { - size_t start = it->first; - if (start >= new_len) continue; - size_t count = std::min(it->second, new_len - start); - size_t tail_start = start + count; - size_t tail_count = new_len - tail_start; - if (tail_count > 0) { - std::memmove(int8_base + start * token_elems, - int8_base + tail_start * token_elems, - tail_count * token_elems); - std::memmove(scale_base + start * token_scales, - scale_base + tail_start * token_scales, - tail_count * token_scales * sizeof(float)); - } - new_len -= count; - } - } - hdr->current_seq_len = new_len; - } - } - } -} - void Model::apply_kv_compress_env_override() { config_.parse_kv_compress_override(std::getenv("CACTUS_KV_COMPRESS_AT"), std::getenv("CACTUS_KV_COMPRESS_TO")); @@ -2988,7 +2896,6 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) // All layers now share the same compressed frame, so the next decode query uses position_ids = B. if (have_new_seq_len) { cache_total_seq_len_ = new_seq_len; - cache_renumbered_ = true; if (per_head_protect) special_rows_.set_tracked_len(new_seq_len); else special_rows_.invalidate(); if (map_valid && canonical_captured) { @@ -3004,7 +2911,6 @@ void Model::compress_kv_cache_keydiff(const cactus::kvcompress::Params& params) } void Model::maybe_roll_compact() { - if (compaction_suppressed_) return; if (!config_.kv_compress || config_.kv_compress_trigger_len <= 0) return; if (cache_total_seq_len_ < static_cast(config_.kv_compress_trigger_len)) return; diff --git a/cactus-engine/src/tokenizer.cpp b/cactus-engine/src/tokenizer.cpp index 8f5930d8e..342e9448a 100644 --- a/cactus-engine/src/tokenizer.cpp +++ b/cactus-engine/src/tokenizer.cpp @@ -471,27 +471,6 @@ std::string Tokenizer::format_gemma4_style(const std::vector& messa result += "\n"; } - auto strip_channel = [](const std::string& text) -> std::string { - const std::string open_tag = "<|channel>"; - const std::string close_tag = ""; - std::string out; - size_t pos = 0; - while (pos < text.size()) { - size_t open_pos = text.find(open_tag, pos); - if (open_pos == std::string::npos) { - out += text.substr(pos); - break; - } - out += text.substr(pos, open_pos - pos); - size_t close_pos = text.find(close_tag, open_pos + open_tag.size()); - if (close_pos == std::string::npos) { - break; - } - pos = close_pos + close_tag.size(); - } - return out; - }; - auto compute_soft_tokens = [&](const std::string& image_path) -> size_t { int w = 0, h = 0, c = 0; unsigned char* data = cactus_image_load(image_path.c_str(), &w, &h, &c, 3); @@ -516,7 +495,7 @@ std::string Tokenizer::format_gemma4_style(const std::vector& messa std::string role = (msg.role == "assistant") ? "model" : msg.role; result += "<|turn>" + role + "\n"; if (role == "model") { - result += strip_channel(msg.content); + result += msg.content; if (!msg.tool_calls.empty()) { for (const auto& tc : msg.tool_calls) { result += format_tool_call_for_prompt(tc.name, tc.arguments, true); diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 9f66f4895..ca64eb549 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -1534,30 +1534,6 @@ inline void parse_function_calls_from_response(const std::string& response_text, } } -inline std::vector> find_channel_token_ranges( - const std::vector& tokens, size_t offset, - uint32_t channel_open_id, uint32_t channel_close_id) { - std::vector> ranges; - size_t pos = 0; - while (pos < tokens.size()) { - if (tokens[pos] != channel_open_id) { - pos++; - continue; - } - - size_t block_start = pos; - pos++; - while (pos < tokens.size() && tokens[pos] != channel_close_id) { - pos++; - } - if (pos < tokens.size()) { - pos++; - } - ranges.push_back({offset + block_start, pos - block_start}); - } - return ranges; -} - inline void strip_tag_blocks(std::string& text, std::string& extracted, const std::string& open_tag, const std::string& close_tag) { std::string result; @@ -1592,7 +1568,7 @@ inline void strip_tag_blocks(std::string& text, std::string& extracted, text = result; } -inline void strip_thinking_block(const std::string& input, std::string& thinking, std::string& content) { +inline void partition_thinking_response(const std::string& input, std::string& thinking, std::string& content) { thinking.clear(); content = input; @@ -1634,13 +1610,17 @@ inline std::string construct_response_json(const std::string& regular_response, float confidence = 0.0f, bool cloud_handoff = false, const std::string& thinking = "", - const std::vector& segments = {}) { + const std::vector& segments = {}, + const std::string& context_response = "") { std::ostringstream json; json << "{"; json << "\"success\":true,"; json << "\"error\":null,"; json << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; json << "\"response\":\"" << escape_json_string(regular_response) << "\","; + if (!context_response.empty()) { + json << "\"context_response\":\"" << escape_json_string(context_response) << "\","; + } if (!thinking.empty()) { json << "\"thinking\":\"" << escape_json_string(thinking) << "\","; } From 524fe9c5f79c54472d80a6f30d0309137244c97b Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 20:33:49 -0700 Subject: [PATCH 47/55] Persist Gemma-4 thinking in chat surfaces; rewrite thinking test - chat.cpp and server.py store context_response (raw assistant text with thinking) as conversation history while still displaying the clean response, so thinking survives the next turn's re-render. - Rewrite test_gemma4_thinking for the new contract: the formatter retains assistant channel/thinking content, the API response stays clean, and re-rendered multi-turn history prefix-matches the cache. Signed-off-by: Noah Cylich --- cactus-engine/tests/chat.cpp | 3 +- cactus-engine/tests/test_gemma4_thinking.cpp | 189 +++++++++++++++++++ python/cactus/server.py | 3 +- 3 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 cactus-engine/tests/test_gemma4_thinking.cpp diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/chat.cpp index a5074e39b..82a349141 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/chat.cpp @@ -680,7 +680,8 @@ int main(int argc, char** argv) { continue; } - std::string assistant = json_string_value(response_json, "response"); + std::string assistant = json_string_value(response_json, "context_response"); + if (assistant.empty()) assistant = json_string_value(response_json, "response"); history.push_back({"assistant", assistant}); current_image.clear(); current_audio.clear(); diff --git a/cactus-engine/tests/test_gemma4_thinking.cpp b/cactus-engine/tests/test_gemma4_thinking.cpp new file mode 100644 index 000000000..5dc84ebb8 --- /dev/null +++ b/cactus-engine/tests/test_gemma4_thinking.cpp @@ -0,0 +1,189 @@ +#include "test_utils.h" +#include "../src/engine.h" +#include "../src/utils.h" +#include +#include +#include +#include + +using namespace cactus::engine; +using cactus::ffi::partition_thinking_response; + +static const char* get_gemma4_model_path() { + const char* path = std::getenv("CACTUS_TEST_GEMMA4_MODEL"); + if (path) return path; + return std::getenv("CACTUS_TEST_MODEL"); +} + +static bool check_partition(const std::string& input, + const std::string& expected_thinking, + const std::string& expected_content) { + std::string thinking, content; + partition_thinking_response(input, thinking, content); + if (thinking != expected_thinking) { + std::cerr << " thinking: '" << thinking << "' != '" << expected_thinking << "'\n"; + return false; + } + if (content != expected_content) { + std::cerr << " content: '" << content << "' != '" << expected_content << "'\n"; + return false; + } + return true; +} + +// partition_thinking_response is display-only: it splits generated text into (thinking, content) +// for the API, without touching the cache. +bool test_partition_thinking_response() { + return check_partition("<|channel>reasonanswer", "reason", "answer") + && check_partition("<|channel>\n reason\n\n\nanswer", "reason", "answer") + && check_partition("no tags here", "", "no tags here") + && check_partition("reasonanswer", "reason", "answer") + && check_partition("<|channel>thought1text1<|channel>thought2text2", + "thought1\nthought2", "text1text2"); +} + +// New contract: the Gemma4 formatter RETAINS prior-turn assistant channel/thinking content in the +// rendered prompt (no strip), matching LiteRT so thinking persists across turns. +bool test_prompt_gemma4_retains_thinking() { + const char* model_path = get_gemma4_model_path(); + if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } + + cactus_model_t model = cactus_init(model_path, nullptr, false); + if (!model) { std::cout << " [SKIP] Could not load model\n"; return true; } + + auto* handle = static_cast(model); + if (handle->model->get_config().model_type != Config::ModelType::GEMMA4) { + std::cout << " [SKIP] Not a Gemma4 model\n"; + cactus_destroy(model); + return true; + } + auto* tok = handle->model->get_tokenizer(); + + std::vector msgs = { + {"user", "hello", "", {}, {}, 0, {}}, + {"assistant", "<|channel>internal reasoningvisible response", "", {}, {}, 0, {}}, + {"user", "followup", "", {}, {}, 0, {}} + }; + + std::string prompt = tok->format_chat_prompt(msgs, true, "", true); + cactus_destroy(model); + + bool has_visible = prompt.find("visible response") != std::string::npos; + bool has_reasoning = prompt.find("internal reasoning") != std::string::npos; + bool has_channel_tags = prompt.find("<|channel>") != std::string::npos + && prompt.find("") != std::string::npos; + + if (!has_visible) std::cerr << " missing visible response in prompt\n"; + if (!has_reasoning) std::cerr << " thinking content not retained in assistant turn\n"; + if (!has_channel_tags) std::cerr << " channel tags not retained in prompt\n"; + + return has_visible && has_reasoning && has_channel_tags; +} + +// API separation is unchanged: response excludes thinking, thinking is populated, and no raw +// channel tags leak into the user-facing response. +bool test_complete_gemma4_thinking_api_clean() { + const char* model_path = get_gemma4_model_path(); + if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } + + cactus_model_t model = cactus_init(model_path, nullptr, false); + if (!model) return false; + + auto* handle = static_cast(model); + if (handle->model->get_config().model_type != Config::ModelType::GEMMA4) { + std::cout << " [SKIP] Not a Gemma4 model\n"; + cactus_destroy(model); + return true; + } + + const char* msgs = R"([{"role": "user", "content": "What is 2+2?"}])"; + char buf[8192]; + + int r = cactus_complete(model, msgs, buf, sizeof(buf), + R"({"max_tokens":128,"enable_thinking_if_supported":true,"telemetry_enabled":false})", + nullptr, nullptr, nullptr, nullptr, 0); + std::string resp(buf); + cactus_destroy(model); + + std::string response = EngineTestUtils::json_string(resp, "response"); + bool ok = r > 0 + && resp.find("\"success\":true") != std::string::npos + && response.find("<|channel>") == std::string::npos + && response.find("") == std::string::npos; + if (!ok) std::cerr << " thinking-enabled completion api not clean: " << resp << "\n"; + return ok; +} + +// Multi-turn: store the raw context_response (thinking retained) for history; the re-rendered +// turn-2 prompt must prefix-match the turn-1 processed tokens (cache reuse) and recall the fact. +bool test_multiturn_thinking_persist() { + const char* model_path = get_gemma4_model_path(); + if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } + + cactus_model_t model = cactus_init(model_path, nullptr, false); + if (!model) { std::cerr << " Failed to load model\n"; return false; } + + auto* handle = static_cast(model); + if (handle->model->get_config().model_type != Config::ModelType::GEMMA4) { + std::cout << " [SKIP] Not a Gemma4 model\n"; + cactus_destroy(model); + return true; + } + + auto* tokenizer = handle->model->get_tokenizer(); + const char* options = R"({"max_tokens":128,"temperature":0,"top_k":1,"enable_thinking_if_supported":true,"telemetry_enabled":false,"auto_handoff":false})"; + const char* turn1_msgs = R"([{"role": "user", "content": "My name is Alice. Please remember this."}])"; + char buf[16384]; + + int r1 = cactus_complete(model, turn1_msgs, buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); + if (r1 <= 0) { std::cerr << " Turn 1 failed\n"; cactus_destroy(model); return false; } + + std::vector processed_after_t1 = handle->processed_tokens; + std::string turn1_json(buf); + std::string context_response = EngineTestUtils::json_string(turn1_json, "context_response"); + if (context_response.empty()) { + std::cerr << " context_response missing from turn 1 result\n"; + cactus_destroy(model); + return false; + } + + std::vector t2_chat = { + {"user", "My name is Alice. Please remember this.", "", {}, {}, 0, {}}, + {"assistant", context_response, "", {}, {}, 0, {}}, + {"user", "What is my name?", "", {}, {}, 0, {}} + }; + std::vector t2_prompt_tokens = tokenizer->encode(tokenizer->format_chat_prompt(t2_chat, true, "", true)); + + bool prefix_ok = (t2_prompt_tokens.size() >= processed_after_t1.size()) && + std::equal(processed_after_t1.begin(), processed_after_t1.end(), t2_prompt_tokens.begin()); + std::cout << " Prefix match (cache reuse): " << (prefix_ok ? "YES" : "NO") << "\n"; + + std::string escaped = EngineTestUtils::escape_json(context_response); + std::string turn2_json = R"([{"role": "user", "content": "My name is Alice. Please remember this."},{"role": "assistant", "content": ")" + + escaped + R"("},{"role": "user", "content": "What is my name?"}])"; + + int r2 = cactus_complete(model, turn2_json.c_str(), buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); + if (r2 <= 0) { std::cerr << " Turn 2 failed\n"; cactus_destroy(model); return false; } + + std::string turn2_result(buf); + std::string turn2_response = EngineTestUtils::json_string(turn2_result, "response"); + bool mentions_alice = turn2_response.find("Alice") != std::string::npos + || turn2_response.find("alice") != std::string::npos; + std::cout << " Turn 2 mentions Alice: " << (mentions_alice ? "YES" : "NO") << "\n"; + + cactus_destroy(model); + + if (!prefix_ok) std::cerr << " FAIL: re-rendered history did not prefix-match cache\n"; + if (!mentions_alice) std::cerr << " FAIL: turn 2 did not recall the name\n"; + return prefix_ok && mentions_alice; +} + +int main() { + TestUtils::TestRunner runner("Gemma4 Thinking Tests"); + runner.run_test("partition_thinking_response", test_partition_thinking_response()); + runner.run_test("prompt_retains_thinking", test_prompt_gemma4_retains_thinking()); + runner.run_test("complete_thinking_api_clean", test_complete_gemma4_thinking_api_clean()); + runner.run_test("multiturn_thinking_persist", test_multiturn_thinking_persist()); + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/python/cactus/server.py b/python/cactus/server.py index cfab62ca2..bd03dc0eb 100644 --- a/python/cactus/server.py +++ b/python/cactus/server.py @@ -333,7 +333,8 @@ def _build_chat_response(result: dict[str, Any], model_id: str, request_id: str) has_tool_calls = bool(tool_calls) prefill = int(result.get("prefill_tokens") or 0) decode = int(result.get("decode_tokens") or 0) - message: dict[str, Any] = {"role": "assistant", "content": None if has_tool_calls else result.get("response", "")} + content = result.get("context_response") or result.get("response", "") + message: dict[str, Any] = {"role": "assistant", "content": None if has_tool_calls else content} if has_tool_calls: message["tool_calls"] = tool_calls return { From 48f270d0bec296a9ae36cd7066a10a86f10c0766 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 20:44:29 -0700 Subject: [PATCH 48/55] Fix review: clean OpenAI response content; drop stale v1 thinking test server.py returned the raw context_response (channel tags) as public message content; return the clean response (context_response is only for stateful chat history). Delete the superseded v1 tests/test_gemma4_thinking.cpp, which still asserted the removed strip contract; the active test lives in cactus-engine/tests. Signed-off-by: Noah Cylich --- python/cactus/server.py | 2 +- tests/test_gemma4_thinking.cpp | 311 --------------------------------- 2 files changed, 1 insertion(+), 312 deletions(-) delete mode 100644 tests/test_gemma4_thinking.cpp diff --git a/python/cactus/server.py b/python/cactus/server.py index bd03dc0eb..7a8e06275 100644 --- a/python/cactus/server.py +++ b/python/cactus/server.py @@ -333,7 +333,7 @@ def _build_chat_response(result: dict[str, Any], model_id: str, request_id: str) has_tool_calls = bool(tool_calls) prefill = int(result.get("prefill_tokens") or 0) decode = int(result.get("decode_tokens") or 0) - content = result.get("context_response") or result.get("response", "") + content = result.get("response", "") message: dict[str, Any] = {"role": "assistant", "content": None if has_tool_calls else content} if has_tool_calls: message["tool_calls"] = tool_calls diff --git a/tests/test_gemma4_thinking.cpp b/tests/test_gemma4_thinking.cpp deleted file mode 100644 index 482715480..000000000 --- a/tests/test_gemma4_thinking.cpp +++ /dev/null @@ -1,311 +0,0 @@ -#include "test_utils.h" -#include "../cactus/ffi/cactus_utils.h" -#include -#include -#include -#include - -using namespace cactus::ffi; -using namespace cactus::engine; - -static const char* get_gemma4_model_path() { - const char* path = std::getenv("CACTUS_TEST_GEMMA4_MODEL"); - if (path) return path; - return std::getenv("CACTUS_TEST_MODEL"); -} - -static bool check_strip(const std::string& input, - const std::string& expected_thinking, - const std::string& expected_content) { - std::string thinking, content; - strip_thinking_block(input, thinking, content); - if (thinking != expected_thinking) { - std::cerr << " thinking: '" << thinking << "' != '" << expected_thinking << "'\n"; - return false; - } - if (content != expected_content) { - std::cerr << " content: '" << content << "' != '" << expected_content << "'\n"; - return false; - } - return true; -} - -static std::string escape_for_json(const std::string& s) { - std::string out; - for (char c : s) { - if (c == '"') out += "\\\""; - else if (c == '\\') out += "\\\\"; - else if (c == '\n') out += "\\n"; - else if (c == '\r') out += "\\r"; - else if (c == '\t') out += "\\t"; - else out += c; - } - return out; -} - -bool test_strip_thinking_channel() { - return check_strip("<|channel>reasonanswer", "reason", "answer") - && check_strip("<|channel>\n reason\n\n\nanswer", "reason", "answer") - && check_strip("no tags here", "", "no tags here") - && check_strip("<|channel>unclosed", "unclosed", "") - && check_strip("reason\n\n\nanswer", "reason", "answer") - && check_strip("<|channel>thought1text1<|channel>thought2text2", - "thought1\nthought2", "text1text2"); -} - -bool test_find_channel_token_ranges() { - constexpr uint32_t OPEN = 100; - constexpr uint32_t CLOSE = 101; - - { - std::vector tokens = {OPEN, 50, 51, 52, CLOSE, 60, 61}; - auto ranges = find_channel_token_ranges(tokens, 0, OPEN, CLOSE); - if (ranges.size() != 1 || ranges[0].first != 0 || ranges[0].second != 5) { - std::cerr << " Single block failed\n"; - return false; - } - } - { - std::vector tokens = {10, OPEN, 50, CLOSE, 20, OPEN, 51, 52, CLOSE, 30}; - auto ranges = find_channel_token_ranges(tokens, 0, OPEN, CLOSE); - if (ranges.size() != 2 || ranges[0].first != 1 || ranges[0].second != 3 - || ranges[1].first != 5 || ranges[1].second != 4) { - std::cerr << " Two blocks failed\n"; - return false; - } - } - { - std::vector tokens = {10, 20, 30}; - if (!find_channel_token_ranges(tokens, 0, OPEN, CLOSE).empty()) { - std::cerr << " No blocks failed\n"; - return false; - } - } - { - std::vector tokens = {OPEN, 50, 51}; - auto ranges = find_channel_token_ranges(tokens, 0, OPEN, CLOSE); - if (ranges.size() != 1 || ranges[0].first != 0 || ranges[0].second != 3) { - std::cerr << " Unclosed block failed\n"; - return false; - } - } - { - std::vector tokens = {OPEN, 50, CLOSE, 60}; - auto ranges = find_channel_token_ranges(tokens, 100, OPEN, CLOSE); - if (ranges.size() != 1 || ranges[0].first != 100 || ranges[0].second != 3) { - std::cerr << " Offset failed\n"; - return false; - } - } - - return true; -} - -bool test_prompt_gemma4_thinking_injection() { - const char* model_path = get_gemma4_model_path(); - if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } - - cactus_model_t model = cactus_init(model_path, nullptr, false); - if (!model) { std::cout << " [SKIP] Could not load model\n"; return true; } - - auto* handle = static_cast(model); - auto* tok = handle->model->get_tokenizer(); - std::vector msgs = {{"user", "hello", "", {}, {}, 0, {}}}; - - std::string enabled = tok->format_chat_prompt(msgs, true, "", true); - std::string disabled = tok->format_chat_prompt(msgs, true, "", false); - cactus_destroy(model); - - bool ok = enabled.find("<|think|>") != std::string::npos - && disabled.find("<|think|>") == std::string::npos - && disabled.find("<|channel>") == std::string::npos - && disabled.find("") == std::string::npos; - if (!ok) { - std::cerr << " thinking injection mismatch in prompt\n"; - std::cerr << " enabled prompt: " << enabled << "\n"; - std::cerr << " disabled prompt: " << disabled << "\n"; - } - return ok; -} - -bool test_prompt_gemma4_assistant_stripping() { - const char* model_path = get_gemma4_model_path(); - if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } - - cactus_model_t model = cactus_init(model_path, nullptr, false); - if (!model) { std::cout << " [SKIP] Could not load model\n"; return true; } - - auto* handle = static_cast(model); - auto* tok = handle->model->get_tokenizer(); - - std::vector msgs = { - {"user", "hello", "", {}, {}, 0, {}}, - {"assistant", "<|channel>internal reasoningvisible response", "", {}, {}, 0, {}}, - {"user", "followup", "", {}, {}, 0, {}} - }; - - std::string prompt = tok->format_chat_prompt(msgs, true, "", true); - cactus_destroy(model); - - bool has_visible = prompt.find("visible response") != std::string::npos; - bool no_reasoning = prompt.find("internal reasoning") == std::string::npos; - bool no_channel_tags = prompt.find("<|channel>") == std::string::npos - && prompt.find("") == std::string::npos; - - if (!has_visible) std::cerr << " missing visible response in prompt\n"; - if (!no_reasoning) std::cerr << " thinking content not stripped from assistant turn\n"; - if (!no_channel_tags) std::cerr << " channel tags not stripped from prompt\n"; - - return has_visible && no_reasoning && no_channel_tags; -} - -bool test_complete_gemma4_thinking_toggle() { - const char* model_path = get_gemma4_model_path(); - if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } - - cactus_model_t model = cactus_init(model_path, nullptr, false); - if (!model) return false; - - auto* handle = static_cast(model); - auto mtype = handle->model->get_config().model_type; - if (mtype != Config::ModelType::GEMMA4) { - std::cout << " [SKIP] Not a Gemma4 model\n"; - cactus_destroy(model); - return true; - } - - const char* msgs = R"([{"role": "user", "content": "What is 2+2?"}])"; - char buf[8192]; - - int r1 = cactus_complete(model, msgs, buf, sizeof(buf), - R"({"max_tokens":128,"enable_thinking_if_supported":true,"telemetry_enabled":false})", - nullptr, nullptr, nullptr, nullptr, 0); - std::string resp1(buf); - bool ok1 = r1 > 0 && resp1.find("\"success\":true") != std::string::npos; - - handle->model->reset_cache(); - handle->processed_tokens.clear(); - - int r2 = cactus_complete(model, msgs, buf, sizeof(buf), - R"({"max_tokens":128,"enable_thinking_if_supported":false,"telemetry_enabled":false})", - nullptr, nullptr, nullptr, nullptr, 0); - std::string resp2(buf); - bool ok2 = r2 > 0 - && resp2.find("\"thinking\"") == std::string::npos - && resp2.find("<|channel>") == std::string::npos - && resp2.find("") == std::string::npos; - - cactus_destroy(model); - if (!ok1) std::cerr << " thinking-enabled completion failed\n"; - if (!ok2) std::cerr << " thinking-disabled completion should not have thinking field\n"; - return ok1 && ok2; -} - -bool test_complete_gemma4_tool_call() { - const char* model_path = get_gemma4_model_path(); - if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } - - cactus_model_t model = cactus_init(model_path, nullptr, false); - if (!model) return false; - - auto* handle = static_cast(model); - auto mtype = handle->model->get_config().model_type; - if (mtype != Config::ModelType::GEMMA4) { - std::cout << " [SKIP] Not a Gemma4 model\n"; - cactus_destroy(model); - return true; - } - - const char* msgs = R"([{"role": "user", "content": "What is the weather in San Francisco?"}])"; - const char* tools = R"([{"type":"function","function":{"name":"get_weather","description":"Get current weather","parameters":{"type":"object","properties":{"location":{"type":"string","description":"City name"}},"required":["location"]}}}])"; - char buf[8192]; - - int r = cactus_complete(model, msgs, buf, sizeof(buf), - R"({"max_tokens":256,"force_tool_call":true,"enable_thinking_if_supported":true,"telemetry_enabled":false})", - tools, nullptr, nullptr, nullptr, 0); - std::string resp(buf); - - cactus_destroy(model); - - bool success = r > 0 && resp.find("\"success\":true") != std::string::npos; - bool has_calls = resp.find("\"function_calls\"") != std::string::npos - && resp.find("get_weather") != std::string::npos; - - if (!success) std::cerr << " tool call completion failed: " << resp << "\n"; - if (!has_calls) std::cerr << " expected get_weather tool call in response: " << resp << "\n"; - return success && has_calls; -} - -bool test_multiturn_cache_reuse() { - const char* model_path = get_gemma4_model_path(); - if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } - - cactus_model_t model = cactus_init(model_path, nullptr, false); - if (!model) { std::cerr << " Failed to load model\n"; return false; } - - auto* handle = static_cast(model); - if (handle->model->get_config().model_type != Config::ModelType::GEMMA4) { - std::cout << " [SKIP] Not a Gemma4 model\n"; - cactus_destroy(model); - return true; - } - - auto* tokenizer = handle->model->get_tokenizer(); - const char* options = R"({"max_tokens":128,"temperature":0,"top_k":1,"enable_thinking_if_supported":true,"telemetry_enabled":false,"auto_handoff":false})"; - const char* turn1_msgs = R"([{"role": "user", "content": "My name is Alice. Please remember this."}])"; - char buf[16384]; - - int r1 = cactus_complete(model, turn1_msgs, buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); - if (r1 <= 0) { std::cerr << " Turn 1 failed\n"; cactus_destroy(model); return false; } - - std::vector processed_after_t1 = handle->processed_tokens; - - std::vector t1_chat = {{"user", "My name is Alice. Please remember this.", "", {}, {}, 0, {}}}; - std::vector t1_prompt_tokens = tokenizer->encode(tokenizer->format_chat_prompt(t1_chat, true, "", true)); - std::vector gen_tokens(processed_after_t1.begin() + t1_prompt_tokens.size(), processed_after_t1.end()); - std::string assistant_text = tokenizer->decode(gen_tokens); - - std::vector t2_chat = { - {"user", "My name is Alice. Please remember this.", "", {}, {}, 0, {}}, - {"assistant", assistant_text, "", {}, {}, 0, {}}, - {"user", "What is my name?", "", {}, {}, 0, {}} - }; - std::vector t2_prompt_tokens = tokenizer->encode(tokenizer->format_chat_prompt(t2_chat, true, "", true)); - - bool prefix_ok = (t2_prompt_tokens.size() >= processed_after_t1.size()) && - std::equal(processed_after_t1.begin(), processed_after_t1.end(), t2_prompt_tokens.begin()); - std::cout << " Prefix match (cache reuse): " << (prefix_ok ? "YES" : "NO") << "\n"; - - size_t new_tokens_for_t2 = t2_prompt_tokens.size() - processed_after_t1.size(); - std::cout << " Turn 2 new tokens: " << new_tokens_for_t2 << " (full prompt: " << t2_prompt_tokens.size() << ")\n"; - - std::string escaped = escape_for_json(assistant_text); - std::string turn2_json = R"([{"role": "user", "content": "My name is Alice. Please remember this."},{"role": "assistant", "content": ")" - + escaped + R"("},{"role": "user", "content": "What is my name?"}])"; - - int r2 = cactus_complete(model, turn2_json.c_str(), buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); - if (r2 <= 0) { std::cerr << " Turn 2 failed\n"; cactus_destroy(model); return false; } - - std::string turn2_response(buf); - bool mentions_alice = turn2_response.find("Alice") != std::string::npos - || turn2_response.find("alice") != std::string::npos; - std::cout << " Turn 2 mentions Alice: " << (mentions_alice ? "YES" : "NO") << "\n"; - - cactus_destroy(model); - - if (!prefix_ok) std::cerr << " FAIL: cache surgery did not produce valid prefix\n"; - return prefix_ok; -} - -int main() { - TestUtils::TestRunner runner("Gemma4 Thinking Tests"); - runner.run_test("strip_thinking_channel", test_strip_thinking_channel()); - runner.run_test("find_channel_token_ranges", test_find_channel_token_ranges()); - runner.run_test("prompt_thinking_injection", test_prompt_gemma4_thinking_injection()); - runner.run_test("prompt_assistant_stripping", test_prompt_gemma4_assistant_stripping()); - runner.run_test("complete_thinking_toggle", test_complete_gemma4_thinking_toggle()); - runner.run_test("complete_tool_call", test_complete_gemma4_tool_call()); - runner.run_test("multiturn_cache_reuse", test_multiturn_cache_reuse()); - runner.print_summary(); - return runner.all_passed() ? 0 : 1; -} From 2b3337173804e2383b5bc2e3d9ac41bd5abd826a Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 20:46:13 -0700 Subject: [PATCH 49/55] Remove stale iOS harness reference to deleted thinking test Signed-off-by: Noah Cylich --- tests/ios/CactusTest/CactusTest/AppDelegate.mm | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/ios/CactusTest/CactusTest/AppDelegate.mm b/tests/ios/CactusTest/CactusTest/AppDelegate.mm index 643d983b6..5bc25fb69 100644 --- a/tests/ios/CactusTest/CactusTest/AppDelegate.mm +++ b/tests/ios/CactusTest/CactusTest/AppDelegate.mm @@ -19,7 +19,6 @@ extern int test_gemma4_audio_main(); extern int test_gemma4_audio_layers_main(); extern int test_gemma4_suite_main(); -extern int test_gemma4_thinking_main(); extern int test_gemma4_vision_main(); extern int test_graph_main(); extern int test_index_main(); @@ -182,7 +181,6 @@ - (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:( if (should_run("gemma4_audio")) test_gemma4_audio_main(); if (should_run("gemma4_audio_layers")) test_gemma4_audio_layers_main(); if (should_run("gemma4_suite")) test_gemma4_suite_main(); - if (should_run("gemma4_thinking")) test_gemma4_thinking_main(); if (should_run("gemma4_vision")) test_gemma4_vision_main(); if (should_run("graph")) test_graph_main(); if (should_run("index")) test_index_main(); From dfbefffb6cb7df8152b384141eeb96b6bf0e06e3 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 20:57:16 -0700 Subject: [PATCH 50/55] Remove channel-token config orphaned by the thinking-strip deletion find_channel_token_ranges was the only reader of channel_open/close_token_id; drop the now-unused config fields and their parse. Revert the one-line server.py refactor to the original inline form. Signed-off-by: Noah Cylich --- cactus-engine/src/engine.h | 2 -- cactus-engine/src/model.cpp | 2 -- python/cactus/server.py | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 83817048e..e63041b25 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -229,8 +229,6 @@ struct Config { uint32_t audio_fft_length = 1024; uint32_t audio_token_id = 0; bool audio_fft_overdrive = false; - uint32_t channel_open_token_id = 100; - uint32_t channel_close_token_id = 101; static bool is_gemma_family(ModelType t) { return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index f5aceb911..7017cfe08 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -3225,8 +3225,6 @@ bool Config::from_json(const std::string& config_path) { audio_fft_length = audio_fft_overdrive ? 1024u : 512u; } else if (key == "audio_token_id") audio_token_id = static_cast(std::stoul(value)); - else if (key == "channel_open_token_id") channel_open_token_id = static_cast(std::stoul(value)); - else if (key == "channel_close_token_id") channel_close_token_id = static_cast(std::stoul(value)); else if (key == "activation_sparsity_ppf") { activation_sparsity_ppf.clear(); std::stringstream ss(value); diff --git a/python/cactus/server.py b/python/cactus/server.py index 7a8e06275..cfab62ca2 100644 --- a/python/cactus/server.py +++ b/python/cactus/server.py @@ -333,8 +333,7 @@ def _build_chat_response(result: dict[str, Any], model_id: str, request_id: str) has_tool_calls = bool(tool_calls) prefill = int(result.get("prefill_tokens") or 0) decode = int(result.get("decode_tokens") or 0) - content = result.get("response", "") - message: dict[str, Any] = {"role": "assistant", "content": None if has_tool_calls else content} + message: dict[str, Any] = {"role": "assistant", "content": None if has_tool_calls else result.get("response", "")} if has_tool_calls: message["tool_calls"] = tool_calls return { From ac780c9c223d8ce57f2aeb50cada6422883dbed5 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 6 Jun 2026 22:01:21 -0700 Subject: [PATCH 51/55] Reproduce Qwen thinking opener in history so KV cache reuses across turns Qwen's chat template injects a thinking opener (\n, or \n\n\n\n when thinking is disabled) only in the generation-prompt branch. The model continues from that opener, so the opener tokens are stored in the KV cache as part of the assistant turn. Re-rendering that turn as history did not reproduce the opener, so the next turn's prompt diverged from the cached tokens at the assistant boundary, prompt_context_matches failed, and every turn did a full re-prefill. Emit the same opener per assistant history turn (selected from whether the content already closed ) so the re-templated history byte-matches the cache. Signed-off-by: Noah Cylich --- cactus-engine/src/tokenizer.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cactus-engine/src/tokenizer.cpp b/cactus-engine/src/tokenizer.cpp index 342e9448a..14f1336d0 100644 --- a/cactus-engine/src/tokenizer.cpp +++ b/cactus-engine/src/tokenizer.cpp @@ -426,6 +426,10 @@ std::string Tokenizer::format_qwen_style(const std::vector& message if (is_lfm2) result += "\n"; } } + // Replicating generation prompt so the kv cache can and is re-used across turns. + if (role == "assistant" && chat_template_.find("") != std::string::npos) { + result += msg.content.find("") != std::string::npos ? "\n" : "\n\n\n\n"; + } result += msg.content; if (role == "assistant" && !msg.tool_calls.empty()) { for (const auto& tc : msg.tool_calls) { From ab128bac46b2ff52c770dd63184959b214b6c10a Mon Sep 17 00:00:00 2001 From: jakmro Date: Mon, 8 Jun 2026 19:39:39 +0200 Subject: [PATCH 52/55] Add cache_context_length parameter to component specs functions --- python/cactus/transpile/model_adapters.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 1f0b1e6c6..0d427dc9f 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -5283,6 +5283,7 @@ def _build_lfm2_causal_lm_component_specs( named_tensors: dict[str, torch.Tensor], weights_dir: str | None = None, components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, ) -> list[ComponentModuleSpec] | None: input_ids = named_tensors.get("input_ids") if input_ids is None: @@ -5623,6 +5624,7 @@ def build_component_module_specs( named_tensors=named_tensors, weights_dir=weights_dir, components=components, + cache_context_length=cache_context_length, ) if family == "parakeet_tdt" and task == "tdt_transcription": from cactus.transpile.tdt_runtime import build_parakeet_tdt_component_specs From 211e5a2c4c40642aeb505ec90d60fdcd54a66af0 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Mon, 8 Jun 2026 10:44:19 -0700 Subject: [PATCH 53/55] Shorten keepsets_from_fp16 comment to one line Signed-off-by: Noah Cylich --- cactus-engine/src/kv_compress.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cactus-engine/src/kv_compress.h b/cactus-engine/src/kv_compress.h index d5d118eff..c33a5e659 100644 --- a/cactus-engine/src/kv_compress.h +++ b/cactus-engine/src/kv_compress.h @@ -70,8 +70,7 @@ void rerope_recent_fp16(uint16_t* key_rows, size_t kv_heads, size_t head_dim, void rerope_recent_int8(int8_t* int8_rows, float* scale_rows, size_t kv_heads, size_t head_dim, size_t group_size, size_t lo, size_t hi, double rope_theta, double delta_pos); -// Per-head keep-sets from POST-RoPE keys. protect_per_head (sized to kv_heads) overrides -// Params::protect per head; empty -> shared protect. +// Per-head keep-sets from POST-RoPE keys. protect_per_head overrides Params::protect; empty -> shared. std::vector> keepsets_from_fp16(const uint16_t* key_rows, size_t n, size_t kv_heads, size_t head_dim, const std::vector& unrope, From 4ea057ef0bb522c26f2348bdb8d9ef8b0a666062 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Mon, 8 Jun 2026 10:50:00 -0700 Subject: [PATCH 54/55] Remove gemma4 context-scaling benchmark script Standalone manual benchmark harness, not part of the test suite or referenced anywhere; out of scope for the PR. Signed-off-by: Noah Cylich --- .../tools/benchmark_gemma4_context_scaling.py | 445 ------------------ 1 file changed, 445 deletions(-) delete mode 100755 python/tests/transpile/tools/benchmark_gemma4_context_scaling.py diff --git a/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py b/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py deleted file mode 100755 index 9a4a90c53..000000000 --- a/python/tests/transpile/tools/benchmark_gemma4_context_scaling.py +++ /dev/null @@ -1,445 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import csv -import json -import shutil -import statistics -import subprocess -from datetime import datetime -from pathlib import Path -from typing import Any - - -LENGTHS = (64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768) -SWEEPS = ( - ("warmup", (64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768)), - ("measured_1", (2048, 4096, 1024, 8192, 512, 16384, 256, 32768, 128, 64)), - ("measured_2", (64, 128, 32768, 256, 16384, 512, 8192, 1024, 4096, 2048)), - ("measured_3", (4096, 1024, 8192, 512, 16384, 256, 32768, 128, 64, 2048)), -) -JSON_FIELDS = ( - "success", - "time_to_first_token_ms", - "total_time_ms", - "prefill_tps", - "prefill_compute_tps", - "prefill_prepare_tps", - "ttft_prompt_tps", - "cache_prime_ms", - "cache_prime_compute_ms", - "cache_state_copy_ms", - "cache_prime_tokens", - "first_decode_ms", - "first_token_from_prefill", - "prefill_padding_tokens", - "prefill_scalar_tail_tokens", - "decode_tps", - "prompt_tokens", - "completion_tokens", - "peak_ram_usage_mb", - "ram_usage_mb", -) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Run Gemma4 E4B context-scaling decode benchmarks through cactus run.", - ) - parser.add_argument( - "--model", - default="weights/gemma-4-e4b-it", - help="Prepared Cactus bundle to benchmark.", - ) - parser.add_argument( - "--out-dir", - type=Path, - default=None, - help="Directory for token files, raw results, logs, JSONL, CSV, and summary.", - ) - parser.add_argument( - "--cactus-bin", - default="cactus", - help="Cactus CLI executable to invoke.", - ) - parser.add_argument( - "--token-id", - type=int, - default=106, - help="Token id repeated to build exact-length prompts.", - ) - parser.add_argument( - "--max-new-tokens", - type=int, - default=32, - help="Number of tokens to generate after each prefill.", - ) - parser.add_argument( - "--skip-build", - action="store_true", - help="Skip the required cactus build step when the caller already built this checkout.", - ) - parser.add_argument( - "--continue-on-error", - action="store_true", - help="Record failed runs and continue instead of stopping at the first failure.", - ) - parser.add_argument( - "--no-plots", - action="store_true", - help="Do not generate PNG plots after the benchmark completes.", - ) - return parser.parse_args() - - -def default_out_dir() -> Path: - timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") - return Path("tmp") / "gemma4-context-scaling" / timestamp - - -def write_token_file(path: Path, *, token_id: int, count: int) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - chunk_size = 4096 - remaining = count - first = True - with path.open("w", encoding="utf-8") as handle: - while remaining > 0: - current = min(chunk_size, remaining) - text = ",".join([str(token_id)] * current) - if not first: - handle.write(",") - handle.write(text) - first = False - remaining -= current - - -def ensure_token_files(out_dir: Path, *, token_id: int) -> dict[int, Path]: - token_dir = out_dir / "tokens" - files: dict[int, Path] = {} - for length in LENGTHS: - path = token_dir / f"tokens_{length}.txt" - write_token_file(path, token_id=token_id, count=length) - files[length] = path - return files - - -def run_command(command: list[str], *, stdout_path: Path, stderr_path: Path) -> int: - with stdout_path.open("w", encoding="utf-8") as stdout, stderr_path.open("w", encoding="utf-8") as stderr: - proc = subprocess.run(command, stdout=stdout, stderr=stderr, text=True) - return int(proc.returncode) - - -def read_result_json(result_path: Path, stdout_path: Path) -> dict[str, Any]: - if result_path.exists(): - return json.loads(result_path.read_text(encoding="utf-8")) - for line in stdout_path.read_text(encoding="utf-8").splitlines(): - stripped = line.strip() - if stripped.startswith("{") and stripped.endswith("}"): - return json.loads(stripped) - raise RuntimeError(f"No result JSON found in {result_path} or {stdout_path}") - - -def decode_ms_per_token(payload: dict[str, Any]) -> float | None: - decode_tps = float(payload.get("decode_tps") or 0.0) - if decode_tps <= 0.0: - return None - return 1000.0 / decode_tps - - -def append_jsonl(path: Path, record: dict[str, Any]) -> None: - with path.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(record, sort_keys=True) + "\n") - - -def write_csv(path: Path, records: list[dict[str, Any]]) -> None: - fieldnames = [ - "phase", - "sweep_index", - "sweep_position", - "context_tokens", - "max_new_tokens", - "returncode", - "decode_ms_per_token", - *JSON_FIELDS, - "result_json", - "stdout_log", - "stderr_log", - ] - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames) - writer.writeheader() - for record in records: - writer.writerow({name: record.get(name) for name in fieldnames}) - - -def write_summary(path: Path, records: list[dict[str, Any]]) -> None: - measured = [record for record in records if str(record.get("phase", "")).startswith("measured")] - fieldnames = [ - "context_tokens", - "runs", - "decode_ms_per_token_mean", - "decode_ms_per_token_stdev", - "decode_tps_mean", - "decode_tps_stdev", - "total_time_ms_mean", - "total_time_ms_stdev", - "peak_ram_usage_mb_mean", - "peak_ram_usage_mb_stdev", - ] - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames) - writer.writeheader() - for length in LENGTHS: - rows = [record for record in measured if int(record.get("context_tokens", -1)) == length] - ms_values = [value for row in rows if (value := _optional_float(row.get("decode_ms_per_token"))) is not None] - tps_values = [value for row in rows if (value := _optional_float(row.get("decode_tps"))) is not None] - total_values = [value for row in rows if (value := _optional_float(row.get("total_time_ms"))) is not None] - ram_values = [value for row in rows if (value := _optional_float(row.get("peak_ram_usage_mb"))) is not None] - writer.writerow({ - "context_tokens": length, - "runs": len(rows), - "decode_ms_per_token_mean": statistics.mean(ms_values) if ms_values else None, - "decode_ms_per_token_stdev": statistics.stdev(ms_values) if len(ms_values) > 1 else 0.0 if ms_values else None, - "decode_tps_mean": statistics.mean(tps_values) if tps_values else None, - "decode_tps_stdev": statistics.stdev(tps_values) if len(tps_values) > 1 else 0.0 if tps_values else None, - "total_time_ms_mean": statistics.mean(total_values) if total_values else None, - "total_time_ms_stdev": statistics.stdev(total_values) if len(total_values) > 1 else 0.0 if total_values else None, - "peak_ram_usage_mb_mean": statistics.mean(ram_values) if ram_values else None, - "peak_ram_usage_mb_stdev": statistics.stdev(ram_values) if len(ram_values) > 1 else 0.0 if ram_values else None, - }) - - -def _float_column(rows: list[dict[str, str]], key: str, *, default: float = 0.0) -> list[float]: - values: list[float] = [] - for row in rows: - raw = row.get(key) - values.append(float(raw) if raw not in {None, ""} else default) - return values - - -def _optional_float(value: object) -> float | None: - if value in {None, ""}: - return None - return float(value) - - -def write_plots(summary_path: Path, out_dir: Path) -> list[Path]: - import matplotlib - matplotlib.use("Agg") - import matplotlib.pyplot as plt - - with summary_path.open(newline="", encoding="utf-8") as handle: - rows = list(csv.DictReader(handle)) - contexts = [int(row["context_tokens"]) for row in rows] - ms_mean = _float_column(rows, "decode_ms_per_token_mean") - ms_std = _float_column(rows, "decode_ms_per_token_stdev") - tps_mean = _float_column(rows, "decode_tps_mean") - tps_std = _float_column(rows, "decode_tps_stdev") - ram_mean = _float_column(rows, "peak_ram_usage_mb_mean") - ram_std = _float_column(rows, "peak_ram_usage_mb_stdev") - total_mean = [value / 1000.0 for value in _float_column(rows, "total_time_ms_mean")] - total_std = [value / 1000.0 for value in _float_column(rows, "total_time_ms_stdev")] - - plt.style.use("seaborn-v0_8-whitegrid") - error_style = { - "ecolor": "#1f1f1f", - "elinewidth": 1.8, - "capthick": 1.8, - "capsize": 6, - "markeredgecolor": "#1f1f1f", - "markeredgewidth": 0.9, - } - - def configure_x_axis(ax) -> None: - ax.set_xscale("log", base=2) - ax.set_xticks(contexts) - ax.set_xticklabels([str(context) for context in contexts], rotation=35, ha="right") - ax.grid(True, which="both", axis="y", alpha=0.35) - ax.grid(True, which="major", axis="x", alpha=0.18) - - def finish(fig, path: Path) -> Path: - fig.tight_layout() - fig.savefig(path, dpi=180) - plt.close(fig) - return path - - paths: list[Path] = [] - - fig, ax = plt.subplots(figsize=(9.5, 5.5)) - ax.errorbar(contexts, ms_mean, yerr=ms_std, marker="o", linewidth=2, color="#2f6f9f", **error_style) - configure_x_axis(ax) - ax.set_xlabel("Prefill context tokens") - ax.set_ylabel("Decode latency (ms/token)") - ax.set_title("Gemma 4 E4B IT decode latency vs context length") - paths.append(finish(fig, out_dir / "decode_ms_per_token.png")) - - fig, ax = plt.subplots(figsize=(9.5, 5.5)) - ax.errorbar(contexts, tps_mean, yerr=tps_std, marker="o", linewidth=2, color="#3d7d54", **error_style) - configure_x_axis(ax) - ax.set_xlabel("Prefill context tokens") - ax.set_ylabel("Decode throughput (tokens/sec)") - ax.set_title("Gemma 4 E4B IT decode throughput vs context length") - paths.append(finish(fig, out_dir / "decode_tps.png")) - - fig, ax1 = plt.subplots(figsize=(9.5, 5.5)) - line1 = ax1.errorbar( - contexts, - ms_mean, - yerr=ms_std, - marker="o", - linewidth=2, - color="#2f6f9f", - label="Decode ms/token", - **error_style, - ) - configure_x_axis(ax1) - ax1.set_xlabel("Prefill context tokens") - ax1.set_ylabel("Decode latency (ms/token)", color="#2f6f9f") - ax1.tick_params(axis="y", labelcolor="#2f6f9f") - ax2 = ax1.twinx() - line2 = ax2.errorbar( - contexts, - ram_mean, - yerr=ram_std, - marker="s", - linewidth=2, - color="#8a5a2b", - label="Peak RAM MB", - **error_style, - ) - ax2.set_ylabel("Peak RAM (MB)", color="#8a5a2b") - ax2.tick_params(axis="y", labelcolor="#8a5a2b") - ax1.set_title("Gemma 4 E4B IT decode latency and RAM vs context length") - ax1.legend([line1, line2], ["Decode ms/token", "Peak RAM MB"], loc="upper left") - paths.append(finish(fig, out_dir / "decode_latency_and_ram.png")) - - fig, ax = plt.subplots(figsize=(9.5, 5.5)) - ax.errorbar(contexts, total_mean, yerr=total_std, marker="o", linewidth=2, color="#7b4f9f", **error_style) - configure_x_axis(ax) - ax.set_yscale("log") - ax.set_xlabel("Prefill context tokens") - ax.set_ylabel("Mean total run time (seconds, log scale)") - ax.set_title("Gemma 4 E4B IT total run time vs context length") - paths.append(finish(fig, out_dir / "total_time_seconds.png")) - - baseline = ms_mean[0] - baseline_std = ms_std[0] - slowdown = [value / baseline for value in ms_mean] - slowdown_std = [] - for value, std in zip(ms_mean, ms_std): - if value <= 0.0 or baseline <= 0.0: - slowdown_std.append(0.0) - else: - relative_std = ((std / value) ** 2 + (baseline_std / baseline) ** 2) ** 0.5 - slowdown_std.append((value / baseline) * relative_std) - fig, ax = plt.subplots(figsize=(9.5, 5.5)) - ax.errorbar(contexts, slowdown, yerr=slowdown_std, marker="o", linewidth=2, color="#b24c4c", **error_style) - ax.axhline(1.0, color="#444444", linewidth=1, linestyle="--", alpha=0.7) - configure_x_axis(ax) - ax.set_xlabel("Prefill context tokens") - ax.set_ylabel("Decode slowdown vs 64-token baseline") - ax.set_title("Gemma 4 E4B IT decode slowdown vs initial speed") - for context, value in zip(contexts, slowdown): - ax.annotate(f"{value:.2f}x", (context, value), textcoords="offset points", xytext=(0, 8), ha="center", fontsize=8) - paths.append(finish(fig, out_dir / "decode_slowdown_vs_64.png")) - - return paths - - -def validate_environment(cactus_bin: str) -> None: - if shutil.which(cactus_bin) is None: - raise RuntimeError(f"Could not find Cactus CLI executable: {cactus_bin}") - - -def main() -> int: - args = parse_args() - validate_environment(args.cactus_bin) - - out_dir = args.out_dir or default_out_dir() - out_dir.mkdir(parents=True, exist_ok=True) - token_files = ensure_token_files(out_dir, token_id=args.token_id) - - metadata = { - "model": args.model, - "token_id": args.token_id, - "max_new_tokens": args.max_new_tokens, - "sweeps": [{"phase": phase, "lengths": list(lengths)} for phase, lengths in SWEEPS], - } - (out_dir / "metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - if not args.skip_build: - build_rc = run_command( - [args.cactus_bin, "build"], - stdout_path=out_dir / "build.stdout.log", - stderr_path=out_dir / "build.stderr.log", - ) - if build_rc != 0: - raise RuntimeError(f"cactus build failed; see {out_dir / 'build.stderr.log'}") - - records: list[dict[str, Any]] = [] - jsonl_path = out_dir / "runs.jsonl" - csv_path = out_dir / "runs.csv" - summary_path = out_dir / "summary.csv" - - for sweep_index, (phase, lengths) in enumerate(SWEEPS): - for sweep_position, length in enumerate(lengths): - prefix = f"{sweep_index:02d}_{phase}_{sweep_position:02d}_{length}" - result_path = out_dir / "results" / f"{prefix}.json" - stdout_path = out_dir / "logs" / f"{prefix}.stdout.log" - stderr_path = out_dir / "logs" / f"{prefix}.stderr.log" - result_path.parent.mkdir(parents=True, exist_ok=True) - stdout_path.parent.mkdir(parents=True, exist_ok=True) - - command = [ - args.cactus_bin, - "run", - args.model, - "--input-ids-file", - str(token_files[length]), - "--max-new-tokens", - str(args.max_new_tokens), - "--result-json", - str(result_path), - ] - print(f"[{phase} {sweep_position + 1}/{len(lengths)}] context={length}", flush=True) - returncode = run_command(command, stdout_path=stdout_path, stderr_path=stderr_path) - - if returncode == 0: - payload = read_result_json(result_path, stdout_path) - else: - payload = {"success": False} - - record = { - "phase": phase, - "sweep_index": sweep_index, - "sweep_position": sweep_position, - "context_tokens": length, - "max_new_tokens": args.max_new_tokens, - "returncode": returncode, - "decode_ms_per_token": decode_ms_per_token(payload), - "result_json": str(result_path), - "stdout_log": str(stdout_path), - "stderr_log": str(stderr_path), - } - for field in JSON_FIELDS: - record[field] = payload.get(field) - records.append(record) - append_jsonl(jsonl_path, record) - write_csv(csv_path, records) - write_summary(summary_path, records) - - if returncode != 0 and not args.continue_on_error: - raise RuntimeError(f"Benchmark run failed for context={length}; see {stderr_path}") - - print(f"wrote {jsonl_path}") - print(f"wrote {csv_path}") - print(f"wrote {summary_path}") - if not args.no_plots: - for path in write_plots(summary_path, out_dir): - print(f"wrote {path}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From b3743905ef25f2fae2c977cf55df40326fa79223 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Mon, 8 Jun 2026 11:05:50 -0700 Subject: [PATCH 55/55] Consolidate gemma4 thinking tests into test_llm Move the four gemma4 thinking tests into test_llm.cpp (next to the prefill cache-reuse tests) and delete the standalone suite. The three model-gated tests run only when the chosen model is Gemma4 and warn-skip otherwise; partition_thinking_response is a pure unit test that always runs. Share a load_gemma4_or_skip helper instead of repeating the gate. Signed-off-by: Noah Cylich --- cactus-engine/tests/test_gemma4_thinking.cpp | 189 ------------------- cactus-engine/tests/test_llm.cpp | 147 +++++++++++++++ 2 files changed, 147 insertions(+), 189 deletions(-) delete mode 100644 cactus-engine/tests/test_gemma4_thinking.cpp diff --git a/cactus-engine/tests/test_gemma4_thinking.cpp b/cactus-engine/tests/test_gemma4_thinking.cpp deleted file mode 100644 index 5dc84ebb8..000000000 --- a/cactus-engine/tests/test_gemma4_thinking.cpp +++ /dev/null @@ -1,189 +0,0 @@ -#include "test_utils.h" -#include "../src/engine.h" -#include "../src/utils.h" -#include -#include -#include -#include - -using namespace cactus::engine; -using cactus::ffi::partition_thinking_response; - -static const char* get_gemma4_model_path() { - const char* path = std::getenv("CACTUS_TEST_GEMMA4_MODEL"); - if (path) return path; - return std::getenv("CACTUS_TEST_MODEL"); -} - -static bool check_partition(const std::string& input, - const std::string& expected_thinking, - const std::string& expected_content) { - std::string thinking, content; - partition_thinking_response(input, thinking, content); - if (thinking != expected_thinking) { - std::cerr << " thinking: '" << thinking << "' != '" << expected_thinking << "'\n"; - return false; - } - if (content != expected_content) { - std::cerr << " content: '" << content << "' != '" << expected_content << "'\n"; - return false; - } - return true; -} - -// partition_thinking_response is display-only: it splits generated text into (thinking, content) -// for the API, without touching the cache. -bool test_partition_thinking_response() { - return check_partition("<|channel>reasonanswer", "reason", "answer") - && check_partition("<|channel>\n reason\n\n\nanswer", "reason", "answer") - && check_partition("no tags here", "", "no tags here") - && check_partition("reasonanswer", "reason", "answer") - && check_partition("<|channel>thought1text1<|channel>thought2text2", - "thought1\nthought2", "text1text2"); -} - -// New contract: the Gemma4 formatter RETAINS prior-turn assistant channel/thinking content in the -// rendered prompt (no strip), matching LiteRT so thinking persists across turns. -bool test_prompt_gemma4_retains_thinking() { - const char* model_path = get_gemma4_model_path(); - if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } - - cactus_model_t model = cactus_init(model_path, nullptr, false); - if (!model) { std::cout << " [SKIP] Could not load model\n"; return true; } - - auto* handle = static_cast(model); - if (handle->model->get_config().model_type != Config::ModelType::GEMMA4) { - std::cout << " [SKIP] Not a Gemma4 model\n"; - cactus_destroy(model); - return true; - } - auto* tok = handle->model->get_tokenizer(); - - std::vector msgs = { - {"user", "hello", "", {}, {}, 0, {}}, - {"assistant", "<|channel>internal reasoningvisible response", "", {}, {}, 0, {}}, - {"user", "followup", "", {}, {}, 0, {}} - }; - - std::string prompt = tok->format_chat_prompt(msgs, true, "", true); - cactus_destroy(model); - - bool has_visible = prompt.find("visible response") != std::string::npos; - bool has_reasoning = prompt.find("internal reasoning") != std::string::npos; - bool has_channel_tags = prompt.find("<|channel>") != std::string::npos - && prompt.find("") != std::string::npos; - - if (!has_visible) std::cerr << " missing visible response in prompt\n"; - if (!has_reasoning) std::cerr << " thinking content not retained in assistant turn\n"; - if (!has_channel_tags) std::cerr << " channel tags not retained in prompt\n"; - - return has_visible && has_reasoning && has_channel_tags; -} - -// API separation is unchanged: response excludes thinking, thinking is populated, and no raw -// channel tags leak into the user-facing response. -bool test_complete_gemma4_thinking_api_clean() { - const char* model_path = get_gemma4_model_path(); - if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } - - cactus_model_t model = cactus_init(model_path, nullptr, false); - if (!model) return false; - - auto* handle = static_cast(model); - if (handle->model->get_config().model_type != Config::ModelType::GEMMA4) { - std::cout << " [SKIP] Not a Gemma4 model\n"; - cactus_destroy(model); - return true; - } - - const char* msgs = R"([{"role": "user", "content": "What is 2+2?"}])"; - char buf[8192]; - - int r = cactus_complete(model, msgs, buf, sizeof(buf), - R"({"max_tokens":128,"enable_thinking_if_supported":true,"telemetry_enabled":false})", - nullptr, nullptr, nullptr, nullptr, 0); - std::string resp(buf); - cactus_destroy(model); - - std::string response = EngineTestUtils::json_string(resp, "response"); - bool ok = r > 0 - && resp.find("\"success\":true") != std::string::npos - && response.find("<|channel>") == std::string::npos - && response.find("") == std::string::npos; - if (!ok) std::cerr << " thinking-enabled completion api not clean: " << resp << "\n"; - return ok; -} - -// Multi-turn: store the raw context_response (thinking retained) for history; the re-rendered -// turn-2 prompt must prefix-match the turn-1 processed tokens (cache reuse) and recall the fact. -bool test_multiturn_thinking_persist() { - const char* model_path = get_gemma4_model_path(); - if (!model_path) { std::cout << " [SKIP] CACTUS_TEST_GEMMA4_MODEL not set\n"; return true; } - - cactus_model_t model = cactus_init(model_path, nullptr, false); - if (!model) { std::cerr << " Failed to load model\n"; return false; } - - auto* handle = static_cast(model); - if (handle->model->get_config().model_type != Config::ModelType::GEMMA4) { - std::cout << " [SKIP] Not a Gemma4 model\n"; - cactus_destroy(model); - return true; - } - - auto* tokenizer = handle->model->get_tokenizer(); - const char* options = R"({"max_tokens":128,"temperature":0,"top_k":1,"enable_thinking_if_supported":true,"telemetry_enabled":false,"auto_handoff":false})"; - const char* turn1_msgs = R"([{"role": "user", "content": "My name is Alice. Please remember this."}])"; - char buf[16384]; - - int r1 = cactus_complete(model, turn1_msgs, buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); - if (r1 <= 0) { std::cerr << " Turn 1 failed\n"; cactus_destroy(model); return false; } - - std::vector processed_after_t1 = handle->processed_tokens; - std::string turn1_json(buf); - std::string context_response = EngineTestUtils::json_string(turn1_json, "context_response"); - if (context_response.empty()) { - std::cerr << " context_response missing from turn 1 result\n"; - cactus_destroy(model); - return false; - } - - std::vector t2_chat = { - {"user", "My name is Alice. Please remember this.", "", {}, {}, 0, {}}, - {"assistant", context_response, "", {}, {}, 0, {}}, - {"user", "What is my name?", "", {}, {}, 0, {}} - }; - std::vector t2_prompt_tokens = tokenizer->encode(tokenizer->format_chat_prompt(t2_chat, true, "", true)); - - bool prefix_ok = (t2_prompt_tokens.size() >= processed_after_t1.size()) && - std::equal(processed_after_t1.begin(), processed_after_t1.end(), t2_prompt_tokens.begin()); - std::cout << " Prefix match (cache reuse): " << (prefix_ok ? "YES" : "NO") << "\n"; - - std::string escaped = EngineTestUtils::escape_json(context_response); - std::string turn2_json = R"([{"role": "user", "content": "My name is Alice. Please remember this."},{"role": "assistant", "content": ")" - + escaped + R"("},{"role": "user", "content": "What is my name?"}])"; - - int r2 = cactus_complete(model, turn2_json.c_str(), buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); - if (r2 <= 0) { std::cerr << " Turn 2 failed\n"; cactus_destroy(model); return false; } - - std::string turn2_result(buf); - std::string turn2_response = EngineTestUtils::json_string(turn2_result, "response"); - bool mentions_alice = turn2_response.find("Alice") != std::string::npos - || turn2_response.find("alice") != std::string::npos; - std::cout << " Turn 2 mentions Alice: " << (mentions_alice ? "YES" : "NO") << "\n"; - - cactus_destroy(model); - - if (!prefix_ok) std::cerr << " FAIL: re-rendered history did not prefix-match cache\n"; - if (!mentions_alice) std::cerr << " FAIL: turn 2 did not recall the name\n"; - return prefix_ok && mentions_alice; -} - -int main() { - TestUtils::TestRunner runner("Gemma4 Thinking Tests"); - runner.run_test("partition_thinking_response", test_partition_thinking_response()); - runner.run_test("prompt_retains_thinking", test_prompt_gemma4_retains_thinking()); - runner.run_test("complete_thinking_api_clean", test_complete_gemma4_thinking_api_clean()); - runner.run_test("multiturn_thinking_persist", test_multiturn_thinking_persist()); - runner.print_summary(); - return runner.all_passed() ? 0 : 1; -} diff --git a/cactus-engine/tests/test_llm.cpp b/cactus-engine/tests/test_llm.cpp index 56382474d..a7fe93cb0 100644 --- a/cactus-engine/tests/test_llm.cpp +++ b/cactus-engine/tests/test_llm.cpp @@ -1,4 +1,5 @@ #include "test_utils.h" +#include "../src/utils.h" #include #include #include @@ -14,9 +15,39 @@ #endif using namespace EngineTestUtils; +using namespace cactus::engine; +using cactus::ffi::partition_thinking_response; static const char* g_model_path = std::getenv("CACTUS_TEST_MODEL"); +static bool check_partition(const std::string& input, + const std::string& expected_thinking, + const std::string& expected_content) { + std::string thinking, content; + partition_thinking_response(input, thinking, content); + if (thinking != expected_thinking) { + std::cerr << " thinking: '" << thinking << "' != '" << expected_thinking << "'\n"; + return false; + } + if (content != expected_content) { + std::cerr << " content: '" << content << "' != '" << expected_content << "'\n"; + return false; + } + return true; +} + +static cactus_model_t load_gemma4_or_skip() { + if (!g_model_path) { std::cout << " [WARN] CACTUS_TEST_MODEL not set; skipping\n"; return nullptr; } + cactus_model_t model = cactus_init(g_model_path, nullptr, false); + if (!model) { std::cout << " [WARN] Could not load model; skipping\n"; return nullptr; } + if (static_cast(model)->model->get_config().model_type != Config::ModelType::GEMMA4) { + std::cout << " [WARN] chosen model is not Gemma4; skipping\n"; + cactus_destroy(model); + return nullptr; + } + return model; +} + static const char* g_options = R"({ "max_tokens": 256, "stop_sequences": ["<|im_end|>", ""], @@ -559,6 +590,118 @@ bool test_1k_context() { }, nullptr, 100); } +bool test_partition_thinking_response() { + return check_partition("<|channel>reasonanswer", "reason", "answer") + && check_partition("<|channel>\n reason\n\n\nanswer", "reason", "answer") + && check_partition("no tags here", "", "no tags here") + && check_partition("reasonanswer", "reason", "answer") + && check_partition("<|channel>thought1text1<|channel>thought2text2", + "thought1\nthought2", "text1text2"); +} + +bool test_prompt_gemma4_retains_thinking() { + cactus_model_t model = load_gemma4_or_skip(); + if (!model) return true; + + auto* handle = static_cast(model); + auto* tok = handle->model->get_tokenizer(); + + std::vector msgs = { + {"user", "hello", "", {}, {}, 0, {}}, + {"assistant", "<|channel>internal reasoningvisible response", "", {}, {}, 0, {}}, + {"user", "followup", "", {}, {}, 0, {}} + }; + + std::string prompt = tok->format_chat_prompt(msgs, true, "", true); + cactus_destroy(model); + + bool has_visible = prompt.find("visible response") != std::string::npos; + bool has_reasoning = prompt.find("internal reasoning") != std::string::npos; + bool has_channel_tags = prompt.find("<|channel>") != std::string::npos + && prompt.find("") != std::string::npos; + + if (!has_visible) std::cerr << " missing visible response in prompt\n"; + if (!has_reasoning) std::cerr << " thinking content not retained in assistant turn\n"; + if (!has_channel_tags) std::cerr << " channel tags not retained in prompt\n"; + + return has_visible && has_reasoning && has_channel_tags; +} + +bool test_complete_gemma4_thinking_api_clean() { + cactus_model_t model = load_gemma4_or_skip(); + if (!model) return true; + + const char* msgs = R"([{"role": "user", "content": "What is 2+2?"}])"; + char buf[8192]; + + int r = cactus_complete(model, msgs, buf, sizeof(buf), + R"({"max_tokens":128,"enable_thinking_if_supported":true,"telemetry_enabled":false})", + nullptr, nullptr, nullptr, nullptr, 0); + std::string resp(buf); + cactus_destroy(model); + + std::string response = EngineTestUtils::json_string(resp, "response"); + bool ok = r > 0 + && resp.find("\"success\":true") != std::string::npos + && response.find("<|channel>") == std::string::npos + && response.find("") == std::string::npos; + if (!ok) std::cerr << " thinking-enabled completion api not clean: " << resp << "\n"; + return ok; +} + +bool test_multiturn_thinking_persist() { + cactus_model_t model = load_gemma4_or_skip(); + if (!model) return true; + + auto* handle = static_cast(model); + auto* tokenizer = handle->model->get_tokenizer(); + const char* options = R"({"max_tokens":128,"temperature":0,"top_k":1,"enable_thinking_if_supported":true,"telemetry_enabled":false,"auto_handoff":false})"; + const char* turn1_msgs = R"([{"role": "user", "content": "My name is Alice. Please remember this."}])"; + char buf[16384]; + + int r1 = cactus_complete(model, turn1_msgs, buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); + if (r1 <= 0) { std::cerr << " Turn 1 failed\n"; cactus_destroy(model); return false; } + + std::vector processed_after_t1 = handle->processed_tokens; + std::string turn1_json(buf); + std::string context_response = EngineTestUtils::json_string(turn1_json, "context_response"); + if (context_response.empty()) { + std::cerr << " context_response missing from turn 1 result\n"; + cactus_destroy(model); + return false; + } + + std::vector t2_chat = { + {"user", "My name is Alice. Please remember this.", "", {}, {}, 0, {}}, + {"assistant", context_response, "", {}, {}, 0, {}}, + {"user", "What is my name?", "", {}, {}, 0, {}} + }; + std::vector t2_prompt_tokens = tokenizer->encode(tokenizer->format_chat_prompt(t2_chat, true, "", true)); + + bool prefix_ok = (t2_prompt_tokens.size() >= processed_after_t1.size()) && + std::equal(processed_after_t1.begin(), processed_after_t1.end(), t2_prompt_tokens.begin()); + std::cout << " Prefix match (cache reuse): " << (prefix_ok ? "YES" : "NO") << "\n"; + + std::string escaped = EngineTestUtils::escape_json(context_response); + std::string turn2_json = R"([{"role": "user", "content": "My name is Alice. Please remember this."},{"role": "assistant", "content": ")" + + escaped + R"("},{"role": "user", "content": "What is my name?"}])"; + + int r2 = cactus_complete(model, turn2_json.c_str(), buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); + if (r2 <= 0) { std::cerr << " Turn 2 failed\n"; cactus_destroy(model); return false; } + + std::string turn2_result(buf); + std::string turn2_response = EngineTestUtils::json_string(turn2_result, "response"); + bool mentions_alice = turn2_response.find("Alice") != std::string::npos + || turn2_response.find("alice") != std::string::npos; + std::cout << " Turn 2 mentions Alice: " << (mentions_alice ? "YES" : "NO") << "\n"; + + cactus_destroy(model); + + if (!prefix_ok) std::cerr << " FAIL: re-rendered history did not prefix-match cache\n"; + if (!mentions_alice) std::cerr << " FAIL: turn 2 did not recall the name\n"; + return prefix_ok && mentions_alice; +} + int main() { TestUtils::TestRunner runner("LLM Tests"); runner.run_test("1k_context", test_1k_context()); @@ -570,6 +713,10 @@ int main() { runner.run_test("tool_calls", test_tool_call()); runner.run_test("tool_multiple_tool_call_invocations", test_multiple_tool_call_invocations()); runner.run_test("tool_calls_with_three_tools", test_tool_call_with_three_tools()); + runner.run_test("partition_thinking_response", test_partition_thinking_response()); + runner.run_test("prompt_retains_thinking", test_prompt_gemma4_retains_thinking()); + runner.run_test("complete_thinking_api_clean", test_complete_gemma4_thinking_api_clean()); + runner.run_test("multiturn_thinking_persist", test_multiturn_thinking_persist()); runner.print_summary(); return runner.all_passed() ? 0 : 1; }