From 0968ae1b30d3481928933db72e9d5945676a5dae Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 12 Jun 2026 13:23:02 -0700 Subject: [PATCH 01/13] cactus-graph: preserve cache-node descriptor across steal_cache_buffer Any second chunked prefill on a gated-deltanet bundle threw a state rank error because moving the cache buffer into the decoder graph left the prefill node's BufferDesc empty. Restore the source to a fresh unallocated descriptor with its original shape and precision so the next execute re-initializes the state exactly as on first run. Signed-off-by: Noah Cylich --- .../tests/test_recurrent_prefill.cpp | 43 +++++++++++++++++++ cactus-graph/src/ops_cache.cpp | 3 ++ cactus-graph/tests/test_cache.cpp | 23 ++++++++++ 3 files changed, 69 insertions(+) create mode 100644 cactus-engine/tests/test_recurrent_prefill.cpp diff --git a/cactus-engine/tests/test_recurrent_prefill.cpp b/cactus-engine/tests/test_recurrent_prefill.cpp new file mode 100644 index 000000000..f4a508e9d --- /dev/null +++ b/cactus-engine/tests/test_recurrent_prefill.cpp @@ -0,0 +1,43 @@ +#include "test_utils.h" +#include +#include +#include + +static const char* g_model_path = std::getenv("CACTUS_TEST_MODEL"); + +static std::string benchmark_tokens_json(cactus_model_t model, const std::vector& ids, size_t max_new) { + std::vector response(1 << 16, 0); + int rc = cactus_benchmark_tokens(model, ids.data(), ids.size(), max_new, response.data(), response.size()); + return rc < 0 ? std::string() : std::string(response.data()); +} + +bool test_repeated_chunked_prefill() { + if (!g_model_path) { std::cout << " [WARN] CACTUS_TEST_MODEL not set; skipping\n"; return true; } + + cactus_model_t model = cactus_init(g_model_path, nullptr, false); + if (!model) { std::cerr << " [✗] model init failed\n"; return false; } + + std::vector ids(600); + for (size_t i = 0; i < ids.size(); i++) ids[i] = static_cast(100 + (i % 200)); + + std::string first = benchmark_tokens_json(model, ids, 4); + std::string second = benchmark_tokens_json(model, ids, 4); + cactus_destroy(model); + + if (first.empty() || first.find("\"success\":true") == std::string::npos) { + std::cerr << " [✗] first chunked prefill failed\n"; + return false; + } + if (second.empty() || second.find("\"success\":true") == std::string::npos) { + std::cerr << " [✗] second chunked prefill failed\n"; + return false; + } + return true; +} + +int main() { + TestUtils::TestRunner runner("Recurrent Prefill Tests"); + runner.run_test("repeated_chunked_prefill", test_repeated_chunked_prefill()); + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 24c0998dc..367bdc453 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -530,7 +530,10 @@ void CactusGraph::steal_cache_buffer(size_t dst_node, CactusGraph& src, size_t s auto& s = src.nodes_[src.node_index_map_.at(src_node)]; // Buffer carries its own runtime precision (may be fp16); only op_type is invariant pre-move. assert(dst->op_type == s->op_type); + auto shape = s->output_buffer.shape; + auto prec = s->output_buffer.precision; dst->output_buffer = std::move(s->output_buffer); + s->output_buffer = BufferDesc(shape, prec); } namespace { diff --git a/cactus-graph/tests/test_cache.cpp b/cactus-graph/tests/test_cache.cpp index 6ed754dd9..b1733d46e 100644 --- a/cactus-graph/tests/test_cache.cpp +++ b/cactus-graph/tests/test_cache.cpp @@ -663,6 +663,28 @@ bool test_recurrent_cache_state_initializes_to_zero() { return true; } +bool test_steal_cache_buffer_preserves_source_descriptor() { + CactusGraph src; + CactusGraph dst; + const std::vector shape{1, 16, 4, 32}; + const size_t elements = 1 * 16 * 4 * 32; + size_t src_state = src.recurrent_cache_state(shape, Precision::FP16); + size_t dst_state = dst.recurrent_cache_state(shape, Precision::FP16); + src.execute(); + dst.steal_cache_buffer(dst_state, src, src_state); + if (!dst.get_output(dst_state)) return false; + if (src.get_output_buffer(src_state).shape != shape) return false; + src.execute(); + const BufferDesc& buf = src.get_output_buffer(src_state); + const __fp16* data = static_cast(buf.get_data()); + if (!data) return false; + if (buf.byte_size != elements * sizeof(__fp16)) return false; + for (size_t i = 0; i < elements; ++i) { + if (static_cast(data[i]) != 0.0f) return false; + } + return true; +} + bool test_recurrent_cache_write_rejects_non_state_input() { CactusGraph g; size_t plain_input = g.input({2, 4}, Precision::FP16); @@ -935,6 +957,7 @@ int main() { runner.run_test("Conv Cache Initialize Resets Dirty Cache", test_conv_cache_initialize_resets_dirty_cache()); runner.run_test("Conv Cache Initialize Output Zero Byte", test_conv_cache_initialize_output_is_zero_byte()); runner.run_test("Recurrent Cache State Initializes to Zero", test_recurrent_cache_state_initializes_to_zero()); + runner.run_test("Steal Cache Buffer Preserves Source Descriptor", test_steal_cache_buffer_preserves_source_descriptor()); runner.run_test("Recurrent Cache Write Carries State", test_recurrent_cache_write_carries_state_across_executions()); runner.run_test("Recurrent Cache Write Rejects Non-State Input", test_recurrent_cache_write_rejects_non_state_input()); runner.run_test("Recurrent Cache Write Rejects Mismatch", test_recurrent_cache_write_rejects_shape_mismatch()); From 6df484ecde2a090aceef2aef8bdf4f5c9f5044bc Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 12 Jun 2026 13:20:17 -0700 Subject: [PATCH 02/13] cactus-engine: clear tool-constraint bias on deactivation Every generation after a force_tools turn on the same handle degenerated into <|tool_call> loops on gemma models, surviving cactus_reset. The deactivating init left the previously computed force bias in place while update() no-ops when inactive, so decode kept force-picking the call-start token. Clear the bias in init and drop the redundant reset in clear_tool_constraints that recomputed it just before deactivation. Signed-off-by: Noah Cylich --- cactus-engine/src/constraints.cpp | 2 +- cactus-engine/tests/test_llm.cpp | 30 ++++++++++ python/tests/test_tool_constraint_state.py | 70 ++++++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 python/tests/test_tool_constraint_state.py diff --git a/cactus-engine/src/constraints.cpp b/cactus-engine/src/constraints.cpp index 8102b03ac..8592f95a3 100644 --- a/cactus-engine/src/constraints.cpp +++ b/cactus-engine/src/constraints.cpp @@ -289,6 +289,7 @@ void ToolCallConstrainer::init(Config::ModelType model_type, } tokenizer_ = tokenizer; generated_text_.clear(); + current_bias_.clear(); brace_depth_ = 0; in_argument_string_ = false; const bool model_supported = is_needle() || Config::is_gemma_family(model_type_); @@ -558,7 +559,6 @@ void Model::set_tool_constraints(const std::vector& tools) { } void Model::clear_tool_constraints() { - tool_constrainer_.reset(); tool_constrainer_.init(config_.model_type, {}, tokenizer_.get()); } diff --git a/cactus-engine/tests/test_llm.cpp b/cactus-engine/tests/test_llm.cpp index fd71b308c..1e2e82a3c 100644 --- a/cactus-engine/tests/test_llm.cpp +++ b/cactus-engine/tests/test_llm.cpp @@ -411,6 +411,35 @@ bool test_prefill() { return all_success && warm_prefilled_less; } +struct LengthTokenizer : public Tokenizer { + std::vector encode(const std::string& text) const override { + return {static_cast(text.size())}; + } + std::string decode(const std::vector&) const override { return std::string(); } + uint32_t get_vocab_size() const override { return 0; } + uint32_t get_unk_token() const override { return 0; } + uint32_t get_bos_token() const override { return 0; } + uint32_t get_eos_token() const override { return 0; } + bool load_vocabulary_with_config(const std::string&, const std::string&, const std::string&) override { return true; } +}; + +bool test_tool_constraint_clear_releases_bias() { + LengthTokenizer tok; + ToolCallConstrainer constrainer; + constrainer.init(Config::ModelType::GEMMA4, {{"get_weather", {"location"}, {"location"}}}, &tok); + if (constrainer.get_bias().empty()) { + std::cerr << " expected bias after activating init\n"; + return false; + } + constrainer.reset(); + constrainer.init(Config::ModelType::GEMMA4, {}, &tok); + if (!constrainer.get_bias().empty()) { + std::cerr << " stale bias survived deactivating init\n"; + return false; + } + return true; +} + bool test_tool_call() { const char* messages = R"([ {"role": "system", "content": "You are a helpful assistant that can use tools."}, @@ -801,6 +830,7 @@ int main() { runner.run_test("tool_calls", test_tool_call()); runner.run_test("tool_multiple_tool_call_invocations", test_multiple_tool_call_invocations()); runner.run_test("tool_calls_with_three_tools", test_tool_call_with_three_tools()); + runner.run_test("tool_constraint_clear_releases_bias", test_tool_constraint_clear_releases_bias()); runner.run_test("partition_thinking_response", test_partition_thinking_response()); runner.run_test("prompt_retains_thinking", test_prompt_gemma4_retains_thinking()); runner.run_test("complete_thinking_api_clean", test_complete_gemma4_thinking_api_clean()); diff --git a/python/tests/test_tool_constraint_state.py b/python/tests/test_tool_constraint_state.py new file mode 100644 index 000000000..aff54124f --- /dev/null +++ b/python/tests/test_tool_constraint_state.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from cactus import cactus_complete, cactus_destroy, cactus_init + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +BUNDLE = PROJECT_ROOT / "weights" / "gemma-4-e2b-it" + +GET_WEATHER = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"}, + }, + "required": ["location"], + }, + }, +} + + +def _valid_bundle(path: Path) -> bool: + return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() + + +@pytest.fixture(scope="module") +def model(): + if not _valid_bundle(BUNDLE): + pytest.skip(f"Live-test model not found: {BUNDLE}") + handle = cactus_init(str(BUNDLE)) + yield handle + cactus_destroy(handle) + + +def test_turn_after_force_tools_is_not_degenerate(model): + tools = [GET_WEATHER] + messages = [ + {"role": "system", "content": "You are a helpful assistant that can use tools."}, + {"role": "user", "content": "What's the weather in Tokyo right now?"}, + ] + forced = cactus_complete( + model, messages, {"force_tools": True, "max_tokens": 200, "temperature": 0}, tools + ) + assert forced.get("success"), forced + assert forced.get("function_calls"), forced + + messages.append({ + "role": "assistant", + "content": forced.get("response", ""), + "tool_calls": [{"type": "function", "function": { + "name": "get_weather", "arguments": {"location": "Tokyo"}, + }}], + }) + messages.append({ + "role": "tool", + "name": "get_weather", + "content": json.dumps({"temperature_c": 18}), + }) + followup = cactus_complete(model, messages, {"max_tokens": 150, "temperature": 0}, tools) + assert followup.get("success"), followup + response = followup.get("response", "") + assert response.strip() + assert "<|tool_call>" not in response From 9902990c79bd918a8d9a652d5708dd75e4435ec5 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 12 Jun 2026 13:45:24 -0700 Subject: [PATCH 03/13] engine+python: golden chat-template render tests vs HF reference Adds a cactus_render_prompt FFI call that exposes the fully rendered chat prompt, plus a ctypes wrapper and export. A golden pytest suite checks engine renders and tokenizations against the HF reference chat templates for gemma-4, qwen3, and lfm2 across system, tool-call, parallel-tool-call, and thinking conversations, plus the gemma-4 image expansion. Aligns the gemma-4 formatter with the reference template: <|think|> gains its trailing newline and the image block drops its surrounding blank lines. Signed-off-by: Noah Cylich --- cactus-engine/cactus_engine.h | 9 + cactus-engine/src/complete.cpp | 30 ++- cactus-engine/src/tokenizer.cpp | 6 +- python/cactus/__init__.py | 1 + python/cactus/bindings/cactus.py | 26 +++ python/tests/test_chat_template_golden.py | 250 ++++++++++++++++++++++ 6 files changed, 315 insertions(+), 7 deletions(-) create mode 100644 python/tests/test_chat_template_golden.py diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index 41af950da..395757c4d 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -65,6 +65,15 @@ CACTUS_FFI_EXPORT int cactus_tokenize( size_t* out_token_len ); +CACTUS_FFI_EXPORT int cactus_render_prompt( + cactus_model_t model, + const char* messages_json, + const char* options_json, + const char* tools_json, + char* prompt_buffer, + size_t buffer_size +); + CACTUS_FFI_EXPORT int cactus_score_window( cactus_model_t model, const uint32_t* tokens, diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index e3ff46097..40bcee884 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -387,6 +387,7 @@ struct PreparedPrompt { std::vector audio_paths; std::vector messages; std::vector tools; + std::string rendered; std::vector tokens; size_t context_token_count = 0; std::vector> images; @@ -584,16 +585,16 @@ PreparedPrompt prepare_prompt( setup_tool_constraints(handle, prompt.tools, prompt.options.force_tools, prompt.options.temperature); } - std::string full_prompt = tokenizer->format_chat_prompt( + prompt.rendered = tokenizer->format_chat_prompt( prompt.messages, add_generation_prompt, formatted_tools, prompt.options.enable_thinking_if_supported ); - if (full_prompt.find("ERROR:") == 0) { - throw std::runtime_error(full_prompt.substr(6)); + if (prompt.rendered.find("ERROR:") == 0) { + throw std::runtime_error(prompt.rendered.substr(6)); } - prompt.tokens = tokenizer->encode(full_prompt); + prompt.tokens = tokenizer->encode(prompt.rendered); prompt.context_token_count = prompt.tokens.size(); prompt.images = images_from_message(prompt.messages); return prompt; @@ -1258,6 +1259,27 @@ int cactus_tokenize( } } +int cactus_render_prompt( + cactus_model_t model, + const char* messages_json, + const char* options_json, + const char* tools_json, + char* prompt_buffer, + size_t buffer_size +) { + if (!model || !messages_json || !prompt_buffer || buffer_size == 0) return -1; + + try { + auto* handle = static_cast(model); + auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, false, true); + if (prompt.rendered.size() >= buffer_size) return -2; + std::strcpy(prompt_buffer, prompt.rendered.c_str()); + return static_cast(prompt.rendered.size()); + } catch (...) { + return -1; + } +} + int cactus_score_window( cactus_model_t model, const uint32_t* tokens, diff --git a/cactus-engine/src/tokenizer.cpp b/cactus-engine/src/tokenizer.cpp index 746a15aa7..ccfea9917 100644 --- a/cactus-engine/src/tokenizer.cpp +++ b/cactus-engine/src/tokenizer.cpp @@ -622,7 +622,7 @@ std::string Tokenizer::format_gemma4_style(const std::vector& messa if (enable_thinking_if_supported || !sys_content.empty() || !tools_json.empty()) { result += "<|turn>system\n"; if (enable_thinking_if_supported) { - result += "<|think|>"; + result += "<|think|>\n"; } result += sys_content; result += tools_json; @@ -681,10 +681,10 @@ std::string Tokenizer::format_gemma4_style(const std::vector& messa for (const auto& image_path : msg.images) { size_t n = compute_soft_tokens(image_path); if (n > 0) { - result += "\n\n<|image>"; + result += "<|image>"; for (size_t j = 0; j < n; j++) result += "<|image|>"; - result += "\n\n"; + result += ""; } } result += msg.content; diff --git a/python/cactus/__init__.py b/python/cactus/__init__.py index 841fd0832..ac35d47ed 100644 --- a/python/cactus/__init__.py +++ b/python/cactus/__init__.py @@ -26,6 +26,7 @@ "cactus_transcribe", "cactus_detect_language", "cactus_tokenize", + "cactus_render_prompt", "cactus_decode_tokens", "cactus_score_window", "cactus_rag_query", diff --git a/python/cactus/bindings/cactus.py b/python/cactus/bindings/cactus.py index 9ecbd9ef3..9a0e53974 100644 --- a/python/cactus/bindings/cactus.py +++ b/python/cactus/bindings/cactus.py @@ -726,6 +726,19 @@ class cactus_tensor_info_t(ctypes.Structure): ] _lib.cactus_tokenize.restype = ctypes.c_int +_bind_optional( + "cactus_render_prompt", + [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_size_t, + ], + ctypes.c_int, +) + _bind_optional( "cactus_decode_tokens", [ @@ -1129,6 +1142,19 @@ def cactus_tokenize(model, text): return list(arr[:n.value]) +def cactus_render_prompt(model, messages, options=None, tools=None): + """Render the chat-template prompt string for messages without generating.""" + if not hasattr(_lib, "cactus_render_prompt"): + raise RuntimeError("cactus_render_prompt is unavailable; rebuild with cactus build --python") + buf = ctypes.create_string_buffer(1 << 20) + rc = _lib.cactus_render_prompt( + model, _to_json(messages), _to_json(options), _to_json(tools), buf, len(buf), + ) + if rc < 0: + raise RuntimeError(_err("Prompt rendering failed")) + return buf.value.decode("utf-8", errors="ignore") + + def cactus_decode_tokens(model, tokens, temperature=0.0, top_p=1.0, top_k=1): """Decode the next token from a sequence. Returns the next token ID as an int.""" if not tokens: diff --git a/python/tests/test_chat_template_golden.py b/python/tests/test_chat_template_golden.py new file mode 100644 index 000000000..1211e5b60 --- /dev/null +++ b/python/tests/test_chat_template_golden.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +os.environ.setdefault("HF_HUB_OFFLINE", "1") +os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") + +transformers = pytest.importorskip("transformers") + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +WEIGHTS = PROJECT_ROOT / "weights" +MONKEY_IMAGE = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" / "test_monkey.png" + +FAMILIES = [ + ("gemma-4-e2b-it", "google/gemma-4-E2B-it"), + ("qwen3-0.6b", "Qwen/Qwen3-0.6B"), + ("lfm2-350m", "LiquidAI/LFM2-350M"), +] +FAMILY_IDS = ["gemma4", "qwen3", "lfm2"] +THINKING_TYPES = {"gemma4", "qwen"} + +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name."}, + }, + "required": ["city"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_time", + "description": "Get the current time in a timezone.", + "parameters": { + "type": "object", + "properties": { + "timezone": {"type": "string", "description": "IANA timezone name."}, + }, + "required": ["timezone"], + }, + }, + }, +] + +SYSTEM_CHAT = [ + {"role": "system", "content": "You are a concise assistant."}, + {"role": "user", "content": "What is the capital of France?"}, + {"role": "assistant", "content": "Paris."}, + {"role": "user", "content": "And of Germany?"}, +] + +TOOLS_USER = [ + {"role": "user", "content": "What is the weather in Paris?"}, +] + +TOOL_CALL_RESPONSE = [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", + "function": {"name": "get_weather", "arguments": {"city": "Paris"}}}, + ], + }, + {"role": "tool", "tool_call_id": "call_1", "name": "get_weather", "content": "Sunny, 22C"}, +] + +PARALLEL_TOOL_CALLS = [ + {"role": "user", "content": "Weather in Paris and time in UTC?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", + "function": {"name": "get_weather", "arguments": {"city": "Paris"}}}, + {"id": "call_2", "type": "function", + "function": {"name": "get_time", "arguments": {"timezone": "UTC"}}}, + ], + }, + {"role": "tool", "tool_call_id": "call_1", "name": "get_weather", "content": "Sunny, 22C"}, + {"role": "tool", "tool_call_id": "call_2", "name": "get_time", "content": "12:00"}, +] + +THINKING_CHAT = [ + {"role": "user", "content": "Briefly explain entropy."}, +] + +CASES = [ + ("system_chat", SYSTEM_CHAT, None, False), + ("tools_user", TOOLS_USER, TOOLS, False), + ("tool_call_response", TOOL_CALL_RESPONSE, TOOLS, False), + ("parallel_tool_calls", PARALLEL_TOOL_CALLS, TOOLS, False), + ("thinking_enabled", THINKING_CHAT, None, True), + ("thinking_disabled", THINKING_CHAT, None, False), +] + + +def _read_model_type(bundle: Path) -> str: + for line in (bundle / "config.txt").read_text(encoding="utf-8").splitlines(): + if line.startswith("model_type="): + return line.split("=", 1)[1].strip() + return "" + + +def _valid_bundle(path: Path) -> bool: + return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() + + +def _pythonic_call(call: dict) -> str: + fn = call["function"] + args = ", ".join(f"{key}={json.dumps(value)}" for key, value in fn["arguments"].items()) + return f"{fn['name']}({args})" + + +def _embed_lfm2_tool_calls(message: dict) -> dict: + calls = ", ".join(_pythonic_call(call) for call in message["tool_calls"]) + embedded = dict(message) + embedded["content"] = message["content"] + f"<|tool_call_start|>[{calls}]<|tool_call_end|>" + del embedded["tool_calls"] + return embedded + + +def _to_hf_messages(messages: list[dict], model_type: str) -> list[dict]: + out = [] + for message in messages: + if message.get("images"): + message = dict(message) + message["content"] = [{"type": "image"} for _ in message.pop("images")] + [ + {"type": "text", "text": message["content"]} + ] + if model_type == "lfm2" and message.get("tool_calls"): + message = _embed_lfm2_tool_calls(message) + out.append(message) + return out + + +_LOADED: dict[str, tuple] = {} + + +@pytest.fixture(scope="session") +def load_family(): + from cactus import cactus_destroy, cactus_init + + def _load(bundle_name: str, hf_id: str): + if bundle_name not in _LOADED: + bundle = WEIGHTS / bundle_name + if not _valid_bundle(bundle): + pytest.skip(f"bundle not prepared: {bundle}") + try: + tokenizer = transformers.AutoTokenizer.from_pretrained(hf_id) + except OSError as exc: + pytest.skip(f"HF tokenizer not cached for {hf_id}: {exc}") + _LOADED[bundle_name] = (cactus_init(str(bundle)), tokenizer, _read_model_type(bundle)) + return _LOADED[bundle_name] + + yield _load + while _LOADED: + model, _, _ = _LOADED.popitem()[1] + cactus_destroy(model) + + +def _assert_render_matches(model, tokenizer, engine_render: str, hf_render: str, label: str): + from cactus import cactus_tokenize + + assert engine_render == hf_render, ( + f"{label}: engine chat-template render diverges from the HF reference template" + ) + engine_ids = cactus_tokenize(model, engine_render) + hf_ids = tokenizer(engine_render, add_special_tokens=False).input_ids + assert engine_ids == hf_ids, ( + f"{label}: engine tokenization of the render diverges from the HF tokenizer" + ) + + +@pytest.mark.parametrize(("case_name", "messages", "tools", "thinking"), CASES, + ids=[case[0] for case in CASES]) +@pytest.mark.parametrize(("bundle_name", "hf_id"), FAMILIES, ids=FAMILY_IDS) +def test_render_matches_hf_template(load_family, bundle_name, hf_id, case_name, messages, + tools, thinking): + from cactus import cactus_render_prompt + + model, tokenizer, model_type = load_family(bundle_name, hf_id) + if case_name.startswith("thinking") and model_type not in THINKING_TYPES: + pytest.skip(f"{model_type} has no thinking template support") + engine_render = cactus_render_prompt( + model, messages, options={"enable_thinking_if_supported": thinking}, tools=tools + ) + hf_render = tokenizer.apply_chat_template( + _to_hf_messages(messages, model_type), + tools=tools, + add_generation_prompt=True, + tokenize=False, + enable_thinking=thinking, + ) + _assert_render_matches(model, tokenizer, engine_render, hf_render, + f"{bundle_name}/{case_name}") + + +def _monkey_soft_tokens(hf_id: str) -> int: + from huggingface_hub.constants import HF_HUB_CACHE + from PIL import Image + from transformers.models.gemma4.image_processing_gemma4 import Gemma4ImageProcessor + + repo_dir = Path(HF_HUB_CACHE) / f"models--{hf_id.replace('/', '--')}" + configs = sorted(repo_dir.glob("snapshots/*/processor_config.json")) + if not configs: + pytest.skip(f"no cached processor_config.json for {hf_id}") + image_processor = Gemma4ImageProcessor( + **json.loads(configs[0].read_text(encoding="utf-8"))["image_processor"] + ) + with Image.open(MONKEY_IMAGE) as image: + return image_processor(image)["num_soft_tokens_per_image"][0] + + +def test_gemma4_image_render_matches_hf_template(load_family): + from cactus import cactus_render_prompt + + bundle_name, hf_id = FAMILIES[0] + model, tokenizer, model_type = load_family(bundle_name, hf_id) + messages = [ + {"role": "user", "content": "What is in this image?", "images": [str(MONKEY_IMAGE)]}, + ] + engine_render = cactus_render_prompt( + model, messages, options={"enable_thinking_if_supported": False} + ) + hf_render = tokenizer.apply_chat_template( + _to_hf_messages(messages, model_type), + add_generation_prompt=True, + tokenize=False, + enable_thinking=False, + ) + soft_tokens = _monkey_soft_tokens(hf_id) + expanded = hf_render.replace( + tokenizer.image_token, + tokenizer.boi_token + tokenizer.image_token * soft_tokens + tokenizer.eoi_token, + ) + _assert_render_matches(model, tokenizer, engine_render, expanded, f"{bundle_name}/image") From 8b9639fb194485ca082df3777ef26ca93b1989b7 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 12 Jun 2026 13:19:45 -0700 Subject: [PATCH 04/13] transpiler: stable cache layer keys across components The internal KV-cache layer key fell through to the node id whenever attention_layer_index was 0, and decoder_media_step graphs never received attention_layer_index hints at all, so components of one bundle disagreed on layer keys and the engine's positional cache-state pairing failed at load time with "prefill and step cache layer mismatch". Derive the key with explicit None checks before falling back to the node id, and include decoder_media_step in the attention-hint assignment. Signed-off-by: Noah Cylich --- python/cactus/transpile/lower.py | 14 +++-- python/cactus/transpile/optimize_graph.py | 2 +- python/tests/test_transpile_layer_keys.py | 68 +++++++++++++++++++++++ 3 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 python/tests/test_transpile_layer_keys.py diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py index e0e7a6604..ceeee03b1 100644 --- a/python/cactus/transpile/lower.py +++ b/python/cactus/transpile/lower.py @@ -247,6 +247,14 @@ def _sequence_dim(layout: str) -> int: return True +def _kv_cache_layer_key(node: IRNode) -> str: + for meta_key in ("attention_layer_index", "layer_index"): + value = node.meta.get(meta_key) + if value is not None: + return str(value) + return str(node.id) + + def _lower_attention_with_internal_kv_cache( g: Graph, ir: IRGraph, @@ -268,11 +276,7 @@ def _lower_attention_with_internal_kv_cache( raise NotImplementedError("cached attention currently expects batch size 1") cache_states: dict[str, tuple[Tensor, Tensor]] = env.setdefault("__internal_kv_cache_states", {}) # type: ignore[assignment] - layer_key = str( - node.meta.get("attention_layer_index") - or node.meta.get("layer_index") - or node.id - ) + layer_key = _kv_cache_layer_key(node) if layer_key not in cache_states: default_cache_len = max(512, int(key_shape[1])) requested_cache_len = node.meta.get("max_cache_seq_len", ir.meta.get("max_cache_seq_len", default_cache_len)) diff --git a/python/cactus/transpile/optimize_graph.py b/python/cactus/transpile/optimize_graph.py index de3116e22..ea2714d0d 100644 --- a/python/cactus/transpile/optimize_graph.py +++ b/python/cactus/transpile/optimize_graph.py @@ -840,7 +840,7 @@ def normalize_cached_decoder_attention_hints(graph: IRGraph) -> bool: def _assign_gemma4_decoder_attention_hints_from_graph_meta(graph: IRGraph) -> bool: component = str(graph.meta.get("component", "") or "").strip().lower() - if component not in {"decoder", "decoder_step", "decoder_prefill_chunk"}: + if component not in {"decoder", "decoder_step", "decoder_prefill_chunk", "decoder_media_step"}: return False layer_types = _graph_layer_types(graph) diff --git a/python/tests/test_transpile_layer_keys.py b/python/tests/test_transpile_layer_keys.py new file mode 100644 index 000000000..1bd157ef0 --- /dev/null +++ b/python/tests/test_transpile_layer_keys.py @@ -0,0 +1,68 @@ +from cactus.transpile.graph_ir import IRGraph +from cactus.transpile.graph_ir import IRNode +from cactus.transpile.graph_ir import IRValue +from cactus.transpile.lower import _kv_cache_layer_key +from cactus.transpile.optimize_graph import normalize_cached_decoder_attention_hints + + +def _decoder_graph(component: str) -> IRGraph: + values: dict[str, IRValue] = {} + nodes: dict[str, IRNode] = {} + for index in range(2): + node_id = "n_scaled_dot_product_attention" + ("" if index == 0 else f"_{index}") + for name in ("query", "key", "value"): + values[f"{name}_{index}"] = IRValue(id=f"{name}_{index}", shape=(1, 8, 1, 64), dtype="fp16") + values[f"out_{index}"] = IRValue(id=f"out_{index}", shape=(1, 8, 1, 64), dtype="fp16", producer=node_id) + nodes[node_id] = IRNode( + id=node_id, + op="scaled_dot_product_attention", + inputs=[f"query_{index}", f"key_{index}", f"value_{index}"], + outputs=[f"out_{index}"], + ) + return IRGraph( + values=values, + nodes=nodes, + order=list(nodes), + inputs=[value_id for value_id in values if not value_id.startswith("out_")], + outputs=["out_1"], + constants={}, + meta={ + "component": component, + "use_internal_kv_cache": True, + "layer_types": ["full_attention"] * 2, + }, + ) + + +def test_kv_cache_layer_key_keeps_zero_layer_index() -> None: + node = IRNode( + id="n_scaled_dot_product_attention", + op="scaled_dot_product_attention", + inputs=["query_0", "key_0", "value_0"], + outputs=["out_0"], + ) + assert _kv_cache_layer_key(node) == "n_scaled_dot_product_attention" + node.meta["layer_index"] = 5 + assert _kv_cache_layer_key(node) == "5" + node.meta["attention_layer_index"] = 0 + assert _kv_cache_layer_key(node) == "0" + + +def test_media_step_receives_attention_layer_indices() -> None: + graph = _decoder_graph("decoder_media_step") + + changed = normalize_cached_decoder_attention_hints(graph) + + assert changed is True + assert graph.nodes["n_scaled_dot_product_attention"].meta["attention_layer_index"] == 0 + assert graph.nodes["n_scaled_dot_product_attention_1"].meta["attention_layer_index"] == 1 + + +def test_step_and_media_step_layer_keys_match() -> None: + keys = [] + for component in ("decoder_step", "decoder_media_step"): + graph = _decoder_graph(component) + normalize_cached_decoder_attention_hints(graph) + keys.append([_kv_cache_layer_key(graph.nodes[node_id]) for node_id in graph.order]) + + assert keys[0] == keys[1] == ["0", "1"] From 7dd78b44e55bfc67e65983be76685c4014b97c05 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 12 Jun 2026 13:35:26 -0700 Subject: [PATCH 05/13] cactus-engine: disable cloud handoff for session after auth failure A stale or revoked API key made every low-confidence turn re-attempt the cloud handoff, burning up to cloud_timeout_ms per turn before falling back to local output. Track http_401/http_403 completion failures on the model handle and drop handoff eligibility for the rest of the session once one is seen; timeouts, 5xx, and connection errors keep retrying as before. The pre-streaming trigger now also respects handoff eligibility, and the live test drives a forced handoff against a local 401 server to verify the second turn stays local. Signed-off-by: Noah Cylich --- cactus-engine/src/complete.cpp | 16 ++++- cactus-engine/src/utils.h | 1 + python/tests/test_cloud_handoff_live.py | 82 +++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 python/tests/test_cloud_handoff_live.py diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 40bcee884..38ea69488 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -788,7 +788,7 @@ int cactus_complete( bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); const bool cloud_disabled = env_flag_enabled("CACTUS_DISABLE_CLOUD_HANDOFF"); - const bool cloud_eligible = !cloud_disabled && + const bool cloud_eligible = !cloud_disabled && !handle->cloud_handoff_disabled && prompt.options.auto_handoff && (!has_images || prompt.options.handoff_with_images); handle->model->reset_handoff_probe_rollout(); const bool defer_local_stream_until_probe = cloud_eligible && handle->model->has_handoff_probe(); @@ -810,6 +810,14 @@ int cactus_complete( return request; }; + auto disable_handoff_on_auth_failure = [&](const std::string& error) { + if (error.rfind("http_401", 0) == 0 || error.rfind("http_403", 0) == 0) { + handle->cloud_handoff_disabled = true; + CACTUS_LOG_WARN("cloud_handoff", "Cloud auth failed (" << error + << "); disabling cloud handoff for this session"); + } + }; + auto return_cloud_completion = [&](const CloudCompletionResult& cloud_result, double ttft_ms, double total_ms, @@ -860,6 +868,7 @@ int cactus_complete( } std::string cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed before local generation: " << cloud_error); + disable_handoff_on_auth_failure(cloud_error); handle_error_response(("cloud handoff failed before local generation: " + cloud_error).c_str(), response_buffer, buffer_size); return -1; @@ -924,7 +933,8 @@ int cactus_complete( entropy.add(first_token_entropy); if (!matches_stop_sequence(generated_tokens, stop_token_sequences)) { - if (!defer_local_stream_until_probe + if (cloud_eligible + && !defer_local_stream_until_probe && !pre_generation_cloud_attempted && confidence < prompt.options.confidence_threshold) { CACTUS_LOG_WARN("cloud_handoff", "Cloud handoff triggered before local streaming; waiting up to " @@ -942,6 +952,7 @@ int cactus_complete( } cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed before local streaming, falling back to local output: " << cloud_error); + disable_handoff_on_auth_failure(cloud_error); } if (callback && !defer_local_stream_until_probe) { @@ -1058,6 +1069,7 @@ int cactus_complete( cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed after probe handoff, falling back to local output: " << cloud_error); + disable_handoff_on_auth_failure(cloud_error); } if (callback && defer_local_stream_until_probe && !primary_response.empty()) { diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 147e2602d..ec69def16 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -82,6 +82,7 @@ struct CactusModelHandle { size_t corpus_embedding_dim = 0; std::vector> tool_embeddings; std::vector tool_texts; + bool cloud_handoff_disabled = false; CactusModelHandle() : should_stop(false) {} }; diff --git a/python/tests/test_cloud_handoff_live.py b/python/tests/test_cloud_handoff_live.py new file mode 100644 index 000000000..d20ac6574 --- /dev/null +++ b/python/tests/test_cloud_handoff_live.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +WEIGHTS = PROJECT_ROOT / "weights" +PREFERRED_BUNDLES = ["qwen3-0.6b", "lfm2-350m"] +OPTIONS = {"max_tokens": 8, "temperature": 0, "confidence_threshold": 1.0} +MESSAGES = [{"role": "user", "content": "What is 2+2?"}] + + +def _find_bundle() -> Path: + for name in PREFERRED_BUNDLES: + candidate = WEIGHTS / name + if (candidate / "config.txt").exists() and (candidate / "components" / "manifest.json").exists(): + return candidate + pytest.skip(f"No live-test bundle found under {WEIGHTS}; expected one of: {PREFERRED_BUNDLES}") + + +class _AuthFailHandler(BaseHTTPRequestHandler): + def do_POST(self): + with self.server.hits_lock: + self.server.hits += 1 + self.rfile.read(int(self.headers.get("Content-Length", 0))) + body = json.dumps({"error": "invalid or expired api key"}).encode() + self.send_response(401) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +def _hits(server) -> int: + with server.hits_lock: + return server.hits + + +def test_auth_failure_disables_cloud_handoff_for_session(monkeypatch, tmp_path): + bundle = _find_bundle() + server = ThreadingHTTPServer(("127.0.0.1", 0), _AuthFailHandler) + server.hits = 0 + server.hits_lock = threading.Lock() + threading.Thread(target=server.serve_forever, daemon=True).start() + + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("CACTUS_CLOUD_API_BASE", f"http://127.0.0.1:{server.server_address[1]}/api/v1") + monkeypatch.setenv("CACTUS_CLOUD_KEY", "stale-test-key") + monkeypatch.setenv("CACTUS_NO_CLOUD_TELE", "1") + monkeypatch.delenv("CACTUS_DISABLE_CLOUD_HANDOFF", raising=False) + + from cactus.bindings.cactus import cactus_complete, cactus_destroy, cactus_init + + model = cactus_init(str(bundle)) + try: + with pytest.raises(RuntimeError): + cactus_complete(model, MESSAGES, options=OPTIONS) + assert _hits(server) == 1 + + start = time.monotonic() + try: + result = cactus_complete(model, MESSAGES, options=OPTIONS) + except RuntimeError: + result = None + elapsed = time.monotonic() - start + + assert _hits(server) == 1 + assert result is not None + assert result["success"] is True + assert elapsed < 5.0 + finally: + cactus_destroy(model) + server.shutdown() + server.server_close() From b0824fdfa6c0f1fcfa5dac5c838f6ac469f8f31f Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 12 Jun 2026 13:30:52 -0700 Subject: [PATCH 06/13] server: send OpenAI-wrapped tool definitions to the engine The engine's parse_tools_json scans for a literal "function" key, so the flat {name, description, parameters} dicts _translate_tools emitted were parsed as zero tools and requests silently lost their tool definitions. Build the wrapped form once and filter it for the tool_choice object branch. Signed-off-by: Noah Cylich --- python/cactus/server.py | 12 +++---- python/tests/test_server.py | 67 +++++++++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/python/cactus/server.py b/python/cactus/server.py index 792fcc671..0697b66ac 100644 --- a/python/cactus/server.py +++ b/python/cactus/server.py @@ -296,17 +296,13 @@ def _flatten_message(msg: ChatMessage) -> dict[str, Any]: def _translate_tools(tools: list[Tool] | None, tool_choice) -> tuple[list[dict[str, Any]] | None, bool]: if not tools or tool_choice == "none": return None, False - if isinstance(tool_choice, ToolChoiceObject): - selected = [ - {"name": t.function.name, "description": t.function.description or "", "parameters": t.function.parameters or {}} - for t in tools - if t.function.name == tool_choice.function.name - ] - return selected or None, True translated = [ - {"name": t.function.name, "description": t.function.description or "", "parameters": t.function.parameters or {}} + {"type": "function", "function": {"name": t.function.name, "description": t.function.description or "", "parameters": t.function.parameters or {}}} for t in tools ] + if isinstance(tool_choice, ToolChoiceObject): + selected = [t for t in translated if t["function"]["name"] == tool_choice.function.name] + return selected or None, True return translated, tool_choice == "required" diff --git a/python/tests/test_server.py b/python/tests/test_server.py index d290a3a3c..3c0780f8e 100644 --- a/python/tests/test_server.py +++ b/python/tests/test_server.py @@ -77,16 +77,27 @@ def fake_complete(handle, messages, options, tools, callback): assert body["usage"] == {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3} +_WRAPPED_LOOKUP = [{"type": "function", "function": {"name": "lookup", "description": "", "parameters": {}}}] + + +def _capturing_complete(captured: dict): + def fake_complete(handle, messages, options, tools, callback): + captured["tools"] = tools + return { + "success": True, + "response": "", + "function_calls": [{"name": "lookup", "arguments": {"q": "x"}}], + "prefill_tokens": 1, + "decode_tokens": 1, + } + return fake_complete + + def test_chat_tool_calls_are_translated(tmp_path: Path, monkeypatch) -> None: app = _app(tmp_path, monkeypatch) monkeypatch.setattr(server, "cactus_reset", lambda handle: None) - monkeypatch.setattr(server, "cactus_complete", lambda *args, **kwargs: { - "success": True, - "response": "", - "function_calls": [{"name": "lookup", "arguments": {"q": "x"}}], - "prefill_tokens": 1, - "decode_tokens": 1, - }) + captured = {} + monkeypatch.setattr(server, "cactus_complete", _capturing_complete(captured)) with TestClient(app) as client: res = client.post("/v1/chat/completions", json={ @@ -101,6 +112,48 @@ def test_chat_tool_calls_are_translated(tmp_path: Path, monkeypatch) -> None: assert choice["message"]["content"] is None assert choice["message"]["tool_calls"][0]["function"]["name"] == "lookup" assert choice["message"]["tool_calls"][0]["function"]["arguments"] == '{"q": "x"}' + assert captured["tools"] == _WRAPPED_LOOKUP + + +def test_streaming_chat_sends_wrapped_tools(tmp_path: Path, monkeypatch) -> None: + app = _app(tmp_path, monkeypatch) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + captured = {} + monkeypatch.setattr(server, "cactus_complete", _capturing_complete(captured)) + + with TestClient(app) as client: + with client.stream("POST", "/v1/chat/completions", json={ + "model": "llm", + "messages": [{"role": "user", "content": "call tool"}], + "tools": [{"type": "function", "function": {"name": "lookup", "parameters": {}}}], + "stream": True, + }) as res: + body = "".join(res.iter_text()) + + assert '"finish_reason": "tool_calls"' in body + assert "data: [DONE]" in body + assert captured["tools"] == _WRAPPED_LOOKUP + + +def test_tool_choice_object_selects_wrapped_tool(tmp_path: Path, monkeypatch) -> None: + app = _app(tmp_path, monkeypatch) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + captured = {} + monkeypatch.setattr(server, "cactus_complete", _capturing_complete(captured)) + + with TestClient(app) as client: + res = client.post("/v1/chat/completions", json={ + "model": "llm", + "messages": [{"role": "user", "content": "call tool"}], + "tools": [ + {"type": "function", "function": {"name": "lookup", "parameters": {}}}, + {"type": "function", "function": {"name": "other", "parameters": {}}}, + ], + "tool_choice": {"type": "function", "function": {"name": "lookup"}}, + }) + + assert res.status_code == 200 + assert captured["tools"] == _WRAPPED_LOOKUP def test_streaming_completion_error_terminates(tmp_path: Path, monkeypatch) -> None: From 68d109fadbf7228459f7313a5f1f025dbede94ca Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 12 Jun 2026 14:56:39 -0700 Subject: [PATCH 07/13] cactus-engine: render gemma3 tool calls with function special tokens Gemma3-family prompts now follow the FunctionGemma chat template: system text and tool declarations render in a developer turn, prior assistant tool calls as call:name{...} blocks, and tool results as blocks, with quoting instead of the gemma4 pipe tags. The tool-call constrainer keys its tags on the model family and forces multi-token tags one token at a time, so force_tools no longer degenerates into marker loops on bundles whose tags are not single special tokens. DSL-to- JSON argument conversion escapes string content so malformed model output cannot corrupt the response payload. Signed-off-by: Noah Cylich --- cactus-engine/src/complete.cpp | 2 +- cactus-engine/src/constraints.cpp | 44 +++++++++--- cactus-engine/src/engine.h | 10 +++ cactus-engine/src/gemma_tools.h | 36 ++++++++-- cactus-engine/src/tokenizer.cpp | 83 +++++++++++++++++------ python/tests/test_chat_template_golden.py | 3 +- 6 files changed, 141 insertions(+), 37 deletions(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 38ea69488..321e83f34 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -578,7 +578,7 @@ PreparedPrompt prepare_prompt( } else if (tokenizer->is_lfm2_family()) { formatted_tools = chat_tools::serialize_tools_lfm2(prompt.tools); } else { - formatted_tools = gemma::format_tools(prompt.tools, true); + formatted_tools = gemma::format_tools(prompt.tools, !Config::is_gemma3_family(prompt.model_type)); } if (apply_tool_constraints) { diff --git a/cactus-engine/src/constraints.cpp b/cactus-engine/src/constraints.cpp index 8592f95a3..599d3e8ed 100644 --- a/cactus-engine/src/constraints.cpp +++ b/cactus-engine/src/constraints.cpp @@ -264,9 +264,12 @@ void ToolCallConstrainer::tokenize_grammar_elements() { escape_tokens_.clear(); gemma_response_start_tokens_.clear(); - add_tokens_for_string(call_start_tag_, gemma_call_start_tokens_); - add_tokens_for_string(call_end_tag_, gemma_call_end_tokens_); - add_tokens_for_string("<|tool_response>", gemma_response_start_tokens_); + call_start_sequence_ = tokenizer_->encode(call_start_tag_); + call_end_sequence_ = tokenizer_->encode(call_end_tag_); + std::vector response_sequence = tokenizer_->encode(response_start_tag_); + if (call_start_sequence_.size() == 1) gemma_call_start_tokens_.insert(call_start_sequence_[0]); + if (call_end_sequence_.size() == 1) gemma_call_end_tokens_.insert(call_end_sequence_[0]); + if (response_sequence.size() == 1) gemma_response_start_tokens_.insert(response_sequence[0]); add_tokens_for_string("call:", gemma_call_prefix_tokens_); add_tokens_for_string("{", open_brace_tokens_); @@ -292,13 +295,21 @@ void ToolCallConstrainer::init(Config::ModelType model_type, current_bias_.clear(); brace_depth_ = 0; in_argument_string_ = false; + forced_tag_progress_ = 0; const bool model_supported = is_needle() || Config::is_gemma_family(model_type_); active_ = model_supported && !function_names_.empty() && tokenizer != nullptr; reset_needle_constraints(); state_ = is_needle() ? State::NEEDLE_START : State::GEMMA_START; - call_start_tag_ = "<|tool_call>"; - call_end_tag_ = ""; + if (Config::is_gemma3_family(model_type_)) { + call_start_tag_ = ""; + call_end_tag_ = ""; + response_start_tag_ = ""; + } else { + call_start_tag_ = "<|tool_call>"; + call_end_tag_ = ""; + response_start_tag_ = "<|tool_response>"; + } if (!active_) { return; @@ -311,7 +322,15 @@ void ToolCallConstrainer::init(Config::ModelType model_type, compute_bias(); } -void ToolCallConstrainer::update(uint32_t /*token_id*/, const std::string& decoded_text) { +void ToolCallConstrainer::advance_forced_tag(const std::vector& sequence, uint32_t token_id) { + if (forced_tag_progress_ < sequence.size() && token_id == sequence[forced_tag_progress_]) { + forced_tag_progress_++; + } else { + forced_tag_progress_ = (!sequence.empty() && token_id == sequence[0]) ? 1 : 0; + } +} + +void ToolCallConstrainer::update(uint32_t token_id, const std::string& decoded_text) { if (!active_) return; generated_text_ += decoded_text; @@ -325,9 +344,11 @@ void ToolCallConstrainer::update(uint32_t /*token_id*/, const std::string& decod switch (state_) { case State::GEMMA_START: + advance_forced_tag(call_start_sequence_, token_id); if (generated_text_.find(call_start_tag_) != std::string::npos) { state_ = State::GEMMA_EXPECT_CALL; generated_text_.clear(); + forced_tag_progress_ = 0; } break; @@ -370,6 +391,7 @@ void ToolCallConstrainer::update(uint32_t /*token_id*/, const std::string& decod if (brace_depth_ == 0) { state_ = State::GEMMA_EXPECT_END; in_argument_string_ = false; + forced_tag_progress_ = 0; break; } } @@ -378,6 +400,7 @@ void ToolCallConstrainer::update(uint32_t /*token_id*/, const std::string& decod break; case State::GEMMA_EXPECT_END: + advance_forced_tag(call_end_sequence_, token_id); if (generated_text_.find(call_end_tag_) != std::string::npos) { state_ = State::DONE; generated_text_.clear(); @@ -452,8 +475,8 @@ void ToolCallConstrainer::compute_bias() { switch (state_) { case State::GEMMA_START: - for (uint32_t t : gemma_call_start_tokens_) { - current_bias_[t] = FORCE_BIAS; + if (forced_tag_progress_ < call_start_sequence_.size()) { + current_bias_[call_start_sequence_[forced_tag_progress_]] = FORCE_BIAS; } for (uint32_t t : open_brace_tokens_) { current_bias_[t] = BLOCK_BIAS; @@ -523,8 +546,8 @@ void ToolCallConstrainer::compute_bias() { break; case State::GEMMA_EXPECT_END: - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = FORCE_BIAS; + if (forced_tag_progress_ < call_end_sequence_.size()) { + current_bias_[call_end_sequence_[forced_tag_progress_]] = FORCE_BIAS; } for (uint32_t t : open_brace_tokens_) { current_bias_[t] = BLOCK_BIAS; @@ -544,6 +567,7 @@ void ToolCallConstrainer::reset() { current_bias_.clear(); brace_depth_ = 0; in_argument_string_ = false; + forced_tag_progress_ = 0; reset_needle_constraints(); state_ = is_needle() ? State::NEEDLE_START : State::GEMMA_START; diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index aeed9d0e5..c332bc447 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -240,6 +240,10 @@ struct Config { return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; } + static bool is_gemma3_family(ModelType t) { + return t == ModelType::GEMMA || t == ModelType::GEMMA3N; + } + bool from_json(const std::string& json_path); std::string to_json() const; // Disable rolling unless 0 < target < trigger (when trigger > 0). @@ -337,6 +341,7 @@ class Tokenizer { virtual bool has_chat_template() const { return has_chat_template_; } bool is_qwen_family() const { return model_type_ == ModelType::QWEN; } bool is_lfm2_family() const { return model_type_ == ModelType::LFM2; } + bool has_function_call_tokens() const { return encode("").size() == 1; } std::string get_default_stop_sequence() const; virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; @@ -535,6 +540,10 @@ class ToolCallConstrainer { std::string call_start_tag_; std::string call_end_tag_; + std::string response_start_tag_; + std::vector call_start_sequence_; + std::vector call_end_sequence_; + size_t forced_tag_progress_ = 0; std::unordered_set all_func_name_tokens_; std::unordered_map> func_name_sequences_; @@ -557,6 +566,7 @@ class ToolCallConstrainer { void tokenize_grammar_elements(); void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); void add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set); + void advance_forced_tag(const std::vector& sequence, uint32_t token_id); void tokenize_function_names(bool quote_names); void init_common_tokens(); diff --git a/cactus-engine/src/gemma_tools.h b/cactus-engine/src/gemma_tools.h index fdded8b7f..9497eb3b4 100644 --- a/cactus-engine/src/gemma_tools.h +++ b/cactus-engine/src/gemma_tools.h @@ -4,9 +4,12 @@ #include #include #include +#include #include #include +#include "json_escape.h" + namespace gemma { inline std::string to_upper(const std::string& s) { @@ -19,6 +22,15 @@ inline std::string escape(const std::string& s) { return "<|\"|>" + s + "<|\"|>"; } +inline std::string use_escape_tags(std::string s) { + size_t pos = 0; + while ((pos = s.find("<|\"|>", pos)) != std::string::npos) { + s.replace(pos, 5, ""); + pos += 8; + } + return s; +} + inline void skip_whitespace(const std::string& json, size_t& pos) { while (pos < json.length() && std::isspace(json[pos])) pos++; } @@ -398,7 +410,8 @@ inline std::string format_tools(const std::vector& tools, bool use params_json = it->second; } - result += format_function_declaration(tool.name, tool.description, params_json); + std::string declaration = format_function_declaration(tool.name, tool.description, params_json); + result += use_pipe_tags ? declaration : use_escape_tags(declaration); result += decl_end; } return result; @@ -429,6 +442,16 @@ inline std::string unescape(const std::string& s) { return result; } +inline std::string to_json_value(const std::string& v) { + if (v == "true" || v == "false" || v == "null") return v; + if (!v.empty()) { + char* end = nullptr; + std::strtod(v.c_str(), &end); + if (end == v.c_str() + v.size()) return v; + } + return "\"" + escape_json_string(v) + "\""; +} + inline std::string args_to_json(const std::string& args_content) { std::string result = "{"; size_t pos = 0; @@ -455,7 +478,7 @@ inline std::string args_to_json(const std::string& args_content) { pos += qtag_len; size_t val_end = find_quote_tag(args_content, pos); if (val_end != std::string::npos) { - value = "\"" + args_content.substr(pos, val_end - pos) + "\""; + value = "\"" + escape_json_string(args_content.substr(pos, val_end - pos)) + "\""; pos = val_end + match_quote_tag(args_content, val_end); } } else if (args_content[pos] == '{') { @@ -493,13 +516,13 @@ inline std::string args_to_json(const std::string& args_content) { arr_pos += aq_len; size_t end = find_quote_tag(arr_content, arr_pos); if (end != std::string::npos) { - value += "\"" + arr_content.substr(arr_pos, end - arr_pos) + "\""; + value += "\"" + escape_json_string(arr_content.substr(arr_pos, end - arr_pos)) + "\""; arr_pos = end + match_quote_tag(arr_content, end); } } else { size_t end = arr_content.find_first_of(",]", arr_pos); if (end == std::string::npos) end = arr_content.length(); - value += arr_content.substr(arr_pos, end - arr_pos); + value += to_json_value(arr_content.substr(arr_pos, end - arr_pos)); arr_pos = end; } } @@ -511,12 +534,13 @@ inline std::string args_to_json(const std::string& args_content) { } value = args_content.substr(val_start, pos - val_start); while (!value.empty() && std::isspace(value.back())) value.pop_back(); + value = to_json_value(value); } } if (!first) result += ","; first = false; - result += "\"" + key + "\":" + value; + result += "\"" + escape_json_string(key) + "\":" + (value.empty() ? "\"\"" : value); } result += "}"; @@ -554,7 +578,7 @@ inline void parse_function_calls(std::string& response, std::vector if (args_content.back() != '}') args_content += "}"; std::string args_json = args_to_json(args_content); - std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + std::string json_call = "{\"name\":\"" + escape_json_string(func_name) + "\",\"arguments\":" + args_json + "}"; function_calls.push_back(json_call); } } else { diff --git a/cactus-engine/src/tokenizer.cpp b/cactus-engine/src/tokenizer.cpp index ccfea9917..cffbbd49d 100644 --- a/cactus-engine/src/tokenizer.cpp +++ b/cactus-engine/src/tokenizer.cpp @@ -33,14 +33,21 @@ std::string format_needle_query_text(const std::vector& messages) { } std::string format_tool_call_for_prompt(const std::string& name, const std::string& arguments, bool gemma4) { + std::string args = arguments.empty() ? "{}" : arguments; + size_t pos = 0; + std::string dsl = gemma::format_argument(args, pos, false); + if (dsl.empty() || dsl.front() != '{') dsl = "{" + dsl + "}"; if (gemma4) { - std::string args = arguments.empty() ? "{}" : arguments; - size_t pos = 0; - std::string dsl = gemma::format_argument(args, pos, /*escape_keys=*/false); - if (dsl.empty() || dsl.front() != '{') dsl = "{" + dsl + "}"; return "<|tool_call>call:" + name + dsl + ""; } - return "\ncall:" + name + "(" + arguments + ")\n"; + return "call:" + name + gemma::use_escape_tags(dsl) + ""; +} + +std::string format_tool_response_for_prompt(const std::string& name, const std::string& content, bool gemma4) { + if (gemma4) { + return "<|tool_response>response:" + name + "{value:<|\"|>" + content + "<|\"|>}"; + } + return "response:" + name + "{value:" + content + "}"; } std::string trim_copy(const std::string& value) { @@ -436,24 +443,63 @@ std::string Tokenizer::format_chat_prompt(const std::vector& messag std::string Tokenizer::format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const { std::string result = ""; - std::string pending_context = tools_json; - for (const auto& msg : messages) { - if (msg.role == "system" || msg.role == "developer") { - pending_context += msg.content + "\n\n"; + if (tools_json.empty() && !has_function_call_tokens()) { + std::string pending_context; + for (const auto& msg : messages) { + if (msg.role == "system" || msg.role == "developer") { + pending_context += msg.content + "\n\n"; + continue; + } + std::string role = msg.role == "assistant" ? "model" : "user"; + std::string content = msg.content; + for (const auto& tc : msg.tool_calls) { + content += "\ncall:" + tc.name + "(" + tc.arguments + ")"; + } + if (role == "user" && !pending_context.empty()) { + content = pending_context + content; + pending_context.clear(); + } + result += "" + role + "\n" + content + "\n"; + } + if (add_generation_prompt) { + result += "model\n"; + } + return result; + } + + size_t first = 0; + std::string sys; + if (!messages.empty() && (messages[0].role == "system" || messages[0].role == "developer")) { + sys = messages[0].content; + first = 1; + } + if (first > 0 || !tools_json.empty()) { + result += "developer\n" + sys + tools_json + "\n"; + } + bool after_tool_response = false; + for (size_t i = first; i < messages.size(); i++) { + const auto& msg = messages[i]; + if (msg.role == "tool") { + result += format_tool_response_for_prompt(msg.name, msg.content, false); + after_tool_response = true; continue; } - std::string role = msg.role == "assistant" ? "model" : "user"; - std::string content = msg.content; + std::string role = msg.role == "assistant" ? "model" : msg.role; + if (!after_tool_response) { + result += "" + role + "\n"; + } + after_tool_response = false; + result += msg.content; for (const auto& tc : msg.tool_calls) { - content += "\ncall:" + tc.name + "(" + tc.arguments + ")"; + result += format_tool_call_for_prompt(tc.name, tc.arguments, false); } - if (role == "user" && !pending_context.empty()) { - content = pending_context + content; - pending_context.clear(); + if (msg.tool_calls.empty()) { + result += "\n"; + } else if (i + 1 == messages.size()) { + result += ""; } - result += "" + role + "\n" + content + "\n"; } - if (add_generation_prompt) { + if (add_generation_prompt && !after_tool_response) { result += "model\n"; } return result; @@ -673,8 +719,7 @@ std::string Tokenizer::format_gemma4_style(const std::vector& messa : (!pending_call_names.empty() ? pending_call_names.front() : std::string("unknown")); if (!pending_call_names.empty()) pending_call_names.erase(pending_call_names.begin()); - result += "<|tool_response>response:" + fn + "{value:<|\"|>" + msg.content - + "<|\"|>}"; + result += format_tool_response_for_prompt(fn, msg.content, true); } else { close_model_turn(); result += "<|turn>" + role + "\n"; diff --git a/python/tests/test_chat_template_golden.py b/python/tests/test_chat_template_golden.py index 1211e5b60..69a19f4e6 100644 --- a/python/tests/test_chat_template_golden.py +++ b/python/tests/test_chat_template_golden.py @@ -19,8 +19,9 @@ ("gemma-4-e2b-it", "google/gemma-4-E2B-it"), ("qwen3-0.6b", "Qwen/Qwen3-0.6B"), ("lfm2-350m", "LiquidAI/LFM2-350M"), + ("functiongemma-270m-it", "google/functiongemma-270m-it"), ] -FAMILY_IDS = ["gemma4", "qwen3", "lfm2"] +FAMILY_IDS = ["gemma4", "qwen3", "lfm2", "functiongemma"] THINKING_TYPES = {"gemma4", "qwen"} TOOLS = [ From bdc74ed5f1f5e92aea8d922533085dccce0b42c4 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 12 Jun 2026 16:17:38 -0700 Subject: [PATCH 08/13] cactus-engine: render qwen think markup only for thinking chat templates format_qwen_style appended the Qwen3-hybrid "\n\n\n\n" generation scaffold (and think-wrapped trailing assistant prefills) for every qwen-family bundle. Non-thinking bundles such as qwen3-vl-2b-instruct have no think handling in their chat template, so the scaffold pushed the model off-distribution: at temperature 0 the second turn of a conversation greedily replayed the previous assistant message byte-for-byte instead of answering the new question. The dequantized bundle reproduces both behaviors in the reference stack: with the scaffold it emits the identical replay, without it it answers the question. Gate the think emission sites on the bundle's chat template actually containing , which keeps hybrid-thinking renders byte-identical, and add a multi-turn distinctness regression test. Signed-off-by: Noah Cylich --- cactus-engine/src/tokenizer.cpp | 5 ++-- cactus-engine/tests/test_llm.cpp | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/cactus-engine/src/tokenizer.cpp b/cactus-engine/src/tokenizer.cpp index cffbbd49d..bea7402cc 100644 --- a/cactus-engine/src/tokenizer.cpp +++ b/cactus-engine/src/tokenizer.cpp @@ -522,6 +522,7 @@ std::string Tokenizer::format_qwen_style(const std::vector& message const std::string& tools_json, bool enable_thinking_if_supported) const { std::string result; const size_t n = messages.size(); + const bool template_has_thinking = !has_chat_template_ || chat_template_.find("") != std::string::npos; size_t first = 0; std::string sys; @@ -570,7 +571,7 @@ std::string Tokenizer::format_qwen_style(const std::vector& message content = lstrip_newlines(content.substr(tpos + 8)); } result += "<|im_start|>assistant\n"; - if (static_cast(i) > last_query_index && (i == n - 1 || !reasoning.empty())) { + if (template_has_thinking && static_cast(i) > last_query_index && (i == n - 1 || !reasoning.empty())) { result += "\n" + reasoning + "\n\n\n" + lstrip_newlines(content); } else { result += content; @@ -591,7 +592,7 @@ std::string Tokenizer::format_qwen_style(const std::vector& message if (add_generation_prompt) { result += "<|im_start|>assistant\n"; - if (!enable_thinking_if_supported) result += "\n\n\n\n"; + if (!enable_thinking_if_supported && template_has_thinking) result += "\n\n\n\n"; } return result; } diff --git a/cactus-engine/tests/test_llm.cpp b/cactus-engine/tests/test_llm.cpp index 1e2e82a3c..e96867347 100644 --- a/cactus-engine/tests/test_llm.cpp +++ b/cactus-engine/tests/test_llm.cpp @@ -731,6 +731,50 @@ bool test_multiturn_thinking_persist() { return prefix_ok && mentions_alice; } +bool test_multiturn_turn2_distinct() { + std::cout << "\n╔══════════════════════════════════════════╗\n" + << "║" << std::setw(42) << std::left << " MULTITURN TURN-2 DISTINCT TEST" << "║\n" + << "╚══════════════════════════════════════════╝\n"; + + if (!g_model_path) { std::cout << " [WARN] CACTUS_TEST_MODEL not set; skipping\n"; return true; } + cactus_model_t model = cactus_init(g_model_path, nullptr, false); + if (!model) { + std::cerr << "[✗] Failed to initialize model\n"; + return false; + } + + const char* options = R"({"max_tokens":150,"temperature":0,"telemetry_enabled":false,"auto_handoff":false})"; + const char* turn1_msgs = R"([ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hi! My name is Ada and I live in Paris. Briefly say hello."} + ])"; + char buf[16384]; + + int r1 = cactus_complete(model, turn1_msgs, buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); + if (r1 <= 0) { std::cerr << " Turn 1 failed\n"; cactus_destroy(model); return false; } + std::string response1 = EngineTestUtils::json_string(std::string(buf), "response"); + std::cout << " Turn 1: " << response1 << "\n"; + + std::string turn2_msgs = R"([ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hi! My name is Ada and I live in Paris. Briefly say hello."}, + {"role": "assistant", "content": ")" + EngineTestUtils::escape_json(response1) + R"("}, + {"role": "user", "content": "What is my name and what city do I live in? Answer in one short sentence."} + ])"; + + int r2 = cactus_complete(model, turn2_msgs.c_str(), buf, sizeof(buf), options, nullptr, nullptr, nullptr, nullptr, 0); + if (r2 <= 0) { std::cerr << " Turn 2 failed\n"; cactus_destroy(model); return false; } + std::string response2 = EngineTestUtils::json_string(std::string(buf), "response"); + std::cout << " Turn 2: " << response2 << "\n"; + + cactus_destroy(model); + + bool distinct = !response1.empty() && !response2.empty() && response2 != response1; + std::cout << " Turn 2 distinct from turn 1: " << (distinct ? "YES" : "NO") << "\n"; + if (!distinct) std::cerr << " FAIL: turn 2 repeated turn 1 verbatim\n"; + return distinct; +} + static std::string benchmark_tokens_json(cactus_model_t model, const std::vector& ids, size_t max_new) { std::vector response(1 << 16, 0); int rc = cactus_benchmark_tokens(model, ids.data(), ids.size(), max_new, response.data(), response.size()); @@ -835,6 +879,7 @@ int main() { runner.run_test("prompt_retains_thinking", test_prompt_gemma4_retains_thinking()); runner.run_test("complete_thinking_api_clean", test_complete_gemma4_thinking_api_clean()); runner.run_test("multiturn_thinking_persist", test_multiturn_thinking_persist()); + runner.run_test("multiturn_turn2_distinct", test_multiturn_turn2_distinct()); runner.print_summary(); return runner.all_passed() ? 0 : 1; } From f2631e6b85b6abd4d2fd06ff414430614d4cffd5 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Fri, 12 Jun 2026 17:11:06 -0700 Subject: [PATCH 09/13] tests: share bundle helpers and deduplicate suite scaffolding Moves bundle discovery into tests/bundles.py, folds the repeated chunked-prefill check into the llm suite reusing its existing helper, drops import-time HF env mutation for local_files_only, and extracts the shared tool-capture setup in the server tests. Signed-off-by: Noah Cylich --- cactus-engine/tests/test_llm.cpp | 7 +-- .../tests/test_recurrent_prefill.cpp | 43 ------------------- python/tests/bundles.py | 32 ++++++++++++++ python/tests/test_chat_template_golden.py | 23 ++-------- python/tests/test_cloud_handoff_live.py | 15 ++----- python/tests/test_server.py | 17 ++++---- python/tests/test_server_live.py | 26 +---------- python/tests/test_tool_constraint_state.py | 10 ++--- 8 files changed, 56 insertions(+), 117 deletions(-) delete mode 100644 cactus-engine/tests/test_recurrent_prefill.cpp create mode 100644 python/tests/bundles.py diff --git a/cactus-engine/tests/test_llm.cpp b/cactus-engine/tests/test_llm.cpp index e96867347..7181c21d8 100644 --- a/cactus-engine/tests/test_llm.cpp +++ b/cactus-engine/tests/test_llm.cpp @@ -811,8 +811,9 @@ bool test_chunked_prefill_padding() { std::string padded = benchmark_tokens_json(model, ids, 4); std::string padded_again = benchmark_tokens_json(model, ids, 4); cactus_destroy(model); - if (padded.empty() || padded.find("\"success\":true") == std::string::npos) { - std::cerr << " [✗] padded benchmark failed at len " << prompt_len << "\n"; + if (padded.empty() || padded.find("\"success\":true") == std::string::npos + || padded_again.empty() || padded_again.find("\"success\":true") == std::string::npos) { + std::cerr << " [✗] repeated chunked benchmark failed at len " << prompt_len << "\n"; return false; } long tail_chunk = static_cast(EngineTestUtils::json_number(padded, "prefill_tail_chunk_tokens", -1)); @@ -822,7 +823,7 @@ bool test_chunked_prefill_padding() { << " pads=" << tail_pads << " scalar=" << scalar << "\n"; if (tail_chunk <= 0) { std::cout << " [WARN] padded tail did not engage (no sliding caches?); skipping\n"; - return true; + continue; } if (tail_pads <= 0 || scalar > 1) { std::cerr << " [✗] unexpected padding telemetry\n"; diff --git a/cactus-engine/tests/test_recurrent_prefill.cpp b/cactus-engine/tests/test_recurrent_prefill.cpp deleted file mode 100644 index f4a508e9d..000000000 --- a/cactus-engine/tests/test_recurrent_prefill.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "test_utils.h" -#include -#include -#include - -static const char* g_model_path = std::getenv("CACTUS_TEST_MODEL"); - -static std::string benchmark_tokens_json(cactus_model_t model, const std::vector& ids, size_t max_new) { - std::vector response(1 << 16, 0); - int rc = cactus_benchmark_tokens(model, ids.data(), ids.size(), max_new, response.data(), response.size()); - return rc < 0 ? std::string() : std::string(response.data()); -} - -bool test_repeated_chunked_prefill() { - if (!g_model_path) { std::cout << " [WARN] CACTUS_TEST_MODEL not set; skipping\n"; return true; } - - cactus_model_t model = cactus_init(g_model_path, nullptr, false); - if (!model) { std::cerr << " [✗] model init failed\n"; return false; } - - std::vector ids(600); - for (size_t i = 0; i < ids.size(); i++) ids[i] = static_cast(100 + (i % 200)); - - std::string first = benchmark_tokens_json(model, ids, 4); - std::string second = benchmark_tokens_json(model, ids, 4); - cactus_destroy(model); - - if (first.empty() || first.find("\"success\":true") == std::string::npos) { - std::cerr << " [✗] first chunked prefill failed\n"; - return false; - } - if (second.empty() || second.find("\"success\":true") == std::string::npos) { - std::cerr << " [✗] second chunked prefill failed\n"; - return false; - } - return true; -} - -int main() { - TestUtils::TestRunner runner("Recurrent Prefill Tests"); - runner.run_test("repeated_chunked_prefill", test_repeated_chunked_prefill()); - runner.print_summary(); - return runner.all_passed() ? 0 : 1; -} diff --git a/python/tests/bundles.py b/python/tests/bundles.py new file mode 100644 index 000000000..2eca102f6 --- /dev/null +++ b/python/tests/bundles.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +WEIGHTS = PROJECT_ROOT / "weights" + + +def _read_model_type(bundle: Path) -> str: + for line in (bundle / "config.txt").read_text(encoding="utf-8").splitlines(): + if line.startswith("model_type="): + return line.split("=", 1)[1].strip() + return "" + + +def _valid_bundle(path: Path) -> bool: + return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() + + +def _find_bundle(preferred: list[str], types: set[str], on_missing=pytest.fail) -> Path: + for name in preferred: + candidate = WEIGHTS / name + if candidate.exists() and _valid_bundle(candidate) and _read_model_type(candidate) in types: + return candidate + if not WEIGHTS.is_dir(): + on_missing(f"Weights directory not found: {WEIGHTS}") + for candidate in sorted(WEIGHTS.iterdir()): + if candidate.is_dir() and _valid_bundle(candidate) and _read_model_type(candidate) in types: + return candidate + on_missing(f"No valid live-test bundle found under {WEIGHTS} for model types: {sorted(types)}") diff --git a/python/tests/test_chat_template_golden.py b/python/tests/test_chat_template_golden.py index 69a19f4e6..cd8cfc346 100644 --- a/python/tests/test_chat_template_golden.py +++ b/python/tests/test_chat_template_golden.py @@ -1,18 +1,14 @@ from __future__ import annotations import json -import os from pathlib import Path import pytest -os.environ.setdefault("HF_HUB_OFFLINE", "1") -os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") - transformers = pytest.importorskip("transformers") -PROJECT_ROOT = Path(__file__).resolve().parents[2] -WEIGHTS = PROJECT_ROOT / "weights" +from .bundles import PROJECT_ROOT, WEIGHTS, _read_model_type, _valid_bundle + MONKEY_IMAGE = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" / "test_monkey.png" FAMILIES = [ @@ -109,17 +105,6 @@ ] -def _read_model_type(bundle: Path) -> str: - for line in (bundle / "config.txt").read_text(encoding="utf-8").splitlines(): - if line.startswith("model_type="): - return line.split("=", 1)[1].strip() - return "" - - -def _valid_bundle(path: Path) -> bool: - return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() - - def _pythonic_call(call: dict) -> str: fn = call["function"] args = ", ".join(f"{key}={json.dumps(value)}" for key, value in fn["arguments"].items()) @@ -161,7 +146,7 @@ def _load(bundle_name: str, hf_id: str): if not _valid_bundle(bundle): pytest.skip(f"bundle not prepared: {bundle}") try: - tokenizer = transformers.AutoTokenizer.from_pretrained(hf_id) + tokenizer = transformers.AutoTokenizer.from_pretrained(hf_id, local_files_only=True) except OSError as exc: pytest.skip(f"HF tokenizer not cached for {hf_id}: {exc}") _LOADED[bundle_name] = (cactus_init(str(bundle)), tokenizer, _read_model_type(bundle)) @@ -194,7 +179,7 @@ def test_render_matches_hf_template(load_family, bundle_name, hf_id, case_name, from cactus import cactus_render_prompt model, tokenizer, model_type = load_family(bundle_name, hf_id) - if case_name.startswith("thinking") and model_type not in THINKING_TYPES: + if case_name.startswith("thinking") and not any(t in model_type for t in THINKING_TYPES): pytest.skip(f"{model_type} has no thinking template support") engine_render = cactus_render_prompt( model, messages, options={"enable_thinking_if_supported": thinking}, tools=tools diff --git a/python/tests/test_cloud_handoff_live.py b/python/tests/test_cloud_handoff_live.py index d20ac6574..3ca32f4f5 100644 --- a/python/tests/test_cloud_handoff_live.py +++ b/python/tests/test_cloud_handoff_live.py @@ -4,25 +4,16 @@ import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path import pytest -PROJECT_ROOT = Path(__file__).resolve().parents[2] -WEIGHTS = PROJECT_ROOT / "weights" +from .bundles import _find_bundle + PREFERRED_BUNDLES = ["qwen3-0.6b", "lfm2-350m"] OPTIONS = {"max_tokens": 8, "temperature": 0, "confidence_threshold": 1.0} MESSAGES = [{"role": "user", "content": "What is 2+2?"}] -def _find_bundle() -> Path: - for name in PREFERRED_BUNDLES: - candidate = WEIGHTS / name - if (candidate / "config.txt").exists() and (candidate / "components" / "manifest.json").exists(): - return candidate - pytest.skip(f"No live-test bundle found under {WEIGHTS}; expected one of: {PREFERRED_BUNDLES}") - - class _AuthFailHandler(BaseHTTPRequestHandler): def do_POST(self): with self.server.hits_lock: @@ -45,7 +36,7 @@ def _hits(server) -> int: def test_auth_failure_disables_cloud_handoff_for_session(monkeypatch, tmp_path): - bundle = _find_bundle() + bundle = _find_bundle(PREFERRED_BUNDLES, {"qwen", "lfm2"}, on_missing=pytest.skip) server = ThreadingHTTPServer(("127.0.0.1", 0), _AuthFailHandler) server.hits = 0 server.hits_lock = threading.Lock() diff --git a/python/tests/test_server.py b/python/tests/test_server.py index 3c0780f8e..d2137a34d 100644 --- a/python/tests/test_server.py +++ b/python/tests/test_server.py @@ -93,11 +93,16 @@ def fake_complete(handle, messages, options, tools, callback): return fake_complete -def test_chat_tool_calls_are_translated(tmp_path: Path, monkeypatch) -> None: +def _tool_capture_app(tmp_path, monkeypatch): app = _app(tmp_path, monkeypatch) monkeypatch.setattr(server, "cactus_reset", lambda handle: None) captured = {} monkeypatch.setattr(server, "cactus_complete", _capturing_complete(captured)) + return app, captured + + +def test_chat_tool_calls_are_translated(tmp_path: Path, monkeypatch) -> None: + app, captured = _tool_capture_app(tmp_path, monkeypatch) with TestClient(app) as client: res = client.post("/v1/chat/completions", json={ @@ -116,10 +121,7 @@ def test_chat_tool_calls_are_translated(tmp_path: Path, monkeypatch) -> None: def test_streaming_chat_sends_wrapped_tools(tmp_path: Path, monkeypatch) -> None: - app = _app(tmp_path, monkeypatch) - monkeypatch.setattr(server, "cactus_reset", lambda handle: None) - captured = {} - monkeypatch.setattr(server, "cactus_complete", _capturing_complete(captured)) + app, captured = _tool_capture_app(tmp_path, monkeypatch) with TestClient(app) as client: with client.stream("POST", "/v1/chat/completions", json={ @@ -136,10 +138,7 @@ def test_streaming_chat_sends_wrapped_tools(tmp_path: Path, monkeypatch) -> None def test_tool_choice_object_selects_wrapped_tool(tmp_path: Path, monkeypatch) -> None: - app = _app(tmp_path, monkeypatch) - monkeypatch.setattr(server, "cactus_reset", lambda handle: None) - captured = {} - monkeypatch.setattr(server, "cactus_complete", _capturing_complete(captured)) + app, captured = _tool_capture_app(tmp_path, monkeypatch) with TestClient(app) as client: res = client.post("/v1/chat/completions", json={ diff --git a/python/tests/test_server_live.py b/python/tests/test_server_live.py index 65fad4a87..4fd3e93ef 100644 --- a/python/tests/test_server_live.py +++ b/python/tests/test_server_live.py @@ -12,25 +12,14 @@ import httpx import pytest -PROJECT_ROOT = Path(__file__).resolve().parents[2] -WEIGHTS = PROJECT_ROOT / "weights" +from .bundles import PROJECT_ROOT, WEIGHTS, _find_bundle, _read_model_type, _valid_bundle + DEFAULT_LLM_BUNDLE = Path("weights/gemma-4-e2b-it") ASSETS = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" LLM_TYPES = {"gemma", "gemma3n", "gemma4", "lfm2", "qwen", "qwen3p5", "needle", "youtu"} STT_TYPES = {"whisper", "parakeet_tdt", "parakeet-tdt"} -def _read_model_type(bundle: Path) -> str: - for line in (bundle / "config.txt").read_text(encoding="utf-8").splitlines(): - if line.startswith("model_type="): - return line.split("=", 1)[1].strip() - return "" - - -def _valid_bundle(path: Path) -> bool: - return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() - - def _require_bundle(relative: Path, types: set[str]) -> Path: candidate = PROJECT_ROOT / relative if not candidate.exists(): @@ -53,17 +42,6 @@ def _require_bundle(relative: Path, types: set[str]) -> Path: return candidate -def _find_bundle(preferred: list[str], types: set[str]) -> Path: - for name in preferred: - candidate = WEIGHTS / name - if candidate.exists() and _valid_bundle(candidate) and _read_model_type(candidate) in types: - return candidate - for candidate in sorted(WEIGHTS.iterdir()): - if candidate.is_dir() and _valid_bundle(candidate) and _read_model_type(candidate) in types: - return candidate - pytest.fail(f"No valid live-test bundle found under {WEIGHTS} for model types: {sorted(types)}") - - def _free_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) diff --git a/python/tests/test_tool_constraint_state.py b/python/tests/test_tool_constraint_state.py index aff54124f..1be5bc994 100644 --- a/python/tests/test_tool_constraint_state.py +++ b/python/tests/test_tool_constraint_state.py @@ -1,14 +1,14 @@ from __future__ import annotations import json -from pathlib import Path import pytest from cactus import cactus_complete, cactus_destroy, cactus_init -PROJECT_ROOT = Path(__file__).resolve().parents[2] -BUNDLE = PROJECT_ROOT / "weights" / "gemma-4-e2b-it" +from .bundles import WEIGHTS, _valid_bundle + +BUNDLE = WEIGHTS / "gemma-4-e2b-it" GET_WEATHER = { "type": "function", @@ -26,10 +26,6 @@ } -def _valid_bundle(path: Path) -> bool: - return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() - - @pytest.fixture(scope="module") def model(): if not _valid_bundle(BUNDLE): From 4c56c4f54f7f16f0b604cc604444b0e264761e94 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 13 Jun 2026 09:14:30 -0700 Subject: [PATCH 10/13] tests: target a non-embedding bundle for the embedding-rejection live test The rejection test ran against the default LLM fixture (gemma-4-e2b-it), which carries a decoder_embed_chunk component and therefore legitimately serves hidden-state embeddings, so the endpoint returned 200 instead of the expected 400. Discover a bundle that lacks both text_embedding and decoder_embed_chunk, skipping when none is present, and share the server startup via a context manager. Signed-off-by: Noah Cylich --- python/tests/test_server_live.py | 64 +++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/python/tests/test_server_live.py b/python/tests/test_server_live.py index 4fd3e93ef..cac599707 100644 --- a/python/tests/test_server_live.py +++ b/python/tests/test_server_live.py @@ -7,6 +7,7 @@ import sys import time from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager from pathlib import Path import httpx @@ -90,15 +91,14 @@ def _wait_ready(proc: subprocess.Popen, base_url: str) -> None: pytest.fail(f"server did not become ready: {last_error}") -@pytest.fixture(scope="module") -def live_server(): - _require_bundle(DEFAULT_LLM_BUNDLE, LLM_TYPES) +@contextmanager +def _serve(bundle: Path): port = _free_port() - proc = _start_server(DEFAULT_LLM_BUNDLE, port) + proc = _start_server(bundle, port) base_url = f"http://127.0.0.1:{port}" try: _wait_ready(proc, base_url) - yield base_url, DEFAULT_LLM_BUNDLE.name + yield base_url, bundle.name finally: proc.terminate() try: @@ -108,6 +108,13 @@ def live_server(): proc.wait(timeout=5) +@pytest.fixture(scope="module") +def live_server(): + _require_bundle(DEFAULT_LLM_BUNDLE, LLM_TYPES) + with _serve(DEFAULT_LLM_BUNDLE) as server: + yield server + + def test_live_models(live_server) -> None: base_url, _ = live_server res = httpx.get(f"{base_url}/v1/models", timeout=10) @@ -425,22 +432,37 @@ def _find_embed_bundle() -> Path: ) +def _supports_embedding(bundle: Path) -> bool: + try: + data = json.loads((bundle / "components" / "manifest.json").read_text(encoding="utf-8")) + except Exception: + return False + names = {str(c.get("component", "")) for c in data.get("components", []) if isinstance(c, dict)} + return "text_embedding" in names or "decoder_embed_chunk" in names + + +def _find_non_embed_llm_bundle() -> Path | None: + for candidate in sorted(WEIGHTS.iterdir()) if WEIGHTS.exists() else []: + if (candidate.is_dir() and _valid_bundle(candidate) + and _read_model_type(candidate) in LLM_TYPES + and not _supports_embedding(candidate)): + return candidate + return None + + @pytest.fixture(scope="module") def embed_server(): - bundle = _find_embed_bundle() - port = _free_port() - proc = _start_server(bundle, port) - base_url = f"http://127.0.0.1:{port}" - try: - _wait_ready(proc, base_url) - yield base_url, bundle.name - finally: - proc.terminate() - try: - proc.wait(timeout=10) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait(timeout=5) + with _serve(_find_embed_bundle()) as server: + yield server + + +@pytest.fixture(scope="module") +def non_embed_server(): + bundle = _find_non_embed_llm_bundle() + if bundle is None: + pytest.skip("No non-embedding LLM bundle under weights/") + with _serve(bundle) as server: + yield server def test_live_embeddings_string(embed_server) -> None: @@ -473,8 +495,8 @@ def test_live_embeddings_list(embed_server) -> None: assert len(data[0]["embedding"]) == len(data[1]["embedding"]) > 0 -def test_live_embeddings_rejects_llm_model(live_server) -> None: - base_url, model = live_server +def test_live_embeddings_rejects_llm_model(non_embed_server) -> None: + base_url, model = non_embed_server res = httpx.post( f"{base_url}/v1/embeddings", json={"model": model, "input": "hello"}, From fa6c08e00faf10d567f554b4651a59b0593dfdf9 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 13 Jun 2026 17:41:38 -1000 Subject: [PATCH 11/13] cactus-convert: down-scale the gemma4 vision embedding projection The gemma4 FP16 weight scaler keyed the 1/16 down-scaling on the filename embed_vision_embedding.weights, but the vision embedding projection is saved as embed_vision_proj.weights, so it was stored unscaled. The gemma4 runtime operates in a 1/16-scaled residual space (token embeddings, norms, etc. are all down-scaled to keep activations in fp16 range), so an unscaled projection left image soft-tokens ~16x too large in the residual stream. This drowned out the per-layer refinement of image features and degraded multimodal reasoning: gemma-4-e4b-it confidently misidentified a monkey as a toucan, while e2b was unaffected only because its projection was converted before the filename was dropped from the scale set (regression in #659). Add the real filename so the projection is down-scaled like every other gemma4 embedding-space weight. Signed-off-by: Noah Cylich --- python/cactus/convert/cactus_adapters/tensor_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cactus/convert/cactus_adapters/tensor_io.py b/python/cactus/convert/cactus_adapters/tensor_io.py index 5afb0df6f..dc59d0aa8 100644 --- a/python/cactus/convert/cactus_adapters/tensor_io.py +++ b/python/cactus/convert/cactus_adapters/tensor_io.py @@ -292,7 +292,7 @@ def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=Fal data = data * GEMMA4_WEIGHT_SCALE elif 'router_scale' in filename: data = data / GEMMA4_WEIGHT_SCALE - elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights'): + elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights', 'embed_vision_proj.weights'): data = data / GEMMA4_WEIGHT_SCALE elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE From f892740175e1ec9b0c6f472a0125c9ab8cc9d8ae Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 13 Jun 2026 17:46:13 -1000 Subject: [PATCH 12/13] cactus-convert: down-scale the gemma4 audio embedding projection Same bug class as the vision embedding projection: the gemma4 audio feature projection (model.embed_audio.embedding_projection.weight -> embed_audio_proj.weights) maps audio soft-tokens into the LM's 1/16-scaled residual space but was stored unscaled (scale_factor 1.0, confirmed: stored norm == HF norm), leaving audio soft-tokens ~16x too large in the residual stream. Add embed_audio_proj and embed_audio_embedding to the gemma4 down-scale set so audio embeddings are scaled like vision embeddings and every other embedding-space weight. Signed-off-by: Noah Cylich --- python/cactus/convert/cactus_adapters/tensor_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cactus/convert/cactus_adapters/tensor_io.py b/python/cactus/convert/cactus_adapters/tensor_io.py index dc59d0aa8..8a8fb078f 100644 --- a/python/cactus/convert/cactus_adapters/tensor_io.py +++ b/python/cactus/convert/cactus_adapters/tensor_io.py @@ -292,7 +292,7 @@ def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=Fal data = data * GEMMA4_WEIGHT_SCALE elif 'router_scale' in filename: data = data / GEMMA4_WEIGHT_SCALE - elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights', 'embed_vision_proj.weights'): + elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights', 'embed_vision_proj.weights', 'embed_audio_embedding.weights', 'embed_audio_proj.weights'): data = data / GEMMA4_WEIGHT_SCALE elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE From 23f2677f2c58e79c965770ea67b6d9931e4744b5 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sat, 13 Jun 2026 18:30:42 -1000 Subject: [PATCH 13/13] Revert "cactus-convert: down-scale the gemma4 audio embedding projection" This reverts commit f892740175e1ec9b0c6f472a0125c9ab8cc9d8ae. --- python/cactus/convert/cactus_adapters/tensor_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cactus/convert/cactus_adapters/tensor_io.py b/python/cactus/convert/cactus_adapters/tensor_io.py index 8a8fb078f..dc59d0aa8 100644 --- a/python/cactus/convert/cactus_adapters/tensor_io.py +++ b/python/cactus/convert/cactus_adapters/tensor_io.py @@ -292,7 +292,7 @@ def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=Fal data = data * GEMMA4_WEIGHT_SCALE elif 'router_scale' in filename: data = data / GEMMA4_WEIGHT_SCALE - elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights', 'embed_vision_proj.weights', 'embed_audio_embedding.weights', 'embed_audio_proj.weights'): + elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights', 'embed_vision_proj.weights'): data = data / GEMMA4_WEIGHT_SCALE elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE