Custom transpiler

.gitignore

@@ -4,6 +4,9 @@
 *.dylib
 *.bin
 *.cg
+*.json
+*.cactus
+*.weights
 
 !cactus-engine/libs/curl/**/*.a
 !android/mbedtls/**/*.a
@@ -29,4 +32,6 @@ python/cactus/bindings/lib/
 site/
 site_docs/
 
+TODO.md
+explain.md
 python/cactus/code/

README.md

@@ -17,20 +17,16 @@ A hybrid edge-cloud AI engine for mobile devices & wearables.
 └─────────────────┘     
          │
 ┌─────────────────┐
-│  Cactus Graph   │ ←── Zero-copy computation graph 
+│  Cactus Graph   │ ←── Zero-copy computation graph
 └─────────────────┘     
          │
 ┌─────────────────┐
 │ Cactus Kernels  │ ←── CPU/GPU kernels for (Apple, Samsung, Pixel, etc.)
 └─────────────────┘     
          │
 ┌─────────────────┐
-│ Cactus Quants   │ ←── Custom rotation-based quantization technique 
+│ Cactus Quants   │ ←── Custom rotation-based quantization technique
 └─────────────────┘  
-         │
-┌─────────────────┐
-│Cactus Transpiler│ ←── Transpiles custom PyTorch model to Cactus.
-└─────────────────┘
 ```
 
 ## Quick Demo (Mac)
@@ -126,7 +122,7 @@ graph.hard_reset();
 - VLM: Gemma-4-E2B-CQ4 (256px image encode time / decode)
 - Transcribe: Parakeet-TDT-0.6B-CQ4 (20s audio end-to-end transcribe time)
 - 1k-Context RAM: peak MB during the LLM benchmark
-- No speculative decode or MTP, pure decode 
+- No speculative decode or MTP, pure decode
 
 Command: `cactus benchmark` [optional `--ios` or `--android`]
 
@@ -141,14 +137,14 @@ Command: `cactus benchmark` [optional `--ios` or `--android`]
 
 N/B: With 1k-context prefill and decode for 100 runs on M5 Max
 - `LFM2.5-VL-1.6B` = 289toks/sec
-- `Qwen3-1.7B` = 155toks/sec 
+- `Qwen3-1.7B` = 155toks/sec
 - `LFM2.5-VL-450m` = 472toks/sec, image encodes in 43ms
 - `LFM22.5-VL-230m` = 555toks/sec
 
-## Output Quality 
+## Output Quality
 
-- Gemma-4-E2B-it accuracy across bit widths, averaged over 3 seeds. 
-- CQ3.26 and CQ2.54 are mixed-precision, CQ2/CQ3/CQ4 are uniformly quantized. 
+- Gemma-4-E2B-it accuracy across bit widths, averaged over 3 seeds.
+- CQ3.26 and CQ2.54 are mixed-precision, CQ2/CQ3/CQ4 are uniformly quantized.
 - Full results in [docs/cactus_quants.md](/docs/cactus_quants.md):
 
 | Task | F16 (Original) | CQ4 | CQ3.26 | CQ2.54 | CQ2 |
@@ -168,10 +164,10 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max
 
 ## Supported Models
 
-- Any HuggingFace model can be converted using `cactus convert [HF-Name]`, though experimental. 
-- Liquid, Gemma. whisper. parakeet and Qwen model families are especially tested. 
+- Any HuggingFace model can be converted using `cactus convert [HF-Name]`, though experimental.
+- Liquid, Gemma. whisper. parakeet and Qwen model families are especially tested.
 - Some models have been pre-uploaded [here](https://huggingface.co/Cactus-Compute), just run `cactus download [HF-Name]`.
-- `cactus run [HF-Name]` albeit first downloads or convert the model if not found. 
+- `cactus run [HF-Name]` albeit first downloads or convert the model if not found.
 
 ## Needle
 
@@ -190,7 +186,6 @@ cactus run Cactus-Compute/needle [--tools my_tools.json]  # OpenAI function-call
 | [Cactus Kernels](/docs/cactus_kernels.md) | C++ | ARM NEON SIMD kernels for matmul, attention, convolution, quantization, DSP, image processing |
 | [Cactus Quants](/docs/cactus_quants.md) | C++ | Rotation-and-codebook quantization from 4-bit to 1-bit for all weight tensors |
 | [Cactus Hybrid](/docs/cactus_hybrid.md) | C/Python | Route hard queries to the cloud automatically based on local model confidence |
-| [Cactus Transpiler](/docs/cactus_transpiler.md) | Python | Convert any PyTorch model to a Cactus runtime graph for on-device inference |
 | [Python Package](/python/) | Python | Python package and CLI |
 
 ## Bindings
@@ -244,13 +239,12 @@ cactus run Cactus-Compute/needle [--tools my_tools.json]  # OpenAI function-call
 │  cactus download [model]             get a bundle (prebuilt, else build)       │
 │    --bits 1|2|3|4|2.54|3.26          CQ quantization (default: 4)              │
 │    --token <token>                   HuggingFace token (gated models)          │
-│    --reconvert                       force local rebuild from source           │
+│    --reconvert                       refresh cached bundle                     │
 │                                                                                │
-│  cactus convert <model> [dir]        HuggingFace -> runnable cactus bundle     │
-│                                      (CQ weights + runtime graph)              │
+│  cactus convert <model> [dir]        HuggingFace -> Cactus CQ weights          │
 │    --bits 1|2|3|4                    CQ quantization (default: 4)              │
 │    --token <token>                   HuggingFace token (gated models)          │
-│    --reconvert                       force local rebuild from source           │
+│    --reconvert                       force weight conversion from source       │
 │    --lora <path>                     merge a LoRA adapter before converting    │
 │    --weights-only                    stop after CQ weights (skip the graph)    │
 │    --artifact-dir <path>             bundle output (default: weights/<model>)  │

apple/CMakeLists.txt

@@ -27,7 +27,7 @@ if(BUILD_SHARED_LIBS)
     )
 
     target_link_libraries(cactus_framework PUBLIC
-        "-framework Accelerate" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders"
+        "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders"
         "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork"
     )
 

bindings/rust/cactus.rs

@@ -7,6 +7,7 @@ pub type CactusTokenCallback = Option<unsafe extern "C" fn(token: *const c_char,
 pub type CactusLogCallback = Option<unsafe extern "C" fn(level: c_int, component: *const c_char, message: *const c_char, user_data: *mut c_void)>;
 
 #[cfg_attr(target_os = "macos", link(name = "Accelerate", kind = "framework"))]
+#[cfg_attr(target_os = "macos", link(name = "CoreML", kind = "framework"))]
 #[cfg_attr(target_os = "macos", link(name = "Metal", kind = "framework"))]
 #[cfg_attr(target_os = "macos", link(name = "MetalPerformanceShaders", kind = "framework"))]
 #[cfg_attr(target_os = "macos", link(name = "Foundation", kind = "framework"))]

blog/lfm2.5_350m.md

@@ -55,7 +55,7 @@ That's two separate shader codepaths to maintain, with different memory models,
 And GPU inference holds a persistent GPU context, which competes with the UI rendering pipeline and drains battery even when the inference workload is light.
 GPUs are also energy-inefficient. On Macs and PCs this barely matters since they're plugged into a power source most of the time, and GPU inference works well there. But mobile devices run on battery, and every watt spent on GPU inference cuts directly into the user's runtime. The GPU path that works fine on a MacBook becomes a battery killer on a phone.
 
-Dedicated AI accelerators are a more interesting target. Apple's Neural Engine and Qualcomm's Hexagon DSP both offer high-throughput inference at good power efficiency. 
+Dedicated AI accelerators are a more interesting target. Apple's Neural Engine and Qualcomm's Hexagon DSP both offer high-throughput inference at good power efficiency.
 The problem is access. Apple's Neural Engine requires conversion into a proprietary model format, which imposes its own quantization and graph constraints.
 You don't control the execution schedule, the memory layout, or the precision semantics. 
 The quantisation techniques available through that toolchain severely lag the state-of-the-art methods used by GGML and Cactus.
@@ -201,4 +201,4 @@ from a Vision Pro to a Raspberry Pi 5, with no special casing and no device-spec
 - [Cactus Engine API Reference](/docs/cactus_engine.md) - Full C API docs for completion, tool calling, and cloud handoff
 - [LFM2-24B-A2B](/blog/lfm2_24b_a2b.md) - Running LFM2 24B MoE locally on Mac for coding use cases
 - [Hybrid Transcription](/blog/hybrid_transcription.md) - On-device/cloud hybrid speech transcription with Cactus
-- [Ridiculously Fast Transcription](/blog/parakeet.md) - 6M tok/sec decode speed with Parakeet CTC 1.1B
+- [Ridiculously Fast Transcription](/blog/parakeet.md) - 6M tok/sec decode speed with Parakeet CTC 1.1B

cactus-engine/CMakeLists.txt

@@ -15,6 +15,7 @@ set(ENGINE_SOURCES
     src/sp.cpp
     src/constraints.cpp
     src/model.cpp
+    src/model_npu.cpp
     src/kv_compress.cpp
     src/engine_image.cpp
     src/index.cpp
@@ -31,6 +32,13 @@ set(ENGINE_SOURCES
     src/stream.cpp
     src/tokenizer.cpp
 )
+if(APPLE)
+    enable_language(OBJCXX)
+    list(APPEND ENGINE_SOURCES src/npu_ane.mm)
+    set_source_files_properties(src/npu_ane.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc")
+else()
+    list(APPEND ENGINE_SOURCES src/npu.cpp)
+endif()
 add_library(cactus_engine STATIC ${ENGINE_SOURCES})
 set_target_properties(cactus_engine PROPERTIES
     POSITION_INDEPENDENT_CODE ON
@@ -47,7 +55,7 @@ target_link_libraries(cactus_engine PUBLIC cactus_graph)
 
 if(APPLE)
     target_link_libraries(cactus_engine PUBLIC
-        "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders"
+        "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders"
         "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork"
     )
 
@@ -134,6 +142,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     add_library(cactus_engine_shared SHARED ${CMAKE_BINARY_DIR}/cactus_engine_shared_stub.cpp)
     set_target_properties(cactus_engine_shared PROPERTIES OUTPUT_NAME cactus_engine)
     add_dependencies(cactus_engine_shared ${_CACTUS_ENGINE_DEPS})
+    set_property(TARGET cactus_engine_shared APPEND PROPERTY LINK_DEPENDS ${_CACTUS_ENGINE_ARCHIVES})
     target_link_libraries(cactus_engine_shared PRIVATE cactus_flags)
 
     if(APPLE)
@@ -147,7 +156,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
             target_link_libraries(cactus_engine_shared PRIVATE "${_curl}")
         endif()
         target_link_libraries(cactus_engine_shared PRIVATE
-            "-framework Accelerate" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders"
+            "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders"
             "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork"
         )
     else()

cactus-engine/src/complete.cpp

@@ -372,6 +372,13 @@ std::vector<std::vector<uint32_t>> build_stop_sequences(
         stop_token_sequences.push_back(tokenizer->encode(stop_seq));
     }
 
+    if (!tokenizer->has_chat_template()) {
+        stop_token_sequences.push_back(tokenizer->encode("\nUser:"));
+        stop_token_sequences.push_back(tokenizer->encode("\nAssistant:"));
+        stop_token_sequences.push_back(tokenizer->encode("\nUser"));
+        stop_token_sequences.push_back(tokenizer->encode("\nAssistant"));
+    }
+
     if (model_type == Config::ModelType::GEMMA4) {
         stop_token_sequences.push_back(tokenizer->encode("<turn|>"));
         if (has_tools) {
@@ -435,6 +442,7 @@ struct PreparedPrompt {
     std::vector<uint32_t> tokens;
     size_t context_token_count = 0;
     std::vector<std::vector<CactusModelHandle::ProcessedImage>> images;
+    bool force_session_reset = false;
 
     std::vector<std::vector<float>> audio_features;
     size_t audio_num_frames = 0;
@@ -575,7 +583,7 @@ PreparedPrompt prepare_prompt(
         }
     }
 
-    if (prompt.model_type == Config::ModelType::GEMMA4) {
+    if (handle->model->audio_preprocess_strategy() == "gemma4") {
         std::vector<size_t> user_indices;
         for (size_t i = 0; i < prompt.messages.size(); i++) {
             if (prompt.messages[i].role == "user") user_indices.push_back(i);
@@ -629,6 +637,23 @@ PreparedPrompt prepare_prompt(
         setup_tool_constraints(handle, prompt.tools, prompt.options.force_tools, prompt.options.temperature);
     }
 
+    if (tokenizer->uses_default_chat_style(prompt.messages, formatted_tools, prompt.options.enable_thinking_if_supported)) {
+        auto last_user = std::find_if(prompt.messages.rbegin(), prompt.messages.rend(), [](const ChatMessage& msg) {
+            return msg.role == "user";
+        });
+        if (last_user != prompt.messages.rend()) {
+            std::vector<ChatMessage> latest_turn;
+            for (const auto& msg : prompt.messages) {
+                if (msg.role == "system" || msg.role == "developer") {
+                    latest_turn.push_back(msg);
+                }
+            }
+            latest_turn.push_back(*last_user);
+            prompt.messages = std::move(latest_turn);
+            prompt.force_session_reset = true;
+        }
+    }
+
     prompt.rendered = tokenizer->format_chat_prompt(
         prompt.messages,
         add_generation_prompt,
@@ -827,6 +852,15 @@ int cactus_complete(
             handle_error_response(err, response_buffer, buffer_size);
             return -1;
         }
+        const auto runtime_model_type = handle->model->get_config().model_type;
+        if (runtime_model_type == Config::ModelType::WHISPER ||
+            runtime_model_type == Config::ModelType::PARAKEET ||
+            runtime_model_type == Config::ModelType::PARAKEET_TDT) {
+            const std::string err = "Model is speech-to-text; use `cactus transcribe` instead of `cactus run`";
+            CACTUS_LOG_ERROR("complete", err);
+            handle_error_response(err, response_buffer, buffer_size);
+            return -1;
+        }
         auto* tokenizer = handle->model->get_tokenizer();
         auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, true, true, pcm_buffer, pcm_buffer_size);
         if (prompt.options.sample_seed != 0) {
@@ -956,6 +990,10 @@ int cactus_complete(
         uint32_t next_token;
         size_t prompt_tokens;
 
+        if (prompt.force_session_reset && !handle->processed_tokens.empty()) {
+            reset_cache(handle);
+        }
+
         if (has_audio && !handle->processed_tokens.empty()) {
             auto& cache = handle->processed_tokens;
             size_t common = 0;
@@ -979,6 +1017,9 @@ int cactus_complete(
         if (!first_token_from_prefill) {
             auto prefill_result = do_prefill(handle, prompt, prompt.tokens);
             prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size();
+            if (handle->model->repetition_penalty_uses_context()) {
+                handle->model->seed_token_history_from_context(prompt.tokens);
+            }
             next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy);
         }