add tool calling support to run env

README.md

@@ -173,6 +173,14 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max
 - Some models have been pre-uploaded [here](https://huggingface.co/Cactus-Compute), just run `cactus download [HF-Name]`.
 - `cactus run [HF-Name]` albeit first downloads or convert the model if not found. 
 
+## Needle
+
+[Needle](https://github.com/cactus-compute/needle) is a 26m parameter model for on-device tool calling:
+
+```bash
+cactus run Cactus-Compute/needle [--tools my_tools.json]  # OpenAI function-calling format; demo toolset by default
+```
+
 ## Learn More
 
 | Reference | Language | Description |
@@ -221,6 +229,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max
 │    --audio <path>                    audio file for audio chat                 │
 │    --system <prompt>                 system prompt                             │
 │    --prompt <text>                   send prompt immediately                   │
+│    --tools <json|file>               tool definitions for tool calling         │
 │    --thinking                        enable thinking/reasoning mode            │
 │    --token <token>                   HuggingFace token (gated models)          │
 │    --reconvert                       force local rebuild from source           │

cactus-engine/src/complete.cpp

@@ -1160,7 +1160,8 @@ int cactus_complete(
                 : "kept local";
         }
 
-        if (prompt.options.force_tools && !prompt.tools.empty() && primary_function_calls.empty()) {
+        if (prompt.options.force_tools && !prompt.tools.empty() && primary_function_calls.empty() &&
+            !parsed_empty_needle_tool_call) {
             CACTUS_LOG_WARN("force_tools", "force_tools was set but no tool call was parsed from the constrained output");
         }
 

cactus-engine/src/model.cpp

@@ -2446,18 +2446,20 @@ bool Model::prepare_encoder_cross_kv_from_text(const std::vector<uint32_t>& toke
     }
     size_t ids_node = static_cast<size_t>(source_encoder_->runtime_input_node_ids[ids_idx]);
     const auto& ids_desc = source_encoder_->graph->get_output_buffer(ids_node);
-    if (tokens.size() > ids_desc.total_size) {
-        CACTUS_LOG_ERROR("model", "source token count exceeds source encoder capacity");
-        return false;
-    }
-    encoder_cross_kv_source_len_ = tokens.size();
-    for (size_t i = 0; i < tokens.size(); ++i) {
+    size_t count = tokens.size();
+    if (count > ids_desc.total_size) {
+        CACTUS_LOG_WARN("model", "source token count " << count
+            << " exceeds source encoder capacity " << ids_desc.total_size << "; truncating");
+        count = ids_desc.total_size;
+    }
+    encoder_cross_kv_source_len_ = count;
+    for (size_t i = 0; i < count; ++i) {
         write_int_input_at(*source_encoder_, "input_ids", i, static_cast<int64_t>(tokens[i]));
     }
 
     int mask_idx = input_index(*source_encoder_, "attention_mask");
     if (mask_idx >= 0) {
-        for (size_t i = 0; i < tokens.size(); ++i) {
+        for (size_t i = 0; i < count; ++i) {
             write_int_input_at(*source_encoder_, "attention_mask", i, 1);
         }
     }

cactus-engine/tests/run.cpp

@@ -687,6 +687,40 @@ std::string json_string_value(const std::string& json, const std::string& key) {
     return unescape_json(json.substr(start, end - start));
 }
 
+std::string json_array_value(const std::string& json, const std::string& key) {
+    std::string needle = "\"" + key + "\":[";
+    size_t start = json.find(needle);
+    if (start == std::string::npos) return {};
+    start += needle.size() - 1;
+    int depth = 0;
+    bool in_string = false;
+    for (size_t i = start; i < json.size(); ++i) {
+        char c = json[i];
+        if (in_string) {
+            if (c == '\\') ++i;
+            else if (c == '"') in_string = false;
+        } else if (c == '"') {
+            in_string = true;
+        } else if (c == '[') {
+            ++depth;
+        } else if (c == ']' && --depth == 0) {
+            return json.substr(start, i - start + 1);
+        }
+    }
+    return {};
+}
+
+std::vector<std::string> tool_names(const std::string& tools_json) {
+    std::vector<std::string> names;
+    size_t pos = tools_json.find("\"function\":");
+    while (pos != std::string::npos) {
+        std::string name = json_string_value(tools_json.substr(pos), "name");
+        if (!name.empty()) names.push_back(name);
+        pos = tools_json.find("\"function\":", pos + 1);
+    }
+    return names;
+}
+
 double json_number_value(const std::string& json, const std::string& key, double fallback = 0.0) {
     std::string needle = "\"" + key + "\":";
     size_t start = json.find(needle);
@@ -743,7 +777,7 @@ std::string build_messages(const std::string& system_prompt,
 void print_usage(const char* argv0) {
     std::cerr << "Usage: " << argv0
               << " <model_path> [--backend cpu|metal] [--system <prompt>] [--image <path>] [--audio <path>]"
-              << " [--prompt <text>] [--input-ids <ids>] [--input-ids-file <path>] [--max-new-tokens <n>]"
+              << " [--prompt <text>] [--tools <json>] [--input-ids <ids>] [--input-ids-file <path>] [--max-new-tokens <n>]"
               << " [--result-json <path>] [--thinking] [--no-cloud-handoff]"
               << " [--confidence-threshold <value>] [--cloud-timeout-ms <ms>] [-h|--help]\n";
 }
@@ -764,6 +798,7 @@ int main(int argc, char** argv) {
     std::string current_image;
     std::string current_audio;
     std::string initial_prompt;
+    std::string tools_json;
     std::string input_ids;
     std::string input_ids_file;
     std::string result_json;
@@ -800,6 +835,9 @@ int main(int argc, char** argv) {
         } else if (arg == "--prompt") {
             if (!need_value("--prompt")) return 1;
             initial_prompt = argv[++i];
+        } else if (arg == "--tools") {
+            if (!need_value("--tools")) return 1;
+            tools_json = argv[++i];
         } else if (arg == "--input-ids") {
             if (!need_value("--input-ids")) return 1;
             input_ids = argv[++i];
@@ -944,6 +982,18 @@ int main(int argc, char** argv) {
     print_command("exit", "quit");
     std::cout << std::right << "\n";
 
+    if (!tools_json.empty()) {
+        const bool tty = stdout_is_terminal();
+        const char* b = tty ? ansi::bold : "";
+        const char* c = tty ? ansi::cyan : "";
+        const char* r = tty ? ansi::reset : "";
+        std::cout << b << "Tools:" << r;
+        auto names = tool_names(tools_json);
+        if (names.empty()) std::cout << " (none)";
+        for (const auto& name : names) std::cout << " " << c << name << r;
+        std::cout << "\n\n";
+    }
+
     std::vector<ChatTurn> history;
     std::vector<uint8_t> current_pcm;
     TokenPrinter printer;
@@ -1046,6 +1096,7 @@ int main(int argc, char** argv) {
             + ",\"auto_handoff\":" + (auto_handoff ? "true" : "false")
             + ",\"confidence_threshold\":" + std::to_string(confidence_threshold)
             + ",\"cloud_timeout_ms\":" + std::to_string(cloud_timeout_ms)
+            + ",\"tool_rag_top_k\":0"
             + ",\"stop_sequences\":[\"<|im_end|>\",\"<end_of_turn>\",\"<turn|>\"]}";
 
         if (!current_image.empty()) std::cout << "[image: " << current_image << "]\n";
@@ -1063,7 +1114,7 @@ int main(int argc, char** argv) {
                                  response.data(),
                                  response.size(),
                                  options.c_str(),
-                                 nullptr,
+                                 tools_json.empty() ? nullptr : tools_json.c_str(),
                                  token_callback,
                                  nullptr,
                                  current_pcm.empty() ? nullptr : current_pcm.data(),
@@ -1074,6 +1125,18 @@ int main(int argc, char** argv) {
         if (!result_json.empty() && !write_text_file(result_json, response_json)) {
             std::cerr << "Failed to write result JSON: " << result_json << "\n";
         }
+        std::string tool_calls = json_array_value(response_json, "function_calls");
+        const bool tty = stdout_is_terminal();
+        if (tool_calls.size() > 2) {
+            const char* b = tty ? ansi::bold : "";
+            const char* y = tty ? ansi::yellow : "";
+            const char* r = tty ? ansi::reset : "";
+            std::cout << "\n" << b << y << "Tool calls:" << r << " " << y << tool_calls << r << "\n";
+        } else if (!tools_json.empty() && json_string_value(response_json, "response").empty()) {
+            const char* d = tty ? ansi::dim : "";
+            const char* r = tty ? ansi::reset : "";
+            std::cout << "\n" << d << "(no tool call)" << r << "\n";
+        }
         bool cloud_handoff = json_bool_value(response_json, "cloud_handoff");
         double confidence = json_number_value(response_json, "confidence", -1.0);
         double threshold = json_number_value(response_json, "confidence_threshold", -1.0);
@@ -1093,6 +1156,7 @@ int main(int argc, char** argv) {
 
         std::string assistant = json_string_value(response_json, "context_response");
         if (assistant.empty()) assistant = json_string_value(response_json, "response");
+        if (assistant.empty() && tool_calls.size() > 2) assistant = tool_calls;
         history.push_back({"assistant", assistant, "", ""});
         current_image.clear();
         current_audio.clear();

python/cactus/cli/__init__.py

@@ -281,6 +281,10 @@ def create_parser():
                             help="System prompt to prepend to all messages")
     run_parser.add_argument("--prompt",
                             help="Initial prompt to send immediately")
+    run_parser.add_argument("--tools", default=None,
+                            help="Tool definitions in the OpenAI function-calling format, as a JSON "
+                                 "array or a path to a JSON file (tool-call models like needle "
+                                 "default to a demo toolset)")
     run_parser.add_argument("--input-ids", default=None,
                             help="Comma-separated token ids for causal-LM bundles")
     run_parser.add_argument("--input-ids-file", default=None,

python/cactus/cli/run.py

@@ -1,7 +1,61 @@
+import json
 from pathlib import Path
 
 from .common import GREEN, RED, apply_runtime_env, launch_binary, print_color
 
+NEEDLE_DEFAULT_TOOLS = [
+    {"type": "function", "function": {
+        "name": "set_timer",
+        "description": "Set a timer for the specified duration or end time.",
+        "parameters": {"type": "object", "properties": {
+            "time_human": {"type": "string", "description": "The duration or target end time in human readable format e.g. '1 hour and 30 minutes', '45 minutes', 'at 13:30'."},
+        }, "required": ["time_human"]},
+    }},
+    {"type": "function", "function": {
+        "name": "send_email",
+        "description": "Send an email to a recipient.",
+        "parameters": {"type": "object", "properties": {
+            "to": {"type": "string", "description": "The recipient's email address."},
+            "subject": {"type": "string", "description": "The email subject line."},
+            "body": {"type": "string", "description": "The email body text."},
+        }, "required": ["to", "subject", "body"]},
+    }},
+    {"type": "function", "function": {
+        "name": "create_note",
+        "description": "Create a new note with the given text.",
+        "parameters": {"type": "object", "properties": {
+            "text": {"type": "string", "description": "The text of the note."},
+            "title": {"type": "string", "description": "Optional title for the note."},
+        }, "required": ["text"]},
+    }},
+]
+
+
+def resolve_tools(spec):
+    """Validate OpenAI function-calling tools from inline JSON or a file path; return compact JSON."""
+    if not spec.lstrip().startswith(("[", "{")):
+        spec = Path(spec).expanduser().read_text()
+    tools = json.loads(spec)
+    if not isinstance(tools, list):
+        raise ValueError("tools must be a JSON array")
+    for tool in tools:
+        fn = tool.get("function") if isinstance(tool, dict) else None
+        params = (fn.get("parameters") or {}) if isinstance(fn, dict) else {}
+        if not (isinstance(fn, dict) and tool.get("type") == "function" and fn.get("name")
+                and isinstance(params, dict)
+                and (not params or (params.get("type") == "object"
+                                    and isinstance(params.get("properties"), dict)))):
+            raise ValueError(
+                'each tool must be {"type": "function", "function": {"name": ..., "description": ..., '
+                '"parameters": {"type": "object", "properties": {...}, "required": [...]}}}')
+    return json.dumps(tools, separators=(",", ":"), ensure_ascii=False)
+
+
+def bundle_model_type(bundle_dir):
+    from .list import _read_config
+    config = Path(bundle_dir) / "config.txt"
+    return _read_config(config).get("model_type") if config.is_file() else None
+
 
 def cmd_run(args) -> int:
     apply_runtime_env(args)
@@ -18,6 +72,12 @@ def cmd_run(args) -> int:
             return 1
     if args.result_json:
         args.result_json = str(Path(args.result_json).expanduser())
+    if args.tools:
+        try:
+            args.tools = resolve_tools(args.tools)
+        except (ValueError, OSError) as exc:
+            print_color(RED, f"Invalid --tools: {exc}")
+            return 1
 
     from .model import TranspileOptions, prepare_bundle
 
@@ -28,10 +88,15 @@ def cmd_run(args) -> int:
     if bundle_dir is None:
         return 1
 
+    if args.tools is None and bundle_model_type(bundle_dir) == "needle":
+        args.tools = json.dumps(NEEDLE_DEFAULT_TOOLS, separators=(",", ":"))
+        print("Using the demo toolset; pass --tools <file> to use your own.")
+
     cmd = [str(bundle_dir)]
     for flag, value in (
         ("--system", args.system),
         ("--prompt", args.prompt),
+        ("--tools", args.tools),
         ("--image", args.image),
         ("--audio", args.audio),
         ("--input-ids", args.input_ids),

python/tests/test_cli_run.py

@@ -1,7 +1,10 @@
+import json
 from argparse import Namespace
 from pathlib import Path
 from types import SimpleNamespace
 
+import pytest
+
 import cactus.cli.common as common_mod
 import cactus.cli.model as model_mod
 import cactus.cli.run as run_mod
@@ -43,6 +46,7 @@ def fake_subprocess_run(cmd):
         reconvert=False,
         system=None,
         prompt="hi",
+        tools=None,
         image=str(image_file),
         audio=str(audio_file),
         input_ids="1,2,3",
@@ -67,3 +71,72 @@ def fake_subprocess_run(cmd):
     assert cmd[cmd.index("--input-ids-file") + 1] == str(input_ids_file)
     assert cmd[cmd.index("--max-new-tokens") + 1] == "4"
     assert cmd[cmd.index("--result-json") + 1] == str(result_json)
+    assert "--tools" not in cmd
+
+
+def test_resolve_tools_accepts_openai_format(tmp_path: Path) -> None:
+    openai_style = ('[{"type": "function", "function": {"name": "get_weather", "parameters": '
+                    '{"type": "object", "properties": {"location": {"type": "string"}}, '
+                    '"required": ["location"]}}}]')
+    resolved = run_mod.resolve_tools(openai_style)
+    assert json.loads(resolved) == json.loads(openai_style)
+
+    tools_file = tmp_path / "tools.json"
+    tools_file.write_text(openai_style, encoding="utf-8")
+    assert run_mod.resolve_tools(str(tools_file)) == resolved
+
+
+def test_resolve_tools_rejects_improper_schema() -> None:
+    for bad in (
+        '{"type": "function"}',
+        '[{"name": "get_weather", "parameters": {}}]',
+        '[{"type": "function", "function": {"parameters": {}}}]',
+        '[{"type": "function", "function": {"name": "set_timer", "parameters": '
+        '{"time_human": {"type": "string", "required": true}}}}]',
+    ):
+        with pytest.raises(ValueError):
+            run_mod.resolve_tools(bad)
+
+
+def test_cmd_run_defaults_needle_bundle_to_demo_tools(monkeypatch, tmp_path: Path) -> None:
+    bundle_dir = tmp_path / "bundle"
+    (bundle_dir / "components").mkdir(parents=True)
+    (bundle_dir / "components" / "manifest.json").write_text("{}", encoding="utf-8")
+    (bundle_dir / "config.txt").write_text("model_type=needle\n", encoding="utf-8")
+
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir(parents=True)
+    (fake_bin / "run").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+    monkeypatch.setattr(common_mod, "BIN_DIR", fake_bin)
+
+    calls = []
+    monkeypatch.setattr(common_mod.subprocess, "run",
+                        lambda cmd: calls.append(cmd) or SimpleNamespace(returncode=0))
+    monkeypatch.setattr(model_mod, "ensure_runnable_bundle", lambda *a, **k: bundle_dir)
+
+    args = Namespace(
+        no_cloud_tele=False,
+        model_id="Cactus-Compute/needle",
+        bits=4,
+        token=None,
+        reconvert=False,
+        system=None,
+        prompt=None,
+        tools=None,
+        image=None,
+        audio=None,
+        input_ids=None,
+        input_ids_file=None,
+        max_new_tokens=None,
+        result_json=None,
+        confidence_threshold=None,
+        cloud_timeout_ms=None,
+        backend=None,
+        thinking=False,
+        no_cloud_handoff=False,
+    )
+
+    assert run_mod.cmd_run(args) == 0
+    cmd = calls[0]
+    tools = cmd[cmd.index("--tools") + 1]
+    assert '"name":"set_timer"' in tools and '"type":"function"' in tools