diff --git a/README.md b/README.md index 00d2c928e..8fa09065a 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,14 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max - Some models have been pre-uploaded [here](https://huggingface.co/Cactus-Compute), just run `cactus download [HF-Name]`. - `cactus run [HF-Name]` albeit first downloads or convert the model if not found. +## Needle + +[Needle](https://github.com/cactus-compute/needle) is a 26m parameter model for on-device tool calling: + +```bash +cactus run Cactus-Compute/needle [--tools my_tools.json] # OpenAI function-calling format; demo toolset by default +``` + ## Learn More | Reference | Language | Description | @@ -221,6 +229,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max │ --audio audio file for audio chat │ │ --system system prompt │ │ --prompt send prompt immediately │ +│ --tools tool definitions for tool calling │ │ --thinking enable thinking/reasoning mode │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 0821cf9bb..e8fe2e33d 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -1160,7 +1160,8 @@ int cactus_complete( : "kept local"; } - if (prompt.options.force_tools && !prompt.tools.empty() && primary_function_calls.empty()) { + if (prompt.options.force_tools && !prompt.tools.empty() && primary_function_calls.empty() && + !parsed_empty_needle_tool_call) { CACTUS_LOG_WARN("force_tools", "force_tools was set but no tool call was parsed from the constrained output"); } diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 8cbb7c1dc..15788bb43 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -2446,18 +2446,20 @@ bool Model::prepare_encoder_cross_kv_from_text(const std::vector& toke } size_t ids_node = static_cast(source_encoder_->runtime_input_node_ids[ids_idx]); const auto& ids_desc = source_encoder_->graph->get_output_buffer(ids_node); - if (tokens.size() > ids_desc.total_size) { - CACTUS_LOG_ERROR("model", "source token count exceeds source encoder capacity"); - return false; - } - encoder_cross_kv_source_len_ = tokens.size(); - for (size_t i = 0; i < tokens.size(); ++i) { + size_t count = tokens.size(); + if (count > ids_desc.total_size) { + CACTUS_LOG_WARN("model", "source token count " << count + << " exceeds source encoder capacity " << ids_desc.total_size << "; truncating"); + count = ids_desc.total_size; + } + encoder_cross_kv_source_len_ = count; + for (size_t i = 0; i < count; ++i) { write_int_input_at(*source_encoder_, "input_ids", i, static_cast(tokens[i])); } int mask_idx = input_index(*source_encoder_, "attention_mask"); if (mask_idx >= 0) { - for (size_t i = 0; i < tokens.size(); ++i) { + for (size_t i = 0; i < count; ++i) { write_int_input_at(*source_encoder_, "attention_mask", i, 1); } } diff --git a/cactus-engine/tests/run.cpp b/cactus-engine/tests/run.cpp index 6e8f520b2..d3de3e731 100644 --- a/cactus-engine/tests/run.cpp +++ b/cactus-engine/tests/run.cpp @@ -687,6 +687,40 @@ std::string json_string_value(const std::string& json, const std::string& key) { return unescape_json(json.substr(start, end - start)); } +std::string json_array_value(const std::string& json, const std::string& key) { + std::string needle = "\"" + key + "\":["; + size_t start = json.find(needle); + if (start == std::string::npos) return {}; + start += needle.size() - 1; + int depth = 0; + bool in_string = false; + for (size_t i = start; i < json.size(); ++i) { + char c = json[i]; + if (in_string) { + if (c == '\\') ++i; + else if (c == '"') in_string = false; + } else if (c == '"') { + in_string = true; + } else if (c == '[') { + ++depth; + } else if (c == ']' && --depth == 0) { + return json.substr(start, i - start + 1); + } + } + return {}; +} + +std::vector tool_names(const std::string& tools_json) { + std::vector names; + size_t pos = tools_json.find("\"function\":"); + while (pos != std::string::npos) { + std::string name = json_string_value(tools_json.substr(pos), "name"); + if (!name.empty()) names.push_back(name); + pos = tools_json.find("\"function\":", pos + 1); + } + return names; +} + double json_number_value(const std::string& json, const std::string& key, double fallback = 0.0) { std::string needle = "\"" + key + "\":"; size_t start = json.find(needle); @@ -743,7 +777,7 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 << " [--backend cpu|metal] [--system ] [--image ] [--audio ]" - << " [--prompt ] [--input-ids ] [--input-ids-file ] [--max-new-tokens ]" + << " [--prompt ] [--tools ] [--input-ids ] [--input-ids-file ] [--max-new-tokens ]" << " [--result-json ] [--thinking] [--no-cloud-handoff]" << " [--confidence-threshold ] [--cloud-timeout-ms ] [-h|--help]\n"; } @@ -764,6 +798,7 @@ int main(int argc, char** argv) { std::string current_image; std::string current_audio; std::string initial_prompt; + std::string tools_json; std::string input_ids; std::string input_ids_file; std::string result_json; @@ -800,6 +835,9 @@ int main(int argc, char** argv) { } else if (arg == "--prompt") { if (!need_value("--prompt")) return 1; initial_prompt = argv[++i]; + } else if (arg == "--tools") { + if (!need_value("--tools")) return 1; + tools_json = argv[++i]; } else if (arg == "--input-ids") { if (!need_value("--input-ids")) return 1; input_ids = argv[++i]; @@ -944,6 +982,18 @@ int main(int argc, char** argv) { print_command("exit", "quit"); std::cout << std::right << "\n"; + if (!tools_json.empty()) { + const bool tty = stdout_is_terminal(); + const char* b = tty ? ansi::bold : ""; + const char* c = tty ? ansi::cyan : ""; + const char* r = tty ? ansi::reset : ""; + std::cout << b << "Tools:" << r; + auto names = tool_names(tools_json); + if (names.empty()) std::cout << " (none)"; + for (const auto& name : names) std::cout << " " << c << name << r; + std::cout << "\n\n"; + } + std::vector history; std::vector current_pcm; TokenPrinter printer; @@ -1046,6 +1096,7 @@ int main(int argc, char** argv) { + ",\"auto_handoff\":" + (auto_handoff ? "true" : "false") + ",\"confidence_threshold\":" + std::to_string(confidence_threshold) + ",\"cloud_timeout_ms\":" + std::to_string(cloud_timeout_ms) + + ",\"tool_rag_top_k\":0" + ",\"stop_sequences\":[\"<|im_end|>\",\"\",\"\"]}"; if (!current_image.empty()) std::cout << "[image: " << current_image << "]\n"; @@ -1063,7 +1114,7 @@ int main(int argc, char** argv) { response.data(), response.size(), options.c_str(), - nullptr, + tools_json.empty() ? nullptr : tools_json.c_str(), token_callback, nullptr, current_pcm.empty() ? nullptr : current_pcm.data(), @@ -1074,6 +1125,18 @@ int main(int argc, char** argv) { if (!result_json.empty() && !write_text_file(result_json, response_json)) { std::cerr << "Failed to write result JSON: " << result_json << "\n"; } + std::string tool_calls = json_array_value(response_json, "function_calls"); + const bool tty = stdout_is_terminal(); + if (tool_calls.size() > 2) { + const char* b = tty ? ansi::bold : ""; + const char* y = tty ? ansi::yellow : ""; + const char* r = tty ? ansi::reset : ""; + std::cout << "\n" << b << y << "Tool calls:" << r << " " << y << tool_calls << r << "\n"; + } else if (!tools_json.empty() && json_string_value(response_json, "response").empty()) { + const char* d = tty ? ansi::dim : ""; + const char* r = tty ? ansi::reset : ""; + std::cout << "\n" << d << "(no tool call)" << r << "\n"; + } bool cloud_handoff = json_bool_value(response_json, "cloud_handoff"); double confidence = json_number_value(response_json, "confidence", -1.0); double threshold = json_number_value(response_json, "confidence_threshold", -1.0); @@ -1093,6 +1156,7 @@ int main(int argc, char** argv) { std::string assistant = json_string_value(response_json, "context_response"); if (assistant.empty()) assistant = json_string_value(response_json, "response"); + if (assistant.empty() && tool_calls.size() > 2) assistant = tool_calls; history.push_back({"assistant", assistant, "", ""}); current_image.clear(); current_audio.clear(); diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index a308d13c3..7ba4ed2bb 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -281,6 +281,10 @@ def create_parser(): help="System prompt to prepend to all messages") run_parser.add_argument("--prompt", help="Initial prompt to send immediately") + run_parser.add_argument("--tools", default=None, + help="Tool definitions in the OpenAI function-calling format, as a JSON " + "array or a path to a JSON file (tool-call models like needle " + "default to a demo toolset)") run_parser.add_argument("--input-ids", default=None, help="Comma-separated token ids for causal-LM bundles") run_parser.add_argument("--input-ids-file", default=None, diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 38078a3a8..338ae21a1 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -1,7 +1,61 @@ +import json from pathlib import Path from .common import GREEN, RED, apply_runtime_env, launch_binary, print_color +NEEDLE_DEFAULT_TOOLS = [ + {"type": "function", "function": { + "name": "set_timer", + "description": "Set a timer for the specified duration or end time.", + "parameters": {"type": "object", "properties": { + "time_human": {"type": "string", "description": "The duration or target end time in human readable format e.g. '1 hour and 30 minutes', '45 minutes', 'at 13:30'."}, + }, "required": ["time_human"]}, + }}, + {"type": "function", "function": { + "name": "send_email", + "description": "Send an email to a recipient.", + "parameters": {"type": "object", "properties": { + "to": {"type": "string", "description": "The recipient's email address."}, + "subject": {"type": "string", "description": "The email subject line."}, + "body": {"type": "string", "description": "The email body text."}, + }, "required": ["to", "subject", "body"]}, + }}, + {"type": "function", "function": { + "name": "create_note", + "description": "Create a new note with the given text.", + "parameters": {"type": "object", "properties": { + "text": {"type": "string", "description": "The text of the note."}, + "title": {"type": "string", "description": "Optional title for the note."}, + }, "required": ["text"]}, + }}, +] + + +def resolve_tools(spec): + """Validate OpenAI function-calling tools from inline JSON or a file path; return compact JSON.""" + if not spec.lstrip().startswith(("[", "{")): + spec = Path(spec).expanduser().read_text() + tools = json.loads(spec) + if not isinstance(tools, list): + raise ValueError("tools must be a JSON array") + for tool in tools: + fn = tool.get("function") if isinstance(tool, dict) else None + params = (fn.get("parameters") or {}) if isinstance(fn, dict) else {} + if not (isinstance(fn, dict) and tool.get("type") == "function" and fn.get("name") + and isinstance(params, dict) + and (not params or (params.get("type") == "object" + and isinstance(params.get("properties"), dict)))): + raise ValueError( + 'each tool must be {"type": "function", "function": {"name": ..., "description": ..., ' + '"parameters": {"type": "object", "properties": {...}, "required": [...]}}}') + return json.dumps(tools, separators=(",", ":"), ensure_ascii=False) + + +def bundle_model_type(bundle_dir): + from .list import _read_config + config = Path(bundle_dir) / "config.txt" + return _read_config(config).get("model_type") if config.is_file() else None + def cmd_run(args) -> int: apply_runtime_env(args) @@ -18,6 +72,12 @@ def cmd_run(args) -> int: return 1 if args.result_json: args.result_json = str(Path(args.result_json).expanduser()) + if args.tools: + try: + args.tools = resolve_tools(args.tools) + except (ValueError, OSError) as exc: + print_color(RED, f"Invalid --tools: {exc}") + return 1 from .model import TranspileOptions, prepare_bundle @@ -28,10 +88,15 @@ def cmd_run(args) -> int: if bundle_dir is None: return 1 + if args.tools is None and bundle_model_type(bundle_dir) == "needle": + args.tools = json.dumps(NEEDLE_DEFAULT_TOOLS, separators=(",", ":")) + print("Using the demo toolset; pass --tools to use your own.") + cmd = [str(bundle_dir)] for flag, value in ( ("--system", args.system), ("--prompt", args.prompt), + ("--tools", args.tools), ("--image", args.image), ("--audio", args.audio), ("--input-ids", args.input_ids), diff --git a/python/tests/test_cli_run.py b/python/tests/test_cli_run.py index 15b976b40..4e8f983ad 100644 --- a/python/tests/test_cli_run.py +++ b/python/tests/test_cli_run.py @@ -1,7 +1,10 @@ +import json from argparse import Namespace from pathlib import Path from types import SimpleNamespace +import pytest + import cactus.cli.common as common_mod import cactus.cli.model as model_mod import cactus.cli.run as run_mod @@ -43,6 +46,7 @@ def fake_subprocess_run(cmd): reconvert=False, system=None, prompt="hi", + tools=None, image=str(image_file), audio=str(audio_file), input_ids="1,2,3", @@ -67,3 +71,72 @@ def fake_subprocess_run(cmd): assert cmd[cmd.index("--input-ids-file") + 1] == str(input_ids_file) assert cmd[cmd.index("--max-new-tokens") + 1] == "4" assert cmd[cmd.index("--result-json") + 1] == str(result_json) + assert "--tools" not in cmd + + +def test_resolve_tools_accepts_openai_format(tmp_path: Path) -> None: + openai_style = ('[{"type": "function", "function": {"name": "get_weather", "parameters": ' + '{"type": "object", "properties": {"location": {"type": "string"}}, ' + '"required": ["location"]}}}]') + resolved = run_mod.resolve_tools(openai_style) + assert json.loads(resolved) == json.loads(openai_style) + + tools_file = tmp_path / "tools.json" + tools_file.write_text(openai_style, encoding="utf-8") + assert run_mod.resolve_tools(str(tools_file)) == resolved + + +def test_resolve_tools_rejects_improper_schema() -> None: + for bad in ( + '{"type": "function"}', + '[{"name": "get_weather", "parameters": {}}]', + '[{"type": "function", "function": {"parameters": {}}}]', + '[{"type": "function", "function": {"name": "set_timer", "parameters": ' + '{"time_human": {"type": "string", "required": true}}}}]', + ): + with pytest.raises(ValueError): + run_mod.resolve_tools(bad) + + +def test_cmd_run_defaults_needle_bundle_to_demo_tools(monkeypatch, tmp_path: Path) -> None: + bundle_dir = tmp_path / "bundle" + (bundle_dir / "components").mkdir(parents=True) + (bundle_dir / "components" / "manifest.json").write_text("{}", encoding="utf-8") + (bundle_dir / "config.txt").write_text("model_type=needle\n", encoding="utf-8") + + fake_bin = tmp_path / "bin" + fake_bin.mkdir(parents=True) + (fake_bin / "run").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + monkeypatch.setattr(common_mod, "BIN_DIR", fake_bin) + + calls = [] + monkeypatch.setattr(common_mod.subprocess, "run", + lambda cmd: calls.append(cmd) or SimpleNamespace(returncode=0)) + monkeypatch.setattr(model_mod, "ensure_runnable_bundle", lambda *a, **k: bundle_dir) + + args = Namespace( + no_cloud_tele=False, + model_id="Cactus-Compute/needle", + bits=4, + token=None, + reconvert=False, + system=None, + prompt=None, + tools=None, + image=None, + audio=None, + input_ids=None, + input_ids_file=None, + max_new_tokens=None, + result_json=None, + confidence_threshold=None, + cloud_timeout_ms=None, + backend=None, + thinking=False, + no_cloud_handoff=False, + ) + + assert run_mod.cmd_run(args) == 0 + cmd = calls[0] + tools = cmd[cmd.index("--tools") + 1] + assert '"name":"set_timer"' in tools and '"type":"function"' in tools