From 7f5cd2fde70fc7967e3fddc3faf71a1e18255834 Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 18 Jun 2026 17:11:13 +0200 Subject: [PATCH 1/5] abc Signed-off-by: jakmro --- .github/workflows/release.yml | 2 +- README.md | 22 +- bindings/python/README.md | 2 +- ..._stream.cpp => test_stream_transcribe.cpp} | 9 +- docs/cactus_engine.md | 2 +- docs/cactus_quants.md | 24 +- docs/choose-bindings.md | 2 +- docs/finetuning.md | 5 - python/cactus/cli/__init__.py | 118 +++--- python/cactus/cli/clean.py | 1 - python/cactus/cli/common.py | 4 +- python/cactus/cli/compile.py | 1 - python/cactus/cli/convert.py | 28 +- python/cactus/cli/download.py | 24 +- python/cactus/cli/list.py | 4 +- python/cactus/cli/model.py | 51 ++- python/cactus/cli/run.py | 23 +- python/cactus/cli/runtime.py | 2 +- python/cactus/cli/serve.py | 18 +- python/cactus/cli/test.py | 10 +- python/cactus/cli/transcribe.py | 21 +- python/cactus/cli/transpile.py | 4 - python/cactus/cli/upload.py | 53 +++ python/cactus/cli/utils.py | 6 +- .../convert/cactus_adapters/config_utils.py | 2 - .../convert/cactus_adapters/tensor_io.py | 2 - .../convert/cactus_adapters/tokenizer.py | 2 +- python/cactus/convert/calibration/hessian.py | 1 - python/cactus/convert/export/reports.py | 2 +- python/cactus/server.py | 38 +- python/cactus/transpile/README.md | 14 +- python/cactus/transpile/hf_model.py | 351 ------------------ python/cactus/transpile/importers.py | 1 - python/cactus/transpile/model_adapters.py | 14 +- python/cactus/transpile/multimodal_runtime.py | 1 - python/cactus/transpile/npu/README.md | 4 +- .../transpile/npu/coremltools_patches.py | 2 - scripts/upload_weights.py | 86 ----- 38 files changed, 270 insertions(+), 686 deletions(-) rename cactus-engine/tests/{test_stream.cpp => test_stream_transcribe.cpp} (94%) create mode 100644 python/cactus/cli/upload.py delete mode 100644 scripts/upload_weights.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 325e2db38..22960b595 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,7 @@ jobs: license "Cactus Compute License" include Language::Python::Virtualenv depends_on "python@3.12" - depends_on macos: :ventura + depends_on macos: :sonoma depends_on arch: :arm64 livecheck do diff --git a/README.md b/README.md index 802b2606f..64480e7c8 100644 --- a/README.md +++ b/README.md @@ -196,37 +196,35 @@ graph.hard_reset(); │ │ │ cactus run [model|path] run a model (downloads if needed) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target accelerator (default: auto) │ +│ --platform auto|cpu|apple target platform (default: auto) │ │ --image image file for VLM inference │ │ --audio audio file for audio chat │ │ --system system prompt │ │ --prompt send prompt immediately │ │ --thinking enable thinking/reasoning mode │ │ --token HuggingFace token (gated models) │ -│ --reconvert force local convert+transpile fallback │ +│ --reconvert force local rebuild from source │ │ │ │ cactus transcribe [model] live microphone transcription with a model│ │ --file audio file to transcribe (WAV) │ │ --language language code (default: en) │ +│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --platform auto|cpu|apple target platform (default: auto) │ │ --token HuggingFace token (gated models) │ -│ --reconvert force reconversion from source │ +│ --reconvert force local rebuild from source │ │ │ -│ cactus download [model] download a pre-built bundle │ +│ cactus download [model] get a bundle (prebuilt, else build) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target accelerator (default: auto) │ +│ --platform auto|cpu|apple target platform (default: auto) │ │ --token HuggingFace token │ │ │ -│ cactus convert [dir] convert HuggingFace weights to CQ │ +│ cactus convert [dir] build a runnable bundle locally │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --platform auto|cpu|apple target platform (default: auto) │ │ --token HuggingFace token │ -│ --reconvert force build from source │ +│ --reconvert force rebuild from source │ │ --lora merge a LoRA adapter before converting │ │ │ -│ cactus transpile build a runnable bundle from CQ weights │ -│ --weights-dir path to CQ weights (default: lookup) │ -│ --task force task type (default: auto) │ -│ --artifact-dir bundle output (default: weights/) │ -│ │ │ cactus serve [model] OpenAI-compatible local HTTP server │ │ --host bind address (default: 127.0.0.1) │ │ --port port (default: 8080) │ diff --git a/bindings/python/README.md b/bindings/python/README.md index fbb1f5fa2..1963c98ed 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -6,5 +6,5 @@ ctypes FFI to `cactus_engine.h`. The binding module lives at It loads `libcactus_engine.{so,dylib}` from `/python/cactus/bindings/lib/` (bundled) or `cactus-engine/build/` (dev build), populated by `cactus build --python`. -For installation, the high-level API, the CLI, the server, and the transpiler, +For installation, the high-level API, the CLI, and the server, see [`/python/README.md`](/python/). diff --git a/cactus-engine/tests/test_stream.cpp b/cactus-engine/tests/test_stream_transcribe.cpp similarity index 94% rename from cactus-engine/tests/test_stream.cpp rename to cactus-engine/tests/test_stream_transcribe.cpp index 3b83b3711..70ddc1c63 100644 --- a/cactus-engine/tests/test_stream.cpp +++ b/cactus-engine/tests/test_stream_transcribe.cpp @@ -104,7 +104,8 @@ static bool test_stream_matches_oneshot() { std::vector pcm = load_wav(std::string(g_assets) + "/test.wav"); if (pcm.size() > 20 * 16000) pcm.resize(20 * 16000); - auto golden = words_of(transcribe_oneshot(model, pcm)); + std::string oneshot_text = transcribe_oneshot(model, pcm); + auto golden = words_of(oneshot_text); if (golden.size() < 4) { cactus_destroy(model); std::cout << " SKIP (test.wav has no usable speech)\n"; return true; } cactus_stream_transcribe_t stream = cactus_stream_transcribe_start(model, nullptr); @@ -135,6 +136,8 @@ static bool test_stream_matches_oneshot() { cactus_destroy(model); auto sw = words_of(streamed); + std::cout << " ONE-SHOT: " << oneshot_text << "\n"; + std::cout << " STREAM: " << streamed << "\n"; double rc = recall(golden, sw), pr = recall(sw, golden); bool ok = rc >= 0.85 && pr >= 0.85; std::cout << " recall=" << std::fixed << std::setprecision(3) << rc << " precision=" << pr @@ -143,8 +146,8 @@ static bool test_stream_matches_oneshot() { } int main() { - TestUtils::TestRunner runner("Stream Tests"); - runner.run_test("stream_matches_oneshot", test_stream_matches_oneshot()); + TestUtils::TestRunner runner("Stream Transcribe Tests"); + runner.run_test("stream_transcribe_matches_oneshot", test_stream_matches_oneshot()); runner.print_summary(); return runner.all_passed() ? 0 : 1; } diff --git a/docs/cactus_engine.md b/docs/cactus_engine.md index 0b8e851e8..46f78d142 100644 --- a/docs/cactus_engine.md +++ b/docs/cactus_engine.md @@ -30,7 +30,7 @@ best build for your host, e.g. Apple Silicon on macOS); pass `--platform cpu` for the generic build. The result can be loaded directly via `cactus_init()`. For models not on Cactus-Compute, build a bundle from source with -`cactus convert ` followed by `cactus transpile `. +`cactus convert `. ## Types diff --git a/docs/cactus_quants.md b/docs/cactus_quants.md index 83ec3b424..d84c51f2a 100644 --- a/docs/cactus_quants.md +++ b/docs/cactus_quants.md @@ -183,28 +183,18 @@ Three patterns are visible: ## Using CQ Weights -CQ weights are produced by `cactus convert`. The runtime bundle (graph + manifest) -is then built from those weights with `cactus transpile`, and run via -`cactus run`: +`cactus convert ` builds a runnable bundle locally; `cactus run` runs a +bundle path or a model id. ```bash -# 1. Convert at 4-bit (default) -cactus convert google/gemma-4-E2B-it ./gemma4-weights - -# 1'. Convert at 2-bit -cactus convert google/gemma-4-E2B-it ./gemma4-weights --bits 2 - -# 2. Build the runtime bundle from the CQ weights -cactus transpile google/gemma-4-E2B-it \ - --weights-dir ./gemma4-weights --artifact-dir ./gemma4-bundle - -# 3. Run (pass either a HF model id or a bundle path) +cactus convert google/gemma-4-E2B-it ./gemma4-bundle +cactus convert google/gemma-4-E2B-it ./gemma4-bundle --bits 2 cactus run ./gemma4-bundle ``` -For models on huggingface.co/Cactus-Compute, you can skip the convert+transpile -steps entirely: `cactus run ` will fetch the pre-built bundle (or fall -back to local convert+transpile if no bundle is published), then run. +For models on huggingface.co/Cactus-Compute, `cactus run ` (or `cactus +download `) fetches the pre-built bundle, building locally if none is +published. The CQ matmul kernels live in `cactus-kernels/src/matmul.cpp`. At inference, the kernel applies a Walsh-Hadamard transform to the FP16 activation (not the weight), diff --git a/docs/choose-bindings.md b/docs/choose-bindings.md index edb5f2526..b37769035 100644 --- a/docs/choose-bindings.md +++ b/docs/choose-bindings.md @@ -23,7 +23,7 @@ Pick the right one for your platform and use case: - **React Native** -- already have a React Native project? Just `npm install` and go - **Flutter** -- cross-platform mobile + Mac with full native bindings - **Kotlin** -- native Android apps or Kotlin Multiplatform - - **Swift** -- native iOS/macOS apps with Apple NPU acceleration + - **Swift** -- native iOS/macOS apps with the Apple Neural Engine !!! tip "Server-side or scripting?" - **Python** -- server-side inference, batch processing, or rapid prototyping diff --git a/docs/finetuning.md b/docs/finetuning.md index 2e846a0b4..ff1ae7fb4 100644 --- a/docs/finetuning.md +++ b/docs/finetuning.md @@ -67,12 +67,7 @@ cactus convert Qwen/Qwen3-0.6B ./my-qwen3-0.6b --lora username/my-lora-adapter ### 4. Run -Build the native library, transpile the weights into a runtime bundle, then run: - ```bash -cactus build -cactus transpile Qwen/Qwen3-0.6B \ - --weights-dir ./my-qwen3-0.6b --artifact-dir ./my-qwen3-0.6b cactus run ./my-qwen3-0.6b ``` diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index c0bc5ba34..956e9fa53 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -13,7 +13,7 @@ _PLATFORM_CHOICES = ("auto", "cpu", *SUPPORTED_PLATFORMS) _PLATFORM_HELP = ( - f"target accelerator: auto = best for this host, e.g. apple on macOS (default); " + f"target platform: auto = best for this host, e.g. apple on macOS (default); " f"cpu = generic ARM; " f"or one of: {', '.join(SUPPORTED_PLATFORMS) if SUPPORTED_PLATFORMS else '(none yet)'}" ) @@ -23,6 +23,7 @@ from .transcribe import cmd_transcribe from .test import cmd_test, COMPONENTS from .convert import cmd_convert, cmd_transpile +from .upload import cmd_upload from .run import cmd_run from .list import cmd_list @@ -96,33 +97,43 @@ def create_parser(): cactus run [model|path] run a model (default: {DEFAULT_MODEL_ID}) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target accelerator (default: auto) + --platform {_PLATFORM_PIPE:<22} target platform (default: auto) --image image file for VLM inference --audio audio file for audio chat --system system prompt --prompt send prompt immediately --thinking enable thinking/reasoning mode --token HuggingFace token (gated models) - --reconvert force local convert+transpile fallback + --reconvert force local rebuild from source cactus transcribe [model] live microphone transcription with a model --file audio file to transcribe (WAV) --language language code (default: en) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target accelerator (default: auto) + --platform {_PLATFORM_PIPE:<22} target platform (default: auto) --token HuggingFace token (gated models) --reconvert force reconversion from source cactus download [model] fetch a prebuilt bundle, else build locally (default: {DEFAULT_MODEL_ID}) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target accelerator (default: auto) - --token HuggingFace token + --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --token HuggingFace token (gated models) + --reconvert force local rebuild from source + + cactus convert [dir] build a runnable bundle locally (skips prebuilt fetch) + --bits 1|2|3|4 CQ quantization (default: 4) + --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --token HuggingFace token (gated models) + --reconvert force local rebuild from source + --lora merge a LoRA adapter before converting cactus serve [model] OpenAI-compatible local HTTP server --host bind address (default: 127.0.0.1) --port port (default: 8080) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target accelerator (default: auto) + --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --token HuggingFace token (gated models) + --reconvert force local rebuild from source cactus list list local converted weights and bundles @@ -136,6 +147,7 @@ def create_parser(): (default: all) --model default: {DEFAULT_TEST_MODEL_ID} --transcription-model default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID} + --reconvert force local rebuild of test models --suite run a single test suite from any component (kernels, graph, or engine) --list list components and suites @@ -144,51 +156,8 @@ def create_parser(): --enable-telemetry send cloud telemetry (off by default) cactus clean delete build artifacts, weights, venv - cactus --help show this help - ----------------------------------------------------------------- - - Advanced / build pipeline — the two manual steps below run - automatically inside run, serve, transcribe and download; reach for - them only to control the build (custom flags, LoRA, NPU, debugging): - - cactus convert [dir] convert HuggingFace weights to CQ - --bits 1|2|3|4 CQ quantization (default: 4) - --token HuggingFace token - --reconvert force build from source - --lora merge a LoRA adapter before converting - - cactus transpile build a runnable bundle from CQ weights - --weights-dir path to CQ weights (default: weights/) - --task task (default: auto, inferred from model config) - --artifact-dir write bundle here (default: weights/) - --prompt representative prompt for shape capture - --system-prompt system prompt for multimodal chat - --enable-thinking enable thinking markers when supported - --input-ids token ids for causal-LM shape capture - --image-file representative image (repeatable) - --audio-file representative audio file (WAV) - --max-new-tokens preallocate decode context for causal LM - --component-pipeline auto|on|off force component-pipeline transpilation - --components subset of components to transpile - --torch-dtype float16 | float32 | bfloat16 - --token HuggingFace token (defaults to $HF_TOKEN) - --trust-remote-code allow HF remote code at transpile time - --local-files-only require model/processor to be local - --allow-unconverted-weights debug-only: skip the CQ-weights check - --execute-after-transpile run a reference execution after lowering - --graph-filename override saved graph filename - --skip-reference-compare skip PyTorch comparison (with --execute-…) - --no-fuse-rms-norm disable RMSNorm fusion - --no-fuse-rope disable RoPE fusion - --no-fuse-attention disable attention fusion - --no-fuse-attention-block disable attention-block fusion - --no-fuse-add-clipped disable add-clipped fusion - --no-fuse-gated-deltanet disable gated DeltaNet fusion - --npu also emit CoreML .mlpackage(s) for NPU - --npu-quantize 0|4|8 force both NPU encoders to this quant - --npu-audio-quantize 0|4|8 audio encoder quant (default int8) - --npu-vision-quantize 0|4|8 vision encoder quant (default fp16) + cactus --help show this help ----------------------------------------------------------------- """ @@ -215,6 +184,8 @@ def create_parser(): download_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", help=_PLATFORM_HELP) download_parser.add_argument("--token", help="HuggingFace token") + download_parser.add_argument("--reconvert", action="store_true", + help="Force local rebuild from source instead of fetching the prebuilt bundle") build_parser = subparsers.add_parser("build", help="Build cactus libraries") build_group = build_parser.add_mutually_exclusive_group() @@ -236,7 +207,7 @@ def create_parser(): help=_PLATFORM_HELP) run_parser.add_argument("--token", help="HuggingFace token") run_parser.add_argument("--reconvert", action="store_true", - help="Force local convert+transpile fallback") + help="Force local rebuild from source") run_parser.add_argument("--image", help="Path to image file for VLM inference (attached to first message)") run_parser.add_argument("--audio", @@ -246,13 +217,13 @@ def create_parser(): run_parser.add_argument("--prompt", help="Initial prompt to send immediately") run_parser.add_argument("--input-ids", default=None, - help="Comma-separated token ids for transpiled causal-LM bundles") + help="Comma-separated token ids for causal-LM bundles") run_parser.add_argument("--input-ids-file", default=None, - help="File containing token ids for transpiled causal-LM bundles") + help="File containing token ids for causal-LM bundles") run_parser.add_argument("--max-new-tokens", type=_positive_int, default=None, - help="Maximum tokens to generate for transpiled causal-LM bundles") + help="Maximum tokens to generate for causal-LM bundles") run_parser.add_argument("--result-json", default=None, - help="Optional path to save transpiled bundle results as JSON") + help="Optional path to save bundle results as JSON") run_parser.add_argument("--thinking", action="store_true", help="Enable thinking/reasoning for models that support it") run_parser.add_argument("--no-cloud-handoff", action="store_true", @@ -274,15 +245,13 @@ def create_parser(): transcribe_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, help="CQ quantization (default: 4)") transcribe_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help="Target accelerator (default: auto)") + help=_PLATFORM_HELP) transcribe_parser.add_argument("--token", help="HuggingFace token") - transcribe_parser.add_argument("--force-handoff", action="store_true", - help=argparse.SUPPRESS) transcribe_parser.add_argument("--reconvert", action="store_true", - help="Force reconversion from source") + help="Force local rebuild from source") serve_parser = subparsers.add_parser("serve", help="OpenAI-compatible local HTTP server") - serve_parser.add_argument("model", nargs="?", default=None, + serve_parser.add_argument("model_id", nargs="?", default=None, type=_hf_id_or_path, help="HuggingFace model id (e.g. openai/whisper-base) or bundle path") serve_parser.add_argument("--host", default="127.0.0.1", @@ -292,7 +261,16 @@ def create_parser(): serve_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, help="CQ quantization (default: 4)") serve_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help="Target accelerator (default: auto)") + help=_PLATFORM_HELP) + serve_parser.add_argument("--token", help="HuggingFace token") + serve_parser.add_argument("--reconvert", action="store_true", + help="Force local rebuild from source") + serve_parser.add_argument("--no-cloud-handoff", action="store_true", + help="Disable automatic cloud handoff for all requests") + serve_parser.add_argument("--confidence-threshold", type=_unit_float, default=None, + help="Confidence threshold below which completions hand off to cloud (1.0 forces cloud handoff)") + serve_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, + help="Maximum time to wait for cloud handoff before falling back locally") test_parser = subparsers.add_parser("test", help="Run the test suite") test_parser.add_argument("--component", choices=COMPONENTS, default="all", @@ -303,6 +281,8 @@ def create_parser(): test_parser.add_argument("--transcription-model", dest="transcription_model_id", default=None, type=_hf_id_or_path, help=f"HF transcription model ID under test (default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID})") + test_parser.add_argument("--reconvert", action="store_true", + help="Force local rebuild of the test models from source") test_parser.add_argument("--suite", default=None, help="Run a single test suite by name; resolved across all components (e.g. llm → engine)") test_parser.add_argument("--list", action="store_true", @@ -324,18 +304,27 @@ def create_parser(): subparsers.add_parser("list", help="List local converted weights and bundles") convert_parser = subparsers.add_parser("convert", - help="Convert HuggingFace weights to CQ format") + help="Build a runnable bundle locally from source (no prebuilt fetch)") convert_parser.add_argument("model_id", type=_hf_id_or_path, help="HuggingFace model id (e.g. openai/whisper-base)") convert_parser.add_argument("output_dir", nargs="?", default=None, help="Output directory (default: weights/)") convert_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, help="CQ quantization bits (default: 4)") + convert_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", + help=_PLATFORM_HELP) convert_parser.add_argument("--token", help="HuggingFace token") convert_parser.add_argument("--lora", help="Path or HF id of a LoRA adapter to merge before converting (requires `peft`)") convert_parser.add_argument("--reconvert", action="store_true", - help="Force conversion from source") + help="Force rebuild from source") + + upload_parser = subparsers.add_parser("upload") + upload_parser.add_argument("model_id", type=_hf_id_or_path) + upload_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4) + upload_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto") + upload_parser.add_argument("--token", default=None) + upload_parser.add_argument("--reconvert", action="store_true") transpile_parser = subparsers.add_parser("transpile", help="Build a runnable bundle from CQ weights") @@ -425,6 +414,7 @@ def create_parser(): "clean": cmd_clean, "convert": cmd_convert, "transpile": cmd_transpile, + "upload": cmd_upload, } diff --git a/python/cactus/cli/clean.py b/python/cactus/cli/clean.py index 5ac65b1c5..be21a9e21 100644 --- a/python/cactus/cli/clean.py +++ b/python/cactus/cli/clean.py @@ -1,7 +1,6 @@ import shutil import subprocess import sys -from pathlib import Path from .common import ( PROJECT_ROOT, diff --git a/python/cactus/cli/common.py b/python/cactus/cli/common.py index 18996d5fb..b1203a755 100644 --- a/python/cactus/cli/common.py +++ b/python/cactus/cli/common.py @@ -38,7 +38,7 @@ def transpiled_root() -> Path: DEFAULT_TEST_TRANSCRIPTION_MODEL_ID = "openai/whisper-base" -# Add a new vendor accelerator by appending its name here. +# Add a new vendor platform by appending its name here. SUPPORTED_PLATFORMS: tuple[str, ...] = ("apple",) @@ -97,6 +97,4 @@ def resolve_binary(name): return path print_color(RED, f"{name} binary not found at {path}.") - if is_repo_checkout(): - print_color(RED, "Run `cactus build` first.") return None diff --git a/python/cactus/cli/compile.py b/python/cactus/cli/compile.py index adafffa5c..8c2e3f7b0 100644 --- a/python/cactus/cli/compile.py +++ b/python/cactus/cli/compile.py @@ -171,7 +171,6 @@ def cmd_build(args): cactus_dir = PROJECT_ROOT / "cactus-engine" lib_path = cactus_dir / "build" / "libcactus_engine.a" - print_color(YELLOW, "Building Cactus library...") if run_command(str(cactus_dir / "build.sh"), cwd=cactus_dir).returncode != 0: print_color(RED, "Failed to build cactus library") return 1 diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index 03fdc1fb1..fb9a8710b 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -3,7 +3,6 @@ from pathlib import Path from .common import GREEN, RED, YELLOW, print_color -from .download import get_weights_dir def _merge_lora_adapter(base_model_id, lora_path, token=None): @@ -42,10 +41,11 @@ def _merge_lora_adapter(base_model_id, lora_path, token=None): def cmd_convert(args): - """Convert a HuggingFace model to CQ format. Does not transpile — - use `cactus transpile` afterwards to build the runtime graph bundle. + """Build a runnable bundle locally: convert HuggingFace weights to CQ and + transpile. Skips the prebuilt-bundle search — checks the local copy first + and builds from source if it is not present. """ - from .model import ensure_weights + from .model import prepare_bundle source_model_id = args.model_id merged_dir = None @@ -56,20 +56,16 @@ def cmd_convert(args): return 1 source_model_id = merged_dir - output_dir = args.output_dir or str(get_weights_dir(args.model_id)) + output_dir = args.output_dir + if merged_dir and not output_dir: + from .download import get_bundle_dir, resolve_platform + output_dir = str(get_bundle_dir(args.model_id, bits=args.bits, + platform=resolve_platform(args.platform))) try: - ensure_weights( - source_model_id, - bits=args.bits, - token=args.token, - reconvert=args.reconvert, - output_dir=output_dir, - ) - return 0 - except RuntimeError as e: - print_color(RED, f"Conversion error: {e}") - return 1 + bundle = prepare_bundle(args, model_id=source_model_id, prebuilt=False, + output_dir=output_dir, fail_prefix="Conversion error") + return 1 if bundle is None else 0 finally: if merged_dir: shutil.rmtree(merged_dir, ignore_errors=True) diff --git a/python/cactus/cli/download.py b/python/cactus/cli/download.py index 5391b20c2..67beb9261 100644 --- a/python/cactus/cli/download.py +++ b/python/cactus/cli/download.py @@ -1,8 +1,8 @@ -"""Download pre-transpiled bundles from huggingface.co/Cactus-Compute.""" +"""Download prebuilt bundles from huggingface.co/Cactus-Compute.""" from pathlib import Path from .common import ( - BLUE, GREEN, RED, + BLUE, GREEN, SUPPORTED_PLATFORMS, print_color, weights_root, ) @@ -22,7 +22,7 @@ def get_bundle_dir(model_id: str, *, bits: int = 4, platform: str | None = None) def _host_platform() -> str | None: - """Best-fit accelerator for the current host: apple on macOS, else generic CPU.""" + """Best-fit platform for the current host: apple on macOS, else generic CPU.""" import platform as _platform if _platform.system() == "Darwin" and "apple" in SUPPORTED_PLATFORMS: @@ -45,8 +45,7 @@ def ensure_model(model_id: str) -> Path: def download_bundle(model_id: str, *, bits: int = 4, platform: str | None = None, - token: str | None = None, output_dir: Path | None = None, - reconvert: bool = False) -> Path: + token: str | None = None, output_dir: Path | None = None) -> Path: from .utils import ( download_cq_archive, list_hf_cq_archives, @@ -70,18 +69,13 @@ def download_bundle(model_id: str, *, bits: int = 4, platform: str | None = None raise RuntimeError(f"no bundles published at {repo_id}") resolution = resolve_archive(repo_id, local_name, archives, bits, platform=platform) - download_cq_archive(resolution, bundle_dir, token=token, revision=revision, reconvert=reconvert) + download_cq_archive(resolution, bundle_dir, token=token, revision=revision) print_color(GREEN, f"Ready at {bundle_dir}") return bundle_dir def cmd_download(args) -> int: - from .model import ensure_runnable_bundle - - try: - platform = resolve_platform(args.platform) - ensure_runnable_bundle(args.model_id, bits=args.bits, platform=platform, token=args.token) - return 0 - except (RuntimeError, OSError, ValueError) as e: - print_color(RED, f"Failed to prepare {args.model_id}: {e}") - return 1 + from .model import prepare_bundle + + bundle = prepare_bundle(args, fail_prefix=f"Failed to prepare {args.model_id}") + return 1 if bundle is None else 0 diff --git a/python/cactus/cli/list.py b/python/cactus/cli/list.py index 8df562cc7..a5d623d38 100644 --- a/python/cactus/cli/list.py +++ b/python/cactus/cli/list.py @@ -103,7 +103,7 @@ def _print_section(title: str, color: str, entries): def cmd_list(_args): roots = (weights_root(), transpiled_root()) converted, transpiled = _categorize(roots) - _print_section("Converted weights (cactus convert)", BLUE, converted) + _print_section("Converted weights", BLUE, converted) print() - _print_section("Transpiled bundles (cactus transpile / download)", CYAN, transpiled) + _print_section("Runnable bundles (cactus convert / download)", CYAN, transpiled) return 0 diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index 1f01c49ac..1270f354a 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, replace from pathlib import Path -from .common import GREEN, PROJECT_ROOT, YELLOW, print_color +from .common import GREEN, PROJECT_ROOT, RED, YELLOW, print_color @@ -43,6 +43,10 @@ def ensure_weights(model_id, *, bits=4, token=None, reconvert=False, output_dir= print_color(GREEN, f"Model weights found at {weights_dir}") return weights_dir + if weights_dir.exists(): + print_color(YELLOW, "Removing incomplete weights from a previous run...") + shutil.rmtree(weights_dir) + return _convert_from_source(model_id, bits=bits, token=token, weights_dir=weights_dir) @@ -162,7 +166,7 @@ def _has_transpiled_bundle(path): def resolve_bundle_dir(model_id): path = Path(model_id).expanduser() - if path.is_dir() and (path / "components" / "manifest.json").exists(): + if path.is_dir() and _has_transpiled_bundle(path): return path return None @@ -187,11 +191,14 @@ class TranspileOptions: def ensure_runnable_bundle(model_id, *, bits=4, platform=None, token=None, - reconvert=False, transpile=None): + reconvert=False, prebuilt=True, output_dir=None, + transpile=None): """Resolve a runnable bundle via the full fallback ladder, building if needed. Rungs, in order: (1) a local bundle path, (2) a cached prior build, (3) a prebuilt bundle on HuggingFace, (4) local convert + transpile. + With prebuilt=False, rung (3) is skipped and the bundle is always built + locally when not already present. Raises RuntimeError if every rung fails. """ from .download import download_bundle, get_bundle_dir @@ -200,14 +207,14 @@ def ensure_runnable_bundle(model_id, *, bits=4, platform=None, token=None, if local is not None: return local - cached = get_bundle_dir(model_id, bits=bits, platform=platform) + cached = Path(output_dir) if output_dir else get_bundle_dir(model_id, bits=bits, platform=platform) if _has_transpiled_bundle(cached) and not reconvert: return cached - if not reconvert: + if prebuilt and not reconvert: try: return download_bundle(model_id, bits=bits, platform=platform, - token=token, reconvert=reconvert) + token=token, output_dir=cached) except (RuntimeError, OSError) as exc: print_color(YELLOW, f"No prebuilt bundle ({exc}); building locally") @@ -218,6 +225,28 @@ def ensure_runnable_bundle(model_id, *, bits=4, platform=None, token=None, output_dir=cached, transpile=opts) +def prepare_bundle(args, *, model_id=None, transpile=None, prebuilt=True, + output_dir=None, fail_prefix="Model setup failed"): + """Resolve the platform from args and return a runnable bundle, with uniform + error handling shared by every model command. Returns the bundle Path, or + None (after printing the error) on failure.""" + from .download import resolve_platform + try: + return ensure_runnable_bundle( + args.model_id if model_id is None else model_id, + bits=getattr(args, "bits", 4), + platform=resolve_platform(getattr(args, "platform", "auto")), + token=getattr(args, "token", None), + reconvert=getattr(args, "reconvert", False), + prebuilt=prebuilt, + output_dir=output_dir, + transpile=transpile, + ) + except (RuntimeError, OSError, ValueError) as exc: + print_color(RED, f"{fail_prefix}: {exc}") + return None + + def ensure_bundle(model_id, *, bits=4, token=None, reconvert=False, output_dir=None, transpile=None): from .download import get_weights_dir @@ -273,9 +302,9 @@ def ensure_bundle(model_id, *, bits=4, token=None, "using bundled tiny test assets.", ) if needs_image and not spec_image_files: - raise RuntimeError("Multimodal transpile requires --image-file for this model.") + raise RuntimeError("Building this multimodal model requires --image-file.") if needs_audio and not spec_audio_file: - raise RuntimeError("Multimodal transpile requires --audio-file for this model.") + raise RuntimeError("Building this multimodal model requires --audio-file.") if effective_component_pipeline == "auto" and spec.force_component_pipeline: effective_component_pipeline = "on" @@ -293,7 +322,7 @@ def ensure_bundle(model_id, *, bits=4, token=None, "using bundled tiny test audio asset.", ) elif spec.task in _AUDIO_TASKS and not spec_audio_file: - raise RuntimeError(f"{spec.task} transpile requires --audio-file.") + raise RuntimeError(f"Building a {spec.task} model requires --audio-file.") effective_max_new_tokens = opts.max_new_tokens or _default_max_new_tokens(spec) @@ -333,7 +362,7 @@ def ensure_bundle(model_id, *, bits=4, token=None, rc = run_transpile(model_id, extra_args=extra_args) if rc != 0: - raise RuntimeError(f"Transpilation failed for {model_id}") + raise RuntimeError(f"Build failed for {model_id}") try: from cactus.convert.handoff_probe import ( @@ -348,5 +377,5 @@ def ensure_bundle(model_id, *, bits=4, token=None, except Exception as e: print_color(YELLOW, f"Warning: failed to package cloud handoff probe: {e}") - print_color(GREEN, f"Model converted and transpiled to {output_dir}") + print_color(GREEN, f"Model built at {output_dir}") return output_dir diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 68228c7c3..f4ea6f1d1 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -2,12 +2,10 @@ import subprocess from pathlib import Path -from .common import GREEN, RED, apply_cloud_api_key_env, print_color, resolve_binary +from .common import GREEN, apply_cloud_api_key_env, print_color, resolve_binary def cmd_run(args) -> int: - from .download import resolve_platform - if args.no_cloud_tele: os.environ["CACTUS_NO_CLOUD_TELE"] = "1" apply_cloud_api_key_env() @@ -19,20 +17,13 @@ def cmd_run(args) -> int: if args.result_json: args.result_json = str(Path(args.result_json).expanduser()) - from .model import TranspileOptions, ensure_runnable_bundle + from .model import TranspileOptions, prepare_bundle - platform = resolve_platform(args.platform) - try: - bundle_dir = ensure_runnable_bundle( - args.model_id, bits=args.bits, platform=platform, - token=args.token, reconvert=args.reconvert, - transpile=TranspileOptions( - image_files=[args.image] if args.image else None, - audio_file=args.audio, - ), - ) - except RuntimeError as exc: - print_color(RED, f"Model setup failed: {exc}") + bundle_dir = prepare_bundle(args, transpile=TranspileOptions( + image_files=[args.image] if args.image else None, + audio_file=args.audio, + )) + if bundle_dir is None: return 1 binary = resolve_binary("run") diff --git a/python/cactus/cli/runtime.py b/python/cactus/cli/runtime.py index bf8ecaaa0..9e162cddb 100644 --- a/python/cactus/cli/runtime.py +++ b/python/cactus/cli/runtime.py @@ -106,7 +106,7 @@ def ensure_python_runtime_library(): ): return library_path - print_color(YELLOW, "Preparing Cactus shared runtime for transpiler...") + print_color(YELLOW, "Preparing Cactus shared runtime...") if not static_lib.exists(): static_lib = ensure_library() _link_python_runtime_library(static_lib=static_lib, library_path=library_path) diff --git a/python/cactus/cli/serve.py b/python/cactus/cli/serve.py index 49356db85..e99a4ed12 100644 --- a/python/cactus/cli/serve.py +++ b/python/cactus/cli/serve.py @@ -25,15 +25,11 @@ def _is_valid_bundle(path: Path) -> bool: def cmd_serve(args): """Start the OpenAI-compatible HTTP server.""" - model_path, model_name = _resolve_model_arg(args.model) - if args.model and model_path is None: - from .model import ensure_runnable_bundle - from .download import resolve_platform - try: - built = ensure_runnable_bundle( - args.model, bits=args.bits, platform=resolve_platform(args.platform)) - except RuntimeError as exc: - print_color(RED, f"Error: could not prepare {args.model}: {exc}") + model_path, model_name = _resolve_model_arg(args.model_id) + if args.model_id and model_path is None: + from .model import prepare_bundle + built = prepare_bundle(args, fail_prefix=f"Error: could not prepare {args.model_id}") + if built is None: return 1 model_path, model_name = built, built.name if model_path is not None and not _is_valid_bundle(model_path): @@ -58,10 +54,12 @@ def cmd_serve(args): weights_root=weights_root(), model_path=model_path, default_model=model_name, + auto_handoff=not args.no_cloud_handoff, + confidence_threshold=args.confidence_threshold, + cloud_timeout_ms=args.cloud_timeout_ms, ) except RuntimeError as exc: print_color(RED, f"Error: {exc}") - print("Prepare a v2 bundle first with `cactus run ` (or `cactus convert ` then `cactus transpile `).") return 1 models = sorted(application.state.registry.models) diff --git a/python/cactus/cli/test.py b/python/cactus/cli/test.py index c9298b772..07504d632 100644 --- a/python/cactus/cli/test.py +++ b/python/cactus/cli/test.py @@ -68,11 +68,11 @@ def cmd_test(args): if "engine" in targets: from .model import ensure_runnable_bundle try: - args.model_id = str(ensure_runnable_bundle(args.model_id or DEFAULT_TEST_MODEL_ID)) - args.transcription_model_id = str( - ensure_runnable_bundle(args.transcription_model_id or DEFAULT_TEST_TRANSCRIPTION_MODEL_ID) - ) - except RuntimeError as exc: + args.model_id = str(ensure_runnable_bundle( + args.model_id or DEFAULT_TEST_MODEL_ID, reconvert=args.reconvert)) + args.transcription_model_id = str(ensure_runnable_bundle( + args.transcription_model_id or DEFAULT_TEST_TRANSCRIPTION_MODEL_ID, reconvert=args.reconvert)) + except (RuntimeError, OSError, ValueError) as exc: print_color(RED, f"Model setup failed: {exc}") return 1 diff --git a/python/cactus/cli/transcribe.py b/python/cactus/cli/transcribe.py index 33af24ae0..79360ce31 100644 --- a/python/cactus/cli/transcribe.py +++ b/python/cactus/cli/transcribe.py @@ -6,8 +6,7 @@ def cmd_transcribe(args): - from .model import ensure_runnable_bundle, TranspileOptions - from .download import resolve_platform + from .model import prepare_bundle, TranspileOptions if args.audio_file: audio_path = Path(args.audio_file).expanduser() @@ -19,24 +18,10 @@ def cmd_transcribe(args): if args.no_cloud_tele: os.environ["CACTUS_NO_CLOUD_TELE"] = "1" - if args.force_handoff: - os.environ["CACTUS_FORCE_HANDOFF"] = "1" - else: - os.environ.pop("CACTUS_FORCE_HANDOFF", None) - apply_cloud_api_key_env() - try: - bundle_dir = ensure_runnable_bundle( - args.model_id, - bits=args.bits, - platform=resolve_platform(args.platform), - token=args.token, - reconvert=args.reconvert, - transpile=TranspileOptions(audio_file=args.audio_file), - ) - except RuntimeError as e: - print_color(RED, f"Model setup failed: {e}") + bundle_dir = prepare_bundle(args, transpile=TranspileOptions(audio_file=args.audio_file)) + if bundle_dir is None: return 1 binary = resolve_binary("transcribe") diff --git a/python/cactus/cli/transpile.py b/python/cactus/cli/transpile.py index 8e8f77cad..fdeb1104d 100644 --- a/python/cactus/cli/transpile.py +++ b/python/cactus/cli/transpile.py @@ -45,10 +45,6 @@ def run_transpile(model_id, *, extra_args, execute_after_transpile=False, if not _extra_args_has_option(extra_args, "--weights-dir"): if _weights_dir_looks_transpile_ready(default_weights_dir): command.extend(["--weights-dir", str(default_weights_dir)]) - elif not allow_unconverted_weights: - print_color(RED, "Error: transpilation requires converted Cactus CQ weights.") - print_color(YELLOW, f"Run conversion first: cactus convert {model_id}") - return 1 if not _extra_args_has_option(extra_args, "--artifact-dir"): command.extend(["--artifact-dir", str(default_weights_dir)]) diff --git a/python/cactus/cli/upload.py b/python/cactus/cli/upload.py new file mode 100644 index 000000000..b0277b737 --- /dev/null +++ b/python/cactus/cli/upload.py @@ -0,0 +1,53 @@ +import os +import tempfile +import zipfile +from pathlib import Path + +from .common import BLUE, GREEN, PROJECT_ROOT, print_color +from .download import get_model_dir_name, resolve_platform +from .model import prepare_bundle +from .utils import suggested_cq_repo, variant_suffix + + +def cmd_upload(args): + """Build a runnable bundle locally and upload it to Cactus-Compute on HuggingFace.""" + token = args.token or os.getenv("HF_TOKEN") + if not token: + raise SystemExit("missing HuggingFace token: pass --token or set HF_TOKEN") + + platform = resolve_platform(args.platform) + repo_id = suggested_cq_repo(args.model_id) + stem = f"{get_model_dir_name(args.model_id)}-{variant_suffix(args.bits, platform)}" + archive_name = f"{stem}.zip" + tag = (PROJECT_ROOT / "CACTUS_VERSION").read_text(encoding="utf-8").strip() + + bundle_dir = prepare_bundle(args, prebuilt=False, fail_prefix=f"Build failed for {args.model_id}") + if bundle_dir is None: + return 1 + + from huggingface_hub import HfApi + + api = HfApi(token=token) + api.create_repo(repo_id=repo_id, repo_type="model", exist_ok=True) + + with tempfile.TemporaryDirectory(prefix="cactus-upload-") as tmp: + archive_path = Path(tmp) / archive_name + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for path in sorted(p for p in bundle_dir.rglob("*") if p.is_file()): + zf.write(path, f"{stem}/{path.relative_to(bundle_dir).as_posix()}") + print_color(BLUE, f"Uploading {archive_name} ({archive_path.stat().st_size / 1e6:.1f} MB) -> {repo_id}") + api.upload_file( + path_or_fileobj=str(archive_path), + path_in_repo=archive_name, + repo_id=repo_id, + repo_type="model", + commit_message=f"Upload {archive_name} ({tag})", + ) + + try: + api.delete_tag(repo_id=repo_id, tag=tag, repo_type="model") + except Exception: + pass + api.create_tag(repo_id=repo_id, tag=tag, repo_type="model", revision="main") + print_color(GREEN, f"Done: {repo_id} @ {tag}") + return 0 diff --git a/python/cactus/cli/utils.py b/python/cactus/cli/utils.py index 9943d0cad..71882d016 100644 --- a/python/cactus/cli/utils.py +++ b/python/cactus/cli/utils.py @@ -316,12 +316,8 @@ def write_download_metadata(output_dir: Path, resolution: CqResolution, archive_ def download_cq_archive(resolution: CqResolution, output_dir: Path, *, token=None, - revision=None, reconvert=False, dry_run=False) -> Path: - if dry_run: - return output_dir + revision=None) -> Path: - if output_dir.exists() and reconvert: - shutil.rmtree(output_dir) if output_dir.exists() and (output_dir / "components" / "manifest.json").exists(): return output_dir if output_dir.exists() and any(output_dir.iterdir()): diff --git a/python/cactus/convert/cactus_adapters/config_utils.py b/python/cactus/convert/cactus_adapters/config_utils.py index b17b7c229..f08c795fc 100644 --- a/python/cactus/convert/cactus_adapters/config_utils.py +++ b/python/cactus/convert/cactus_adapters/config_utils.py @@ -1,6 +1,4 @@ -import json import math -from typing import Optional, Any, Dict def cfg_get(c, key, default=None): diff --git a/python/cactus/convert/cactus_adapters/tensor_io.py b/python/cactus/convert/cactus_adapters/tensor_io.py index cb38aa5c1..ba978c6aa 100644 --- a/python/cactus/convert/cactus_adapters/tensor_io.py +++ b/python/cactus/convert/cactus_adapters/tensor_io.py @@ -1,7 +1,5 @@ import numpy as np import struct -from pathlib import Path -from typing import Optional, Dict, Any, List from .weight_patterns import EMBED_NAMES from ..model_adapters.naming import GEMMA4_WEIGHT_SCALE, gemma4_scale_factor diff --git a/python/cactus/convert/cactus_adapters/tokenizer.py b/python/cactus/convert/cactus_adapters/tokenizer.py index 06280e6ad..46ee4c86b 100644 --- a/python/cactus/convert/cactus_adapters/tokenizer.py +++ b/python/cactus/convert/cactus_adapters/tokenizer.py @@ -491,7 +491,7 @@ def add_additional_special_token(token_str): else: f.write("has_chat_template=false\n") if len(tool_tokens) > 0: - f.write(f"has_tool_support=true\n") + f.write("has_tool_support=true\n") f.write(f"tool_token_count={len(tool_tokens)}\n") diff --git a/python/cactus/convert/calibration/hessian.py b/python/cactus/convert/calibration/hessian.py index ad6ffa165..62e109a8d 100644 --- a/python/cactus/convert/calibration/hessian.py +++ b/python/cactus/convert/calibration/hessian.py @@ -266,7 +266,6 @@ def tensor_cache_save(path: Path | None, obj) -> None: add_time("preprocess_cache_write_seconds", time.perf_counter() - t0) def run_inputs(inputs, context: str): - nonlocal forwards try: model_dtype = next(model.parameters()).dtype except Exception: diff --git a/python/cactus/convert/export/reports.py b/python/cactus/convert/export/reports.py index 9b759fd53..47e35f73a 100644 --- a/python/cactus/convert/export/reports.py +++ b/python/cactus/convert/export/reports.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from collections import Counter, defaultdict +from collections import Counter from pathlib import Path from typing import Any diff --git a/python/cactus/server.py b/python/cactus/server.py index c8f9206a6..a802efb97 100644 --- a/python/cactus/server.py +++ b/python/cactus/server.py @@ -24,7 +24,7 @@ cactus_reset, cactus_transcribe, ) -from .cli.download import get_weights_dir +from .cli.download import get_model_dir_name, get_weights_dir from .cli.common import DEFAULT_MODEL_ID, PROJECT_ROOT, is_repo_checkout LOGGER = logging.getLogger(__name__) @@ -111,15 +111,32 @@ def _discover(self, root: Path) -> None: if info is not None: self.models[info.id] = info - def require(self, model_id: str) -> ModelInfo: + def _resolve(self, model_id: str) -> ModelInfo | None: + """Match an exact bundle id, or fall back to the HuggingFace id / bare + stem (e.g. 'google/gemma-4-E2B-it' -> 'gemma-4-e2b-it-cq4-apple') when it + is unambiguous.""" info = self.models.get(model_id) + if info is not None: + return info + stem = get_model_dir_name(model_id) + if stem in self.models: + return self.models[stem] + variants = [m for m in self.models.values() if m.id.startswith(f"{stem}-")] + return variants[0] if len(variants) == 1 else None + + def require(self, model_id: str) -> ModelInfo: + info = self._resolve(model_id) if info is None: - raise HTTPException(status_code=404, detail=f"Model '{model_id}' is not available") + available = ", ".join(sorted(self.models)) or "none" + raise HTTPException( + status_code=404, + detail=f"Model '{model_id}' is not available (available: {available})", + ) return info def default_llm(self, preferred: str | None = None) -> ModelInfo: if preferred: - info = self.models.get(preferred) + info = self._resolve(preferred) if info is None: raise RuntimeError(f"Requested model '{preferred}' is not a valid v2 Cactus bundle") if info.model_type not in LLM_MODEL_TYPES: @@ -481,6 +498,9 @@ def create_app( default_model: str | None = None, max_warm: int = 2, preload: bool = True, + auto_handoff: bool = True, + confidence_threshold: float | None = None, + cloud_timeout_ms: int | None = None, ) -> FastAPI: root = Path(weights_root) if weights_root is not None else ( PROJECT_ROOT / "weights" if is_repo_checkout() else Path.home() / ".cache" / "cactus" / "weights" @@ -515,6 +535,9 @@ async def lifespan(app: FastAPI): app.state.manager = manager app.state.default_model = selected.id app.state.default_stt_model = registry.default_stt().id if registry.default_stt() else None + app.state.auto_handoff = auto_handoff + app.state.confidence_threshold = confidence_threshold + app.state.cloud_timeout_ms = cloud_timeout_ms @app.get("/v1/models") async def list_models(request: Request): @@ -530,6 +553,13 @@ async def chat_completions(request: Request, req: ChatRequest): messages = [_flatten_message(m) for m in req.messages] tools, force_tools = _translate_tools(req.tools, req.tool_choice) options = _chat_options(req) + state = request.app.state + if not state.auto_handoff: + options["auto_handoff"] = False + if state.confidence_threshold is not None: + options["confidence_threshold"] = state.confidence_threshold + if state.cloud_timeout_ms is not None: + options["cloud_timeout_ms"] = state.cloud_timeout_ms if force_tools: options["force_tools"] = True request_id = f"chatcmpl-{uuid.uuid4().hex[:29]}" diff --git a/python/cactus/transpile/README.md b/python/cactus/transpile/README.md index a0a059658..01ada18a1 100644 --- a/python/cactus/transpile/README.md +++ b/python/cactus/transpile/README.md @@ -16,8 +16,9 @@ This README is a code-first walkthrough of the actual flow in the repo today. ## Related Entry Points -These files are outside `python/cactus/transpile`, but they are the public top-level -entry points for the transpiler: +These files are outside `python/cactus/transpile`, but they are the top-level +entry points for the transpiler (driven by `cactus convert` for end users; the +`cactus transpile` command is hidden from the CLI menu and used internally): - `python/cactus/cli/transpile.py` - `python/cactus/transpile/hf_model.py` @@ -52,8 +53,9 @@ def cmd_transpile(args): Two important details: - `cactus transpile` saves artifacts and skips execution by default. -- Normal transpilation requires converted Cactus CQ weights. Run `cactus convert` - first, or pass `--weights-dir` to an existing converted folder. +- Transpilation reads converted Cactus CQ weights from the default weights dir + or `--weights-dir`. (`cactus convert` runs convert + transpile together; this + internal command is the transpile-only step.) - The CLI only has an explicit `--execute-after-transpile` flag; most other transpile options are forwarded as unknown args to `hf_model.py`. @@ -665,7 +667,7 @@ cactus run The runtime reads `components/manifest.json` from the bundle directory, mmaps the `.weights` files, and loads the serialized `.cactus` graphs. Per-platform -`.mlpackage` files (Apple CoreML / future vendor accelerators) are dispatched +`.mlpackage` files (Apple CoreML / future vendor platforms) are dispatched to the matching hardware when present. ## Major Situations And How They Are Handled @@ -844,7 +846,7 @@ Notes: ## 1. Basic Transpile -The simplest public command is: +The simplest invocation (internal command, not in the CLI menu) is: ```bash cactus transpile diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py index 4151ccb4a..b1cb6e52c 100644 --- a/python/cactus/transpile/hf_model.py +++ b/python/cactus/transpile/hf_model.py @@ -30,7 +30,6 @@ from cactus.transpile.audio_preprocess import generic_log_mel_features as _generic_log_mel_features from cactus.transpile.audio_preprocess import load_audio_waveform as _load_audio_waveform from cactus.transpile.audio_preprocess import prepare_cactus_audio_features -from cactus.transpile.audio_preprocess import prepare_native_gemma4_audio_features from cactus.transpile.capture_pytorch import capture_model from cactus.transpile.canonicalize.cleanup import canonicalize_exported_graph from cactus.transpile.component_partition import extract_component_subgraphs @@ -1973,16 +1972,6 @@ def _multimodal_context_token_floor(model_type: str = "") -> int: return default_context -_GEMMA4_MULTIMODAL_INPUT_ORDER = ( - "input_ids", - "attention_mask", - "token_type_ids", - "pixel_values", - "pixel_position_ids", - "input_features", - "input_features_mask", -) - _LFM2_VL_MULTIMODAL_INPUT_ORDER = ( "input_ids", "attention_mask", @@ -2000,147 +1989,6 @@ def _multimodal_context_token_floor(model_type: str = "") -> int: ) -def _normalize_multimodal_prompt( - prompt: str, - *, - image_token: str | None, - num_images: int, - audio_token: str | None, - num_audio_segments: int, -) -> str: - normalized = prompt.strip() - prefixes: list[str] = [] - - if image_token and num_images > 0: - image_count = normalized.count(image_token) - if image_count < num_images: - prefixes.append(" ".join(image_token for _ in range(num_images - image_count))) - if audio_token and num_audio_segments > 0: - audio_count = normalized.count(audio_token) - if audio_count < num_audio_segments: - prefixes.append(" ".join(audio_token for _ in range(num_audio_segments - audio_count))) - - if prefixes: - prefix = "\n".join(part for part in prefixes if part) - if normalized: - return f"{prefix}\n{normalized}" - return prefix - return normalized - - -def _build_gemma4_chat_prompt( - *, - prompt: str, - image_token: str | None, - num_images: int, - audio_token: str | None, - num_audio_segments: int, - system_prompt: str = "", - enable_thinking_if_supported: bool = False, -) -> str: - result = "" - normalized_system = system_prompt.strip() - if enable_thinking_if_supported or normalized_system: - result += "<|turn>system\n" - if enable_thinking_if_supported: - result += "<|think|>" - result += normalized_system - result += "\n" - - result += "<|turn>user\n" - if image_token and num_images > 0: - for _ in range(num_images): - # Gemma4Processor expands the raw placeholder into the native - # <|image> ... soft-token region. Supplying wrappers here - # double-wraps the prompt and shifts the multimodal prefill. - result += f"\n\n{image_token}\n\n" - result += prompt - if audio_token and num_audio_segments > 0: - result += "".join( - audio_token - for _ in range(num_audio_segments) - ) - result += "\n" - result += "<|turn>model\n" - return result - - -def _gemma4_split_cactus_newline_token_merges(batch: object) -> None: - """Match native Cactus Gemma4 tokenization for newline runs. - - The HF fast tokenizer merges ``\n\n`` and ``\n\n\n`` into dedicated tokens - (108 and 109). Cactus' converted BPE tokenizer emits repeated newline token - 107 instead. Gemma4's multimodal prefill is sensitive enough that this one - token of drift changes the first generated token, so normalize only this - Gemma4-specific quirk after the processor has expanded image/audio regions. - """ - - input_ids = batch.get("input_ids") if hasattr(batch, "get") else None - if not isinstance(input_ids, torch.Tensor) or input_ids.ndim != 2: - return - - expansions = { - 108: (107, 107), - 109: (107, 107, 107), - } - if not any(int(token) in expansions for token in input_ids.reshape(-1).tolist()): - return - - lengths: list[list[int]] = [] - max_len = 0 - for row in input_ids.detach().cpu().tolist(): - row_lengths = [len(expansions.get(int(token), (int(token),))) for token in row] - lengths.append(row_lengths) - max_len = max(max_len, sum(row_lengths)) - - for key in ("input_ids", "attention_mask", "token_type_ids"): - value = batch.get(key) if hasattr(batch, "get") else None - if not isinstance(value, torch.Tensor) or value.ndim != 2: - continue - pad_value = 0 - expanded = torch.full( - (value.shape[0], max_len), - pad_value, - dtype=value.dtype, - device=value.device, - ) - for row_idx, row in enumerate(value.detach().cpu().tolist()): - out: list[int] = [] - for token_idx, item in enumerate(row): - if key == "input_ids": - out.extend(expansions.get(int(item), (int(item),))) - else: - out.extend([int(item)] * lengths[row_idx][token_idx]) - expanded[row_idx, : len(out)] = torch.tensor(out, dtype=value.dtype, device=value.device) - batch[key] = expanded - - -def _build_gemma4_chat_messages( - *, - prompt: str, - num_images: int, - num_audio_segments: int, - system_prompt: str = "", - enable_thinking_if_supported: bool = False, -) -> list[dict[str, object]]: - messages: list[dict[str, object]] = [] - normalized_system = system_prompt.strip() - if normalized_system or enable_thinking_if_supported: - system_text = normalized_system - if enable_thinking_if_supported: - system_text = f"<|think|>{system_text}" - messages.append({"role": "system", "content": system_text}) - - user_content: list[dict[str, str]] = [] - for _ in range(num_images): - user_content.append({"type": "image"}) - user_content.append({"type": "text", "text": prompt.strip()}) - for _ in range(num_audio_segments): - user_content.append({"type": "audio"}) - messages.append({"role": "user", "content": user_content}) - return messages - - def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: if not image_files: return [] @@ -2159,94 +2007,6 @@ def _load_image_inputs(image_files: tuple[str, ...]) -> list[object]: return images -def _get_processor_image_attr(processor: object, name: str, default: object) -> object: - image_processor = getattr(processor, "image_processor", None) - if image_processor is not None and hasattr(image_processor, name): - return getattr(image_processor, name) - if isinstance(image_processor, dict) and name in image_processor: - return image_processor[name] - return default - - -def _prepare_gemma4_native_image_tensors( - processor: object, - image_files: tuple[str, ...], -) -> tuple[torch.Tensor, torch.Tensor] | None: - if not image_files: - return None - try: - from PIL import Image # type: ignore - except Exception as exc: # pragma: no cover - raise RuntimeError(f"Pillow is required for Gemma4 native image preprocessing: {exc}") from exc - - patch_size = int(_get_processor_image_attr(processor, "patch_size", 16)) - pooling_kernel_size = int(_get_processor_image_attr(processor, "pooling_kernel_size", 3)) - max_soft_tokens = int(_get_processor_image_attr(processor, "max_soft_tokens", 280)) - rescale_factor = float(_get_processor_image_attr(processor, "rescale_factor", 1.0 / 255.0)) - max_patches = max_soft_tokens * pooling_kernel_size * pooling_kernel_size - side_multiple = pooling_kernel_size * patch_size - patch_dim = 3 * patch_size * patch_size - if patch_size <= 0 or pooling_kernel_size <= 0 or max_patches <= 0: - return None - - try: - resample_bilinear = Image.Resampling.BILINEAR - except AttributeError: # pragma: no cover - resample_bilinear = Image.BILINEAR - - pixel_batches: list[np.ndarray] = [] - position_batches: list[np.ndarray] = [] - for image_file in image_files: - path = Path(image_file).resolve() - with Image.open(path) as image: - rgb = image.convert("RGB") - width, height = rgb.size - target_pixels = float(max_patches * patch_size * patch_size) - factor = float(np.sqrt(target_pixels / max(1.0, float(width * height)))) - target_h = int(np.floor(factor * height / side_multiple)) * side_multiple - target_w = int(np.floor(factor * width / side_multiple)) * side_multiple - if target_h == 0: - target_h = side_multiple - if target_w == 0: - target_w = side_multiple - if (target_w, target_h) != rgb.size: - rgb = rgb.resize((target_w, target_h), resample=resample_bilinear) - array = np.asarray(rgb, dtype=np.float32) * rescale_factor - - patch_h = target_h // patch_size - patch_w = target_w // patch_size - num_patches = patch_h * patch_w - if num_patches > max_patches: - raise RuntimeError( - f"Gemma4 native image preprocessing produced {num_patches} patches, " - f"but max_patches={max_patches}" - ) - chw = np.transpose(array, (2, 0, 1)) - patches = ( - chw.reshape(3, patch_h, patch_size, patch_w, patch_size) - .transpose(1, 3, 2, 4, 0) - .reshape(num_patches, patch_dim) - ) - - padded_patches = np.zeros((max_patches, patch_dim), dtype=np.float32) - padded_patches[:num_patches] = patches - positions = np.full((max_patches, 2), -1, dtype=np.int64) - valid_positions = np.zeros((num_patches, 2), dtype=np.int64) - for patch_y in range(patch_h): - row_start = patch_y * patch_w - valid_positions[row_start: row_start + patch_w, 0] = np.arange(patch_w, dtype=np.int64) - valid_positions[row_start: row_start + patch_w, 1] = patch_y - positions[:num_patches] = valid_positions - - pixel_batches.append(padded_patches) - position_batches.append(positions) - - return ( - torch.from_numpy(np.stack(pixel_batches, axis=0)), - torch.from_numpy(np.stack(position_batches, axis=0)), - ) - - def _prepare_gemma4_multimodal_inputs( processor: object | None, *, @@ -2276,117 +2036,6 @@ def _prepare_gemma4_multimodal_inputs( metadata=shared.metadata, ) - images = _load_image_inputs(image_files) - audio_waveforms: list[np.ndarray] = [] - sample_rate: int | None = None - if audio_file: - sample_rate = _resolve_audio_sample_rate(processor) - audio_waveforms.append(_load_audio_waveform(audio_file, target_sample_rate=sample_rate)) - - image_token = getattr(processor, "image_token", None) - audio_token = getattr(processor, "audio_token", None) - processor_prompt = _normalize_multimodal_prompt( - prompt, - image_token=image_token if isinstance(image_token, str) else None, - num_images=len(images), - audio_token=audio_token if isinstance(audio_token, str) else None, - num_audio_segments=len(audio_waveforms), - ) - normalized_prompt = processor_prompt - if use_gemma4_chat_template: - normalized_prompt = prompt.strip() - # Match the native Cactus Gemma4 prompt builder instead of the HF chat - # template. The HF template omits the begin/end multimodal wrapper tokens, - # which shifts the multimodal prefill by four tokens for image+audio prompts. - processor_prompt = _build_gemma4_chat_prompt( - prompt=normalized_prompt, - image_token=image_token if isinstance(image_token, str) else None, - num_images=len(images), - audio_token=audio_token if isinstance(audio_token, str) else None, - num_audio_segments=len(audio_waveforms), - system_prompt=system_prompt, - enable_thinking_if_supported=enable_thinking_if_supported, - ) - - batch = processor( - text=processor_prompt, - images=images or None, - audio=audio_waveforms or None, - return_tensors="pt", - ) - if use_gemma4_chat_template: - _gemma4_split_cactus_newline_token_merges(batch) - native_image_tensors = _prepare_gemma4_native_image_tensors(processor, image_files) - if native_image_tensors is not None: - batch["pixel_values"], batch["pixel_position_ids"] = native_image_tensors - if audio_file and isinstance(batch.get("input_features"), torch.Tensor): - feature_tensor = batch["input_features"] - expected_mels = int(feature_tensor.shape[-1]) - try: - native_audio, native_audio_mask, native_audio_frames = prepare_native_gemma4_audio_features( - audio_file, - expected_mels=expected_mels, - torch_dtype=torch_dtype, - ) - except Exception as exc: - print(f"note=falling back to processor gemma4 audio features: {exc}") - batch["input_features"] = feature_tensor.to(dtype=torch_dtype) - fallback_mask = batch.get("input_features_mask") - if isinstance(fallback_mask, torch.Tensor): - batch["input_features_mask"] = fallback_mask.to(dtype=torch.bool) - native_audio_frames = int(fallback_mask.to(dtype=torch.int32).sum().item()) - else: - native_audio_frames = int(feature_tensor.shape[1]) - batch["input_features_mask"] = torch.ones( - (int(feature_tensor.shape[0]), native_audio_frames), - dtype=torch.bool, - ) - batch["native_audio_frames"] = native_audio_frames - else: - batch["input_features"] = native_audio - batch["input_features_mask"] = native_audio_mask - batch["native_audio_frames"] = native_audio_frames - - ordered_keys = [ - key - for key in _GEMMA4_MULTIMODAL_INPUT_ORDER - if isinstance(batch.get(key), torch.Tensor) - ] - if not ordered_keys: - raise RuntimeError("Gemma4 multimodal processor did not return any tensor inputs") - - tensors: list[torch.Tensor] = [] - for key in ordered_keys: - value = batch[key] - if not isinstance(value, torch.Tensor): - continue - if torch.is_floating_point(value): - value = value.to(dtype=torch_dtype) - tensors.append(value) - - metadata: dict[str, object] = { - "prompt": normalized_prompt, - "processor_prompt": processor_prompt, - "image_files": [str(Path(path).resolve()) for path in image_files], - "input_shapes": { - name: list(tensor.shape) - for name, tensor in zip(ordered_keys, tensors) - }, - } - if audio_file: - metadata["audio_file"] = str(Path(audio_file).resolve()) - if sample_rate is not None: - metadata["sample_rate"] = sample_rate - native_audio_frames = batch.get("native_audio_frames") - if isinstance(native_audio_frames, int): - metadata["native_audio_frames"] = native_audio_frames - - return PreparedInputs( - names=tuple(ordered_keys[: len(tensors)]), - tensors=tuple(tensors), - metadata=metadata, - ) - def _prepare_lfm2_vl_multimodal_inputs( processor: object | None, diff --git a/python/cactus/transpile/importers.py b/python/cactus/transpile/importers.py index d606d32f5..495b1dfbd 100644 --- a/python/cactus/transpile/importers.py +++ b/python/cactus/transpile/importers.py @@ -1397,7 +1397,6 @@ def import_diff(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, concat_shapes = [tuple(int(v) for v in source_shape)] def _append_concat_input(arg: Any, label: str) -> None: - nonlocal concat_inputs, concat_shapes if arg is None: return arg_value_id = value_id(arg, ctx) diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index daac679ce..f72c21b97 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -1632,7 +1632,6 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch. inputs_embeds = backbone.embed_tokens(input_ids) text_position_ids = position_ids.to(dtype=torch.int64) - seq_len = int(inputs_embeds.shape[1]) causal_mask = None hidden_states = inputs_embeds @@ -4760,7 +4759,7 @@ def _build_qwen_causal_lm_component_specs( }, metadata={"family": family, "task": "causal_lm_logits"}, )) - if "decoder_prefill_chunk" in requested_set: + if "decoder_embed_chunk" in requested_set: specs.append(ComponentModuleSpec( component="decoder_embed_chunk", module=decoder_embed_chunk, @@ -6307,17 +6306,6 @@ def canonicalize_model_interface( if not resolved_input_names: resolved_input_names = ("input_ids", "attention_mask") adapter_factory = NomicTextEmbeddingAdapter - elif task == "audio_classification_logits": - if not resolved_input_names: - resolved_input_names = _infer_input_names( - model, - preferred=("input_values", "input_features", "attention_mask"), - ) - adapter_factory = lambda inner_model: AudioClassificationLogitsAdapter( # type: ignore[assignment] - inner_model, - input_names=resolved_input_names, - family=family, - ) else: raise NotImplementedError(f"unsupported task={task}") diff --git a/python/cactus/transpile/multimodal_runtime.py b/python/cactus/transpile/multimodal_runtime.py index 672d849f5..3747afcf6 100644 --- a/python/cactus/transpile/multimodal_runtime.py +++ b/python/cactus/transpile/multimodal_runtime.py @@ -8,7 +8,6 @@ from cactus.transpile.audio_preprocess import audio_duration_limit_seconds from cactus.transpile.audio_preprocess import prepare_native_gemma4_audio_features from cactus.transpile.media_limits import resize_static_image -from cactus.transpile.runtime_support import ensure_transformers_supports_gemma4 from cactus.transpile.runtime_support import patch_torch_flex_attention_compat from cactus.transpile.runtime_support import patch_transformers_torchvision_probe from cactus.transpile.runtime_support import PreparedInputs diff --git a/python/cactus/transpile/npu/README.md b/python/cactus/transpile/npu/README.md index 5c2a9c6ab..781c88271 100644 --- a/python/cactus/transpile/npu/README.md +++ b/python/cactus/transpile/npu/README.md @@ -3,8 +3,8 @@ Emits CoreML `.mlpackage`s so the runtime engine can dispatch through the Apple Neural Engine. Triggered by `--npu` on the underlying transpile script (`python -m cactus.transpile.hf_model --npu …`), or programmatically via -`TranspileOptions(npu=True)` in `cli/model.py`. The flag is not yet -surfaced on the user-facing `cactus transpile` CLI. Runtime side: +`TranspileOptions(npu=True)` in `cli/model.py`, and via `--npu` on the +internal `cactus transpile` command (hidden from the CLI menu). Runtime side: `cactus-engine/src/model_npu.cpp`. **Scope: audio + vision encoders only.** Text-decoder prefill is diff --git a/python/cactus/transpile/npu/coremltools_patches.py b/python/cactus/transpile/npu/coremltools_patches.py index 44cdd1730..9bb0de1cd 100644 --- a/python/cactus/transpile/npu/coremltools_patches.py +++ b/python/cactus/transpile/npu/coremltools_patches.py @@ -31,7 +31,6 @@ def _patch_register_func_allow_dunder() -> None: if getattr(cls.register_func, "_cactus_dunder_patched", False): return - original = cls.register_func def register_func(self, func=None, torch_alias=None, override=False): f_name = func.__name__ @@ -84,7 +83,6 @@ def new_ones(context, node): def _register_logical_and_op() -> None: from coremltools.converters.mil.frontend.torch.torch_op_registry import ( _TORCH_OPS_REGISTRY, - register_torch_op, ) from coremltools.converters.mil import Builder as mb diff --git a/scripts/upload_weights.py b/scripts/upload_weights.py deleted file mode 100644 index 101c6a0d3..000000000 --- a/scripts/upload_weights.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -"""Convert, transpile, package, and upload a Cactus weight bundle to huggingface.co/Cactus-Compute. - - python scripts/upload_weights.py google/gemma-4-E2B-it --acceleration apple --token hf_xxx - python scripts/upload_weights.py openai/whisper-base --acceleration cpu --token hf_xxx -""" -from __future__ import annotations - -import argparse -import os -import sys -import tempfile -import zipfile -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(REPO_ROOT / "python")) - -from cactus.cli.download import ( # noqa: E402 - get_bundle_dir, - get_model_dir_name, - resolve_platform, -) -from cactus.cli.model import TranspileOptions, ensure_bundle # noqa: E402 -from cactus.cli.utils import ( # noqa: E402 - suggested_cq_repo, - variant_suffix, -) - - -def main() -> int: - p = argparse.ArgumentParser(description="Convert, transpile, and upload a Cactus weight bundle to HuggingFace.") - p.add_argument("model", help="model id, e.g. google/gemma-4-E2B-it") - p.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, help="CQ quantization bits (default: 4)") - p.add_argument("--acceleration", default="cpu", help="'cpu' or 'apple' (default: cpu)") - p.add_argument("--token", default=os.getenv("HF_TOKEN"), help="HuggingFace token") - args = p.parse_args() - - if not args.token: - raise SystemExit("missing HuggingFace token: pass --token or set HF_TOKEN") - - platform = resolve_platform(args.acceleration) - repo_id = suggested_cq_repo(args.model) - stem = f"{get_model_dir_name(args.model)}-{variant_suffix(args.bits, platform)}" - archive_name = f"{stem}.zip" - tag = (REPO_ROOT / "CACTUS_VERSION").read_text(encoding="utf-8").strip() - - bundle_dir = get_bundle_dir(args.model, bits=args.bits, platform=platform) - ensure_bundle( - args.model, - bits=args.bits, - token=args.token, - output_dir=bundle_dir, - transpile=TranspileOptions(npu=platform is not None), - ) - - from huggingface_hub import HfApi - - api = HfApi(token=args.token) - api.create_repo(repo_id=repo_id, repo_type="model", exist_ok=True) - - with tempfile.TemporaryDirectory(prefix="cactus-upload-") as tmp: - archive_path = Path(tmp) / archive_name - with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: - for path in sorted(p for p in bundle_dir.rglob("*") if p.is_file()): - zf.write(path, f"{stem}/{path.relative_to(bundle_dir).as_posix()}") - print(f"uploading {archive_name} ({archive_path.stat().st_size / 1e6:.1f} MB) -> {repo_id}") - api.upload_file( - path_or_fileobj=str(archive_path), - path_in_repo=archive_name, - repo_id=repo_id, - repo_type="model", - commit_message=f"Upload {archive_name} ({tag})", - ) - - try: - api.delete_tag(repo_id=repo_id, tag=tag, repo_type="model") - except Exception: - pass - api.create_tag(repo_id=repo_id, tag=tag, repo_type="model", revision="main") - print(f"done: {repo_id} @ {tag}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 033c4f39dce96c9a98f391180c7b637ca3c322cc Mon Sep 17 00:00:00 2001 From: jakmro Date: Fri, 19 Jun 2026 03:45:13 +0200 Subject: [PATCH 2/5] fixes Signed-off-by: jakmro --- docs/cactus_transpiler.md | 4 +- python/cactus/cli/__init__.py | 82 ++++++++------------- python/cactus/cli/auth.py | 6 +- python/cactus/cli/common.py | 29 +++++++- python/cactus/cli/config_utils.py | 10 ++- python/cactus/cli/model.py | 23 +++--- python/cactus/cli/run.py | 22 +++--- python/cactus/cli/serve.py | 9 +-- python/cactus/cli/test.py | 18 +++-- python/cactus/cli/transcribe.py | 17 +---- python/cactus/server.py | 13 ++-- python/tests/bundles.py | 19 ++++- python/tests/test_chat_template_golden.py | 7 +- python/tests/test_cli_run.py | 2 +- python/tests/test_cli_transpile_defaults.py | 80 ++++++++++++++------ python/tests/test_model.py | 8 +- python/tests/test_server_live.py | 20 ++++- python/tests/test_tool_constraint_state.py | 7 +- 18 files changed, 226 insertions(+), 150 deletions(-) diff --git a/docs/cactus_transpiler.md b/docs/cactus_transpiler.md index a0dbd7773..991a729a3 100644 --- a/docs/cactus_transpiler.md +++ b/docs/cactus_transpiler.md @@ -459,7 +459,7 @@ huggingface.co/Cactus-Compute) or a local path to a bundle directory. | Option | Description | |--------|-------------| | `--bits 1\|2\|3\|4` | CQ quantization bits when downloading (default: 4) | -| `--platform auto\|cpu\|apple` | Target accelerator (default: auto — best for the host) | +| `--platform auto\|cpu\|apple` | Target platform (default: auto — best for the host) | | `--token ` | HuggingFace token (gated models) | | `--prompt ` | Input prompt | | `--input-ids ` | Comma-separated token IDs | @@ -473,7 +473,7 @@ huggingface.co/Cactus-Compute) or a local path to a bundle directory. | `--confidence-threshold ` | Confidence threshold below which to hand off to cloud | | `--cloud-timeout-ms ` | Max wait time for cloud handoff | | `--no-cloud-tele` | Disable cloud telemetry (write to cache only) | -| `--reconvert` | Force local convert+transpile fallback | +| `--reconvert` | Force local rebuild from source | --- diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 956e9fa53..1d156f8d6 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -39,6 +39,19 @@ def _telemetry_parent(): return p +def _build_parent(): + """Bundle-build flags shared by every command that prepares a model.""" + p = argparse.ArgumentParser(add_help=False) + p.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, + help="CQ quantization (default: 4)") + p.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", + help=_PLATFORM_HELP) + p.add_argument("--token", help="HuggingFace token") + p.add_argument("--reconvert", action="store_true", + help="Force local rebuild from source") + return p + + def _positive_int(value): n = int(value) if n <= 0: @@ -112,7 +125,7 @@ def create_parser(): --bits 1|2|3|4 CQ quantization (default: 4) --platform {_PLATFORM_PIPE:<22} target platform (default: auto) --token HuggingFace token (gated models) - --reconvert force reconversion from source + --reconvert force local rebuild from source cactus download [model] fetch a prebuilt bundle, else build locally (default: {DEFAULT_MODEL_ID}) --bits 1|2|3|4 CQ quantization (default: 4) @@ -134,6 +147,9 @@ def create_parser(): --platform {_PLATFORM_PIPE:<22} target platform (default: auto) --token HuggingFace token (gated models) --reconvert force local rebuild from source + --no-cloud-handoff disable automatic cloud handoff + --confidence-threshold <0..1> handoff to cloud below this confidence + --cloud-timeout-ms max wait for cloud handoff before local fallback cactus list list local converted weights and bundles @@ -147,6 +163,9 @@ def create_parser(): (default: all) --model default: {DEFAULT_TEST_MODEL_ID} --transcription-model default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID} + --bits 1|2|3|4 CQ quantization (default: 4) + --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --token HuggingFace token (gated models) --reconvert force local rebuild of test models --suite run a single test suite from any component (kernels, graph, or engine) @@ -175,17 +194,11 @@ def create_parser(): parser._action_groups = [] download_parser = subparsers.add_parser("download", - help="Download a pre-built bundle from huggingface.co/Cactus-Compute") + help="Download a pre-built bundle from huggingface.co/Cactus-Compute", + parents=[_build_parent()]) download_parser.add_argument("model_id", nargs="?", default=DEFAULT_MODEL_ID, type=_hf_id_or_path, help=f"HuggingFace model id (default: {DEFAULT_MODEL_ID})") - download_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, - help="CQ quantization bits (default: 4)") - download_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help=_PLATFORM_HELP) - download_parser.add_argument("--token", help="HuggingFace token") - download_parser.add_argument("--reconvert", action="store_true", - help="Force local rebuild from source instead of fetching the prebuilt bundle") build_parser = subparsers.add_parser("build", help="Build cactus libraries") build_group = build_parser.add_mutually_exclusive_group() @@ -197,17 +210,10 @@ def create_parser(): help="Build shared library for Python FFI") run_parser = subparsers.add_parser("run", help="Run a model (downloads bundle if needed)", - parents=[_telemetry_parent()]) + parents=[_telemetry_parent(), _build_parent()]) run_parser.add_argument("model_id", nargs="?", default=DEFAULT_MODEL_ID, type=_hf_id_or_path, help=f"HuggingFace model id or local bundle path (default: {DEFAULT_MODEL_ID})") - run_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, - help="CQ quantization bits (default: 4)") - run_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help=_PLATFORM_HELP) - run_parser.add_argument("--token", help="HuggingFace token") - run_parser.add_argument("--reconvert", action="store_true", - help="Force local rebuild from source") run_parser.add_argument("--image", help="Path to image file for VLM inference (attached to first message)") run_parser.add_argument("--audio", @@ -234,7 +240,7 @@ def create_parser(): help="Maximum time to wait for cloud handoff before falling back locally") transcribe_parser = subparsers.add_parser("transcribe", help="Transcribe audio with a model", - parents=[_telemetry_parent()]) + parents=[_telemetry_parent(), _build_parent()]) transcribe_parser.add_argument("model_id", nargs="?", default=DEFAULT_TRANSCRIPTION_MODEL_ID, type=_hf_id_or_path, help=f"HuggingFace model id (default: {DEFAULT_TRANSCRIPTION_MODEL_ID})") @@ -242,15 +248,9 @@ def create_parser(): help="Audio file to transcribe (WAV)") transcribe_parser.add_argument("--language", default="en", help="Language code (default: en)") - transcribe_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, - help="CQ quantization (default: 4)") - transcribe_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help=_PLATFORM_HELP) - transcribe_parser.add_argument("--token", help="HuggingFace token") - transcribe_parser.add_argument("--reconvert", action="store_true", - help="Force local rebuild from source") - - serve_parser = subparsers.add_parser("serve", help="OpenAI-compatible local HTTP server") + + serve_parser = subparsers.add_parser("serve", help="OpenAI-compatible local HTTP server", + parents=[_telemetry_parent(), _build_parent()]) serve_parser.add_argument("model_id", nargs="?", default=None, type=_hf_id_or_path, help="HuggingFace model id (e.g. openai/whisper-base) or bundle path") @@ -258,13 +258,6 @@ def create_parser(): help="Bind address (default: 127.0.0.1)") serve_parser.add_argument("--port", type=_port_int, default=8080, help="Port (default: 8080)") - serve_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, - help="CQ quantization (default: 4)") - serve_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help=_PLATFORM_HELP) - serve_parser.add_argument("--token", help="HuggingFace token") - serve_parser.add_argument("--reconvert", action="store_true", - help="Force local rebuild from source") serve_parser.add_argument("--no-cloud-handoff", action="store_true", help="Disable automatic cloud handoff for all requests") serve_parser.add_argument("--confidence-threshold", type=_unit_float, default=None, @@ -272,7 +265,8 @@ def create_parser(): serve_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, help="Maximum time to wait for cloud handoff before falling back locally") - test_parser = subparsers.add_parser("test", help="Run the test suite") + test_parser = subparsers.add_parser("test", help="Run the test suite", + parents=[_build_parent()]) test_parser.add_argument("--component", choices=COMPONENTS, default="all", help="Component to test (default: all)") test_parser.add_argument("--model", dest="model_id", default=None, @@ -281,8 +275,6 @@ def create_parser(): test_parser.add_argument("--transcription-model", dest="transcription_model_id", default=None, type=_hf_id_or_path, help=f"HF transcription model ID under test (default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID})") - test_parser.add_argument("--reconvert", action="store_true", - help="Force local rebuild of the test models from source") test_parser.add_argument("--suite", default=None, help="Run a single test suite by name; resolved across all components (e.g. llm → engine)") test_parser.add_argument("--list", action="store_true", @@ -304,27 +296,17 @@ def create_parser(): subparsers.add_parser("list", help="List local converted weights and bundles") convert_parser = subparsers.add_parser("convert", - help="Build a runnable bundle locally from source (no prebuilt fetch)") + help="Build a runnable bundle locally from source (no prebuilt fetch)", + parents=[_build_parent()]) convert_parser.add_argument("model_id", type=_hf_id_or_path, help="HuggingFace model id (e.g. openai/whisper-base)") convert_parser.add_argument("output_dir", nargs="?", default=None, help="Output directory (default: weights/)") - convert_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, - help="CQ quantization bits (default: 4)") - convert_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help=_PLATFORM_HELP) - convert_parser.add_argument("--token", help="HuggingFace token") convert_parser.add_argument("--lora", help="Path or HF id of a LoRA adapter to merge before converting (requires `peft`)") - convert_parser.add_argument("--reconvert", action="store_true", - help="Force rebuild from source") - upload_parser = subparsers.add_parser("upload") + upload_parser = subparsers.add_parser("upload", parents=[_build_parent()]) upload_parser.add_argument("model_id", type=_hf_id_or_path) - upload_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4) - upload_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto") - upload_parser.add_argument("--token", default=None) - upload_parser.add_argument("--reconvert", action="store_true") transpile_parser = subparsers.add_parser("transpile", help="Build a runnable bundle from CQ weights") diff --git a/python/cactus/cli/auth.py b/python/cactus/cli/auth.py index 39c01214f..38989fba7 100644 --- a/python/cactus/cli/auth.py +++ b/python/cactus/cli/auth.py @@ -14,7 +14,11 @@ def cmd_auth(args): print_color(GREEN, "API key cleared.") return 0 - api_key = config.get_api_key() + try: + api_key = config.get_api_key() + except (OSError, ValueError): + print_color(YELLOW, "Existing config is unreadable; set or clear a key to reset it.") + api_key = None if api_key: print(f"Current API key: {mask_key(api_key)}") diff --git a/python/cactus/cli/common.py b/python/cactus/cli/common.py index b1203a755..71601e4cb 100644 --- a/python/cactus/cli/common.py +++ b/python/cactus/cli/common.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +import subprocess import sys from pathlib import Path @@ -72,11 +73,37 @@ def mask_key(key): def apply_cloud_api_key_env() -> None: from .config_utils import CactusConfig - api_key = CactusConfig().get_api_key() + try: + api_key = CactusConfig().get_api_key() + except (OSError, ValueError): + return if api_key: os.environ["CACTUS_CLOUD_KEY"] = api_key +def apply_runtime_env(args) -> None: + """Prepare the process env for an inference command: honour --no-cloud-tele + and load the stored cloud API key.""" + if getattr(args, "no_cloud_tele", False): + os.environ["CACTUS_NO_CLOUD_TELE"] = "1" + apply_cloud_api_key_env() + + +def is_valid_bundle(path) -> bool: + """A runnable v2 bundle has both config.txt and the components manifest.""" + path = Path(path) + return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() + + +def launch_binary(name, *args) -> int: + """Resolve a bundled binary and exec it with str-coerced args. Returns its + exit code, or 1 if the binary is unavailable.""" + binary = resolve_binary(name) + if binary is None: + return 1 + return subprocess.run([str(binary), *(str(a) for a in args)]).returncode + + def _auto_build_binaries() -> bool: """Build the native runtime and CLI binaries, equivalent to `cactus build`.""" from argparse import Namespace diff --git a/python/cactus/cli/config_utils.py b/python/cactus/cli/config_utils.py index 24c09c1a1..6b6ecd154 100644 --- a/python/cactus/cli/config_utils.py +++ b/python/cactus/cli/config_utils.py @@ -34,6 +34,12 @@ def save_config(self, config): pass raise + def _load_or_empty(self): + try: + return self.load_config() + except (OSError, ValueError): + return {} + def get_api_key(self): env_key = os.getenv("CACTUS_CLOUD_KEY") or os.getenv("CACTUS_CLOUD_API_KEY") if env_key: @@ -41,11 +47,11 @@ def get_api_key(self): return self.load_config().get("api_key") or None def set_api_key(self, key): - config = self.load_config() + config = self._load_or_empty() config["api_key"] = key self.save_config(config) def clear_api_key(self): - config = self.load_config() + config = self._load_or_empty() config.pop("api_key", None) self.save_config(config) diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index 1270f354a..5e59f27f0 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -30,10 +30,10 @@ def _convert_from_source(model_id, *, bits, token, weights_dir): return weights_dir -def ensure_weights(model_id, *, bits=4, token=None, reconvert=False, output_dir=None): - from .download import get_weights_dir +def ensure_weights(model_id, *, bits=4, platform=None, token=None, reconvert=False, output_dir=None): + from .download import get_bundle_dir - weights_dir = Path(output_dir) if output_dir else get_weights_dir(model_id) + weights_dir = Path(output_dir) if output_dir else get_bundle_dir(model_id, bits=bits, platform=platform) if reconvert and weights_dir.exists(): print_color(YELLOW, "Removing cached weights for reconversion...") @@ -207,6 +207,9 @@ def ensure_runnable_bundle(model_id, *, bits=4, platform=None, token=None, if local is not None: return local + if str(model_id).startswith(("/", "./", "../", "~")) and not Path(model_id).expanduser().exists(): + raise RuntimeError(f"path not found: {model_id}") + cached = Path(output_dir) if output_dir else get_bundle_dir(model_id, bits=bits, platform=platform) if _has_transpiled_bundle(cached) and not reconvert: return cached @@ -221,8 +224,8 @@ def ensure_runnable_bundle(model_id, *, bits=4, platform=None, token=None, opts = transpile or TranspileOptions() if platform == "apple" and not opts.npu: opts = replace(opts, npu=True) - return ensure_bundle(model_id, bits=bits, token=token, reconvert=reconvert, - output_dir=cached, transpile=opts) + return ensure_bundle(model_id, bits=bits, platform=platform, token=token, + reconvert=reconvert, output_dir=cached, transpile=opts) def prepare_bundle(args, *, model_id=None, transpile=None, prebuilt=True, @@ -247,21 +250,21 @@ def prepare_bundle(args, *, model_id=None, transpile=None, prebuilt=True, return None -def ensure_bundle(model_id, *, bits=4, token=None, +def ensure_bundle(model_id, *, bits=4, platform=None, token=None, reconvert=False, output_dir=None, transpile=None): - from .download import get_weights_dir + from .download import get_bundle_dir from .transpile import run_transpile from cactus.transpile.component_plan import infer_component_plan_from_output opts = transpile or TranspileOptions() if output_dir is not None: - output_dir = Path(output_dir) + output_dir = Path(output_dir).expanduser().resolve() else: - output_dir = get_weights_dir(model_id) + output_dir = get_bundle_dir(model_id, bits=bits, platform=platform) ensure_weights( - model_id, bits=bits, token=token, + model_id, bits=bits, platform=platform, token=token, reconvert=reconvert, output_dir=output_dir, ) diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index f4ea6f1d1..e7bd3814a 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -1,19 +1,21 @@ -import os -import subprocess from pathlib import Path -from .common import GREEN, apply_cloud_api_key_env, print_color, resolve_binary +from .common import GREEN, RED, apply_runtime_env, launch_binary, print_color def cmd_run(args) -> int: - if args.no_cloud_tele: - os.environ["CACTUS_NO_CLOUD_TELE"] = "1" - apply_cloud_api_key_env() + apply_runtime_env(args) if args.image: args.image = str(Path(args.image).expanduser()) + if not Path(args.image).is_file(): + print_color(RED, f"Image not found: {args.image}") + return 1 if args.audio: args.audio = str(Path(args.audio).expanduser()) + if not Path(args.audio).is_file(): + print_color(RED, f"Audio not found: {args.audio}") + return 1 if args.result_json: args.result_json = str(Path(args.result_json).expanduser()) @@ -26,11 +28,7 @@ def cmd_run(args) -> int: if bundle_dir is None: return 1 - binary = resolve_binary("run") - if binary is None: - return 1 - - cmd = [str(binary), str(bundle_dir)] + cmd = [str(bundle_dir)] for flag, value in ( ("--system", args.system), ("--prompt", args.prompt), @@ -52,4 +50,4 @@ def cmd_run(args) -> int: print_color(GREEN, f"Running: {bundle_dir}") print() - return subprocess.run(cmd).returncode + return launch_binary("run", *cmd) diff --git a/python/cactus/cli/serve.py b/python/cactus/cli/serve.py index e99a4ed12..0e5b59e35 100644 --- a/python/cactus/cli/serve.py +++ b/python/cactus/cli/serve.py @@ -1,6 +1,6 @@ from pathlib import Path -from .common import BLUE, GREEN, RED, YELLOW, print_color, weights_root +from .common import BLUE, GREEN, RED, YELLOW, apply_runtime_env, is_valid_bundle, print_color, weights_root from .download import get_weights_dir @@ -19,12 +19,9 @@ def _resolve_model_arg(model: str | None) -> tuple[Path | None, str | None]: return None, model -def _is_valid_bundle(path: Path) -> bool: - return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() - - def cmd_serve(args): """Start the OpenAI-compatible HTTP server.""" + apply_runtime_env(args) model_path, model_name = _resolve_model_arg(args.model_id) if args.model_id and model_path is None: from .model import prepare_bundle @@ -32,7 +29,7 @@ def cmd_serve(args): if built is None: return 1 model_path, model_name = built, built.name - if model_path is not None and not _is_valid_bundle(model_path): + if model_path is not None and not is_valid_bundle(model_path): print_color(RED, f"Error: not a valid v2 Cactus bundle: {model_path}") print("Expected config.txt and components/manifest.json.") return 1 diff --git a/python/cactus/cli/test.py b/python/cactus/cli/test.py index 07504d632..8c64ba7ea 100644 --- a/python/cactus/cli/test.py +++ b/python/cactus/cli/test.py @@ -66,15 +66,17 @@ def cmd_test(args): targets = ("kernels", "graph", "engine") if args.component == "all" else (args.component,) if "engine" in targets: - from .model import ensure_runnable_bundle - try: - args.model_id = str(ensure_runnable_bundle( - args.model_id or DEFAULT_TEST_MODEL_ID, reconvert=args.reconvert)) - args.transcription_model_id = str(ensure_runnable_bundle( - args.transcription_model_id or DEFAULT_TEST_TRANSCRIPTION_MODEL_ID, reconvert=args.reconvert)) - except (RuntimeError, OSError, ValueError) as exc: - print_color(RED, f"Model setup failed: {exc}") + from .model import prepare_bundle + model_dir = prepare_bundle(args, model_id=args.model_id or DEFAULT_TEST_MODEL_ID, + fail_prefix="Model setup failed") + if model_dir is None: return 1 + args.model_id = str(model_dir) + stt_dir = prepare_bundle(args, model_id=args.transcription_model_id or DEFAULT_TEST_TRANSCRIPTION_MODEL_ID, + fail_prefix="Model setup failed") + if stt_dir is None: + return 1 + args.transcription_model_id = str(stt_dir) apply_cloud_api_key_env() env = os.environ.copy() diff --git a/python/cactus/cli/transcribe.py b/python/cactus/cli/transcribe.py index 79360ce31..a4d00acea 100644 --- a/python/cactus/cli/transcribe.py +++ b/python/cactus/cli/transcribe.py @@ -1,8 +1,6 @@ -import os -import subprocess from pathlib import Path -from .common import apply_cloud_api_key_env, print_color, resolve_binary, RED, GREEN +from .common import apply_runtime_env, launch_binary, print_color, RED, GREEN def cmd_transcribe(args): @@ -15,20 +13,13 @@ def cmd_transcribe(args): return 1 args.audio_file = str(audio_path) - if args.no_cloud_tele: - os.environ["CACTUS_NO_CLOUD_TELE"] = "1" - - apply_cloud_api_key_env() + apply_runtime_env(args) bundle_dir = prepare_bundle(args, transpile=TranspileOptions(audio_file=args.audio_file)) if bundle_dir is None: return 1 - binary = resolve_binary("transcribe") - if binary is None: - return 1 - - cmd = [str(binary), str(bundle_dir)] + cmd = [str(bundle_dir)] if args.audio_file: cmd.append(args.audio_file) if args.language: @@ -37,4 +28,4 @@ def cmd_transcribe(args): print_color(GREEN, f"Starting Cactus transcription with model: {args.model_id}") print() - return subprocess.run(cmd).returncode + return launch_binary("transcribe", *cmd) diff --git a/python/cactus/server.py b/python/cactus/server.py index a802efb97..18d23dc4b 100644 --- a/python/cactus/server.py +++ b/python/cactus/server.py @@ -4,6 +4,7 @@ import asyncio import json import logging +import re import time import uuid from contextlib import asynccontextmanager @@ -25,7 +26,7 @@ cactus_transcribe, ) from .cli.download import get_model_dir_name, get_weights_dir -from .cli.common import DEFAULT_MODEL_ID, PROJECT_ROOT, is_repo_checkout +from .cli.common import DEFAULT_MODEL_ID, is_valid_bundle, weights_root as default_weights_root LOGGER = logging.getLogger(__name__) @@ -82,9 +83,7 @@ def _supports_embedding(model_dir: Path) -> bool: def _info_for_dir(cls, path: Path) -> ModelInfo | None: display_id = path.expanduser().name resolved = path.expanduser().resolve() - if not (resolved / "config.txt").exists(): - return None - if not (resolved / "components" / "manifest.json").exists(): + if not is_valid_bundle(resolved): return None context_raw = cls._read_config_field(resolved, "context_length") try: @@ -121,7 +120,7 @@ def _resolve(self, model_id: str) -> ModelInfo | None: stem = get_model_dir_name(model_id) if stem in self.models: return self.models[stem] - variants = [m for m in self.models.values() if m.id.startswith(f"{stem}-")] + variants = [m for m in self.models.values() if re.match(rf"{re.escape(stem)}-cq\d", m.id)] return variants[0] if len(variants) == 1 else None def require(self, model_id: str) -> ModelInfo: @@ -502,9 +501,7 @@ def create_app( confidence_threshold: float | None = None, cloud_timeout_ms: int | None = None, ) -> FastAPI: - root = Path(weights_root) if weights_root is not None else ( - PROJECT_ROOT / "weights" if is_repo_checkout() else Path.home() / ".cache" / "cactus" / "weights" - ) + root = Path(weights_root) if weights_root is not None else default_weights_root() registry = ModelRegistry(root, extra_model=model_path) if default_model is not None: selected = registry.models.get(default_model) diff --git a/python/tests/bundles.py b/python/tests/bundles.py index 2eca102f6..058620410 100644 --- a/python/tests/bundles.py +++ b/python/tests/bundles.py @@ -19,11 +19,24 @@ def _valid_bundle(path: Path) -> bool: return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() +def _iter_bundle_candidates(name: str): + """Yield the bare bundle dir for `name` plus any suffixed `name-cq*` variants + (e.g. `gemma-4-e2b-it-cq4`, `gemma-4-e2b-it-cq4-apple`) so callers that know a + model's bare stem still find bundles built under the suffixed convention.""" + bare = WEIGHTS / name + if bare.exists(): + yield bare + if WEIGHTS.is_dir(): + for candidate in sorted(WEIGHTS.glob(f"{name}-cq*")): + if candidate.is_dir(): + yield candidate + + def _find_bundle(preferred: list[str], types: set[str], on_missing=pytest.fail) -> Path: for name in preferred: - candidate = WEIGHTS / name - if candidate.exists() and _valid_bundle(candidate) and _read_model_type(candidate) in types: - return candidate + for candidate in _iter_bundle_candidates(name): + if _valid_bundle(candidate) and _read_model_type(candidate) in types: + return candidate if not WEIGHTS.is_dir(): on_missing(f"Weights directory not found: {WEIGHTS}") for candidate in sorted(WEIGHTS.iterdir()): diff --git a/python/tests/test_chat_template_golden.py b/python/tests/test_chat_template_golden.py index cd8cfc346..78077dd2c 100644 --- a/python/tests/test_chat_template_golden.py +++ b/python/tests/test_chat_template_golden.py @@ -7,7 +7,7 @@ transformers = pytest.importorskip("transformers") -from .bundles import PROJECT_ROOT, WEIGHTS, _read_model_type, _valid_bundle +from .bundles import PROJECT_ROOT, WEIGHTS, _iter_bundle_candidates, _read_model_type, _valid_bundle MONKEY_IMAGE = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" / "test_monkey.png" @@ -142,7 +142,10 @@ def load_family(): def _load(bundle_name: str, hf_id: str): if bundle_name not in _LOADED: - bundle = WEIGHTS / bundle_name + bundle = next( + (c for c in _iter_bundle_candidates(bundle_name) if _valid_bundle(c)), + WEIGHTS / bundle_name, + ) if not _valid_bundle(bundle): pytest.skip(f"bundle not prepared: {bundle}") try: diff --git a/python/tests/test_cli_run.py b/python/tests/test_cli_run.py index 660ffc84c..5c4c0d7e5 100644 --- a/python/tests/test_cli_run.py +++ b/python/tests/test_cli_run.py @@ -32,7 +32,7 @@ def fake_subprocess_run(cmd): calls.append(cmd) return SimpleNamespace(returncode=0) - monkeypatch.setattr(run_mod.subprocess, "run", fake_subprocess_run) + monkeypatch.setattr(common_mod.subprocess, "run", fake_subprocess_run) monkeypatch.setattr(model_mod, "ensure_runnable_bundle", lambda *a, **k: bundle_dir) args = Namespace( diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py index fe9021911..e821cb6a4 100644 --- a/python/tests/test_cli_transpile_defaults.py +++ b/python/tests/test_cli_transpile_defaults.py @@ -68,42 +68,78 @@ def test_cache_context_length_rejects_non_positive_explicit_value() -> None: ) -def test_cmd_convert_does_not_transpile(monkeypatch, tmp_path: Path) -> None: - """`cactus convert` does CQ quantization only; the runtime graph is built - by `cactus transpile` per docs/cactus_transpiler.md.""" +def test_cmd_convert_builds_full_bundle(monkeypatch, tmp_path: Path) -> None: + """`cactus convert` builds a runnable bundle locally (convert + transpile) + and skips the prebuilt-bundle fetch.""" parser = cli.create_parser() - args = parser.parse_args(["convert", "google/gemma-4-E2B-it", str(tmp_path / "out"), "--reconvert"]) + out_dir = tmp_path / "out" + args = parser.parse_args(["convert", "google/gemma-4-E2B-it", str(out_dir)]) - cq_calls: list[list[str]] = [] + import cactus.cli.model as model_mod + import cactus.cli.download as download_mod - def _fake_cq_main(command): - cq_calls.append(list(command)) - out_dir = tmp_path / "out" - out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / "config.txt").write_text("model_type=gemma4\n", encoding="utf-8") - return 0 + built: dict[str, object] = {} - import cactus.convert.cli as cq_cli - monkeypatch.setattr(cq_cli, "main", _fake_cq_main) + def _fake_ensure_bundle(model_id, **kwargs): + built["model_id"] = model_id + built["output_dir"] = kwargs.get("output_dir") + return kwargs.get("output_dir") - # cmd_convert should call ensure_weights, never ensure_bundle (the latter - # transpiles as well). Patch ensure_bundle to fail loudly if invoked. - import cactus.cli.model as model_mod + monkeypatch.setattr(model_mod, "ensure_bundle", _fake_ensure_bundle) - def _fail_if_called(*a, **kw): - raise AssertionError("cmd_convert must not invoke ensure_bundle (would transpile)") + def _fail_download(*a, **kw): + raise AssertionError("cactus convert must not fetch a prebuilt bundle") - monkeypatch.setattr(model_mod, "ensure_bundle", _fail_if_called) + monkeypatch.setattr(download_mod, "download_bundle", _fail_download) rc = convert_cli.cmd_convert(args) assert rc == 0 - assert len(cq_calls) == 1 - assert "--out" in cq_calls[0] + assert built["model_id"] == "google/gemma-4-E2B-it" + assert Path(str(built["output_dir"])).resolve() == out_dir.resolve() + + +def test_ensure_bundle_resolves_relative_output_dir(monkeypatch, tmp_path: Path) -> None: + """A relative output_dir must reach the transpile subprocess (which runs with + cwd=PROJECT_ROOT) as an absolute path resolved against the caller's cwd — + otherwise the in-process CQ-weights step and the transpiler disagree on where + the weights live, and the build fails with 'weights_dir does not exist'.""" + import cactus.cli.model as model_mod + from cactus.cli import transpile as transpile_mod + import cactus.transpile.component_plan as cp + + monkeypatch.chdir(tmp_path) + captured: dict = {} + + def fake_ensure_weights(model_id, **kwargs): + out = Path(kwargs["output_dir"]) + out.mkdir(parents=True, exist_ok=True) + captured["weights_out"] = kwargs["output_dir"] + return out + + def fake_run_transpile(model_id, *, extra_args, **kwargs): + captured["extra"] = list(extra_args) + return 0 + + plan = SimpleNamespace(task="causal_lm_logits", components=(), default_max_new_tokens=8, + needs_image=False, needs_audio=False, force_component_pipeline=False) + + monkeypatch.setattr(model_mod, "ensure_weights", fake_ensure_weights) + monkeypatch.setattr(model_mod, "_has_transpiled_bundle", lambda p: False) + monkeypatch.setattr(transpile_mod, "run_transpile", fake_run_transpile) + monkeypatch.setattr(cp, "infer_component_plan_from_output", lambda *a, **k: plan) + + model_mod.ensure_bundle("some/model", output_dir="rel-out") + + weights_out = Path(str(captured["weights_out"])) + assert weights_out.is_absolute() + assert weights_out == (tmp_path / "rel-out").resolve() + wd = captured["extra"][captured["extra"].index("--weights-dir") + 1] + assert Path(wd).is_absolute() def test_cli_registers_transpile_command() -> None: - """`cactus transpile` is a first-class subcommand per docs/cactus_transpiler.md.""" + """`cactus transpile` stays a registered subcommand (hidden from the CLI menu).""" parser = cli.create_parser() args = parser.parse_args(["transpile", "google/gemma-4-E2B-it", "--weights-dir", "/tmp/x", "--task", "causal_lm_logits"]) diff --git a/python/tests/test_model.py b/python/tests/test_model.py index 543243ae7..b6b859ad7 100644 --- a/python/tests/test_model.py +++ b/python/tests/test_model.py @@ -17,8 +17,10 @@ cactus_transcribe, ) from cactus.cli.model import ensure_bundle +from cactus.cli.download import resolve_platform +_PLATFORM = resolve_platform("auto") _ASSETS_DIR = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" _TEST_IMAGE = _ASSETS_DIR / "test_monkey.png" _TEST_AUDIO = _ASSETS_DIR / "test.wav" @@ -27,7 +29,7 @@ class TestVLMModel(unittest.TestCase): @classmethod def setUpClass(cls): - cls.weights_dir = ensure_bundle("LiquidAI/LFM2-VL-450M") + cls.weights_dir = ensure_bundle("LiquidAI/LFM2-VL-450M", platform=_PLATFORM) cls.model = cactus_init(str(cls.weights_dir), None, False) @classmethod @@ -64,7 +66,7 @@ class TestWhisperModel(unittest.TestCase): @classmethod def setUpClass(cls): - cls.weights_dir = ensure_bundle("openai/whisper-small") + cls.weights_dir = ensure_bundle("openai/whisper-small", platform=_PLATFORM) cls.model = cactus_init(str(cls.weights_dir), None, False) @classmethod @@ -105,7 +107,7 @@ class TestNomicEmbedding(unittest.TestCase): @classmethod def setUpClass(cls): - cls.weights_dir = ensure_bundle(cls.MODEL_ID) + cls.weights_dir = ensure_bundle(cls.MODEL_ID, platform=_PLATFORM) cls.model = cactus_init(str(cls.weights_dir), None, False) @classmethod diff --git a/python/tests/test_server_live.py b/python/tests/test_server_live.py index cac599707..d75f2b738 100644 --- a/python/tests/test_server_live.py +++ b/python/tests/test_server_live.py @@ -13,14 +13,25 @@ import httpx import pytest -from .bundles import PROJECT_ROOT, WEIGHTS, _find_bundle, _read_model_type, _valid_bundle +from .bundles import ( + PROJECT_ROOT, WEIGHTS, _find_bundle, _iter_bundle_candidates, + _read_model_type, _valid_bundle, +) -DEFAULT_LLM_BUNDLE = Path("weights/gemma-4-e2b-it") ASSETS = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" LLM_TYPES = {"gemma", "gemma3n", "gemma4", "lfm2", "qwen", "qwen3p5", "needle", "youtu"} STT_TYPES = {"whisper", "parakeet_tdt", "parakeet-tdt"} +def _default_llm_bundle() -> Path: + """Locate the canonical gemma-4-e2b-it LLM bundle under whichever convention + it was built with (bare `gemma-4-e2b-it` or suffixed `...-cq4[-apple]`).""" + for candidate in _iter_bundle_candidates("gemma-4-e2b-it"): + if _valid_bundle(candidate) and _read_model_type(candidate) in LLM_TYPES: + return candidate + return WEIGHTS / "gemma-4-e2b-it" # fall through so _require_bundle reports it + + def _require_bundle(relative: Path, types: set[str]) -> Path: candidate = PROJECT_ROOT / relative if not candidate.exists(): @@ -110,8 +121,9 @@ def _serve(bundle: Path): @pytest.fixture(scope="module") def live_server(): - _require_bundle(DEFAULT_LLM_BUNDLE, LLM_TYPES) - with _serve(DEFAULT_LLM_BUNDLE) as server: + bundle = _default_llm_bundle() + _require_bundle(bundle, LLM_TYPES) + with _serve(bundle) as server: yield server diff --git a/python/tests/test_tool_constraint_state.py b/python/tests/test_tool_constraint_state.py index 1be5bc994..9fe53d807 100644 --- a/python/tests/test_tool_constraint_state.py +++ b/python/tests/test_tool_constraint_state.py @@ -6,9 +6,12 @@ from cactus import cactus_complete, cactus_destroy, cactus_init -from .bundles import WEIGHTS, _valid_bundle +from .bundles import WEIGHTS, _iter_bundle_candidates, _valid_bundle -BUNDLE = WEIGHTS / "gemma-4-e2b-it" +BUNDLE = next( + (c for c in _iter_bundle_candidates("gemma-4-e2b-it") if _valid_bundle(c)), + WEIGHTS / "gemma-4-e2b-it", +) GET_WEATHER = { "type": "function", From 1f9ed983878c5375bb078f5b4ee6eea18d4611b0 Mon Sep 17 00:00:00 2001 From: jakmro Date: Fri, 19 Jun 2026 13:22:14 +0200 Subject: [PATCH 3/5] clean list command Signed-off-by: jakmro --- python/cactus/cli/list.py | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/python/cactus/cli/list.py b/python/cactus/cli/list.py index a5d623d38..5b024a2fa 100644 --- a/python/cactus/cli/list.py +++ b/python/cactus/cli/list.py @@ -2,7 +2,7 @@ import stat as _stat import struct -from .common import BLUE, CYAN, print_color, transpiled_root, weights_root +from .common import CYAN, print_color, transpiled_root, weights_root _PREC_TO_BITS = {3: 1, 4: 2, 5: 3, 6: 4} @@ -73,8 +73,8 @@ def _dir_size(path): return total -def _categorize(roots): - converted, transpiled = [], [] +def _collect(roots): + models = [] for root in roots: if not root.is_dir(): continue @@ -82,28 +82,19 @@ def _categorize(roots): if not p.is_dir() or not (p / "config.txt").is_file(): continue cfg = _read_config(p / "config.txt") - entry = (p, cfg.get("model_type", "?"), _quant_label(p), _dir_size(p)) - if (p / "components" / "manifest.json").is_file(): - transpiled.append(entry) - else: - converted.append(entry) - return converted, transpiled - - -def _print_section(title: str, color: str, entries): - print_color(color, title) - if not entries: - print(" (none)") - return - width = max(len(p.name) for p, _, _, _ in entries) - for p, model_type, quant, size in entries: - print(f" {p.name:<{width}} {model_type:<12} {quant:<5} {_human_size(size):>10} {p.parent}") + models.append((p, cfg.get("model_type", "?"), _quant_label(p), _dir_size(p))) + return models def cmd_list(_args): - roots = (weights_root(), transpiled_root()) - converted, transpiled = _categorize(roots) - _print_section("Converted weights", BLUE, converted) - print() - _print_section("Runnable bundles (cactus convert / download)", CYAN, transpiled) + models = _collect((weights_root(), transpiled_root())) + print_color(CYAN, "Local models (cactus convert / download)") + if not models: + print(" (none)") + return 0 + name_w = max(len(p.name) for p, _, _, _ in models) + type_w = max(len("type"), max(len(t) for _, t, _, _ in models)) + print(f" {'name':<{name_w}} {'type':<{type_w}} {'quant':<5} {'size':>10} location") + for p, model_type, quant, size in models: + print(f" {p.name:<{name_w}} {model_type:<{type_w}} {quant:<5} {_human_size(size):>10} {p.parent}") return 0 From 682d988811c32c8eb94c0bf9beed4550b5856a00 Mon Sep 17 00:00:00 2001 From: jakmro Date: Fri, 19 Jun 2026 13:31:13 +0200 Subject: [PATCH 4/5] align menu Signed-off-by: jakmro --- README.md | 20 ++++++++++++++++---- python/cactus/cli/__init__.py | 4 ++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 64480e7c8..c2b9697e5 100644 --- a/README.md +++ b/README.md @@ -216,20 +216,28 @@ graph.hard_reset(); │ cactus download [model] get a bundle (prebuilt, else build) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ │ --platform auto|cpu|apple target platform (default: auto) │ -│ --token HuggingFace token │ +│ --token HuggingFace token (gated models) │ +│ --reconvert force local rebuild from source │ │ │ │ cactus convert [dir] build a runnable bundle locally │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ │ --platform auto|cpu|apple target platform (default: auto) │ -│ --token HuggingFace token │ -│ --reconvert force rebuild from source │ +│ --token HuggingFace token (gated models) │ +│ --reconvert force local rebuild from source │ │ --lora merge a LoRA adapter before converting │ │ │ │ cactus serve [model] OpenAI-compatible local HTTP server │ │ --host bind address (default: 127.0.0.1) │ │ --port port (default: 8080) │ +│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --platform auto|cpu|apple target platform (default: auto) │ +│ --token HuggingFace token (gated models) │ +│ --reconvert force local rebuild from source │ +│ --no-cloud-handoff disable automatic cloud handoff │ +│ --confidence-threshold <0..1> handoff to cloud below this confidence │ +│ --cloud-timeout-ms max wait for cloud handoff │ │ │ -│ cactus list list local converted weights and bundles │ +│ cactus list list downloaded models │ │ │ │ cactus build build cactus libraries │ │ --apple Apple (iOS/macOS) │ @@ -241,6 +249,10 @@ graph.hard_reset(); │ (default: all) │ │ --model default: LiquidAI/LFM2-VL-450M │ │ --transcription-model default: openai/whisper-base │ +│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --platform auto|cpu|apple target platform (default: auto) │ +│ --token HuggingFace token (gated models) │ +│ --reconvert force local rebuild of test models │ │ --suite run a single test suite by name │ │ (resolved across components, │ │ e.g. llm → engine) │ diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 1d156f8d6..28a7114ba 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -151,7 +151,7 @@ def create_parser(): --confidence-threshold <0..1> handoff to cloud below this confidence --cloud-timeout-ms max wait for cloud handoff before local fallback - cactus list list local converted weights and bundles + cactus list list downloaded models cactus build build cactus libraries --apple Apple (iOS/macOS) @@ -293,7 +293,7 @@ def create_parser(): clean_parser = subparsers.add_parser("clean", help="Delete build artifacts, downloaded weights, and venv") clean_parser.add_argument("--yes", "-y", action="store_true", help="Skip confirmation prompt") - subparsers.add_parser("list", help="List local converted weights and bundles") + subparsers.add_parser("list", help="List downloaded models") convert_parser = subparsers.add_parser("convert", help="Build a runnable bundle locally from source (no prebuilt fetch)", From 65d6033aadb42829adf7024f1a7e47fbfb579903 Mon Sep 17 00:00:00 2001 From: jakmro Date: Fri, 19 Jun 2026 13:35:33 +0200 Subject: [PATCH 5/5] title rename Signed-off-by: jakmro --- python/cactus/cli/list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cactus/cli/list.py b/python/cactus/cli/list.py index 5b024a2fa..24cac392a 100644 --- a/python/cactus/cli/list.py +++ b/python/cactus/cli/list.py @@ -88,7 +88,7 @@ def _collect(roots): def cmd_list(_args): models = _collect((weights_root(), transpiled_root())) - print_color(CYAN, "Local models (cactus convert / download)") + print_color(CYAN, "Available models") if not models: print(" (none)") return 0