diff --git a/README.md b/README.md index f3d9ddb6b..42c442b95 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max │ --clear remove saved key │ │ │ │ cactus run [model|path] run a model (downloads if needed) │ -│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │ │ --backend cpu|metal inference backend (default: auto) │ │ --image image file for VLM inference │ │ --audio audio file for audio chat │ @@ -227,12 +227,12 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max │ cactus transcribe [model] live microphone transcription with a model│ │ --file audio file to transcribe (WAV) │ │ --language language code (default: en) │ -│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ │ │ cactus download [model] get a bundle (prebuilt, else build) │ -│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ │ @@ -248,7 +248,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max │ cactus serve [model] OpenAI-compatible local HTTP server │ │ --host bind address (default: 127.0.0.1) │ │ --port port (default: 8080) │ -│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │ │ --backend cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ @@ -258,7 +258,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max │ │ │ cactus code run the AI coding agent (TUI / print) │ │ --serve-model auto-start a server with this model │ -│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │ │ --backend cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ @@ -282,7 +282,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max │ (default: all) │ │ --model default: google/gemma-4-E2B-it │ │ --transcription-model default: nvidia/parakeet-tdt-0.6b-v3 │ -│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │ │ --backend cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild of test models │ @@ -297,7 +297,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max │ cactus benchmark run the engine benchmark suite │ │ --model default: google/gemma-4-E2B-it │ │ --transcription-model default: nvidia/parakeet-tdt-0.6b-v3 │ -│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │ │ --backend cpu|metal inference backend (default: auto) │ │ --ios run on connected iPhone │ │ --android run on connected Android │ diff --git a/docs/cactus_transpiler.md b/docs/cactus_transpiler.md index 784def696..de0a7e601 100644 --- a/docs/cactus_transpiler.md +++ b/docs/cactus_transpiler.md @@ -449,7 +449,7 @@ huggingface.co/Cactus-Compute) or a local path to a bundle directory. | Option | Description | |--------|-------------| -| `--bits 1\|2\|3\|4` | CQ quantization bits when downloading (default: 4) | +| `--bits 1\|2\|3\|4\|2.54\|3.26` | CQ quantization bits when downloading (default: 4) | | `--token ` | HuggingFace token (gated models) | | `--prompt ` | Input prompt | | `--input-ids ` | Comma-separated token IDs | diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index fd7a005bc..a308d13c3 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -21,6 +21,9 @@ from .auth import cmd_auth from .clean import cmd_clean from .code import cmd_code +from .utils import bits_arg, ALLOWED_BITS + +_BITS_METAVAR = "{" + ",".join(str(b) for b in ALLOWED_BITS) + "}" def _telemetry_parent(): @@ -31,11 +34,16 @@ def _telemetry_parent(): return p -def _build_parent(): - """Bundle-build flags shared by every command that prepares a model.""" +def _build_parent(mixed: bool = True): + """Bundle-build flags shared by model-preparing commands + (mixed=False limits --bits to uniform 1-4 for build-only commands).""" p = argparse.ArgumentParser(add_help=False) - p.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, - help="CQ quantization (default: 4)") + if mixed: + p.add_argument("--bits", type=bits_arg, default=4, metavar=_BITS_METAVAR, + help="CQ quantization: uniform 1-4 or gemma-4 mixed 3.26/2.54 (default: 4)") + else: + p.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, + help="CQ quantization (default: 4)") p.add_argument("--token", help="HuggingFace token") p.add_argument("--reconvert", action="store_true", help="Force local rebuild from source") @@ -117,7 +125,7 @@ def create_parser(): --clear remove saved key cactus run [model|path] run a model (default: {DEFAULT_MODEL_ID}) - --bits 1|2|3|4 CQ quantization (default: 4) + --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) --image image file for VLM inference --audio audio file for audio chat --system system prompt @@ -129,12 +137,12 @@ def create_parser(): cactus transcribe [model] live microphone transcription with a model --file audio file to transcribe (WAV) --language language code (default: en) - --bits 1|2|3|4 CQ quantization (default: 4) + --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) --token HuggingFace token (gated models) --reconvert force local rebuild from source cactus download [model] fetch a prebuilt bundle, else build locally (default: {DEFAULT_MODEL_ID}) - --bits 1|2|3|4 CQ quantization (default: 4) + --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) --token HuggingFace token (gated models) --reconvert force local rebuild from source @@ -147,7 +155,7 @@ def create_parser(): cactus serve [model] OpenAI-compatible local HTTP server --host bind address (default: 127.0.0.1) --port port (default: 8080) - --bits 1|2|3|4 CQ quantization (default: 4) + --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) --token HuggingFace token (gated models) --reconvert force local rebuild from source --no-cloud-handoff disable automatic cloud handoff @@ -166,7 +174,7 @@ def create_parser(): (default: all) --model default: {DEFAULT_TEST_MODEL_ID} --transcription-model default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID} - --bits 1|2|3|4 CQ quantization (default: 4) + --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) --token HuggingFace token (gated models) --reconvert force local rebuild of test models --suite run a single test suite from any @@ -179,7 +187,7 @@ def create_parser(): cactus benchmark run the engine benchmark suite --model default: {DEFAULT_TEST_MODEL_ID} --transcription-model default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID} - --bits 1|2|3|4 CQ quantization (default: 4) + --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) --backend cpu|metal inference backend (default: auto) --ios run on connected iPhone --android run on connected Android @@ -443,7 +451,7 @@ def create_parser(): help="KV cache context length for cached decode graphs (default: model config)") upload_parser = subparsers.add_parser("upload", - parents=[_build_parent()], + parents=[_build_parent(mixed=False)], help="Build a runnable bundle locally and upload it to Cactus-Compute on HuggingFace") upload_parser.add_argument("model_id", type=_hf_id_or_path, help="HuggingFace model id (e.g. openai/whisper-base)") diff --git a/python/cactus/cli/download.py b/python/cactus/cli/download.py index be4a8f13c..110eded47 100644 --- a/python/cactus/cli/download.py +++ b/python/cactus/cli/download.py @@ -15,7 +15,7 @@ def get_weights_dir(model_id: str) -> Path: return weights_root() / get_model_dir_name(model_id) -def get_bundle_dir(model_id: str, *, bits: int = 4) -> Path: +def get_bundle_dir(model_id: str, *, bits: int | float = 4) -> Path: from .utils import variant_suffix return weights_root() / f"{get_model_dir_name(model_id)}-{variant_suffix(bits)}" @@ -24,7 +24,7 @@ def ensure_model(model_id: str) -> Path: return download_bundle(model_id) -def download_bundle(model_id: str, *, bits: int = 4, +def download_bundle(model_id: str, *, bits: int | float = 4, token: str | None = None, output_dir: Path | None = None) -> Path: from .utils import ( download_cq_archive, diff --git a/python/cactus/cli/list.py b/python/cactus/cli/list.py index 24cac392a..973a9a767 100644 --- a/python/cactus/cli/list.py +++ b/python/cactus/cli/list.py @@ -36,12 +36,14 @@ def _cq_precision(weights_file): def _quant_label(model_dir, *, sample_cap=64, scan_cap=2000): - """Infer a model's CQ quantization level from its tensor headers. - - Scans `.weights` files (top-level converted weights, then nested - bundle components) and returns the dominant CQ level, e.g. "CQ4". - Returns "—" if no CQ-quantized tensors are present. - """ + """Infer a model's CQ variant from the `-cqN` dir suffix (the only way to + recover a mixed variant like CQ3.26), else the dominant CQ level in the + tensor headers. Returns "—" if no CQ-quantized tensors are present.""" + from .utils import parse_cq_variant + + variant = parse_cq_variant(model_dir.name) + if variant is not None: + return f"CQ{variant}" candidates = itertools.chain( model_dir.glob("*.weights"), (model_dir / "components").rglob("*.weights"), @@ -94,7 +96,8 @@ def cmd_list(_args): return 0 name_w = max(len(p.name) for p, _, _, _ in models) type_w = max(len("type"), max(len(t) for _, t, _, _ in models)) - print(f" {'name':<{name_w}} {'type':<{type_w}} {'quant':<5} {'size':>10} location") + quant_w = max(len("quant"), max(len(q) for _, _, q, _ in models)) + print(f" {'name':<{name_w}} {'type':<{type_w}} {'quant':<{quant_w}} {'size':>10} location") for p, model_type, quant, size in models: - print(f" {p.name:<{name_w}} {model_type:<{type_w}} {quant:<5} {_human_size(size):>10} {p.parent}") + print(f" {p.name:<{name_w}} {model_type:<{type_w}} {quant:<{quant_w}} {_human_size(size):>10} {p.parent}") return 0 diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index 75455937f..1a8a4a508 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -13,6 +13,11 @@ def _convert_from_source(model_id, *, bits, token, weights_dir, skip_model_load=False): """Download from HuggingFace and run CQ conversion.""" + if bits not in (1, 2, 3, 4): + raise SystemExit( + f"CQ{bits} is a mixed-precision variant, available only as a prebuilt " + f"download; local conversion supports uniform bits 1-4" + ) from .common import convert_toolchain_error err = convert_toolchain_error() if err: diff --git a/python/cactus/cli/utils.py b/python/cactus/cli/utils.py index b05f6c608..418085700 100644 --- a/python/cactus/cli/utils.py +++ b/python/cactus/cli/utils.py @@ -1,5 +1,6 @@ from __future__ import annotations +import argparse import hashlib import json import os @@ -15,18 +16,36 @@ from typing import Iterable -CQ_VARIANT_RE = re.compile(r"-cq([1-4])$", re.IGNORECASE) +CQ_VARIANT_RE = re.compile(r"-cq(\d+(?:\.\d+)?)$", re.IGNORECASE) ARCHIVE_SUFFIXES = (".zip", ".tar", ".tar.gz", ".tgz") +ALLOWED_BITS = (1, 2, 3, 4, 2.54, 3.26) -def variant_suffix(bits: int) -> str: - return f"cq{bits}" + +def normalize_bits(bits: str | int | float) -> int | float: + value = float(bits) + return int(value) if value == int(value) else value + + +def variant_suffix(bits: int | float) -> str: + return f"cq{normalize_bits(bits)}" + + +def bits_arg(value: str) -> int | float: + try: + bits = normalize_bits(value) + except (TypeError, ValueError): + raise argparse.ArgumentTypeError(f"invalid --bits value {value!r}") + if bits not in ALLOWED_BITS: + allowed = ", ".join(str(b) for b in ALLOWED_BITS) + raise argparse.ArgumentTypeError(f"unsupported --bits {value!r}; choose from: {allowed}") + return bits @dataclass(frozen=True) class CqArchive: filename: str - bits: int + bits: int | float size: int | None = None sha256: str | None = None @@ -45,7 +64,7 @@ class CqResolution: def suggested_cq_repo(model_id: str) -> str: name = str(model_id).strip().replace("_", "-").split("/")[-1] - name = re.sub(r"-cq\d*$", "", name, flags=re.IGNORECASE) + name = re.sub(r"-cq\d*(?:\.\d+)?$", "", name, flags=re.IGNORECASE) return f"Cactus-Compute/{name}" @@ -56,12 +75,12 @@ def archive_stem(filename: str) -> str: return filename -def parse_cq_variant(filename: str) -> int | None: +def parse_cq_variant(filename: str) -> int | float | None: stem = archive_stem(Path(filename).name) m = CQ_VARIANT_RE.search(stem) if not m: return None - return int(m.group(1)) + return normalize_bits(m.group(1)) def is_supported_archive(filename: str) -> bool: @@ -82,7 +101,7 @@ def archives_from_repo_files(repo_files: Iterable[str], sizes: dict[str, int] | continue archives.append(CqArchive(filename=filename, bits=bits, size=sizes.get(filename), sha256=sha256s.get(filename))) - return tuple(sorted(archives, key=lambda a: a.bits)) + return tuple(sorted(archives, key=lambda a: (isinstance(a.bits, float), a.bits))) VERSION_TAG_RE = re.compile(r"^v?(\d+(?:\.\d+){0,2})$", re.IGNORECASE) @@ -140,7 +159,7 @@ def resolve_weight_revision(repo_id: str, runtime_version: str | None = None, def resolve_archive(repo_id: str, local_name: str, archives: Iterable[CqArchive], - bits: int) -> CqResolution: + bits: int | float) -> CqResolution: available = tuple(archives) if not available: raise RuntimeError(f"No CQ archives found in {repo_id}")