Skip to content

add wider cq support #763

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jakmro merged 1 commit into from
Jul 11, 2026
Merged

add wider cq support #763

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max
│ --clear remove saved key │
│ │
│ cactus run [model|path] run a model (downloads if needed) │
│ --bits 1|2|3|4 CQ quantization (default: 4) │
│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │
│ --backend cpu|metal inference backend (default: auto) │
│ --image <path> image file for VLM inference │
│ --audio <path> audio file for audio chat │
Expand All @@ -227,12 +227,12 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max
│ cactus transcribe [model] live microphone transcription with a model│
│ --file <audio.wav> audio file to transcribe (WAV) │
│ --language <code> language code (default: en) │
│ --bits 1|2|3|4 CQ quantization (default: 4) │
│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │
│ --token <token> HuggingFace token (gated models) │
│ --reconvert force local rebuild from source │
│ │
│ cactus download [model] get a bundle (prebuilt, else build) │
│ --bits 1|2|3|4 CQ quantization (default: 4) │
│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │
│ --token <token> HuggingFace token (gated models) │
│ --reconvert force local rebuild from source │
│ │
Expand All @@ -248,7 +248,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max
│ cactus serve [model] OpenAI-compatible local HTTP server │
│ --host <addr> bind address (default: 127.0.0.1) │
│ --port <port> port (default: 8080) │
│ --bits 1|2|3|4 CQ quantization (default: 4) │
│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │
│ --backend cpu|metal inference backend (default: auto) │
│ --token <token> HuggingFace token (gated models) │
│ --reconvert force local rebuild from source │
Expand All @@ -258,7 +258,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max
│ │
│ cactus code run the AI coding agent (TUI / print) │
│ --serve-model <id> auto-start a server with this model │
│ --bits 1|2|3|4 CQ quantization (default: 4) │
│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │
│ --backend cpu|metal inference backend (default: auto) │
│ --token <token> HuggingFace token (gated models) │
│ --reconvert force local rebuild from source │
Expand All @@ -282,7 +282,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max
│ (default: all) │
│ --model <hf-id> default: google/gemma-4-E2B-it │
│ --transcription-model <hf-id> default: nvidia/parakeet-tdt-0.6b-v3 │
│ --bits 1|2|3|4 CQ quantization (default: 4) │
│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │
│ --backend cpu|metal inference backend (default: auto) │
│ --token <token> HuggingFace token (gated models) │
│ --reconvert force local rebuild of test models │
Expand All @@ -297,7 +297,7 @@ N/B: With 1k-context prefill and decode for 100 runs on M5 Max
│ cactus benchmark run the engine benchmark suite │
│ --model <hf-id> default: google/gemma-4-E2B-it │
│ --transcription-model <hf-id> default: nvidia/parakeet-tdt-0.6b-v3 │
│ --bits 1|2|3|4 CQ quantization (default: 4) │
│ --bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4) │
│ --backend cpu|metal inference backend (default: auto) │
│ --ios run on connected iPhone │
│ --android run on connected Android │
Expand Down
2 changes: 1 addition & 1 deletion docs/cactus_transpiler.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ huggingface.co/Cactus-Compute) or a local path to a bundle directory.

| Option | Description |
|--------|-------------|
| `--bits 1\|2\|3\|4` | CQ quantization bits when downloading (default: 4) |
| `--bits 1\|2\|3\|4\|2.54\|3.26` | CQ quantization bits when downloading (default: 4) |
| `--token <token>` | HuggingFace token (gated models) |
| `--prompt <text>` | Input prompt |
| `--input-ids <ids>` | Comma-separated token IDs |
Expand Down
30 changes: 19 additions & 11 deletions python/cactus/cli/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
from .auth import cmd_auth
from .clean import cmd_clean
from .code import cmd_code
from .utils import bits_arg, ALLOWED_BITS

_BITS_METAVAR = "{" + ",".join(str(b) for b in ALLOWED_BITS) + "}"


def _telemetry_parent():
Expand All @@ -31,11 +34,16 @@ def _telemetry_parent():
return p


def _build_parent():
"""Bundle-build flags shared by every command that prepares a model."""
def _build_parent(mixed: bool = True):
"""Bundle-build flags shared by model-preparing commands
(mixed=False limits --bits to uniform 1-4 for build-only commands)."""
p = argparse.ArgumentParser(add_help=False)
p.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4,
help="CQ quantization (default: 4)")
if mixed:
p.add_argument("--bits", type=bits_arg, default=4, metavar=_BITS_METAVAR,
help="CQ quantization: uniform 1-4 or gemma-4 mixed 3.26/2.54 (default: 4)")
else:
p.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4,
help="CQ quantization (default: 4)")
p.add_argument("--token", help="HuggingFace token")
p.add_argument("--reconvert", action="store_true",
help="Force local rebuild from source")
Expand Down Expand Up @@ -117,7 +125,7 @@ def create_parser():
--clear remove saved key

cactus run [model|path] run a model (default: {DEFAULT_MODEL_ID})
--bits 1|2|3|4 CQ quantization (default: 4)
--bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4)
--image <path> image file for VLM inference
--audio <path> audio file for audio chat
--system <prompt> system prompt
Expand All @@ -129,12 +137,12 @@ def create_parser():
cactus transcribe [model] live microphone transcription with a model
--file <audio.wav> audio file to transcribe (WAV)
--language <code> language code (default: en)
--bits 1|2|3|4 CQ quantization (default: 4)
--bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4)
--token <token> HuggingFace token (gated models)
--reconvert force local rebuild from source

cactus download [model] fetch a prebuilt bundle, else build locally (default: {DEFAULT_MODEL_ID})
--bits 1|2|3|4 CQ quantization (default: 4)
--bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4)
--token <token> HuggingFace token (gated models)
--reconvert force local rebuild from source

Expand All @@ -147,7 +155,7 @@ def create_parser():
cactus serve [model] OpenAI-compatible local HTTP server
--host <addr> bind address (default: 127.0.0.1)
--port <port> port (default: 8080)
--bits 1|2|3|4 CQ quantization (default: 4)
--bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4)
--token <token> HuggingFace token (gated models)
--reconvert force local rebuild from source
--no-cloud-handoff disable automatic cloud handoff
Expand All @@ -166,7 +174,7 @@ def create_parser():
(default: all)
--model <hf-id> default: {DEFAULT_TEST_MODEL_ID}
--transcription-model <hf-id> default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID}
--bits 1|2|3|4 CQ quantization (default: 4)
--bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4)
--token <token> HuggingFace token (gated models)
--reconvert force local rebuild of test models
--suite <name> run a single test suite from any
Expand All @@ -179,7 +187,7 @@ def create_parser():
cactus benchmark run the engine benchmark suite
--model <hf-id> default: {DEFAULT_TEST_MODEL_ID}
--transcription-model <hf-id> default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID}
--bits 1|2|3|4 CQ quantization (default: 4)
--bits 1|2|3|4|2.54|3.26 CQ quantization (default: 4)
--backend cpu|metal inference backend (default: auto)
--ios run on connected iPhone
--android run on connected Android
Expand Down Expand Up @@ -443,7 +451,7 @@ def create_parser():
help="KV cache context length for cached decode graphs (default: model config)")

upload_parser = subparsers.add_parser("upload",
parents=[_build_parent()],
parents=[_build_parent(mixed=False)],
help="Build a runnable bundle locally and upload it to Cactus-Compute on HuggingFace")
upload_parser.add_argument("model_id", type=_hf_id_or_path,
help="HuggingFace model id (e.g. openai/whisper-base)")
Expand Down
4 changes: 2 additions & 2 deletions python/cactus/cli/download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def get_weights_dir(model_id: str) -> Path:
return weights_root() / get_model_dir_name(model_id)


def get_bundle_dir(model_id: str, *, bits: int = 4) -> Path:
def get_bundle_dir(model_id: str, *, bits: int | float = 4) -> Path:
from .utils import variant_suffix
return weights_root() / f"{get_model_dir_name(model_id)}-{variant_suffix(bits)}"

Expand All @@ -24,7 +24,7 @@ def ensure_model(model_id: str) -> Path:
return download_bundle(model_id)


def download_bundle(model_id: str, *, bits: int = 4,
def download_bundle(model_id: str, *, bits: int | float = 4,
token: str | None = None, output_dir: Path | None = None) -> Path:
from .utils import (
download_cq_archive,
Expand Down
19 changes: 11 additions & 8 deletions python/cactus/cli/list.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,14 @@ def _cq_precision(weights_file):


def _quant_label(model_dir, *, sample_cap=64, scan_cap=2000):
"""Infer a model's CQ quantization level from its tensor headers.

Scans `.weights` files (top-level converted weights, then nested
bundle components) and returns the dominant CQ level, e.g. "CQ4".
Returns "—" if no CQ-quantized tensors are present.
"""
"""Infer a model's CQ variant from the `-cqN` dir suffix (the only way to
recover a mixed variant like CQ3.26), else the dominant CQ level in the
tensor headers. Returns "—" if no CQ-quantized tensors are present."""
from .utils import parse_cq_variant

variant = parse_cq_variant(model_dir.name)
if variant is not None:
return f"CQ{variant}"
candidates = itertools.chain(
model_dir.glob("*.weights"),
(model_dir / "components").rglob("*.weights"),
Expand Down Expand Up @@ -94,7 +96,8 @@ def cmd_list(_args):
return 0
name_w = max(len(p.name) for p, _, _, _ in models)
type_w = max(len("type"), max(len(t) for _, t, _, _ in models))
print(f" {'name':<{name_w}} {'type':<{type_w}} {'quant':<5} {'size':>10} location")
quant_w = max(len("quant"), max(len(q) for _, _, q, _ in models))
print(f" {'name':<{name_w}} {'type':<{type_w}} {'quant':<{quant_w}} {'size':>10} location")
for p, model_type, quant, size in models:
print(f" {p.name:<{name_w}} {model_type:<{type_w}} {quant:<5} {_human_size(size):>10} {p.parent}")
print(f" {p.name:<{name_w}} {model_type:<{type_w}} {quant:<{quant_w}} {_human_size(size):>10} {p.parent}")
return 0
5 changes: 5 additions & 0 deletions python/cactus/cli/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@

def _convert_from_source(model_id, *, bits, token, weights_dir, skip_model_load=False):
"""Download from HuggingFace and run CQ conversion."""
if bits not in (1, 2, 3, 4):
raise SystemExit(
f"CQ{bits} is a mixed-precision variant, available only as a prebuilt "
f"download; local conversion supports uniform bits 1-4"
)
from .common import convert_toolchain_error
err = convert_toolchain_error()
if err:
Expand Down
37 changes: 28 additions & 9 deletions python/cactus/cli/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import argparse
import hashlib
import json
import os
Expand All @@ -15,18 +16,36 @@
from typing import Iterable


CQ_VARIANT_RE = re.compile(r"-cq([1-4])$", re.IGNORECASE)
CQ_VARIANT_RE = re.compile(r"-cq(\d+(?:\.\d+)?)$", re.IGNORECASE)
ARCHIVE_SUFFIXES = (".zip", ".tar", ".tar.gz", ".tgz")

ALLOWED_BITS = (1, 2, 3, 4, 2.54, 3.26)

def variant_suffix(bits: int) -> str:
return f"cq{bits}"

def normalize_bits(bits: str | int | float) -> int | float:
value = float(bits)
return int(value) if value == int(value) else value


def variant_suffix(bits: int | float) -> str:
return f"cq{normalize_bits(bits)}"


def bits_arg(value: str) -> int | float:
try:
bits = normalize_bits(value)
except (TypeError, ValueError):
raise argparse.ArgumentTypeError(f"invalid --bits value {value!r}")
if bits not in ALLOWED_BITS:
allowed = ", ".join(str(b) for b in ALLOWED_BITS)
raise argparse.ArgumentTypeError(f"unsupported --bits {value!r}; choose from: {allowed}")
return bits


@dataclass(frozen=True)
class CqArchive:
filename: str
bits: int
bits: int | float
size: int | None = None
sha256: str | None = None

Expand All @@ -45,7 +64,7 @@ class CqResolution:

def suggested_cq_repo(model_id: str) -> str:
name = str(model_id).strip().replace("_", "-").split("/")[-1]
name = re.sub(r"-cq\d*$", "", name, flags=re.IGNORECASE)
name = re.sub(r"-cq\d*(?:\.\d+)?$", "", name, flags=re.IGNORECASE)
return f"Cactus-Compute/{name}"


Expand All @@ -56,12 +75,12 @@ def archive_stem(filename: str) -> str:
return filename


def parse_cq_variant(filename: str) -> int | None:
def parse_cq_variant(filename: str) -> int | float | None:
stem = archive_stem(Path(filename).name)
m = CQ_VARIANT_RE.search(stem)
if not m:
return None
return int(m.group(1))
return normalize_bits(m.group(1))


def is_supported_archive(filename: str) -> bool:
Expand All @@ -82,7 +101,7 @@ def archives_from_repo_files(repo_files: Iterable[str], sizes: dict[str, int] |
continue
archives.append(CqArchive(filename=filename, bits=bits,
size=sizes.get(filename), sha256=sha256s.get(filename)))
return tuple(sorted(archives, key=lambda a: a.bits))
return tuple(sorted(archives, key=lambda a: (isinstance(a.bits, float), a.bits)))


VERSION_TAG_RE = re.compile(r"^v?(\d+(?:\.\d+){0,2})$", re.IGNORECASE)
Expand Down Expand Up @@ -140,7 +159,7 @@ def resolve_weight_revision(repo_id: str, runtime_version: str | None = None,


def resolve_archive(repo_id: str, local_name: str, archives: Iterable[CqArchive],
bits: int) -> CqResolution:
bits: int | float) -> CqResolution:
available = tuple(archives)
if not available:
raise RuntimeError(f"No CQ archives found in {repo_id}")
Expand Down
Loading