diff --git a/python/cactus/cli/compile.py b/python/cactus/cli/compile.py index a91b8774b..adafffa5c 100644 --- a/python/cactus/cli/compile.py +++ b/python/cactus/cli/compile.py @@ -46,6 +46,8 @@ def _detect_sdl2() -> tuple[list[str], list[str]]: is_darwin = platform.system() == "Darwin" if is_darwin: + if not check_command("brew"): + return [], [] sdl2_check = subprocess.run(["brew", "list", "sdl2"], capture_output=True) if sdl2_check.returncode == 0: sdl2_prefix_result = subprocess.run( diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index a24e4d6d0..2d7b04469 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -1,6 +1,7 @@ """Model resolution, weight management, and bundle preparation.""" from __future__ import annotations +import os import shutil from dataclasses import dataclass from pathlib import Path @@ -21,7 +22,8 @@ def _convert_from_source(model_id, *, bits, token, weights_dir): "--bits", str(bits), ] if token: - cq_args.extend(["--token", token]) + os.environ["HF_TOKEN"] = token + os.environ["HUGGING_FACE_HUB_TOKEN"] = token cq_main(cq_args) print_color(GREEN, f"Model converted and ready at {weights_dir}") diff --git a/python/cactus/convert/cactus_adapters/tokenizer.py b/python/cactus/convert/cactus_adapters/tokenizer.py index f041ce616..5acc680c8 100644 --- a/python/cactus/convert/cactus_adapters/tokenizer.py +++ b/python/cactus/convert/cactus_adapters/tokenizer.py @@ -159,9 +159,10 @@ def convert_hf_tokenizer(tokenizer, output_dir, token=None, model_id=None, label id_to_token[token_id] = token_str else: vocab = tokenizer.get_vocab() - id_to_token = [""] * len(vocab) + vocab_size = (max(vocab.values()) + 1) if vocab else 0 + id_to_token = [""] * vocab_size for token_str, token_id in vocab.items(): - if token_id < len(id_to_token): + if token_id < vocab_size: id_to_token[token_id] = token_str # vocab.txt is written later, after special tokens are collected @@ -261,8 +262,8 @@ def write_merges_file(merges_list): if not any(item["token"] == token_str and item["id"] == int(token_id) for item in additional_special_tokens): additional_special_tokens.append({"token": token_str, "id": int(token_id)}) - model_type = model_name_l or getattr(tokenizer, 'name_or_path', '').lower() - if 'gemma' in model_type: + name_or_path_l = model_name_l or getattr(tokenizer, 'name_or_path', '').lower() + if 'gemma' in (model_type or '') or 'gemma' in name_or_path_l: gemma_special_tokens = { '': None, '': None, diff --git a/python/cactus/convert/cli.py b/python/cactus/convert/cli.py index 68c4f4916..20f8dc94e 100644 --- a/python/cactus/convert/cli.py +++ b/python/cactus/convert/cli.py @@ -3,6 +3,7 @@ import argparse import json import shutil +from collections.abc import Mapping from dataclasses import replace from pathlib import Path from typing import Any @@ -83,7 +84,7 @@ def _load_hf(model_id_or_path: str, device: str): low_cpu_mem_usage=True, local_files_only=Path(model_id_or_path).exists(), ) - except Exception: + except Exception as primary_exc: try: model = AutoModel.from_pretrained( model_id_or_path, @@ -92,7 +93,8 @@ def _load_hf(model_id_or_path: str, device: str): low_cpu_mem_usage=True, local_files_only=Path(model_id_or_path).exists(), ) - except Exception: + except Exception as auto_exc: + print(f"warning: could not load model object ({model_cls.__name__}: {primary_exc}; AutoModel: {auto_exc})") model = None if model is None: return cfg, processor, model @@ -155,6 +157,8 @@ def _load_checkpoint_state_dict(model_id_or_path: str) -> dict[str, Any] | None: raise RuntimeError(f"duplicate tensor key {key!r} across checkpoint shards") state[key] = tensor return state + except RuntimeError: + raise except Exception: return None safetensor_files = sorted(root.glob("*.safetensors")) @@ -284,7 +288,7 @@ def _collect_manifest_token_ids( if text: try: encoded = processor(text, return_tensors=None) - ids = encoded.get("input_ids", encoded) if isinstance(encoded, dict) else encoded + ids = encoded.get("input_ids", encoded) if isinstance(encoded, Mapping) else encoded if ids and isinstance(ids[0], list): ids = ids[0] out.extend(int(x) for x in ids) @@ -440,6 +444,12 @@ def convert(args: argparse.Namespace) -> None: target_modules.add(target) pending.append((name, tensor, match, policy)) + if target_modules and model is None and not args.hessian_cache_in: + msg = "model object failed to load; GPTQ Hessian calibration is disabled and use_gptq tensors will fall back to RTN" + if args.strict: + raise RuntimeError(msg) + print(f"warning: {msg}") + if args.hessian_cache_in: hessian_stats = _load_hessian_artifacts(Path(args.hessian_cache_in)) else: @@ -537,7 +547,7 @@ def convert(args: argparse.Namespace) -> None: _save_fallback_tensor(emit_tensor, out_path, "FP16", family) status = "fallback" if match.recognized else "unrecognized" precision = "FP16" - emit_policy = type(emit_policy)(emit_policy.action, precision, emit_policy.bits, emit_policy.component, emit_policy.use_gptq, emit_policy.rotation, str(exc)) + emit_policy = replace(emit_policy, action="fallback", precision="FP16", fallback_reason=str(exc)) rows.append({ "source_name": name, "hf_name": match.hf_name or name, diff --git a/python/cactus/convert/model_adapters/adapters.py b/python/cactus/convert/model_adapters/adapters.py index fbc032aa0..5277f66f2 100644 --- a/python/cactus/convert/model_adapters/adapters.py +++ b/python/cactus/convert/model_adapters/adapters.py @@ -132,11 +132,19 @@ def scale_factor(self, output_name: str) -> float: class Gemma4Adapter(FamilyAdapter): family = "gemma4" + def __init__(self) -> None: + super().__init__() + self.kv_shared_from_layer = 15 + def runtime_config(self, cfg: Any) -> dict[str, Any]: text_cfg = _cfg_get(cfg, "text_config", None) base_cfg = text_cfg if text_cfg is not None else cfg config = extract_base_config(base_cfg, cfg) config.update(extract_complex_gemma_config(base_cfg, cfg)) + num_layers = int(config.get("num_layers", 0) or 0) + num_kv_shared_layers = int(config.get("num_kv_shared_layers", 0) or 0) + if num_layers > 0 and num_kv_shared_layers > 0: + self.kv_shared_from_layer = max(0, num_layers - num_kv_shared_layers) vision_cfg = _cfg_get(cfg, "vision_config", None) if vision_cfg: config.update(extract_vision_config(cfg, vision_cfg)) @@ -169,7 +177,7 @@ def policy(self, match: NameMatch, shape: tuple[int, ...], requested_bits: int) layer = int(parts[3]) if parts[:3] == ["model", "language_model", "layers"] else None except Exception: layer = None - if layer is not None and layer >= 15: + if layer is not None and layer >= self.kv_shared_from_layer: return replace(policy, use_gptq=False, fallback_reason=policy.fallback_reason or "shared KV tensor has no per-layer hook module") return policy @@ -266,7 +274,7 @@ def transform_tensor(self, match: NameMatch, tensor: Any) -> tuple[Any, str]: if len(_tensor_shape(tensor)) != 4: return tensor, "none" shape = _tensor_shape(tensor) - is_hwio_k3 = shape[1] == 3 and shape[2] == 3 and shape[3] >= 1 + is_hwio_k3 = shape[1] == 3 and shape[2] == 3 and shape[3] == 1 is_hwio_pw = shape[1] == 1 and shape[2] == 1 and shape[3] > 1 if not (is_hwio_k3 or is_hwio_pw): return tensor, "none"