diff --git a/README.md b/README.md index 5ab4db76e..d36a5d2da 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ cactus run Cactus-Compute/needle [--tools my_tools.json] # OpenAI function-call │ │ │ cactus convert [dir] HuggingFace -> Cactus CQ weights │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --calibration-manifest representative data for GPTQ calibration │ │ --token HuggingFace token (gated models) │ │ --reconvert force weight conversion from source │ │ --lora merge a LoRA adapter before converting │ diff --git a/docs/cactus_quants.md b/docs/cactus_quants.md index 5b8e96536..e652f633c 100644 --- a/docs/cactus_quants.md +++ b/docs/cactus_quants.md @@ -193,6 +193,28 @@ cactus convert google/gemma-4-E2B-it ./gemma4-weights --bits 2 cactus run google/gemma-4-E2B-it ``` +For a finetuned checkpoint, provide representative data so GPTQ-aware CQ tensors +are calibrated against the checkpoint's activation distribution: + +```json +{ + "language": {"path": "calibration.jsonl"} +} +``` + +```bash +cactus convert ./finetuned-model ./finetuned-cq4 --bits 4 \ + --calibration-manifest calibration-manifest.json +``` + +Language calibration JSONL rows may contain `messages`, `prompt_text` plus +`completion_text`, or `prompt` plus `completion`. Without representative samples, +GPTQ-eligible tensors use calibration-free RTN when samples are missing or the +collected Hessian cannot be applied. The converter reports both fallback cases in the +terminal and records their extent in `conversion_summary.json`; affected tensors and +their reasons are identified in `conversion_manifest.json`. Supplying a manifest when +the output bundle already exists is an error unless `--reconvert` is also passed. + For models on huggingface.co/Cactus-Compute, `cactus run ` (or `cactus download `) fetches the pre-built bundle. diff --git a/docs/cactus_transpiler.md b/docs/cactus_transpiler.md index de0a7e601..6c0b85ae2 100644 --- a/docs/cactus_transpiler.md +++ b/docs/cactus_transpiler.md @@ -419,6 +419,7 @@ slot, so single-stream decode stays lean while fixed-batch decode (`decode_batch | Option | Description | |--------|-------------| | `--bits 1\|2\|3\|4` | CQ quantization bits (default: 4) | +| `--calibration-manifest ` | Representative calibration data for GPTQ-aware CQ conversion | | `--weights-only` | Stop after CQ quantization; skip the runtime graph | | `--weights-dir ` | Path to converted CQ weights (default: `weights/`) | | `--task ` | Force task type (default: `auto` — inferred from model config). Choices: `causal_lm_logits`, `multimodal_causal_lm_logits`, `ctc_logits`, `encoder_hidden_states`, `seq2seq_transcription`, `tdt_transcription` | diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index e841b7d36..fc9d187b6 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -366,6 +366,10 @@ def create_parser(): help="Output directory (default: weights/)") convert_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, help="CQ quantization bits (default: 4)") + convert_parser.add_argument( + "--calibration-manifest", + help="JSON manifest of representative calibration data for GPTQ-aware CQ conversion", + ) convert_parser.add_argument("--token", help="HuggingFace token") convert_parser.add_argument("--lora", help="Path or HF id of a LoRA adapter to merge before converting (requires `peft`)") diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index 396f70623..e6a6142aa 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -170,6 +170,7 @@ def cmd_convert(args): reconvert=args.reconvert, output_dir=output_dir, skip_model_load=bool(getattr(args, "skip_model_load", False)), + calibration_manifest=getattr(args, "calibration_manifest", None), ) if getattr(args, "weights_only", False): return 0 diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index fd930632e..44c8d48e4 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -36,7 +36,15 @@ def package_handoff_probe(output_dir, model_id): print_color(YELLOW, f"Warning: failed to package cloud handoff probe: {exc}") -def _convert_from_source(model_id, *, bits, token, weights_dir, skip_model_load=False): +def _convert_from_source( + model_id, + *, + bits, + token, + weights_dir, + skip_model_load=False, + calibration_manifest=None, +): """Download from HuggingFace and run CQ conversion.""" if bits not in (1, 2, 3, 4): raise SystemExit( @@ -57,6 +65,8 @@ def _convert_from_source(model_id, *, bits, token, weights_dir, skip_model_load= ] if skip_model_load: cq_args.append("--skip-model-load") + if calibration_manifest: + cq_args.extend(["--calibration-manifest", str(calibration_manifest)]) if token: os.environ["HF_TOKEN"] = token os.environ["HUGGING_FACE_HUB_TOKEN"] = token @@ -66,7 +76,16 @@ def _convert_from_source(model_id, *, bits, token, weights_dir, skip_model_load= return weights_dir -def ensure_weights(model_id, *, bits=4, token=None, reconvert=False, output_dir=None, skip_model_load=False): +def ensure_weights( + model_id, + *, + bits=4, + token=None, + reconvert=False, + output_dir=None, + skip_model_load=False, + calibration_manifest=None, +): from .download import get_bundle_dir weights_dir = Path(output_dir) if output_dir else get_bundle_dir(model_id, bits=bits) @@ -76,6 +95,12 @@ def ensure_weights(model_id, *, bits=4, token=None, reconvert=False, output_dir= shutil.rmtree(weights_dir) if weights_dir.exists() and (weights_dir / "config.txt").exists(): + if calibration_manifest is not None: + raise RuntimeError( + "A calibration manifest is only applied while converting weights, but a cached " + f"bundle already exists at {weights_dir}. Pass --reconvert to rebuild it with " + "calibration, or choose a different output directory." + ) print_color(GREEN, f"Model weights found at {weights_dir}") return weights_dir @@ -89,6 +114,7 @@ def ensure_weights(model_id, *, bits=4, token=None, reconvert=False, output_dir= token=token, weights_dir=weights_dir, skip_model_load=skip_model_load, + calibration_manifest=calibration_manifest, ) diff --git a/python/cactus/convert/cli.py b/python/cactus/convert/cli.py index 60a85bce8..bc8b2d5d3 100644 --- a/python/cactus/convert/cli.py +++ b/python/cactus/convert/cli.py @@ -531,6 +531,7 @@ def convert(args: argparse.Namespace) -> None: emit_source_names = emission.source_names or source_names status = emit_policy.action precision = emit_policy.precision + gptq_expected = bool(emit_policy.action == "convert" and emit_policy.use_gptq) gptq_used = False hessian_missing_reason = None module_name = adapter.module_target_name(name, model) or _module_name(name) @@ -560,6 +561,11 @@ def convert(args: argparse.Namespace) -> None: input_scale=input_scale, ) cq = _scale_cq_norms(cq, adapter.scale_factor(out_path.name)) + if gptq_expected and not cq.gptq_used: + if hessian_missing_reason is None: + hessian_missing_reason = "collected GPTQ Hessian could not be applied" + if args.strict: + raise RuntimeError(f"{name}: {hessian_missing_reason} ({module_name})") if getattr(emit_policy, "layout", "row_major") == "interleaved_4row": cq = replace(cq, interleaved_4row=True) write_cq_tensor(out_path, cq) @@ -593,6 +599,7 @@ def convert(args: argparse.Namespace) -> None: "required": bool(emit_policy.action != "ignored"), "fallback_reason": emit_policy.fallback_reason, "hessian_samples": int(hessian_samples.get(module_name, 0)), + "gptq_expected": gptq_expected, "gptq_used": bool(gptq_used), "bytes": _tensor_bytes(out_path), "scale_factor": float(adapter.scale_factor(out_path.name)) if out_path else 1.0, diff --git a/python/cactus/convert/export/reports.py b/python/cactus/convert/export/reports.py index 47e35f73a..96e9e80fd 100644 --- a/python/cactus/convert/export/reports.py +++ b/python/cactus/convert/export/reports.py @@ -33,6 +33,17 @@ def write_reports(out_dir: Path, rows: list[dict[str, Any]]) -> dict[str, Any]: by_component = Counter(r["component"] for r in rows) by_precision = Counter(r["precision"] for r in rows) fallback = Counter(r.get("fallback_reason") or "" for r in rows if r["status"] in {"fallback", "unrecognized"}) + gptq_expected = [r for r in rows if r.get("gptq_expected")] + gptq_used = [r for r in gptq_expected if r.get("gptq_used")] + gptq_uncalibrated = [r for r in gptq_expected if not r.get("gptq_used")] + gptq_zero_samples = [ + r for r in gptq_uncalibrated + if int(r.get("hessian_samples", 0) or 0) <= 0 + ] + gptq_unusable_hessian = [ + r for r in gptq_uncalibrated + if int(r.get("hessian_samples", 0) or 0) > 0 + ] total_bytes = sum(int(r.get("bytes", 0) or 0) for r in rows) summary = { "total_tensors": len(rows), @@ -40,6 +51,13 @@ def write_reports(out_dir: Path, rows: list[dict[str, Any]]) -> dict[str, Any]: "counts_by_component": dict(by_component), "counts_by_precision": dict(by_precision), "fallback_reasons": dict(fallback), + "gptq": { + "expected_tensors": len(gptq_expected), + "calibrated_tensors": len(gptq_used), + "uncalibrated_tensors": len(gptq_uncalibrated), + "zero_sample_tensors": len(gptq_zero_samples), + "unusable_hessian_tensors": len(gptq_unusable_hessian), + }, "total_bytes": total_bytes, } (out_dir / "conversion_summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8") @@ -52,3 +70,18 @@ def print_summary(summary: dict[str, Any]) -> None: for key in ["converted", "fallback", "ignored", "unrecognized"]: print(f"{key:13s} {summary.get('counts_by_status', {}).get(key, 0)}") print(f"{'total bytes':13s} {summary.get('total_bytes', 0)}") + gptq = summary.get("gptq", {}) + expected = int(gptq.get("expected_tensors", 0) or 0) + calibrated = int(gptq.get("calibrated_tensors", 0) or 0) + uncalibrated = int(gptq.get("uncalibrated_tensors", 0) or 0) + zero_samples = int(gptq.get("zero_sample_tensors", 0) or 0) + unusable_hessian = int(gptq.get("unusable_hessian_tensors", 0) or 0) + if expected: + print(f"{'GPTQ calibrated':13s} {calibrated}/{expected}") + if uncalibrated: + print( + f"\nWARNING: {uncalibrated} GPTQ-eligible CQ tensors used RTN fallback " + f"({zero_samples} zero-sample, {unusable_hessian} unusable Hessian). " + "Provide --calibration-manifest with representative data; see " + "conversion_manifest.json for affected tensors." + ) diff --git a/python/cactus/convert/tests/test_hessian.py b/python/cactus/convert/tests/test_hessian.py index 7b6b0a12c..79f0c11fc 100644 --- a/python/cactus/convert/tests/test_hessian.py +++ b/python/cactus/convert/tests/test_hessian.py @@ -1,9 +1,13 @@ from __future__ import annotations +import numpy as np import torch from cactus.convert.calibration.hessian import HessianStats, collect_manifest_hessians from cactus.convert.cli import _load_hessian_artifacts, _save_hessian_artifacts +from cactus.convert.quantization.cq import quantize_hadamard +from cactus.models.needle.configuration_needle import NeedleConfig +from cactus.models.needle.modeling_needle import NeedleForCausalLM class TinyModel(torch.nn.Module): @@ -54,3 +58,55 @@ def test_hessian_artifact_cache_roundtrip(tmp_path): assert loaded.samples == {"layers.0": 7} assert torch.equal(loaded.hessians["layers.0"], torch.eye(2)) assert torch.equal(loaded.diag["layers.0"], torch.ones(2)) + + +def test_needle_manifest_hessian_is_used_by_gptq(tmp_path): + rows = tmp_path / "language.jsonl" + rows.write_text('{"prompt_text":"hello","completion_text":"world"}\n', encoding="utf-8") + manifest = {"language": {"path": str(rows)}} + config = NeedleConfig( + vocab_size=16, + hidden_size=64, + d_model=64, + num_attention_heads=2, + num_key_value_heads=1, + num_encoder_layers=1, + num_decoder_layers=1, + torch_dtype="float32", + ) + model = NeedleForCausalLM(config).float() + target = "model.encoder.layers.0.self_attn.q_proj" + + stats = collect_manifest_hessians( + model, + TinyTokenizer(), + manifest, + {"language": 1}, + {target}, + "cpu", + ) + + assert stats.errors == {} + assert stats.unresolved_targets == [] + assert stats.samples[target] > 0 + assert stats.hessians[target].shape == (64, 64) + + result = quantize_hadamard( + model.state_dict()[f"{target}.weight"], + bits=4, + hessian=stats.hessians[target].numpy(), + use_gptq=True, + ) + + assert result.gptq_used is True + + +def test_hadamard_rejects_mismatched_hessian_for_gptq(): + result = quantize_hadamard( + torch.randn(4, 64), + bits=4, + hessian=np.eye(32, dtype=np.float32), + use_gptq=True, + ) + + assert result.gptq_used is False diff --git a/python/cactus/convert/tests/test_reports.py b/python/cactus/convert/tests/test_reports.py index 625170e4d..046549b41 100644 --- a/python/cactus/convert/tests/test_reports.py +++ b/python/cactus/convert/tests/test_reports.py @@ -3,7 +3,7 @@ import json from pathlib import Path -from cactus.convert.export.reports import write_reports +from cactus.convert.export.reports import print_summary, write_reports def test_weights_manifest_includes_bias_tensors(tmp_path: Path) -> None: @@ -34,3 +34,60 @@ def test_weights_manifest_includes_bias_tensors(tmp_path: Path) -> None: assert "decoder.layer_0_mlp_fc1.bias" in output_names assert "decoder.layer_0_mlp_fc1.weights" in output_names + + +def test_reports_make_all_gptq_fallbacks_visible(tmp_path: Path, capsys) -> None: + summary = write_reports( + tmp_path, + [ + { + "source_name": "layers.0.q_proj.weight", + "output_file": "model.layer_0_attention_q_proj.weights", + "shape": [4, 4], + "precision": "CQ4", + "status": "converted", + "component": "language", + "gptq_expected": True, + "gptq_used": False, + "hessian_samples": 0, + }, + { + "source_name": "layers.0.k_proj.weight", + "output_file": "model.layer_0_attention_k_proj.weights", + "shape": [4, 4], + "precision": "CQ4", + "status": "converted", + "component": "language", + "gptq_expected": True, + "gptq_used": True, + "hessian_samples": 8, + }, + { + "source_name": "layers.0.v_proj.weight", + "output_file": "model.layer_0_attention_v_proj.weights", + "shape": [4, 4], + "precision": "CQ4", + "status": "converted", + "component": "language", + "gptq_expected": True, + "gptq_used": False, + "hessian_samples": 8, + }, + ], + ) + + assert summary["gptq"] == { + "expected_tensors": 3, + "calibrated_tensors": 1, + "uncalibrated_tensors": 2, + "zero_sample_tensors": 1, + "unusable_hessian_tensors": 1, + } + saved = json.loads((tmp_path / "conversion_summary.json").read_text()) + assert saved["gptq"] == summary["gptq"] + + print_summary(summary) + output = capsys.readouterr().out + assert "GPTQ calibrated 1/3" in output + assert "WARNING: 2 GPTQ-eligible CQ tensors used RTN fallback" in output + assert "1 zero-sample, 1 unusable Hessian" in output diff --git a/python/tests/test_cli_calibration.py b/python/tests/test_cli_calibration.py new file mode 100644 index 000000000..2ac895a03 --- /dev/null +++ b/python/tests/test_cli_calibration.py @@ -0,0 +1,111 @@ +import sys +import types +from pathlib import Path + +import pytest + +from cactus import cli +from cactus.cli import convert as convert_cli +from cactus.cli import model as model_cli + + +def test_convert_parser_accepts_calibration_manifest(tmp_path: Path) -> None: + manifest = tmp_path / "calibration.json" + + args = cli.create_parser().parse_args([ + "convert", + "org/model", + "--calibration-manifest", + str(manifest), + ]) + + assert args.calibration_manifest == str(manifest) + + +def test_convert_command_forwards_calibration_manifest(monkeypatch, tmp_path: Path) -> None: + manifest = tmp_path / "calibration.json" + output = tmp_path / "weights" + args = cli.create_parser().parse_args([ + "convert", + "org/model", + str(output), + "--weights-only", + "--calibration-manifest", + str(manifest), + ]) + + weight_calls: list[dict] = [] + + def fake_ensure_weights(model_id, **kwargs): + weight_calls.append({"model_id": model_id, **kwargs}) + return output + + fake_transpiler = types.ModuleType("cactus.cli.transpiler") + fake_transpiler.build_transpiled_bundle = lambda *args, **kwargs: output + fake_transpiler.parse_modalities = lambda value: value + fake_transpiler.resolve_transpile_config = lambda *args, **kwargs: None + monkeypatch.setitem(sys.modules, "cactus.cli.transpiler", fake_transpiler) + monkeypatch.setattr(model_cli, "ensure_weights", fake_ensure_weights) + monkeypatch.setattr(model_cli, "package_handoff_probe", lambda *args: None) + + assert convert_cli.cmd_convert(args) == 0 + assert weight_calls == [{ + "model_id": "org/model", + "bits": 4, + "token": None, + "reconvert": False, + "output_dir": str(output), + "skip_model_load": False, + "calibration_manifest": str(manifest), + }] + + +def test_convert_from_source_forwards_calibration_manifest(monkeypatch, tmp_path: Path) -> None: + from cactus.convert import cli as cq_cli + + calls: list[list[str]] = [] + monkeypatch.setattr("cactus.cli.common.convert_toolchain_error", lambda: None) + monkeypatch.setattr(cq_cli, "main", lambda args: calls.append(args)) + + manifest = tmp_path / "calibration.json" + output = tmp_path / "weights" + model_cli._convert_from_source( + "org/model", + bits=4, + token=None, + weights_dir=output, + calibration_manifest=manifest, + ) + + assert calls == [[ + "convert", + "--model", + "org/model", + "--out", + str(output), + "--bits", + "4", + "--calibration-manifest", + str(manifest), + ]] + + +def test_cached_bundle_rejects_ignored_calibration_manifest(tmp_path: Path) -> None: + output = tmp_path / "weights" + output.mkdir() + (output / "config.txt").write_text("model_type=test\n", encoding="utf-8") + + with pytest.raises(RuntimeError, match="--reconvert"): + model_cli.ensure_weights( + "org/model", + output_dir=output, + calibration_manifest=tmp_path / "calibration.json", + ) + + +def test_cached_bundle_is_reused_without_calibration_manifest(tmp_path: Path) -> None: + output = tmp_path / "weights" + output.mkdir() + (output / "config.txt").write_text("model_type=test\n", encoding="utf-8") + + assert model_cli.ensure_weights("org/model", output_dir=output) == output