From e1ec55005202dffc588e6c83d9c4b0ecc924c3e4 Mon Sep 17 00:00:00 2001 From: rachitj27 Date: Thu, 3 Sep 2026 11:49:27 -0700 Subject: [PATCH 1/4] fix(gptq): use Cholesky factor of inverse Hessian for block updates The per-group GPTQ correction reused the single global inverse Hessian for every 128-column block. OBS requires the inverse of the *remaining* columns, which changes after each block is quantized. The upper Cholesky factor U of H^-1 has the property that U[rest, rest] is the factor of the correctly updated inverse at every step, so the block update becomes solve_triangular(U[B,B], U[B,rest]) with no refactorisation. Also computes the inverse and factorisation in float64 so ill-conditioned Hessians do not silently disable GPTQ. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01A8jVC79MsJ4PX472pAZ1Qk Signed-off-by: rachitj27 --- python/cactus/convert/quantization/cq.py | 40 ++++++++++++++++++------ 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/python/cactus/convert/quantization/cq.py b/python/cactus/convert/quantization/cq.py index 2d3942389..46162f86d 100644 --- a/python/cactus/convert/quantization/cq.py +++ b/python/cactus/convert/quantization/cq.py @@ -230,13 +230,29 @@ def _prepare_input_scale(input_scale: np.ndarray | None, k: int) -> np.ndarray: return np.clip(scale, 1e-8, 65504.0).astype(np.float32, copy=False) -def _gptq_correct_group(work: np.ndarray, recon: np.ndarray, h_inv: np.ndarray | None, start: int, stop: int) -> None: - if h_inv is None or stop >= work.shape[1]: +def _gptq_correct_group(work: np.ndarray, recon: np.ndarray, h_inv_chol: np.ndarray | None, start: int, stop: int) -> None: + # GPTQ block update using the upper Cholesky factor U of H^-1 (H^-1 == U.T @ U). + # + # OBS says that after quantizing block B the remaining columns should be corrected with + # W[:, rest] -= (W_B - Q_B) @ inv(Hinv[B, B]) @ Hinv[B, rest] + # where Hinv is the inverse Hessian of the *not yet quantized* columns. That inverse changes + # after every block (it becomes the Schur complement Hinv[rest,rest] - Hinv[rest,B] inv(Hinv[B,B]) Hinv[B,rest]), + # so reusing the single global inverse for every block is wrong for all but the first block. + # Writing Hinv = U.T @ U with U upper triangular and processing columns left to right, the + # updated inverse for the remaining columns is exactly U[rest, rest].T @ U[rest, rest], i.e. the + # trailing block of the global factor. Hence inv(Hinv[B,B]) @ Hinv[B,rest] == inv(U[B,B]) @ U[B,rest] + # holds at every step with no refactorisation. + if h_inv_chol is None or stop >= work.shape[1]: return try: - m_bb = h_inv[start:stop, start:stop].astype(np.float32, copy=False) - m_bs = h_inv[start:stop, stop:].astype(np.float32, copy=False) - update = np.linalg.solve(m_bb + np.eye(m_bb.shape[0], dtype=np.float32) * 1e-6, m_bs) + u_bb = h_inv_chol[start:stop, start:stop].astype(np.float32, copy=False) + u_bs = h_inv_chol[start:stop, stop:].astype(np.float32, copy=False) + try: + from scipy.linalg import solve_triangular + + update = solve_triangular(u_bb, u_bs, lower=False) + except ImportError: + update = np.linalg.solve(u_bb, u_bs) work[:, stop:] -= (work[:, start:stop] - recon) @ update except Exception: return @@ -266,7 +282,7 @@ def quantize_hadamard( groups = k // GROUP_SIZE indices = np.zeros((n, k), dtype=np.uint8) norms = np.zeros((n, groups), dtype=np.float16) - h_inv = None + h_inv_chol = None if use_gptq and hessian is not None: try: h = np.asarray(hessian, dtype=np.float32) @@ -277,9 +293,13 @@ def quantize_hadamard( s = np.clip(input_scale.astype(np.float32), 1e-6, None) h = h / (s[:, None] * s[None, :]) h = h + np.eye(h.shape[0], dtype=np.float32) * (0.01 * np.mean(np.diag(h)) + 1e-6) - h_inv = np.linalg.inv(h) + # Invert and factor in float64: a float32 inverse of an ill-conditioned Hessian can + # lose positive-definiteness and make the Cholesky fail (silently disabling GPTQ). + h_inv = np.linalg.inv(h.astype(np.float64)) + # Upper Cholesky factor: h_inv == h_inv_chol.T @ h_inv_chol. See _gptq_correct_group. + h_inv_chol = np.linalg.cholesky(h_inv).T.astype(np.float32) except Exception: - h_inv = None + h_inv_chol = None for g in range(groups): start, stop = g * GROUP_SIZE, (g + 1) * GROUP_SIZE group = work[:, start:stop] @@ -289,8 +309,8 @@ def quantize_hadamard( recon = (codebook[idx] @ rot.T) * row_norms[:, None] indices[:, start:stop] = idx norms[:, g] = row_norms.astype(np.float16) - _gptq_correct_group(work, recon, h_inv, start, stop) - return CQTensor(indices=indices, norms=norms, input_scale=input_scale.astype(np.float16), bits=bits, gptq_used=bool(use_gptq and h_inv is not None)) + _gptq_correct_group(work, recon, h_inv_chol, start, stop) + return CQTensor(indices=indices, norms=norms, input_scale=input_scale.astype(np.float16), bits=bits, gptq_used=bool(use_gptq and h_inv_chol is not None)) def quantize_orthogonal(weight, bits: int = 4, seed: int = 1234, input_scale: np.ndarray | None = None) -> CQTensor: From 310872b0dda8c23b7c22b3feb84ed842a76b2470 Mon Sep 17 00:00:00 2001 From: rachitj27 Date: Thu, 3 Sep 2026 11:50:08 -0700 Subject: [PATCH 2/4] fix(gptq): reject non-finite Cholesky factor instead of using it np.linalg.cholesky on a NaN/inf inverse can return a NaN factor without raising, which would have enabled GPTQ with garbage corrections. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01A8jVC79MsJ4PX472pAZ1Qk Signed-off-by: rachitj27 --- python/cactus/convert/quantization/cq.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/cactus/convert/quantization/cq.py b/python/cactus/convert/quantization/cq.py index 46162f86d..466982c29 100644 --- a/python/cactus/convert/quantization/cq.py +++ b/python/cactus/convert/quantization/cq.py @@ -298,6 +298,9 @@ def quantize_hadamard( h_inv = np.linalg.inv(h.astype(np.float64)) # Upper Cholesky factor: h_inv == h_inv_chol.T @ h_inv_chol. See _gptq_correct_group. h_inv_chol = np.linalg.cholesky(h_inv).T.astype(np.float32) + # LAPACK can return a NaN/inf factor without raising (e.g. NaN Hessian); reject it. + if not np.all(np.isfinite(h_inv_chol)): + h_inv_chol = None except Exception: h_inv_chol = None for g in range(groups): From 1b224ec9ee8a89e986b2eb8586d38abc9d8ca0f5 Mon Sep 17 00:00:00 2001 From: rachitj27 Date: Thu, 3 Sep 2026 11:50:17 -0700 Subject: [PATCH 3/4] test(gptq): cover Cholesky block update, calibrated error, and bad Hessians - block update matches an explicit per-step recompute of the remaining column inverse (the Schur-complement reference) - GPTQ lowers calibrated output error vs plain Hadamard CQ4 - negative-definite and NaN Hessians disable GPTQ cleanly Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01A8jVC79MsJ4PX472pAZ1Qk Signed-off-by: rachitj27 --- python/cactus/convert/tests/test_cq.py | 86 ++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/python/cactus/convert/tests/test_cq.py b/python/cactus/convert/tests/test_cq.py index 2cd816e25..9e0337449 100644 --- a/python/cactus/convert/tests/test_cq.py +++ b/python/cactus/convert/tests/test_cq.py @@ -155,3 +155,89 @@ def test_pointwise_conv1d_int8_preserves_rank3_shape(tmp_path): assert scales_bytes == 3 * 2 * 2 assert group_size == GROUP_SIZE assert num_groups == 2 + + +def _correlated_inputs(rng: np.random.Generator, samples: int, k: int, rank: int = 32) -> np.ndarray: + latent = rng.standard_normal((samples, rank), dtype=np.float32) + mixing = rng.standard_normal((rank, k), dtype=np.float32) + noise = 0.1 * rng.standard_normal((samples, k), dtype=np.float32) + return (latent @ mixing + noise).astype(np.float32) + + +def _damp_like_cq(h: np.ndarray) -> np.ndarray: + return h + np.eye(h.shape[0], dtype=np.float32) * (0.01 * np.mean(np.diag(h)) + 1e-6) + + +def test_gptq_cholesky_block_update_matches_explicit_obs(): + from cactus.convert.quantization import cq as cq_mod + + rng = np.random.default_rng(10) + k = 3 * cq_mod.GROUP_SIZE + x = _correlated_inputs(rng, 2000, k) + h = _damp_like_cq((x.T @ x / x.shape[0]).astype(np.float32)) + h64 = h.astype(np.float64) + + build_factor = getattr(cq_mod, "_gptq_cholesky_factor", None) + factor = build_factor(h) if build_factor is not None else np.linalg.cholesky(np.linalg.inv(h64)).T.astype(np.float32) + + work = rng.standard_normal((8, k), dtype=np.float32) + work_ref = work.astype(np.float64) + for g in range(k // cq_mod.GROUP_SIZE): + start, stop = g * cq_mod.GROUP_SIZE, (g + 1) * cq_mod.GROUP_SIZE + recon = np.round(work[:, start:stop], 1).astype(np.float32) + cq_mod._gptq_correct_group(work, recon, factor, start, stop) + if stop < k: + # Reference: the inverse Hessian restricted to the still-unquantized columns, + # recomputed from scratch each step (equal to the Schur-updated OBS inverse). + hinv_cur = np.linalg.inv(h64[start:, start:]) + gsz = stop - start + update_ref = np.linalg.solve(hinv_cur[:gsz, :gsz], hinv_cur[:gsz, gsz:]) + err = work_ref[:, start:stop] - recon.astype(np.float64) + work_ref[:, stop:] -= err @ update_ref + np.testing.assert_allclose(work[:, stop:], work_ref[:, stop:], rtol=1e-4, atol=1e-4) + work_ref[:, start:stop] = work[:, start:stop] + + +def test_gptq_reduces_calibrated_error(tmp_path): + rng = np.random.default_rng(11) + n, k = 16, 384 + w = rng.standard_normal((n, k), dtype=np.float32) + x = _correlated_inputs(rng, 4096, k) + h = (x.T @ x / x.shape[0]).astype(np.float32) + + plain = quantize_hadamard(w, bits=4) + gptq = quantize_hadamard(w, bits=4, hessian=h, use_gptq=True) + assert not plain.gptq_used + assert gptq.gptq_used + + def reconstruct(cq, name): + path = tmp_path / name + write_cq_tensor(path, cq) + return dequantize_cq_file(path, read_header(path), torch.float32, 4).numpy() + + w_plain = reconstruct(plain, "plain.weights") + w_gptq = reconstruct(gptq, "gptq.weights") + err_plain = np.linalg.norm(x @ (w - w_plain).T) + err_gptq = np.linalg.norm(x @ (w - w_gptq).T) + assert np.isfinite(err_gptq) + assert err_gptq < err_plain, (err_gptq, err_plain) + + +def test_gptq_disabled_on_bad_hessian(): + rng = np.random.default_rng(12) + w = rng.standard_normal((4, 256), dtype=np.float32) + baseline = quantize_hadamard(w, bits=4) + + negative = quantize_hadamard(w, bits=4, hessian=-np.eye(256, dtype=np.float32) * 1e6, use_gptq=True) + assert not negative.gptq_used + np.testing.assert_array_equal(negative.indices, baseline.indices) + + +def test_gptq_disabled_on_nan_hessian(): + rng = np.random.default_rng(12) + w = rng.standard_normal((4, 256), dtype=np.float32) + baseline = quantize_hadamard(w, bits=4) + nan_h = np.full((256, 256), np.nan, dtype=np.float32) + broken = quantize_hadamard(w, bits=4, hessian=nan_h, use_gptq=True) + np.testing.assert_array_equal(broken.indices, baseline.indices) + assert not broken.gptq_used From d034d74e862dc354d5f454e38246bfaba217f7f1 Mon Sep 17 00:00:00 2001 From: rachitj27 Date: Thu, 3 Sep 2026 12:09:02 -0700 Subject: [PATCH 4/4] refactor(gptq): drop comments, name the Cholesky helper per CONTRIBUTING Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01A8jVC79MsJ4PX472pAZ1Qk Signed-off-by: rachitj27 --- python/cactus/convert/quantization/cq.py | 48 ++++++++++-------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/python/cactus/convert/quantization/cq.py b/python/cactus/convert/quantization/cq.py index 466982c29..c3ffa23e0 100644 --- a/python/cactus/convert/quantization/cq.py +++ b/python/cactus/convert/quantization/cq.py @@ -230,29 +230,26 @@ def _prepare_input_scale(input_scale: np.ndarray | None, k: int) -> np.ndarray: return np.clip(scale, 1e-8, 65504.0).astype(np.float32, copy=False) -def _gptq_correct_group(work: np.ndarray, recon: np.ndarray, h_inv_chol: np.ndarray | None, start: int, stop: int) -> None: - # GPTQ block update using the upper Cholesky factor U of H^-1 (H^-1 == U.T @ U). - # - # OBS says that after quantizing block B the remaining columns should be corrected with - # W[:, rest] -= (W_B - Q_B) @ inv(Hinv[B, B]) @ Hinv[B, rest] - # where Hinv is the inverse Hessian of the *not yet quantized* columns. That inverse changes - # after every block (it becomes the Schur complement Hinv[rest,rest] - Hinv[rest,B] inv(Hinv[B,B]) Hinv[B,rest]), - # so reusing the single global inverse for every block is wrong for all but the first block. - # Writing Hinv = U.T @ U with U upper triangular and processing columns left to right, the - # updated inverse for the remaining columns is exactly U[rest, rest].T @ U[rest, rest], i.e. the - # trailing block of the global factor. Hence inv(Hinv[B,B]) @ Hinv[B,rest] == inv(U[B,B]) @ U[B,rest] - # holds at every step with no refactorisation. - if h_inv_chol is None or stop >= work.shape[1]: +def _upper_cholesky_of_inverse(h: np.ndarray) -> np.ndarray | None: + h_inv = np.linalg.inv(h.astype(np.float64)) + upper = np.linalg.cholesky(h_inv).T.astype(np.float32) + if not np.all(np.isfinite(upper)): + return None + return upper + + +def _gptq_correct_group(work: np.ndarray, recon: np.ndarray, h_inv_upper: np.ndarray | None, start: int, stop: int) -> None: + if h_inv_upper is None or stop >= work.shape[1]: return try: - u_bb = h_inv_chol[start:stop, start:stop].astype(np.float32, copy=False) - u_bs = h_inv_chol[start:stop, stop:].astype(np.float32, copy=False) + block = h_inv_upper[start:stop, start:stop] + block_to_rest = h_inv_upper[start:stop, stop:] try: from scipy.linalg import solve_triangular - update = solve_triangular(u_bb, u_bs, lower=False) + update = solve_triangular(block, block_to_rest, lower=False) except ImportError: - update = np.linalg.solve(u_bb, u_bs) + update = np.linalg.solve(block, block_to_rest) work[:, stop:] -= (work[:, start:stop] - recon) @ update except Exception: return @@ -282,7 +279,7 @@ def quantize_hadamard( groups = k // GROUP_SIZE indices = np.zeros((n, k), dtype=np.uint8) norms = np.zeros((n, groups), dtype=np.float16) - h_inv_chol = None + h_inv_upper = None if use_gptq and hessian is not None: try: h = np.asarray(hessian, dtype=np.float32) @@ -293,16 +290,9 @@ def quantize_hadamard( s = np.clip(input_scale.astype(np.float32), 1e-6, None) h = h / (s[:, None] * s[None, :]) h = h + np.eye(h.shape[0], dtype=np.float32) * (0.01 * np.mean(np.diag(h)) + 1e-6) - # Invert and factor in float64: a float32 inverse of an ill-conditioned Hessian can - # lose positive-definiteness and make the Cholesky fail (silently disabling GPTQ). - h_inv = np.linalg.inv(h.astype(np.float64)) - # Upper Cholesky factor: h_inv == h_inv_chol.T @ h_inv_chol. See _gptq_correct_group. - h_inv_chol = np.linalg.cholesky(h_inv).T.astype(np.float32) - # LAPACK can return a NaN/inf factor without raising (e.g. NaN Hessian); reject it. - if not np.all(np.isfinite(h_inv_chol)): - h_inv_chol = None + h_inv_upper = _upper_cholesky_of_inverse(h) except Exception: - h_inv_chol = None + h_inv_upper = None for g in range(groups): start, stop = g * GROUP_SIZE, (g + 1) * GROUP_SIZE group = work[:, start:stop] @@ -312,8 +302,8 @@ def quantize_hadamard( recon = (codebook[idx] @ rot.T) * row_norms[:, None] indices[:, start:stop] = idx norms[:, g] = row_norms.astype(np.float16) - _gptq_correct_group(work, recon, h_inv_chol, start, stop) - return CQTensor(indices=indices, norms=norms, input_scale=input_scale.astype(np.float16), bits=bits, gptq_used=bool(use_gptq and h_inv_chol is not None)) + _gptq_correct_group(work, recon, h_inv_upper, start, stop) + return CQTensor(indices=indices, norms=norms, input_scale=input_scale.astype(np.float16), bits=bits, gptq_used=bool(use_gptq and h_inv_upper is not None)) def quantize_orthogonal(weight, bits: int = 4, seed: int = 1234, input_scale: np.ndarray | None = None) -> CQTensor: