From 9ecfb17e08b8f29f383bf265cbcb549ff119a220 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sun, 14 Jun 2026 12:27:26 -1000 Subject: [PATCH 1/2] convert: fold gemma4 audio projection scaling into the weight reparam The gemma4 audio soft-token 1/16 scaling lived in the transpile graph (_gemma4_compute_native_like_audio_features) while the equivalent vision scaling lived in the conversion-time weight reparam. The audio embedder projection is a bias-free Linear with no post-projection norm, so scaling its weight is identical to scaling its output. Add embed_audio_proj to the gemma4 div set and drop the runtime multiply so all multimodal projection scaling is applied in one place, matching vision. Verified by E2E: e2b transcribes test.wav cleanly and identifies the monkey image. Signed-off-by: Noah Cylich --- python/cactus/convert/cactus_adapters/tensor_io.py | 2 +- python/cactus/transpile/model_adapters.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/python/cactus/convert/cactus_adapters/tensor_io.py b/python/cactus/convert/cactus_adapters/tensor_io.py index dc59d0aa8..fbdd93a7c 100644 --- a/python/cactus/convert/cactus_adapters/tensor_io.py +++ b/python/cactus/convert/cactus_adapters/tensor_io.py @@ -292,7 +292,7 @@ def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=Fal data = data * GEMMA4_WEIGHT_SCALE elif 'router_scale' in filename: data = data / GEMMA4_WEIGHT_SCALE - elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights', 'embed_vision_proj.weights'): + elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights', 'embed_vision_proj.weights', 'embed_audio_proj.weights'): data = data / GEMMA4_WEIGHT_SCALE elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index b09357e42..b6e2cee0d 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -1094,10 +1094,6 @@ def _gemma4_compute_native_like_audio_features( if not callable(getattr(embed_audio, "forward", None)): raise TypeError("Gemma4 audio embedder is not callable") projected = embed_audio(inputs_embeds=audio_encodings) - # Native Cactus Gemma4 scales audio soft tokens before merging them into - # the language stream. HF leaves this implicit in its fp16 path, but the - # quantized Cactus decoder expects the native-scaled representation. - projected = projected * (1.0 / 16.0) # The transpiled bundle is shape-specialized from representative media. # Native Gemma4 audio preprocessing emits an unpadded feature tensor for # that media, so the post-subsampling sequence is already the real token From 0939e3b512630ac4e63b6d75a13365cc2ab8ea9e Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Sun, 14 Jun 2026 12:38:16 -1000 Subject: [PATCH 2/2] convert: drop dead embed_vision_embedding entry from gemma4 scaling gemma4's Gemma4MultimodalEmbedder deletes the hard embedding table, so no embed_vision_embedding.weights tensor is ever produced. Remove the name from the gemma4 div set in tensor_io.py and from GEMMA4_DIV_BASENAMES; both were no-ops. The real vision weight (embed_vision_proj) stays scaled. Signed-off-by: Noah Cylich --- python/cactus/convert/cactus_adapters/tensor_io.py | 2 +- python/cactus/convert/model_adapters/naming.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/cactus/convert/cactus_adapters/tensor_io.py b/python/cactus/convert/cactus_adapters/tensor_io.py index fbdd93a7c..8bfc632f1 100644 --- a/python/cactus/convert/cactus_adapters/tensor_io.py +++ b/python/cactus/convert/cactus_adapters/tensor_io.py @@ -292,7 +292,7 @@ def save_tensor_with_header(tensor, output_path, precision='INT8', transpose=Fal data = data * GEMMA4_WEIGHT_SCALE elif 'router_scale' in filename: data = data / GEMMA4_WEIGHT_SCALE - elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_embedding.weights', 'embed_vision_proj.weights', 'embed_audio_proj.weights'): + elif filename in ('token_embeddings.weights', 'output_weight.weights', 'embed_vision_proj.weights', 'embed_audio_proj.weights'): data = data / GEMMA4_WEIGHT_SCALE elif filename == 'output_norm.weights': data = data * GEMMA4_WEIGHT_SCALE diff --git a/python/cactus/convert/model_adapters/naming.py b/python/cactus/convert/model_adapters/naming.py index d854b1a02..e96f568fb 100644 --- a/python/cactus/convert/model_adapters/naming.py +++ b/python/cactus/convert/model_adapters/naming.py @@ -23,7 +23,6 @@ class NameMatch: GEMMA4_DIV_BASENAMES = { "token_embeddings", "output_weight", - "embed_vision_embedding", }