diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp
index 456d471aa..e66a0e312 100644
--- a/cactus-engine/src/model.cpp
+++ b/cactus-engine/src/model.cpp
@@ -1187,7 +1187,6 @@ void Model::move_cache_states(Component& source, Component& target, size_t logic
                 logical_current != std::numeric_limits<size_t>::max()) {
                 auto* meta = static_cast<uint64_t*>(target.graph->get_output(static_cast<size_t>(dst_node)));
                 if (meta && logical_current < meta[0]) {
-                    meta[4] = std::min<uint64_t>(meta[4], logical_current);
                     meta[0] = logical_current;
                 }
             }
@@ -1202,7 +1201,6 @@ void Model::set_cache_current_len(Component& comp, size_t len) {
             if (comp.graph->get_node_op_type(static_cast<size_t>(node_id)) != OpType::KV_CACHE_STATE) continue;
             auto* meta = static_cast<uint64_t*>(comp.graph->get_output(static_cast<size_t>(node_id)));
             if (!meta || len >= meta[0]) continue;
-            meta[4] = std::min<uint64_t>(meta[4], len);
             meta[0] = len;
         }
     }
diff --git a/cactus-engine/tests/run.cpp b/cactus-engine/tests/run.cpp
index e412cdce8..359e4bd3c 100644
--- a/cactus-engine/tests/run.cpp
+++ b/cactus-engine/tests/run.cpp
@@ -247,6 +247,9 @@ struct TokenPrinter {
     std::string pending;
     bool in_code_fence = false;
     bool color = false;
+    bool suppress_thinking_stream = false;
+    bool show_thinking = false;
+    std::string stream_carry;
     bool label_printed = false;
     std::thread spinner_thread;
     std::atomic<bool> spinner_running{false};
@@ -261,10 +264,12 @@ struct TokenPrinter {
         in_code_fence = false;
         label_printed = false;
         color = stdout_is_terminal();
+        suppress_thinking_stream = false;
+        stream_carry.clear();
     }
 
     void start_thinking() {
-        if (!color) return;
+        if (!color || spinner_running.load()) return;
         spinner_running = true;
         spinner_thread = std::thread([this]() {
             const char* frames[] = {"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"};
@@ -434,25 +439,61 @@ struct TokenPrinter {
     }
 
     void on_token(const char* text) {
-        print_label();
         if (!saw_first) {
             first = std::chrono::steady_clock::now();
             saw_first = true;
         }
         ++count;
         if (!text) return;
-        pending += text;
+        feed(text);
+        std::cout << std::flush;
+    }
+
+    void feed(const std::string& chunk) {
+        static const std::vector<std::string> markers = {
+            "<|channel>", "<channel|>", "<|turn>", "<turn|>", "<|think|>", "<think|>",
+        };
+        stream_carry += chunk;
+        size_t i = 0;
+        while (i < stream_carry.size()) {
+            const std::string* matched = nullptr;
+            bool maybe_partial = false;
+            for (const auto& m : markers) {
+                if (stream_carry.compare(i, m.size(), m) == 0) { matched = &m; break; }
+                size_t avail = stream_carry.size() - i;
+                if (avail < m.size() && stream_carry.compare(i, avail, m, 0, avail) == 0) maybe_partial = true;
+            }
+            if (matched) {
+                if (*matched == "<|channel>") { suppress_thinking_stream = true; if (!show_thinking) start_thinking(); }
+                else if (*matched == "<channel|>") { suppress_thinking_stream = false; stop_thinking(); }
+                i += matched->size();
+                continue;
+            }
+            if (maybe_partial) break;
+            if (!suppress_thinking_stream || show_thinking) emit_visible(stream_carry[i]);
+            ++i;
+        }
+        stream_carry.erase(0, i);
+    }
+
+    void emit_visible(char c) {
+        print_label();
+        pending += c;
         size_t nl;
         while ((nl = pending.find('\n')) != std::string::npos) {
             render_line(pending.substr(0, nl));
             std::cout << "\n";
             pending.erase(0, nl + 1);
         }
-        std::cout << std::flush;
     }
 
     // Flush the trailing partial line once generation completes (no trailing newline).
     void finish() {
+        stop_thinking();
+        if (!stream_carry.empty() && (!suppress_thinking_stream || show_thinking)) {
+            for (char c : stream_carry) emit_visible(c);
+        }
+        stream_carry.clear();
         print_label();
         if (!pending.empty()) {
             render_line(pending);
@@ -893,6 +934,7 @@ int main(int argc, char** argv) {
     std::vector<uint8_t> current_pcm;
     TokenPrinter printer;
     g_printer = &printer;
+    printer.show_thinking = thinking;
     bool auto_send = !initial_prompt.empty() || !current_audio.empty() || !current_image.empty();
 
     while (true) {
@@ -990,7 +1032,7 @@ int main(int argc, char** argv) {
             + ",\"auto_handoff\":" + (auto_handoff ? "true" : "false")
             + ",\"confidence_threshold\":" + std::to_string(confidence_threshold)
             + ",\"cloud_timeout_ms\":" + std::to_string(cloud_timeout_ms)
-            + ",\"stop_sequences\":[\"<|im_end|>\",\"<end_of_turn>\"]}";
+            + ",\"stop_sequences\":[\"<|im_end|>\",\"<end_of_turn>\",\"<turn|>\"]}";
 
         if (!current_image.empty()) std::cout << "[image: " << current_image << "]\n";
         if (!current_audio.empty()) std::cout << "[audio: " << current_audio << "]\n";
diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h
index 55aa7076b..cd37fd4d2 100644
--- a/cactus-graph/cactus_graph.h
+++ b/cactus-graph/cactus_graph.h
@@ -1073,7 +1073,7 @@ CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk(
 CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated(
     cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices,
     const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights,
-    size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out);
+    size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out);
 CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated(
     cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices,
     const cactus_node_t* w1_weights, const cactus_node_t* w2_weights,
diff --git a/cactus-graph/src/builder.cpp b/cactus-graph/src/builder.cpp
index 88778a168..7e82ed65b 100644
--- a/cactus-graph/src/builder.cpp
+++ b/cactus-graph/src/builder.cpp
@@ -1220,7 +1220,12 @@ size_t CactusGraph::add_node(OpType op_type, const std::vector<size_t>& inputs,
     }
 
     Precision result_precision = params.output_precision;
-    if (op_type == OpType::PRECISION_CAST || op_type == OpType::EMBEDDING) {
+    if (op_type == OpType::PRECISION_CAST ||
+        op_type == OpType::EMBEDDING ||
+        op_type == OpType::TOPK ||
+        op_type == OpType::SCATTER_TOPK ||
+        op_type == OpType::SAMPLE ||
+        op_type == OpType::GAUSSIAN_TOPK) {
         result_precision = params.output_precision;
     } else if (!inputs.empty()) {
         result_precision = nodes_[node_index_map_[inputs[0]]]->output_buffer.precision;
@@ -1305,8 +1310,12 @@ size_t CactusGraph::altup_correct(size_t coefs, size_t innovation, const size_t*
 
 size_t CactusGraph::gaussian_topk(size_t input, float ppf) {
     const auto& in_buf = get_output_buffer(input);
+    if (in_buf.precision != Precision::FP16) {
+        throw std::runtime_error("gaussian_topk only supports FP16 input");
+    }
     OpParams params;
     params.scalar = ppf;
+    params.output_precision = in_buf.precision;
     return add_node(OpType::GAUSSIAN_TOPK, {input}, in_buf.shape, params);
 }
 
diff --git a/cactus-graph/src/graph_ffi.cpp b/cactus-graph/src/graph_ffi.cpp
index 685b6aa3d..74ce9a8e6 100644
--- a/cactus-graph/src/graph_ffi.cpp
+++ b/cactus-graph/src/graph_ffi.cpp
@@ -1341,7 +1341,7 @@ int cactus_graph_gaussian_topk(cactus_graph_t graph, cactus_node_t input, float
 
 int cactus_graph_moe_layer_gated(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices,
                                  const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights,
-                                 size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out) {
+                                 size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out) {
     if (!graph || !w1_weights || !w3_weights || !w2_weights || !out) return fail_invalid("Invalid args to cactus_graph_moe_layer_gated");
     try {
         std::vector<size_t> w1(num_experts), w3(num_experts), w2(num_experts);
@@ -1350,7 +1350,7 @@ int cactus_graph_moe_layer_gated(cactus_graph_t graph, cactus_node_t hidden, cac
             w3[i] = static_cast<size_t>(w3_weights[i]);
             w2[i] = static_cast<size_t>(w2_weights[i]);
         }
-        *out = static_cast<cactus_node_t>(as_graph(graph)->graph.moe_layer(static_cast<size_t>(hidden), static_cast<size_t>(routing_probs), static_cast<size_t>(topk_indices), w1, w3, w2, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor));
+        *out = static_cast<cactus_node_t>(as_graph(graph)->graph.moe_layer(static_cast<size_t>(hidden), static_cast<size_t>(routing_probs), static_cast<size_t>(topk_indices), w1, w3, w2, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor, static_cast<Activation>(activation)));
         return 0;
     } catch (const std::exception& e) {
         last_error_message = e.what();
diff --git a/cactus-graph/src/ops_nn.cpp b/cactus-graph/src/ops_nn.cpp
index d2c84d198..bee9810ba 100644
--- a/cactus-graph/src/ops_nn.cpp
+++ b/cactus-graph/src/ops_nn.cpp
@@ -83,6 +83,7 @@ void compute_matmul_node(GraphNode& node, const std::vector<std::unique_ptr<Grap
 namespace {
     thread_local std::vector<__fp16> moe_compact_hidden_buf;
     thread_local std::vector<__fp16> moe_gate_buf;
+    thread_local std::vector<__fp16> moe_gate_pad_buf;
     thread_local std::vector<__fp16> moe_up_buf;
     thread_local std::vector<__fp16> moe_expert_out_buf;
     thread_local std::vector<size_t> moe_expert_offsets_buf;
@@ -185,18 +186,21 @@ void compute_moe_layer_node(GraphNode& node, const std::vector<std::unique_ptr<G
     size_t* expert_offsets = moe_expert_offsets_buf.data(); 
     size_t* expert_tokens_flat = moe_expert_tokens_buf.data();  
 
+    auto expert_index = [&](float raw_idx) -> size_t {
+        if (!std::isfinite(raw_idx) || raw_idx < 0.0f) {
+            throw std::runtime_error("moe_layer got invalid expert index");
+        }
+        size_t idx = static_cast<size_t>(raw_idx + 0.5f);
+        if (idx >= num_experts) {
+            throw std::runtime_error("moe_layer got expert index out of range");
+        }
+        return idx;
+    };
+
     std::memset(expert_offsets, 0, (num_experts + 1) * sizeof(size_t));
     for (size_t tok = 0; tok < token_count; ++tok) {
         for (size_t k = 0; k < top_k; ++k) {
-            float raw_idx = topk_idx[tok * top_k + k];
-            if (!std::isfinite(raw_idx)) {
-                throw std::runtime_error("moe_layer got non-finite expert index");
-            }
-            size_t idx = static_cast<size_t>(raw_idx + 0.5f);
-            if (idx >= num_experts) {
-                throw std::runtime_error("moe_layer got expert index out of range");
-            }
-            expert_offsets[idx + 1]++;
+            expert_offsets[expert_index(topk_idx[tok * top_k + k]) + 1]++;
         }
     }
     
@@ -210,7 +214,7 @@ void compute_moe_layer_node(GraphNode& node, const std::vector<std::unique_ptr<G
 
     for (size_t tok = 0; tok < token_count; ++tok) {
         for (size_t k = 0; k < top_k; ++k) {
-            size_t idx = static_cast<size_t>(topk_idx[tok * top_k + k] + 0.5f);
+            size_t idx = expert_index(topk_idx[tok * top_k + k]);
             expert_tokens_flat[moe_write_cursors[idx]++] = tok;
         }
     }
@@ -220,7 +224,7 @@ void compute_moe_layer_node(GraphNode& node, const std::vector<std::unique_ptr<G
         for (size_t tok = 0; tok < token_count; ++tok) {
             float sum_probs = 0.0f;
             for (size_t k = 0; k < top_k; ++k) {
-                size_t idx = static_cast<size_t>(topk_idx[tok * top_k + k] + 0.5f);
+                size_t idx = expert_index(topk_idx[tok * top_k + k]);
                 sum_probs += routing_prob(tok, idx);
             }
             routing_denom[tok] = sum_probs + eps;
@@ -283,7 +287,23 @@ void compute_moe_layer_node(GraphNode& node, const std::vector<std::unique_ptr<G
             cactus_multiply_f16(gate, up, gate, selected_count * expert_intermediate_dim);
         }
 
-        moe_matmul(gate, selected_count, expert_intermediate_dim, w2_buffer, expert_out, hidden_dim);
+        const size_t w2_k = w2_buffer.shape.size() == 2 ? w2_buffer.shape[1] : 0;
+        if (w2_k < expert_intermediate_dim) {
+            throw std::runtime_error("moe_layer down-proj weight K smaller than expert intermediate dim");
+        }
+        const __fp16* w2_input = gate;
+        if (w2_k != expert_intermediate_dim) {
+            if (moe_gate_pad_buf.size() < selected_count * w2_k) moe_gate_pad_buf.resize(selected_count * w2_k);
+            std::memset(moe_gate_pad_buf.data(), 0, selected_count * w2_k * sizeof(__fp16));
+            for (size_t i = 0; i < selected_count; ++i) {
+                std::memcpy(moe_gate_pad_buf.data() + i * w2_k,
+                            gate + i * expert_intermediate_dim,
+                            expert_intermediate_dim * sizeof(__fp16));
+            }
+            w2_input = moe_gate_pad_buf.data();
+        }
+
+        moe_matmul(w2_input, selected_count, w2_k, w2_buffer, expert_out, hidden_dim);
 
         for (size_t i = 0; i < selected_count; ++i) {
             const size_t tok = selected_tokens[i];
@@ -934,6 +954,3 @@ void compute_groupnorm_node(GraphNode& node, const std::vector<std::unique_ptr<G
         }
     }
 }
-
-
-
diff --git a/python/cactus/bindings/cactus.py b/python/cactus/bindings/cactus.py
index 327afdb5b..27e85f84d 100644
--- a/python/cactus/bindings/cactus.py
+++ b/python/cactus/bindings/cactus.py
@@ -608,7 +608,7 @@ class cactus_tensor_info_t(ctypes.Structure):
 _lib.cactus_graph_moe_layer_gated.argtypes = [
     cactus_graph_t, cactus_node_t, cactus_node_t, cactus_node_t,
     ctypes.POINTER(cactus_node_t), ctypes.POINTER(cactus_node_t), ctypes.POINTER(cactus_node_t),
-    ctypes.c_size_t, ctypes.c_size_t, ctypes.c_bool, ctypes.c_float, ctypes.c_float, ctypes.POINTER(cactus_node_t)
+    ctypes.c_size_t, ctypes.c_size_t, ctypes.c_bool, ctypes.c_float, ctypes.c_float, ctypes.c_int32, ctypes.POINTER(cactus_node_t)
 ]
 _lib.cactus_graph_moe_layer_gated.restype = ctypes.c_int
 _bind_optional(
@@ -2560,7 +2560,7 @@ def gaussian_topk(self, x, ppf):
         return self._tensor_from_node(out.value)
 
     def moe_layer_gated(self, hidden, routing_probs, topk_indices, w1_weights, w3_weights, w2_weights,
-                        num_experts, num_experts_per_tok, normalize_routing=True, epsilon=1e-6, routed_scaling_factor=1.0):
+                        num_experts, num_experts_per_tok, normalize_routing=True, epsilon=1e-6, routed_scaling_factor=1.0, activation=0):
         hidden = self._ensure_tensor(hidden)
         routing_probs = self._ensure_tensor(routing_probs)
         topk_indices = self._ensure_tensor(topk_indices)
@@ -2572,7 +2572,7 @@ def moe_layer_gated(self, hidden, routing_probs, topk_indices, w1_weights, w3_we
             self.h, cactus_node_t(hidden.id), cactus_node_t(routing_probs.id), cactus_node_t(topk_indices.id),
             w1, w3, w2, ctypes.c_size_t(int(num_experts)), ctypes.c_size_t(int(num_experts_per_tok)),
             ctypes.c_bool(bool(normalize_routing)), ctypes.c_float(float(epsilon)),
-            ctypes.c_float(float(routed_scaling_factor)), ctypes.byref(out)
+            ctypes.c_float(float(routed_scaling_factor)), ctypes.c_int32(int(activation)), ctypes.byref(out)
         )
         if rc != 0:
             raise RuntimeError(_err("graph_moe_layer_gated failed"))
diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py
index d5cb6c6bc..79ee701af 100644
--- a/python/cactus/cli/__init__.py
+++ b/python/cactus/cli/__init__.py
@@ -209,6 +209,7 @@ def create_parser():
     --torch-dtype <dtype>              float16 | float32 | bfloat16
     --trust-remote-code                allow HF remote code during the build
     --local-files-only                 require model/processor to be local
+    --low-memory-load                  graph capture with meta tensors
     --allow-unconverted-weights        debug-only: skip the CQ-weights check
     --execute-after-transpile          run a reference execution after building
     --graph-filename <name>            override saved graph filename
@@ -354,6 +355,10 @@ def create_parser():
                                 help="Path or HF id of a LoRA adapter to merge before converting (requires `peft`)")
     convert_parser.add_argument("--reconvert", action="store_true",
                                 help="Force conversion from source")
+    convert_parser.add_argument("--skip-model-load", action="store_true",
+                                help="Convert directly from checkpoint tensors without loading the full HF model object")
+    convert_parser.add_argument("--low-memory-load", action="store_true",
+                                help="Avoid loading checkpoint tensors during graph capture")
     convert_parser.add_argument("--weights-only", action="store_true",
                                 help="Only quantize weights; skip building the runtime graph bundle")
     convert_parser.add_argument("--dynamic-batch", action="store_true",
diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py
index 24b22e2df..654396a8c 100644
--- a/python/cactus/cli/convert.py
+++ b/python/cactus/cli/convert.py
@@ -65,6 +65,7 @@ def cmd_convert(args):
             token=args.token,
             reconvert=args.reconvert,
             output_dir=output_dir,
+            skip_model_load=bool(getattr(args, "skip_model_load", False)),
         )
         if getattr(args, "weights_only", False):
             return 0
@@ -83,6 +84,7 @@ def cmd_transpile(args):
     """Build the runtime graph bundle from already-converted CQ weights."""
     from .transpile import run_transpile
     from .model import _default_multimodal_assets
+    from cactus.transpile.component_plan import infer_component_plan_from_output
 
     extra_args = []
     if args.weights_dir:
@@ -100,11 +102,21 @@ def cmd_transpile(args):
 
     image_files = list(args.image_file or [])
     audio_file = args.audio_file
-    if not image_files or not audio_file:
+    weights_dir = Path(args.weights_dir).expanduser() if args.weights_dir else get_weights_dir(args.model_id)
+    from .model import _AUDIO_TASKS
+    plan = infer_component_plan_from_output(str(weights_dir), model_id=args.model_id)
+    default_needs_image = bool(
+        plan.needs_image if plan is not None else args.task == "multimodal_causal_lm_logits"
+    )
+    default_needs_audio = bool(
+        plan.needs_audio if plan is not None
+        else (args.task in _AUDIO_TASKS or args.task == "multimodal_causal_lm_logits")
+    )
+    if (default_needs_image and not image_files) or (default_needs_audio and not audio_file):
         default_images, default_audio = _default_multimodal_assets()
-        if not image_files:
+        if default_needs_image and not image_files:
             image_files = default_images
-        if not audio_file and default_audio:
+        if default_needs_audio and not audio_file and default_audio:
             audio_file = default_audio
 
     for img in image_files:
@@ -125,6 +137,8 @@ def cmd_transpile(args):
         extra_args.append("--trust-remote-code")
     if args.local_files_only:
         extra_args.append("--local-files-only")
+    if getattr(args, "low_memory_load", False):
+        extra_args.append("--low-memory-load")
     if args.artifact_dir:
         extra_args.extend(["--artifact-dir", args.artifact_dir])
     if args.graph_filename:
diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py
index 5e59f27f0..0bcf8ab81 100644
--- a/python/cactus/cli/model.py
+++ b/python/cactus/cli/model.py
@@ -11,7 +11,7 @@
 
 
 
-def _convert_from_source(model_id, *, bits, token, weights_dir):
+def _convert_from_source(model_id, *, bits, token, weights_dir, skip_model_load=False):
     """Download from HuggingFace and run CQ conversion."""
     print_color(YELLOW, f"Converting {model_id} from HuggingFace source...")
     from ..convert.cli import main as cq_main
@@ -21,6 +21,8 @@ def _convert_from_source(model_id, *, bits, token, weights_dir):
         "--out", str(weights_dir),
         "--bits", str(bits),
     ]
+    if skip_model_load:
+        cq_args.append("--skip-model-load")
     if token:
         os.environ["HF_TOKEN"] = token
         os.environ["HUGGING_FACE_HUB_TOKEN"] = token
@@ -30,7 +32,7 @@ def _convert_from_source(model_id, *, bits, token, weights_dir):
     return weights_dir
 
 
-def ensure_weights(model_id, *, bits=4, platform=None, token=None, reconvert=False, output_dir=None):
+def ensure_weights(model_id, *, bits=4, platform=None, token=None, reconvert=False, output_dir=None, skip_model_load=False):
     from .download import get_bundle_dir
 
     weights_dir = Path(output_dir) if output_dir else get_bundle_dir(model_id, bits=bits, platform=platform)
@@ -47,7 +49,13 @@ def ensure_weights(model_id, *, bits=4, platform=None, token=None, reconvert=Fal
         print_color(YELLOW, "Removing incomplete weights from a previous run...")
         shutil.rmtree(weights_dir)
 
-    return _convert_from_source(model_id, bits=bits, token=token, weights_dir=weights_dir)
+    return _convert_from_source(
+        model_id,
+        bits=bits,
+        token=token,
+        weights_dir=weights_dir,
+        skip_model_load=skip_model_load,
+    )
 
 
 
@@ -251,7 +259,8 @@ def prepare_bundle(args, *, model_id=None, transpile=None, prebuilt=True,
 
 
 def ensure_bundle(model_id, *, bits=4, platform=None, token=None,
-                  reconvert=False, output_dir=None, transpile=None):
+                  reconvert=False, output_dir=None, transpile=None,
+                  skip_model_load=False):
     from .download import get_bundle_dir
     from .transpile import run_transpile
     from cactus.transpile.component_plan import infer_component_plan_from_output
@@ -266,6 +275,7 @@ def ensure_bundle(model_id, *, bits=4, platform=None, token=None,
     ensure_weights(
         model_id, bits=bits, platform=platform, token=token,
         reconvert=reconvert, output_dir=output_dir,
+        skip_model_load=skip_model_load,
     )
 
     if _has_transpiled_bundle(output_dir):
diff --git a/python/cactus/convert/cactus_adapters/config_utils.py b/python/cactus/convert/cactus_adapters/config_utils.py
index f08c795fc..012f658d7 100644
--- a/python/cactus/convert/cactus_adapters/config_utils.py
+++ b/python/cactus/convert/cactus_adapters/config_utils.py
@@ -490,6 +490,12 @@ def extract_complex_gemma_config(cfg, root_config):
         cfg_get(root_config, 'num_global_key_value_heads', None))
     expert_intermediate_size = cfg_get(cfg, 'expert_intermediate_size',
         cfg_get(root_config, 'expert_intermediate_size', None))
+    if expert_intermediate_size is None:
+        expert_intermediate_size = cfg_get(cfg, 'moe_intermediate_size',
+            cfg_get(root_config, 'moe_intermediate_size', None))
+    top_k_experts = cfg_get(cfg, 'top_k_experts', cfg_get(root_config, 'top_k_experts', None))
+    if top_k_experts is None:
+        top_k_experts = cfg_get(cfg, 'num_experts_per_tok', cfg_get(root_config, 'num_experts_per_tok', None))
     attention_k_eq_v = bool(cfg_get(cfg, 'attention_k_eq_v',
         cfg_get(root_config, 'attention_k_eq_v', False)))
     vocab_size_per_layer_input = cfg_get(cfg, 'vocab_size_per_layer_input',
@@ -513,6 +519,9 @@ def extract_complex_gemma_config(cfg, root_config):
         'attention_k_eq_v': attention_k_eq_v,
         'enable_moe_block': enable_moe_block,
     }
+    if top_k_experts is not None:
+        result['num_top_experts'] = int(top_k_experts)
+        result['num_experts_per_tok'] = int(top_k_experts)
     if global_head_dim is not None:
         result['global_head_dim'] = int(global_head_dim)
     if num_global_kv_heads is not None:
diff --git a/python/cactus/convert/cli.py b/python/cactus/convert/cli.py
index 20f8dc94e..128901d09 100644
--- a/python/cactus/convert/cli.py
+++ b/python/cactus/convert/cli.py
@@ -44,7 +44,7 @@ def _hf_cache_dir() -> str | None:
     return os.environ.get("HF_HUB_CACHE") or os.environ.get("HF_HOME") or None
 
 
-def _load_hf(model_id_or_path: str, device: str):
+def _load_hf(model_id_or_path: str, device: str, *, skip_model_load: bool = False):
     import logging, warnings
     logging.getLogger("transformers").setLevel(logging.ERROR)
     warnings.filterwarnings("ignore", message=".*You are using a model of type.*")
@@ -72,6 +72,8 @@ def _load_hf(model_id_or_path: str, device: str):
     family = detect_family(cfg, "auto")
     adapter = adapter_for_family(family)
     processor = adapter.load_processor(model_id_or_path)
+    if skip_model_load:
+        return cfg, processor, None
     model_cls = adapter.model_class(cfg)
     model_type = str(cfg_get(cfg, "model_type", "") or "").lower()
     if isinstance(cfg, dict) and model_type == "parakeet_tdt":
@@ -397,7 +399,7 @@ def convert(args: argparse.Namespace) -> None:
         shutil.rmtree(out_dir)
     out_dir.mkdir(parents=True, exist_ok=True)
 
-    cfg, processor, model = _load_hf(args.model, args.device)
+    cfg, processor, model = _load_hf(args.model, args.device, skip_model_load=bool(args.skip_model_load))
     runtime_source = ensure_parakeet_tdt_nemo_source(args.model, cache_dir=_hf_cache_dir()) or args.model
     family = detect_family(cfg, args.model_family)
     adapter = adapter_for_family(family)
@@ -597,6 +599,7 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--model-family", default="auto", choices=sorted(SUPPORTED_FAMILIES))
     p.add_argument("--strict", action="store_true")
     p.add_argument("--force", action="store_true")
+    p.add_argument("--skip-model-load", action="store_true")
     p.add_argument("--max-language-examples", type=int)
     p.add_argument("--max-vision-examples", type=int)
     p.add_argument("--max-audio-examples", type=int)
diff --git a/python/cactus/convert/model_adapters/adapters.py b/python/cactus/convert/model_adapters/adapters.py
index a29592d6a..41eaa86dc 100644
--- a/python/cactus/convert/model_adapters/adapters.py
+++ b/python/cactus/convert/model_adapters/adapters.py
@@ -3,6 +3,7 @@
 from dataclasses import dataclass, field, replace
 import json
 from pathlib import Path
+import re
 from typing import Any
 
 import numpy as np
@@ -16,6 +17,7 @@
     extract_audio_config,
     extract_base_config,
     extract_complex_gemma_config,
+    extract_lfm2_config,
     extract_parakeet_config,
     extract_parakeet_tdt_config,
     extract_vision_config,
@@ -136,6 +138,48 @@ def __init__(self) -> None:
         super().__init__()
         self.kv_shared_from_layer = 15
 
+    def normalize_state_dict(self, state_dict: dict[str, Any]) -> NormalizedState:
+        out = dict(state_dict)
+        provenance: dict[str, TensorProvenance] = {}
+        down_pattern = re.compile(
+            r"(?:(?:model|language_model|model\.language_model)\.)?layers\.(\d+)\.(moe|experts)\.down_proj"
+        )
+        for source_name, tensor in list(state_dict.items()):
+            match = down_pattern.fullmatch(source_name)
+            if match is None:
+                continue
+            shape = _tensor_shape(tensor)
+            if len(shape) != 3:
+                continue
+            block_name = match.group(2)
+            scale_name = source_name[: -len(".down_proj")] + ".per_expert_scale"
+            if block_name == "experts":
+                prefix = source_name[: -len(".experts.down_proj")]
+                scale_name = f"{prefix}.router.per_expert_scale"
+            scale = state_dict.get(scale_name)
+            if scale is None:
+                raise ValueError(
+                    f"gemma4 MoE down_proj {source_name!r} is missing per-expert scale {scale_name!r}"
+                )
+            scale_shape = _tensor_shape(scale)
+            if len(scale_shape) != 1 or int(shape[0]) != int(scale_shape[0]):
+                raise ValueError(
+                    f"gemma4 MoE down_proj {source_name!r} scale {scale_name!r} shape {scale_shape} "
+                    f"does not match {int(shape[0])} experts"
+                )
+            if torch is not None and isinstance(tensor, torch.Tensor):
+                scale_tensor = scale if isinstance(scale, torch.Tensor) else torch.as_tensor(scale)
+                scale_value = scale_tensor.to(device=tensor.device, dtype=tensor.dtype).reshape(-1, 1, 1)
+                out[source_name] = (tensor * scale_value).contiguous()
+            else:
+                out[source_name] = (np.asarray(tensor) * np.asarray(scale).reshape(-1, 1, 1)).copy()
+            provenance[source_name] = TensorProvenance(
+                [source_name, scale_name],
+                "gemma4_moe_down_proj_per_expert_scale",
+                "hf_key",
+            )
+        return NormalizedState(out, provenance)
+
     def runtime_config(self, cfg: Any) -> dict[str, Any]:
         text_cfg = _cfg_get(cfg, "text_config", None)
         base_cfg = text_cfg if text_cfg is not None else cfg
@@ -167,10 +211,20 @@ def scale_factor(self, output_name: str) -> float:
         return gemma4_scale_factor(output_name)
 
     def policy(self, match: NameMatch, shape: tuple[int, ...], requested_bits: int) -> TensorPolicy:
+        if match.output_name and (
+            "moe_router" in match.output_name
+            or "moe_expert_bias" in match.output_name
+            or "moe_per_expert_scale" in match.output_name
+            or match.source_name.endswith(".router.proj.weight")
+            or match.source_name.endswith(".router.scale")
+        ):
+            return TensorPolicy("fallback", "FP16", None, match.component, False, "none", "moe routing tensor")
         policy = super().policy(match, shape, requested_bits)
         name = match.source_name
         if name == "model.embed_vision.embedding_projection.weight":
             return policy
+        if match.output_name and "moe_expert_" in match.output_name:
+            return replace(policy, use_gptq=False)
         if ".self_attn." in name and (name.endswith(".k_proj.weight") or name.endswith(".v_proj.weight")):
             parts = name.split(".")
             try:
@@ -181,7 +235,94 @@ def policy(self, match: NameMatch, shape: tuple[int, ...], requested_bits: int)
                 return replace(policy, use_gptq=False, fallback_reason=policy.fallback_reason or "shared KV tensor has no per-layer hook module")
         return policy
 
+    def name_tensor(self, source_name: str, tensor: Any, num_layers: int | None) -> NameMatch:
+        match = re.fullmatch(
+            r"(?:(?:model|language_model|model\.language_model)\.)?layers\.(\d+)\.(?:moe|experts)\.(gate_up_proj|down_proj)",
+            source_name,
+        )
+        if match is not None:
+            layer_index = int(match.group(1))
+            kind = match.group(2)
+            if kind == "gate_up_proj":
+                output_name = f"layer_{layer_index}_moe_expert_{{channel}}_gate_up.weights"
+            else:
+                output_name = f"layer_{layer_index}_moe_expert_{{channel}}_w2.weights"
+            return NameMatch(source_name, output_name, "language", True, hf_name=source_name, adapter_name=source_name)
+        return super().name_tensor(source_name, tensor, num_layers)
+
+    def expand_tensor(self, match: NameMatch, tensor: Any) -> list[TensorEmission]:
+        if match.output_name is None:
+            return []
+        source_match = re.fullmatch(
+            r"(?:(?:model|language_model|model\.language_model)\.)?layers\.(\d+)\.(?:moe|experts)\.(gate_up_proj|down_proj)",
+            match.source_name,
+        )
+        if source_match is None:
+            return super().expand_tensor(match, tensor)
+
+        layer_index = int(source_match.group(1))
+        kind = source_match.group(2)
+        shape = _tensor_shape(tensor)
+        if len(shape) != 3:
+            raise ValueError(f"Gemma4-MoE expert tensor expects rank 3, got {shape}")
+        num_experts = int(shape[0])
+        emissions: list[TensorEmission] = []
+
+        def _piece(value, expert_idx: int, start: int | None = None, end: int | None = None):
+            if torch is not None and isinstance(value, torch.Tensor):
+                selected = value[expert_idx] if start is None else value[expert_idx, start:end, :]
+                return selected.contiguous()
+            array = np.asarray(value)
+            selected = array[expert_idx] if start is None else array[expert_idx, start:end, :]
+            return selected.copy()
+
+        def _source_names(weight_name: str, expert_idx: int) -> list[str]:
+            return [
+                f"model.language_model.layers.{layer_index}.moe.{weight_name}.{expert_idx}",
+                f"model.layers.{layer_index}.moe.{weight_name}.{expert_idx}",
+                f"layers.{layer_index}.moe.{weight_name}.{expert_idx}",
+                f"backbone.layers.{layer_index}.moe.{weight_name}.{expert_idx}",
+            ]
+
+        if kind == "gate_up_proj":
+            if int(shape[1]) % 2 != 0:
+                raise ValueError(f"Gemma4-MoE gate_up_proj second dim must be even, got {shape}")
+            intermediate = int(shape[1]) // 2
+            for expert_idx in range(num_experts):
+                emissions.append(TensorEmission(
+                    f"layer_{layer_index}_moe_expert_{expert_idx}_w1.weights",
+                    _piece(tensor, expert_idx, 0, intermediate),
+                    "gemma4_moe_gate_up_split_gate",
+                    source_names=_source_names("w1_weights", expert_idx),
+                    qdq_restore="runtime_key",
+                ))
+                emissions.append(TensorEmission(
+                    f"layer_{layer_index}_moe_expert_{expert_idx}_w3.weights",
+                    _piece(tensor, expert_idx, intermediate, int(shape[1])),
+                    "gemma4_moe_gate_up_split_up",
+                    source_names=_source_names("w3_weights", expert_idx),
+                    qdq_restore="runtime_key",
+                ))
+            return emissions
+
+        for expert_idx in range(num_experts):
+            emissions.append(TensorEmission(
+                f"layer_{layer_index}_moe_expert_{expert_idx}_w2.weights",
+                _piece(tensor, expert_idx),
+                "gemma4_moe_down_proj_split_scaled",
+                source_names=_source_names("w2_weights", expert_idx),
+                qdq_restore="runtime_key",
+            ))
+        return emissions
+
     def module_target_name(self, source_name: str, _model: Any) -> str | None:
+        if (
+            ".moe.gate_up_proj" in source_name
+            or ".moe.down_proj" in source_name
+            or ".experts.gate_up_proj" in source_name
+            or ".experts.down_proj" in source_name
+        ):
+            return None
         target = super().module_target_name(source_name, _model)
         if target and source_name.startswith("model.vision_tower."):
             modules = dict(_model.named_modules()) if _model is not None else {}
@@ -196,6 +337,20 @@ def module_target_name(self, source_name: str, _model: Any) -> str | None:
 class QwenAdapter(FamilyAdapter):
     family = "qwen"
 
+    def runtime_config(self, cfg: Any) -> dict[str, Any]:
+        config = super().runtime_config(cfg)
+        model_type = str(_cfg_get(cfg, "model_type", "") or "").lower()
+        if model_type == "qwen2_moe" or _cfg_get(cfg, "moe_intermediate_size", None) is not None:
+            config.update({
+                "moe_intermediate_size": int(_cfg_get(cfg, "moe_intermediate_size", 0) or 0),
+                "shared_expert_intermediate_size": int(_cfg_get(cfg, "shared_expert_intermediate_size", 0) or 0),
+                "decoder_sparse_step": int(_cfg_get(cfg, "decoder_sparse_step", 0) or 0),
+                "num_experts": int(_cfg_get(cfg, "num_experts", config.get("num_experts", 0)) or 0),
+                "num_experts_per_tok": int(_cfg_get(cfg, "num_experts_per_tok", config.get("num_experts_per_tok", 0)) or 0),
+                "norm_topk_prob": bool(_cfg_get(cfg, "norm_topk_prob", False)),
+            })
+        return config
+
     def model_class(self, cfg: Any):
         from transformers import AutoModel, AutoModelForCausalLM
 
@@ -210,6 +365,115 @@ def model_class(self, cfg: Any):
                 return AutoModel
         return AutoModelForCausalLM
 
+    def policy(self, match: NameMatch, shape: tuple[int, ...], requested_bits: int) -> TensorPolicy:
+        if match.output_name and (
+            "moe_router" in match.output_name
+            or "moe_expert_" in match.output_name
+            or "shared_expert_gate" in match.output_name
+        ):
+            return TensorPolicy("fallback", "FP16", None, match.component, False, "none", "moe routing/expert tensor")
+        return super().policy(match, shape, requested_bits)
+
+    def module_target_name(self, source_name: str, _model: Any) -> str | None:
+        if ".mlp.experts.gate_up_proj" in source_name or ".mlp.experts.down_proj" in source_name:
+            return None
+        return super().module_target_name(source_name, _model)
+
+    def name_tensor(self, source_name: str, tensor: Any, num_layers: int | None) -> NameMatch:
+        match = re.fullmatch(
+            r"(?:(?:model|language_model|model\.language_model)\.)?layers\.(\d+)\.mlp\.experts\.(gate_up_proj|down_proj)",
+            source_name,
+        )
+        if match is not None:
+            layer_index = int(match.group(1))
+            kind = match.group(2)
+            if kind == "gate_up_proj":
+                output_name = f"layer_{layer_index}_moe_expert_{{channel}}_gate_up.weights"
+            else:
+                output_name = f"layer_{layer_index}_moe_expert_{{channel}}_w2.weights"
+            return NameMatch(source_name, output_name, "language", True, hf_name=source_name, adapter_name=source_name)
+
+        router_match = re.fullmatch(
+            r"(?:(?:model|language_model|model\.language_model)\.)?layers\.(\d+)\.mlp\.gate\.weight",
+            source_name,
+        )
+        if router_match is not None:
+            layer_index = int(router_match.group(1))
+            return NameMatch(
+                source_name,
+                f"layer_{layer_index}_moe_router.weights",
+                "language",
+                True,
+                hf_name=source_name,
+                adapter_name=source_name,
+            )
+
+        return super().name_tensor(source_name, tensor, num_layers)
+
+    def expand_tensor(self, match: NameMatch, tensor: Any) -> list[TensorEmission]:
+        if match.output_name is None:
+            return []
+        source_match = re.fullmatch(
+            r"(?:(?:model|language_model|model\.language_model)\.)?layers\.(\d+)\.mlp\.experts\.(gate_up_proj|down_proj)",
+            match.source_name,
+        )
+        if source_match is None:
+            return super().expand_tensor(match, tensor)
+
+        layer_index = int(source_match.group(1))
+        kind = source_match.group(2)
+        shape = _tensor_shape(tensor)
+        if len(shape) != 3:
+            raise ValueError(f"Qwen2-MoE expert tensor expects rank 3, got {shape}")
+        num_experts = int(shape[0])
+        emissions: list[TensorEmission] = []
+
+        def _piece(value, expert_idx: int, start: int | None = None, end: int | None = None):
+            if torch is not None and isinstance(value, torch.Tensor):
+                selected = value[expert_idx] if start is None else value[expert_idx, start:end, :]
+                return selected.contiguous()
+            array = np.asarray(value)
+            selected = array[expert_idx] if start is None else array[expert_idx, start:end, :]
+            return selected.copy()
+
+        def _source_names(weight_name: str, expert_idx: int) -> list[str]:
+            return [
+                f"model.layers.{layer_index}.mlp.{weight_name}.{expert_idx}",
+                f"layers.{layer_index}.mlp.{weight_name}.{expert_idx}",
+                f"backbone.layers.{layer_index}.mlp.{weight_name}.{expert_idx}",
+            ]
+
+        if kind == "gate_up_proj":
+            if int(shape[1]) % 2 != 0:
+                raise ValueError(f"Qwen2-MoE gate_up_proj second dim must be even, got {shape}")
+            intermediate = int(shape[1]) // 2
+            for expert_idx in range(num_experts):
+                emissions.append(TensorEmission(
+                    f"layer_{layer_index}_moe_expert_{expert_idx}_w1.weights",
+                    _piece(tensor, expert_idx, 0, intermediate),
+                    "qwen2_moe_gate_up_split_gate",
+                    source_names=_source_names("w1_weights", expert_idx),
+                    qdq_restore="runtime_key",
+                ))
+                emissions.append(TensorEmission(
+                    f"layer_{layer_index}_moe_expert_{expert_idx}_w3.weights",
+                    _piece(tensor, expert_idx, intermediate, int(shape[1])),
+                    "qwen2_moe_gate_up_split_up",
+                    source_names=_source_names("w3_weights", expert_idx),
+                    qdq_restore="runtime_key",
+                ))
+            return emissions
+
+        for expert_idx in range(num_experts):
+            emissions.append(TensorEmission(
+                f"layer_{layer_index}_moe_expert_{expert_idx}_w2.weights",
+                _piece(tensor, expert_idx),
+                "qwen2_moe_down_proj_split",
+                source_names=_source_names("w2_weights", expert_idx),
+                qdq_restore="runtime_key",
+            ))
+        return emissions
+
 
 class WhisperAdapter(FamilyAdapter):
     family = "whisper"
@@ -435,6 +699,10 @@ def normalize_state_dict(self, state_dict: dict[str, Any]) -> NormalizedState:
 class Lfm2Adapter(FamilyAdapter):
     family = "lfm2"
 
+    def __init__(self) -> None:
+        super().__init__()
+        self.num_experts = 0
+
     def model_class(self, cfg: Any):
         from transformers import AutoModelForCausalLM
 
@@ -449,6 +717,12 @@ def model_class(self, cfg: Any):
                 pass
         return AutoModelForCausalLM
 
+    def runtime_config(self, cfg: Any) -> dict[str, Any]:
+        config = super().runtime_config(cfg)
+        config.update(extract_lfm2_config(cfg))
+        self.num_experts = int(_cfg_get(cfg, "num_experts", config.get("num_experts", 0)) or 0)
+        return config
+
     def load_processor(self, model_id_or_path: str):
         processor = super().load_processor(model_id_or_path)
         if processor is not None and hasattr(processor, "tokenizer"):
@@ -484,16 +758,105 @@ def load_processor(self, model_id_or_path: str):
         return Lfm2VlProcessor(image_processor=image_processor, tokenizer=tokenizer)
 
     def policy(self, match: NameMatch, shape: tuple[int, ...], requested_bits: int) -> TensorPolicy:
+        if match.output_name and (
+            "moe_router" in match.output_name
+            or "moe_expert_bias" in match.output_name
+        ):
+            return TensorPolicy("fallback", "FP16", None, match.component, False, "none", "moe routing tensor")
         policy = super().policy(match, shape, requested_bits)
         if match.source_name.endswith("vision_model.embeddings.patch_embedding.weight"):
             return replace(policy, use_gptq=False, fallback_reason=policy.fallback_reason or "vision patch embedding has no linear Hessian target")
+        if match.output_name and "moe_expert_" in match.output_name:
+            return replace(policy, use_gptq=False)
         return policy
 
     def module_target_name(self, source_name: str, _model: Any) -> str | None:
+        if ".feed_forward.experts.gate_up_proj" in source_name or ".feed_forward.experts.down_proj" in source_name:
+            return None
         if source_name.startswith("model.vision_tower.vision_model."):
             source_name = source_name.replace("model.vision_tower.vision_model.", "model.vision_tower.", 1)
         return super().module_target_name(source_name, _model)
 
+    def name_tensor(self, source_name: str, tensor: Any, num_layers: int | None) -> NameMatch:
+        match = re.fullmatch(
+            r"(?:(?:model|language_model|model\.language_model)\.)?layers\.(\d+)\.feed_forward\.experts\.(gate_up_proj|down_proj)",
+            source_name,
+        )
+        if match is not None:
+            layer_index = int(match.group(1))
+            kind = match.group(2)
+            if kind == "gate_up_proj":
+                output_name = f"layer_{layer_index}_moe_expert_{{channel}}_gate_up.weights"
+            else:
+                output_name = f"layer_{layer_index}_moe_expert_{{channel}}_w2.weights"
+            return NameMatch(source_name, output_name, "language", True, hf_name=source_name, adapter_name=source_name)
+        return super().name_tensor(source_name, tensor, num_layers)
+
+    def expand_tensor(self, match: NameMatch, tensor: Any) -> list[TensorEmission]:
+        if match.output_name is None:
+            return []
+        source_match = re.fullmatch(
+            r"(?:(?:model|language_model|model\.language_model)\.)?layers\.(\d+)\.feed_forward\.experts\.(gate_up_proj|down_proj)",
+            match.source_name,
+        )
+        if source_match is None:
+            return super().expand_tensor(match, tensor)
+
+        layer_index = int(source_match.group(1))
+        kind = source_match.group(2)
+        shape = _tensor_shape(tensor)
+        if len(shape) != 3:
+            raise ValueError(f"LFM2-MoE expert tensor expects rank 3, got {shape}")
+        num_experts = int(shape[0])
+        self.num_experts = self.num_experts or num_experts
+        emissions: list[TensorEmission] = []
+
+        def _piece(value, expert_idx: int, start: int | None = None, end: int | None = None):
+            if torch is not None and isinstance(value, torch.Tensor):
+                selected = value[expert_idx] if start is None else value[expert_idx, start:end, :]
+                return selected.contiguous()
+            array = np.asarray(value)
+            selected = array[expert_idx] if start is None else array[expert_idx, start:end, :]
+            return selected.copy()
+
+        def _source_names(weight_name: str, expert_idx: int) -> list[str]:
+            return [
+                f"model.layers.{layer_index}.feed_forward.{weight_name}.{expert_idx}",
+                f"layers.{layer_index}.feed_forward.{weight_name}.{expert_idx}",
+                f"backbone.layers.{layer_index}.feed_forward.{weight_name}.{expert_idx}",
+            ]
+
+        if kind == "gate_up_proj":
+            if int(shape[1]) % 2 != 0:
+                raise ValueError(f"LFM2-MoE gate_up_proj second dim must be even, got {shape}")
+            intermediate = int(shape[1]) // 2
+            for expert_idx in range(num_experts):
+                emissions.append(TensorEmission(
+                    f"layer_{layer_index}_moe_expert_{expert_idx}_w1.weights",
+                    _piece(tensor, expert_idx, 0, intermediate),
+                    "lfm2_moe_gate_up_split_gate",
+                    source_names=_source_names("w1_weights", expert_idx),
+                    qdq_restore="runtime_key",
+                ))
+                emissions.append(TensorEmission(
+                    f"layer_{layer_index}_moe_expert_{expert_idx}_w3.weights",
+                    _piece(tensor, expert_idx, intermediate, int(shape[1])),
+                    "lfm2_moe_gate_up_split_up",
+                    source_names=_source_names("w3_weights", expert_idx),
+                    qdq_restore="runtime_key",
+                ))
+            return emissions
+
+        for expert_idx in range(num_experts):
+            emissions.append(TensorEmission(
+                f"layer_{layer_index}_moe_expert_{expert_idx}_w2.weights",
+                _piece(tensor, expert_idx),
+                "lfm2_moe_down_proj_split",
+                source_names=_source_names("w2_weights", expert_idx),
+                qdq_restore="runtime_key",
+            ))
+        return emissions
+
 
 class Gemma3Adapter(FamilyAdapter):
     family = "gemma3"
diff --git a/python/cactus/convert/tests/test_gemma4_adapter.py b/python/cactus/convert/tests/test_gemma4_adapter.py
new file mode 100644
index 000000000..4fbfefaa3
--- /dev/null
+++ b/python/cactus/convert/tests/test_gemma4_adapter.py
@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import torch
+
+from cactus.convert.model_adapters.adapters import adapter_for_family
+
+
+def test_gemma4_moe_down_proj_folds_per_expert_scale_before_split():
+    adapter = adapter_for_family("gemma4")
+    down = torch.arange(2 * 4 * 3, dtype=torch.float16).reshape(2, 4, 3)
+    scale = torch.tensor([2.0, 0.5], dtype=torch.float16)
+
+    normalized = adapter.normalize_state_dict({
+        "model.language_model.layers.5.moe.down_proj": down,
+        "model.language_model.layers.5.moe.per_expert_scale": scale,
+    })
+    scaled = normalized.state_dict["model.language_model.layers.5.moe.down_proj"]
+    match = adapter.name_tensor(
+        "model.language_model.layers.5.moe.down_proj",
+        scaled,
+        num_layers=30,
+    )
+    emissions = adapter.expand_tensor(match, scaled)
+
+    assert torch.equal(emissions[0].tensor, down[0] * scale[0])
+    assert torch.equal(emissions[1].tensor, down[1] * scale[1])
+    assert [emission.output_name for emission in emissions] == [
+        "layer_5_moe_expert_0_w2.weights",
+        "layer_5_moe_expert_1_w2.weights",
+    ]
+    assert "model.language_model.layers.5.moe.w2_weights.0" in emissions[0].source_names
+    assert normalized.provenance["model.language_model.layers.5.moe.down_proj"].source_names == [
+        "model.language_model.layers.5.moe.down_proj",
+        "model.language_model.layers.5.moe.per_expert_scale",
+    ]
+
+
+def test_gemma4_a4b_experts_down_proj_folds_router_per_expert_scale():
+    adapter = adapter_for_family("gemma4")
+    down = torch.arange(2 * 4 * 3, dtype=torch.float16).reshape(2, 4, 3)
+    scale = torch.tensor([2.0, 0.5], dtype=torch.float16)
+
+    normalized = adapter.normalize_state_dict({
+        "model.language_model.layers.5.experts.down_proj": down,
+        "model.language_model.layers.5.router.per_expert_scale": scale,
+    })
+    scaled = normalized.state_dict["model.language_model.layers.5.experts.down_proj"]
+    match = adapter.name_tensor(
+        "model.language_model.layers.5.experts.down_proj",
+        scaled,
+        num_layers=30,
+    )
+    emissions = adapter.expand_tensor(match, scaled)
+
+    assert torch.equal(emissions[0].tensor, down[0] * scale[0])
+    assert torch.equal(emissions[1].tensor, down[1] * scale[1])
+    assert [emission.output_name for emission in emissions] == [
+        "layer_5_moe_expert_0_w2.weights",
+        "layer_5_moe_expert_1_w2.weights",
+    ]
+    assert "model.language_model.layers.5.moe.w2_weights.0" in emissions[0].source_names
+
+
+def test_gemma4_moe_gate_up_splits_into_runtime_experts():
+    adapter = adapter_for_family("gemma4")
+    tensor = torch.arange(2 * 6 * 4, dtype=torch.float16).reshape(2, 6, 4)
+
+    match = adapter.name_tensor(
+        "model.language_model.layers.5.moe.gate_up_proj",
+        tensor,
+        num_layers=30,
+    )
+    emissions = adapter.expand_tensor(match, tensor)
+
+    assert [emission.output_name for emission in emissions] == [
+        "layer_5_moe_expert_0_w1.weights",
+        "layer_5_moe_expert_0_w3.weights",
+        "layer_5_moe_expert_1_w1.weights",
+        "layer_5_moe_expert_1_w3.weights",
+    ]
+    assert torch.equal(emissions[0].tensor, tensor[0, :3, :])
+    assert torch.equal(emissions[1].tensor, tensor[0, 3:, :])
+    assert "model.language_model.layers.5.moe.w1_weights.0" in emissions[0].source_names
+    assert "model.language_model.layers.5.moe.w3_weights.1" in emissions[3].source_names
+
+
+def test_gemma4_a4b_experts_gate_up_splits_into_runtime_experts():
+    adapter = adapter_for_family("gemma4")
+    tensor = torch.arange(2 * 6 * 4, dtype=torch.float16).reshape(2, 6, 4)
+
+    match = adapter.name_tensor(
+        "model.language_model.layers.5.experts.gate_up_proj",
+        tensor,
+        num_layers=30,
+    )
+    emissions = adapter.expand_tensor(match, tensor)
+
+    assert [emission.output_name for emission in emissions] == [
+        "layer_5_moe_expert_0_w1.weights",
+        "layer_5_moe_expert_0_w3.weights",
+        "layer_5_moe_expert_1_w1.weights",
+        "layer_5_moe_expert_1_w3.weights",
+    ]
+    assert torch.equal(emissions[0].tensor, tensor[0, :3, :])
+    assert torch.equal(emissions[3].tensor, tensor[1, 3:, :])
+
+
+def test_gemma4_moe_router_policy_stays_fp16():
+    adapter = adapter_for_family("gemma4")
+    tensor = torch.randn(128, 2816, dtype=torch.float16)
+    match = adapter.name_tensor(
+        "model.language_model.layers.5.router.proj.weight",
+        tensor,
+        num_layers=30,
+    )
+    policy = adapter.policy(match, tuple(tensor.shape), requested_bits=4)
+
+    assert match.output_name == "layer_5_router_proj.weights"
+    assert policy.action == "fallback"
+    assert policy.precision == "FP16"
diff --git a/python/cactus/convert/tests/test_lfm2_adapter.py b/python/cactus/convert/tests/test_lfm2_adapter.py
index 49d5798c9..1a11f66ff 100644
--- a/python/cactus/convert/tests/test_lfm2_adapter.py
+++ b/python/cactus/convert/tests/test_lfm2_adapter.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import torch
+
 from cactus.convert.model_adapters.adapters import adapter_for_family
 
 
@@ -59,3 +61,46 @@ def test_lfm2_processor_fallback_handles_tokenizers_backend(tmp_path):
     assert isinstance(processor, Lfm2VlProcessor)
     assert processor.image_token == "<image>"
     assert processor.image_token_id == 3
+
+
+def test_lfm2_moe_packed_gate_up_splits_into_runtime_experts():
+    adapter = adapter_for_family("lfm2")
+    tensor = torch.arange(2 * 6 * 4, dtype=torch.float16).reshape(2, 6, 4)
+
+    match = adapter.name_tensor(
+        "model.layers.5.feed_forward.experts.gate_up_proj",
+        tensor,
+        num_layers=24,
+    )
+    emissions = adapter.expand_tensor(match, tensor)
+
+    assert [emission.output_name for emission in emissions] == [
+        "layer_5_moe_expert_0_w1.weights",
+        "layer_5_moe_expert_0_w3.weights",
+        "layer_5_moe_expert_1_w1.weights",
+        "layer_5_moe_expert_1_w3.weights",
+    ]
+    assert torch.equal(emissions[0].tensor, tensor[0, :3, :])
+    assert torch.equal(emissions[1].tensor, tensor[0, 3:, :])
+    assert "model.layers.5.feed_forward.w1_weights.0" in emissions[0].source_names
+    assert "model.layers.5.feed_forward.w3_weights.1" in emissions[3].source_names
+
+
+def test_lfm2_moe_packed_down_splits_into_runtime_experts():
+    adapter = adapter_for_family("lfm2")
+    tensor = torch.arange(2 * 4 * 3, dtype=torch.float16).reshape(2, 4, 3)
+
+    match = adapter.name_tensor(
+        "model.layers.5.feed_forward.experts.down_proj",
+        tensor,
+        num_layers=24,
+    )
+    emissions = adapter.expand_tensor(match, tensor)
+
+    assert [emission.output_name for emission in emissions] == [
+        "layer_5_moe_expert_0_w2.weights",
+        "layer_5_moe_expert_1_w2.weights",
+    ]
+    assert torch.equal(emissions[0].tensor, tensor[0])
+    assert torch.equal(emissions[1].tensor, tensor[1])
+    assert "model.layers.5.feed_forward.w2_weights.0" in emissions[0].source_names
diff --git a/python/cactus/convert/tests/test_qwen_adapter.py b/python/cactus/convert/tests/test_qwen_adapter.py
new file mode 100644
index 000000000..e3e3e48f9
--- /dev/null
+++ b/python/cactus/convert/tests/test_qwen_adapter.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import torch
+
+from cactus.convert.model_adapters.adapters import adapter_for_family
+
+
+def test_qwen2_moe_packed_gate_up_splits_into_runtime_experts():
+    adapter = adapter_for_family("qwen")
+    tensor = torch.arange(2 * 6 * 4, dtype=torch.float16).reshape(2, 6, 4)
+
+    match = adapter.name_tensor(
+        "model.layers.5.mlp.experts.gate_up_proj",
+        tensor,
+        num_layers=24,
+    )
+    emissions = adapter.expand_tensor(match, tensor)
+
+    assert [emission.output_name for emission in emissions] == [
+        "layer_5_moe_expert_0_w1.weights",
+        "layer_5_moe_expert_0_w3.weights",
+        "layer_5_moe_expert_1_w1.weights",
+        "layer_5_moe_expert_1_w3.weights",
+    ]
+    assert torch.equal(emissions[0].tensor, tensor[0, :3, :])
+    assert torch.equal(emissions[1].tensor, tensor[0, 3:, :])
+    assert "model.layers.5.mlp.w1_weights.0" in emissions[0].source_names
+    assert "model.layers.5.mlp.w3_weights.1" in emissions[3].source_names
+
+
+def test_qwen2_moe_packed_down_splits_into_runtime_experts():
+    adapter = adapter_for_family("qwen")
+    tensor = torch.arange(2 * 4 * 3, dtype=torch.float16).reshape(2, 4, 3)
+
+    match = adapter.name_tensor(
+        "model.layers.5.mlp.experts.down_proj",
+        tensor,
+        num_layers=24,
+    )
+    emissions = adapter.expand_tensor(match, tensor)
+
+    assert [emission.output_name for emission in emissions] == [
+        "layer_5_moe_expert_0_w2.weights",
+        "layer_5_moe_expert_1_w2.weights",
+    ]
+    assert torch.equal(emissions[0].tensor, tensor[0])
+    assert torch.equal(emissions[1].tensor, tensor[1])
+    assert "model.layers.5.mlp.w2_weights.0" in emissions[0].source_names
+
+
+def test_qwen2_moe_router_uses_runtime_router_name_and_fp16_policy():
+    adapter = adapter_for_family("qwen")
+    tensor = torch.randn(60, 2048, dtype=torch.float16)
+
+    match = adapter.name_tensor(
+        "model.layers.5.mlp.gate.weight",
+        tensor,
+        num_layers=24,
+    )
+    policy = adapter.policy(match, tuple(tensor.shape), requested_bits=4)
+
+    assert match.output_name == "layer_5_moe_router.weights"
+    assert policy.action == "fallback"
+    assert policy.precision == "FP16"
diff --git a/python/cactus/transpile/aten_ops.py b/python/cactus/transpile/aten_ops.py
index 90cb587ec..9096b8515 100644
--- a/python/cactus/transpile/aten_ops.py
+++ b/python/cactus/transpile/aten_ops.py
@@ -44,9 +44,16 @@ def canonical_torch_op(target: Any) -> str:
     "aten.view.default": "reshape",
     "aten.reshape.default": "reshape",
     "aten._unsafe_view.default": "reshape",
+    "aten.new_empty.default": "new_empty",
     "aten.flatten.using_ints": "flatten",
     "aten.t.default": "transpose",
     "aten.gelu.erf": "gelu_erf",
+    "cactus_transpile.lfm2_moe_layer_gated.default": "lfm2_moe_layer_gated",
+    "lfm2_moe_layer_gated.default": "lfm2_moe_layer_gated",
+    "cactus_transpile.qwen2_moe_layer_gated.default": "qwen2_moe_layer_gated",
+    "qwen2_moe_layer_gated.default": "qwen2_moe_layer_gated",
+    "cactus_transpile.gemma4_moe_layer_gated.default": "gemma4_moe_layer_gated",
+    "gemma4_moe_layer_gated.default": "gemma4_moe_layer_gated",
     # Keep the legacy importer key until the importer itself is renamed.
     "aten.diff.default": "aten.diff.default",
 }
diff --git a/python/cactus/transpile/canonicalize/cleanup.py b/python/cactus/transpile/canonicalize/cleanup.py
index b7f4de2aa..c68e00447 100644
--- a/python/cactus/transpile/canonicalize/cleanup.py
+++ b/python/cactus/transpile/canonicalize/cleanup.py
@@ -129,6 +129,9 @@
     "attention_block",
     "conv_module",
     "dense_mlp_tq_fused",
+    "lfm2_moe_layer_gated",
+    "qwen2_moe_layer_gated",
+    "gemma4_moe_layer_gated",
     "rope",
 }
 
diff --git a/python/cactus/transpile/component_plan.py b/python/cactus/transpile/component_plan.py
index c9e63cabd..5c8fdd751 100644
--- a/python/cactus/transpile/component_plan.py
+++ b/python/cactus/transpile/component_plan.py
@@ -26,6 +26,8 @@ def _plan_from_profile(
 ) -> ComponentPlan | None:
     if profile.default_task is None or not profile.default_components:
         return None
+    has_vision: bool | None = None
+    has_audio: bool | None = None
     if config is not None and profile.needs_image:
         has_vision = _has_dict_config(config, "vision_config", "visual_config", "image_config")
         if not has_vision:
@@ -42,12 +44,19 @@ def _plan_from_profile(
                 components=("decoder", "decoder_step"),
                 force_component_pipeline=True,
             )
+    if config is not None and profile.needs_audio:
+        has_audio = _has_dict_config(config, "audio_config", "speech_config", "acoustic_config")
+    components = profile.default_components
+    needs_audio = profile.needs_audio
+    if profile.needs_image and profile.needs_audio and has_vision and has_audio is False:
+        components = tuple(component for component in components if component != "audio_encoder")
+        needs_audio = False
     return ComponentPlan(
         task=profile.default_task,
-        components=profile.default_components,
+        components=components,
         default_max_new_tokens=profile.default_max_new_tokens,
         needs_image=profile.needs_image,
-        needs_audio=profile.needs_audio,
+        needs_audio=needs_audio,
         force_component_pipeline=profile.force_component_pipeline,
     )
 
diff --git a/python/cactus/transpile/hf_model.py b/python/cactus/transpile/hf_model.py
index a4b3461aa..14e444ca6 100644
--- a/python/cactus/transpile/hf_model.py
+++ b/python/cactus/transpile/hf_model.py
@@ -161,7 +161,7 @@ def _serialize_json_compatible(value: Any) -> Any:
             "dtype": str(value.dtype),
             "shape": list(value.shape),
         }
-        if value.numel() == 1:
+        if value.numel() == 1 and not value.is_meta:
             payload["data"] = value.detach().cpu().reshape(-1)[0].item()
         return payload
     if isinstance(value, np.ndarray):
@@ -1011,6 +1011,105 @@ def _parse_dtype(name: str) -> torch.dtype:
     return mapping[normalized]
 
 
+def _to_meta_tensor(value: torch.Tensor) -> torch.Tensor:
+    if value.is_meta:
+        return value
+    return torch.empty_strided(
+        tuple(value.shape),
+        tuple(value.stride()),
+        dtype=value.dtype,
+        device="meta",
+        requires_grad=value.requires_grad,
+    )
+
+
+def _prepared_inputs_for_meta_capture(prepared: PreparedInputs) -> PreparedInputs:
+    return PreparedInputs(
+        names=prepared.names,
+        tensors=tuple(_to_meta_tensor(tensor) for tensor in prepared.tensors),
+        metadata=prepared.metadata,
+    )
+
+
+def _materialize_meta_rope_buffers(model: torch.nn.Module, *, dtype: torch.dtype) -> None:
+    def _compute_rope(
+        compute_default,
+        config: object,
+        *,
+        layer_type: str | None = None,
+        rope_init_fn=None,
+        rope_type: str | None = None,
+    ) -> torch.Tensor:
+        fn = rope_init_fn or compute_default
+        kwargs: dict[str, object] = {"device": torch.device("cpu")}
+        if layer_type is not None:
+            kwargs["layer_type"] = layer_type
+            if layer_type == "full_attention" and rope_type == "proportional":
+                kwargs["head_dim_key"] = "global_head_dim"
+        try:
+            materialized, _ = fn(config, **kwargs)
+        except TypeError:
+            materialized, _ = fn(config=config, **kwargs)
+        return materialized.to(dtype=dtype)
+
+    for module in model.modules():
+        compute_default = getattr(module, "compute_default_rope_parameters", None)
+        config = getattr(module, "config", None)
+        if not callable(compute_default) or config is None:
+            continue
+        buffers = getattr(module, "_buffers", None)
+        if not isinstance(buffers, dict):
+            continue
+        inv_freq = buffers.get("inv_freq")
+        if isinstance(inv_freq, torch.Tensor) and inv_freq.is_meta:
+            materialized = _compute_rope(compute_default, config)
+            buffers["inv_freq"] = materialized
+            original = buffers.get("original_inv_freq")
+            if isinstance(original, torch.Tensor) and original.is_meta:
+                buffers["original_inv_freq"] = materialized.clone()
+
+        rope_init_fns = getattr(module, "rope_init_fns", None)
+        rope_types = getattr(module, "rope_type", None)
+        layer_types = getattr(module, "layer_types", ())
+        if not isinstance(layer_types, (set, tuple, list)):
+            continue
+        for layer_type in layer_types:
+            if not isinstance(layer_type, str):
+                continue
+            key = f"{layer_type}_inv_freq"
+            inv_freq = buffers.get(key)
+            if not isinstance(inv_freq, torch.Tensor) or not inv_freq.is_meta:
+                continue
+            rope_init_fn = rope_init_fns.get(layer_type) if isinstance(rope_init_fns, dict) else None
+            rope_type = rope_types.get(layer_type) if isinstance(rope_types, dict) else None
+            materialized = _compute_rope(
+                compute_default,
+                config,
+                layer_type=layer_type,
+                rope_init_fn=rope_init_fn,
+                rope_type=rope_type if isinstance(rope_type, str) else None,
+            )
+            buffers[key] = materialized
+            original = buffers.get(f"{layer_type}_original_inv_freq")
+            if isinstance(original, torch.Tensor) and original.is_meta:
+                buffers[f"{layer_type}_original_inv_freq"] = materialized.clone()
+
+
+def _materialize_meta_scalar_buffers(model: torch.nn.Module, *, dtype: torch.dtype) -> None:
+    for module in model.modules():
+        buffers = getattr(module, "_buffers", None)
+        if not isinstance(buffers, dict):
+            continue
+        embed_scale = buffers.get("embed_scale")
+        scalar_embed_scale = getattr(module, "scalar_embed_scale", None)
+        if isinstance(embed_scale, torch.Tensor) and embed_scale.is_meta and scalar_embed_scale is not None:
+            buffers["embed_scale"] = torch.tensor(
+                float(scalar_embed_scale),
+                dtype=embed_scale.dtype if embed_scale.dtype.is_floating_point else dtype,
+                device="cpu",
+            )
+
+
 class TranspileWrapper(torch.nn.Module):
     def __init__(self, adapter_module: torch.nn.Module, *, weights_dir: str | None = None):
         super().__init__()
@@ -1048,6 +1147,8 @@ def _tie_lfm2_vl_lm_head_if_needed(model: torch.nn.Module) -> str | None:
         return None
     if tuple(lm_head.weight.shape) != tuple(embed_tokens.weight.shape):
         return None
+    if lm_head.weight.is_meta or embed_tokens.weight.is_meta:
+        return None
     if lm_head.weight.data_ptr() == embed_tokens.weight.data_ptr():
         return None
     lm_head.weight = embed_tokens.weight
@@ -2268,10 +2369,12 @@ def _prepare_qwen3_5_multimodal_inputs(
     )
 
 
-def _load_model_source(model_id: str, *, local_files_only: bool) -> str:
+def _load_model_source(model_id: str, *, local_files_only: bool, require_weights: bool = True) -> str:
     local_snapshot = _resolve_local_snapshot(model_id)
     if local_snapshot and _snapshot_has_model_weights(local_snapshot):
         return local_snapshot
+    if local_snapshot and not require_weights:
+        return local_snapshot
     if local_snapshot and local_files_only:
         raise RuntimeError(
             f"Found local snapshot for {model_id}, but it is incomplete and has no model weights:\n"
@@ -2290,6 +2393,7 @@ def _load_transformers_bundle(
     token: str | None,
     trust_remote_code: bool,
     local_files_only: bool,
+    low_memory_load: bool,
 ):
     config = _load_config_json(model_id)
     config_model_type = str(config.get("model_type", "") or "").lower()
@@ -2313,6 +2417,7 @@ def _load_transformers_bundle(
 
     try:
         from transformers import AutoFeatureExtractor  # type: ignore
+        from transformers import AutoConfig  # type: ignore
         from transformers import AutoModel  # type: ignore
         from transformers import AutoModelForCTC  # type: ignore
         from transformers import AutoModelForCausalLM  # type: ignore
@@ -2324,7 +2429,11 @@ def _load_transformers_bundle(
     except Exception as exc:  # pragma: no cover
         raise RuntimeError(f"transformers is not available: {exc}") from exc
 
-    model_source = _load_model_source(model_id, local_files_only=local_files_only)
+    model_source = _load_model_source(
+        model_id,
+        local_files_only=local_files_only,
+        require_weights=not low_memory_load,
+    )
     source_candidates = []
     for candidate in (model_source, model_id):
         if candidate not in source_candidates:
@@ -2336,6 +2445,37 @@ def _load_transformers_bundle(
     if token:
         common_kwargs["token"] = token
 
+    hf_config = None
+    if low_memory_load:
+        if task == "tdt_transcription":
+            raise RuntimeError("--low-memory-load is not supported for TDT transcription models")
+        try:
+            hf_config = AutoConfig.from_pretrained(model_source, **common_kwargs)
+        except Exception as exc:
+            raise RuntimeError(
+                f"--low-memory-load requires loading the HF config without checkpoint weights: {exc}"
+            ) from exc
+
+    def _from_config(loader, *, trust_remote_code_override: bool | None = None):
+        if hf_config is None:
+            raise RuntimeError("internal error: low-memory config was not loaded")
+        with torch.device("meta"):
+            try:
+                model = loader.from_config(
+                    hf_config,
+                    trust_remote_code=(
+                        trust_remote_code
+                        if trust_remote_code_override is None
+                        else bool(trust_remote_code_override)
+                    ),
+                )
+            except TypeError:
+                model = loader.from_config(hf_config)
+        model.to(dtype=torch_dtype)
+        _materialize_meta_rope_buffers(model, dtype=torch_dtype)
+        _materialize_meta_scalar_buffers(model, dtype=torch_dtype)
+        return model.eval()
+
     if task == "tdt_transcription":
         model = load_tdt_local_model(model_source, torch_dtype=torch_dtype).eval()
         return model_source, None, model, config
@@ -2358,13 +2498,16 @@ def _load_transformers_bundle(
                 f"Could not load tokenizer for {model_id}:\n"
                 + "\n".join(tokenizer_errors)
             )
-        model = AutoModel.from_pretrained(
-            model_source,
-            dtype=torch_dtype,
-            device_map=None,
-            low_cpu_mem_usage=True,
-            **remote_kwargs,
-        ).eval()
+        if low_memory_load:
+            model = _from_config(AutoModel, trust_remote_code_override=True)
+        else:
+            model = AutoModel.from_pretrained(
+                model_source,
+                dtype=torch_dtype,
+                device_map=None,
+                low_cpu_mem_usage=True,
+                **remote_kwargs,
+            ).eval()
         return model_source, tokenizer, model, config
 
     if task == "causal_lm_logits":
@@ -2382,32 +2525,41 @@ def _load_transformers_bundle(
                 + "\n".join(tokenizer_errors)
             )
         if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}:
-            model = AutoModelForImageTextToText.from_pretrained(
-                model_source,
-                dtype=torch_dtype,
-                device_map=None,
-                low_cpu_mem_usage=True,
-                **common_kwargs,
-            ).eval()
+            if low_memory_load:
+                model = _from_config(AutoModelForImageTextToText)
+            else:
+                model = AutoModelForImageTextToText.from_pretrained(
+                    model_source,
+                    dtype=torch_dtype,
+                    device_map=None,
+                    low_cpu_mem_usage=True,
+                    **common_kwargs,
+                ).eval()
             tie_note = _tie_lfm2_vl_lm_head_if_needed(model)
             if tie_note:
                 print(f"note={tie_note}")
         elif config_model_type in {"qwen3_5", "qwen3_vl"} and isinstance(config.get("text_config"), dict):
-            model = AutoModelForImageTextToText.from_pretrained(
-                model_source,
-                dtype=torch_dtype,
-                device_map=None,
-                low_cpu_mem_usage=True,
-                **common_kwargs,
-            ).eval()
+            if low_memory_load:
+                model = _from_config(AutoModelForImageTextToText)
+            else:
+                model = AutoModelForImageTextToText.from_pretrained(
+                    model_source,
+                    dtype=torch_dtype,
+                    device_map=None,
+                    low_cpu_mem_usage=True,
+                    **common_kwargs,
+                ).eval()
         else:
-            model = AutoModelForCausalLM.from_pretrained(
-                model_source,
-                dtype=torch_dtype,
-                device_map=None,
-                low_cpu_mem_usage=True,
-                **common_kwargs,
-            ).eval()
+            if low_memory_load:
+                model = _from_config(AutoModelForCausalLM)
+            else:
+                model = AutoModelForCausalLM.from_pretrained(
+                    model_source,
+                    dtype=torch_dtype,
+                    device_map=None,
+                    low_cpu_mem_usage=True,
+                    **common_kwargs,
+                ).eval()
         return model_source, tokenizer, model, config
     if task == "multimodal_causal_lm_logits":
         processor = None
@@ -2442,25 +2594,31 @@ def _load_transformers_bundle(
             )
 
         if config_model_type in {"lfm2_vl", "qwen3_5", "qwen3_vl"}:
-            model = AutoModelForImageTextToText.from_pretrained(
-                model_source,
-                dtype=torch_dtype,
-                device_map=None,
-                low_cpu_mem_usage=True,
-                **common_kwargs,
-            ).eval()
+            if low_memory_load:
+                model = _from_config(AutoModelForImageTextToText)
+            else:
+                model = AutoModelForImageTextToText.from_pretrained(
+                    model_source,
+                    dtype=torch_dtype,
+                    device_map=None,
+                    low_cpu_mem_usage=True,
+                    **common_kwargs,
+                ).eval()
             tie_note = _tie_lfm2_vl_lm_head_if_needed(model)
             if tie_note:
                 print(f"note={tie_note}")
         else:
-            model = AutoModelForCausalLM.from_pretrained(
-                model_source,
-                dtype=torch_dtype,
-                device_map=None,
-                low_cpu_mem_usage=True,
-                **common_kwargs,
-            ).eval()
-        if config_model_type == "gemma4":
+            if low_memory_load:
+                model = _from_config(AutoModelForCausalLM)
+            else:
+                model = AutoModelForCausalLM.from_pretrained(
+                    model_source,
+                    dtype=torch_dtype,
+                    device_map=None,
+                    low_cpu_mem_usage=True,
+                    **common_kwargs,
+                ).eval()
+        if config_model_type == "gemma4" and not low_memory_load:
             repair_result = _repair_gemma4_checkpoint_weights(model, model_source)
             if repair_result.get("applied"):
                 missing = repair_result.get("missing_keys", [])
@@ -2504,13 +2662,16 @@ def _load_transformers_bundle(
     load_errors: list[str] = []
     for loader in model_loaders:
         try:
-            model = loader.from_pretrained(
-                model_source,
-                dtype=torch_dtype,
-                device_map=None,
-                low_cpu_mem_usage=True,
-                **common_kwargs,
-            ).eval()
+            if low_memory_load:
+                model = _from_config(loader)
+            else:
+                model = loader.from_pretrained(
+                    model_source,
+                    dtype=torch_dtype,
+                    device_map=None,
+                    low_cpu_mem_usage=True,
+                    **common_kwargs,
+                ).eval()
             return model_source, processor, model, config
         except Exception as exc:
             load_errors.append(f"{loader.__name__}: {exc}")
@@ -2747,6 +2908,15 @@ def main() -> int:
         action="store_true",
         help="Require the model/processor to already exist locally.",
     )
+    parser.add_argument(
+        "--low-memory-load",
+        action="store_true",
+        help=(
+            "Instantiate the HF model on the meta device for graph capture instead of "
+            "loading checkpoint tensors into RAM. Requires converted CQ weights and "
+            "only supports bundle generation, not execution or NPU emission."
+        ),
+    )
     parser.add_argument(
         "--weights-dir",
         default="",
@@ -2852,6 +3022,15 @@ def main() -> int:
             "\n"
             "For compiler-only debugging, pass --allow-unconverted-weights."
         )
+    if args.low_memory_load:
+        if validated_weights_dir is None:
+            raise RuntimeError("--low-memory-load requires --weights-dir with converted Cactus CQ weights")
+        if args.npu:
+            raise RuntimeError("--low-memory-load cannot emit NPU .mlpackage files because CoreML export needs real weights")
+        if not args.skip_execute:
+            print("note=low_memory_load_forces_skip_execute=true")
+            args.skip_execute = True
+        args.skip_reference_compare = True
 
     image_files = tuple(str(path) for path in args.image_file if str(path).strip())
     if args.task == "auto":
@@ -2879,6 +3058,7 @@ def main() -> int:
         token=args.token,
         trust_remote_code=args.trust_remote_code,
         local_files_only=args.local_files_only,
+        low_memory_load=bool(args.low_memory_load),
     )
     preprocessor_config = _load_optional_json(model_source, "preprocessor_config.json")
     if not preprocessor_config:
@@ -2966,7 +3146,7 @@ def main() -> int:
             weights_dir=weights_dir,
         )
         prime_static_features = getattr(canonical.module, "prime_static_multimodal_features", None)
-        if callable(prime_static_features):
+        if callable(prime_static_features) and not args.low_memory_load:
             prime_static_features(*prepared.tensors)
     elif task == "tdt_transcription":
         if not args.audio_file:
@@ -3021,6 +3201,13 @@ def main() -> int:
             task=task,
             input_names=prepared.names,
         )
+    defer_low_memory_meta_inputs = (
+        bool(args.low_memory_load)
+        and task == "multimodal_causal_lm_logits"
+        and config_model_type == "gemma4"
+    )
+    if args.low_memory_load and not defer_low_memory_meta_inputs:
+        prepared = _prepared_inputs_for_meta_capture(prepared)
     requested_components = None
     if args.components:
         requested_components = tuple(
@@ -3095,6 +3282,8 @@ def main() -> int:
 
     if canonical is None:
         raise RuntimeError(f"task={task} requires a component pipeline but none was selected")
+    if args.low_memory_load and defer_low_memory_meta_inputs:
+        prepared = _prepared_inputs_for_meta_capture(prepared)
     wrapper = TranspileWrapper(canonical.module, weights_dir=weights_dir).eval()
 
     print(f"model_id={args.model_id}")
diff --git a/python/cactus/transpile/importers.py b/python/cactus/transpile/importers.py
index 495b1dfbd..9666f7c03 100644
--- a/python/cactus/transpile/importers.py
+++ b/python/cactus/transpile/importers.py
@@ -112,6 +112,13 @@ def extract_literals(value: Any) -> Any:
     return None
 
 
+def require_literal_bool(arg: Any, ctx: "ImportContext", name: str) -> bool:
+    literal = extract_literals(arg)
+    if not isinstance(literal, (bool, int)):
+        ctx.fail(f"{name} must be a boolean literal, got {literal!r}")
+    return bool(literal)
+
+
 def make_prefixed_node(node: Any, prefix: str) -> Any:
     return SimpleNamespace(
         name=f"{prefix}{node.name}",
@@ -365,7 +372,10 @@ def import_get_attr(
     source_name: str | None = None,
 ) -> None:
     add_value_if_missing(ir, IRValue(id=value_id(node, ctx), shape=shape, dtype=dtype, producer=None))
-    ir.constants[value_id(node, ctx)] = value.detach().cpu() if isinstance(value, torch.Tensor) else value
+    if isinstance(value, torch.Tensor):
+        ir.constants[value_id(node, ctx)] = value if value.is_meta else value.detach().cpu()
+    else:
+        ir.constants[value_id(node, ctx)] = value
     ir.values[value_id(node, ctx)].meta["source_name"] = source_name
     if source_name is not None:
         bindings = ir.meta.setdefault("weight_bindings", {})
@@ -1654,6 +1664,32 @@ def import_ones(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int,
     register_node(ir, ir_node, shape=shape or output_shape, dtype=node_dtype)
 
 
+def import_new_empty(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None:
+    output_shape = _try_materialize_shape(shape)
+    if output_shape is None and len(node.args) > 1:
+        output_shape = _extract_int_sequence_literal(node.args[1])
+    if output_shape is None:
+        ctx.fail(f"unsupported new_empty shape for {torch_op}: {node.args!r}")
+    elif math.prod(output_shape) != 0:
+        ctx.fail(f"new_empty requires an empty (zero-numel) shape, got {output_shape} for {torch_op}")
+
+    node_dtype = dtype
+    if "dtype" in getattr(node, "kwargs", {}) and extract_literals(node.kwargs["dtype"]) is not None:
+        node_dtype = dtype_to_ir(extract_literals(node.kwargs["dtype"]))
+    if node_dtype is None:
+        node_dtype = "fp32"
+
+    ir_node = IRNode(
+        id=node_id(node),
+        op="new_empty",
+        inputs=[],
+        outputs=[value_id(node, ctx)],
+        attrs={"shape": output_shape, "dtype": node_dtype},
+        meta=_base_meta(shape or output_shape, node_dtype, torch_op, node),
+    )
+    register_node(ir, ir_node, shape=shape or output_shape, dtype=node_dtype)
+
+
 def import_layer_norm(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[int, ...] | None, dtype: str | None, torch_op: str) -> None:
     if not node.args:
         ctx.fail(f"unsupported layer_norm signature for {torch_op}: {node.args!r}")
@@ -1773,6 +1809,57 @@ def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i
     register_node(ir, ir_node, shape=shape, dtype=dtype)
 
 
+def _import_moe_layer_gated(ir, node, ctx, *, shape, dtype, torch_op, op, has_expert_bias, activation):
+    num_leading = 3 if has_expert_bias else 2
+    num_args = num_leading + 8 + (1 if has_expert_bias else 0)
+    if len(node.args) != num_args:
+        ctx.fail(f"{op} expected {num_args} args, got {len(node.args)}")
+    w1_inputs = extract_input_ids(node.args[num_leading], ctx)
+    w3_inputs = extract_input_ids(node.args[num_leading + 1], ctx)
+    w2_inputs = extract_input_ids(node.args[num_leading + 2], ctx)
+    num_experts = int(extract_literals(node.args[num_leading + 3]))
+    num_experts_per_tok = int(extract_literals(node.args[num_leading + 4]))
+    if len(w1_inputs) != num_experts or len(w3_inputs) != num_experts or len(w2_inputs) != num_experts:
+        ctx.fail(
+            f"{op} expert input count mismatch: "
+            f"num_experts={num_experts} w1={len(w1_inputs)} w3={len(w3_inputs)} w2={len(w2_inputs)}"
+        )
+    flags = num_leading + 5
+    attrs = {"num_experts": num_experts, "num_experts_per_tok": num_experts_per_tok}
+    if has_expert_bias:
+        attrs["use_expert_bias"] = require_literal_bool(node.args[flags], ctx, "use_expert_bias")
+        flags += 1
+    attrs["normalize_routing"] = require_literal_bool(node.args[flags], ctx, "normalize_routing")
+    attrs["epsilon"] = float(extract_literals(node.args[flags + 1]))
+    attrs["routed_scaling_factor"] = float(extract_literals(node.args[flags + 2]))
+    attrs["activation"] = activation
+    leading_inputs = [value_id(node.args[i], ctx) for i in range(num_leading)]
+    ir_node = IRNode(
+        id=node_id(node),
+        op=op,
+        inputs=[*leading_inputs, *w1_inputs, *w3_inputs, *w2_inputs],
+        outputs=[value_id(node, ctx)],
+        attrs=attrs,
+        meta=_base_meta(shape, dtype, torch_op, node),
+    )
+    register_node(ir, ir_node, shape=shape, dtype=dtype)
+
+
+def import_lfm2_moe_layer_gated(ir, node, ctx, *, shape, dtype, torch_op):
+    _import_moe_layer_gated(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op,
+                            op="lfm2_moe_layer_gated", has_expert_bias=True, activation="silu")
+
+
+def import_qwen2_moe_layer_gated(ir, node, ctx, *, shape, dtype, torch_op):
+    _import_moe_layer_gated(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op,
+                            op="qwen2_moe_layer_gated", has_expert_bias=False, activation="silu")
+
+
+def import_gemma4_moe_layer_gated(ir, node, ctx, *, shape, dtype, torch_op):
+    _import_moe_layer_gated(ir, node, ctx, shape=shape, dtype=dtype, torch_op=torch_op,
+                            op="gemma4_moe_layer_gated", has_expert_bias=False, activation="gelu")
+
+
 OP_IMPORTERS = {
     "arange": import_arange,
     "identity": import_identity,
@@ -1851,6 +1938,7 @@ def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i
     "unbind": import_unbind,
     "aten.unbind.int": import_unbind,
     "ones": import_ones,
+    "new_empty": import_new_empty,
     "pad": import_pad,
     "pow": import_pow,
     "aten.diff.default": import_diff,
@@ -1866,4 +1954,7 @@ def import_getitem(ir: IRGraph, node: Any, ctx: ImportContext, *, shape: tuple[i
     "batch_norm": import_batch_norm,
     "contiguous": import_contiguous,
     "getitem": import_getitem,
+    "lfm2_moe_layer_gated": import_lfm2_moe_layer_gated,
+    "qwen2_moe_layer_gated": import_qwen2_moe_layer_gated,
+    "gemma4_moe_layer_gated": import_gemma4_moe_layer_gated,
 }
diff --git a/python/cactus/transpile/lower.py b/python/cactus/transpile/lower.py
index 4d195310f..fab23d6ad 100644
--- a/python/cactus/transpile/lower.py
+++ b/python/cactus/transpile/lower.py
@@ -607,6 +607,22 @@ def _lookup_weight_binding(value: IRValue) -> WeightBinding | None:
     return None
 
 
+GEMMA4_RESIDUAL_DOWNSCALE = 16.0
+
+
+def _activation_enum(name: object, *, default: int = 0) -> int:
+    value = str(name or "").strip().lower()
+    if value in {"gelu", "gelu_tanh", "gelu_pytorch_tanh"}:
+        return 1
+    if value in {"gelu_erf", "gelu_none"}:
+        return 2
+    if value == "relu":
+        return 3
+    if value == "silu":
+        return 0
+    return default
+
+
 def _debug_mmap_binding(value_id: str, binding: WeightBinding) -> None:
     if os.environ.get("CACTUS_TRANSPILER_DEBUG_MMAP") != "1":
         return
@@ -666,6 +682,18 @@ def _matmul_with_quantized_rhs_legalization(
 ) -> Tensor:
     if _is_quantized_runtime_dtype(rhs.dtype) and lhs.dtype == Graph.FP32:
         lhs = g.precision_cast(lhs, Graph.FP16)
+    if (
+        _is_quantized_runtime_dtype(rhs.dtype)
+        and pretransposed_rhs
+        and len(lhs.shape) >= 2
+        and len(rhs.shape) == 2
+        and int(rhs.shape[-1]) > int(lhs.shape[-1])
+    ):
+        pad = int(rhs.shape[-1]) - int(lhs.shape[-1])
+        pad_shape = tuple(int(dim) for dim in lhs.shape[:-1]) + (pad,)
+        pad_dtype = torch.float16 if lhs.dtype == Graph.FP16 else torch.float32
+        zeros = _materialize_constant_tensor(g, torch.zeros(pad_shape, dtype=pad_dtype))
+        lhs = g.cat([lhs, zeros], axis=len(lhs.shape) - 1)
     return g.matmul(lhs, rhs, pretransposed_rhs=pretransposed_rhs, output_dtype=output_dtype)
 
 
@@ -1110,6 +1138,66 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) ->
             out = g.precision_cast(out, output_dtype)
         return [out]
 
+    if op in ("lfm2_moe_layer_gated", "qwen2_moe_layer_gated", "gemma4_moe_layer_gated"):
+        num_experts = int(node.attrs["num_experts"])
+        num_experts_per_tok = int(node.attrs["num_experts_per_tok"])
+        w1_start = 3 if op == "lfm2_moe_layer_gated" else 2
+        expected_inputs = w1_start + 3 * num_experts
+        if len(node.inputs) != expected_inputs:
+            raise NotImplementedError(
+                f"{op} expects {expected_inputs} inputs for {num_experts} experts, got {len(node.inputs)}"
+            )
+        hidden = _ensure_tensor_dtype(g, _tensor(env, node.inputs[0]), Graph.FP16)
+        router_logits = _ensure_tensor_dtype(g, _tensor(env, node.inputs[1]), Graph.FP16)
+
+        if op == "lfm2_moe_layer_gated":
+            routing_probs = g.sigmoid(router_logits)
+            topk_source = routing_probs
+            if bool(node.attrs.get("use_expert_bias", False)):
+                expert_bias = _ensure_tensor_dtype(g, _tensor(env, node.inputs[2]), routing_probs.dtype)
+                topk_source = _lower_binary_op(g, routing_probs, expert_bias, "add")
+            topk_indices = _ensure_tensor_dtype(g, g.index(g.topk(topk_source, num_experts_per_tok), 0, axis=0), Graph.FP32)
+            normalize_default = False
+            activation = None
+        elif op == "qwen2_moe_layer_gated":
+            routing_probs = g.softmax(router_logits, axis=-1)
+            topk_indices = _ensure_tensor_dtype(g, g.index(g.topk(routing_probs, num_experts_per_tok), 0, axis=0), Graph.FP32)
+            normalize_default = False
+            activation = None
+        else:
+            hidden = g.scalar_multiply(hidden, float(GEMMA4_RESIDUAL_DOWNSCALE))
+            router_logits = g.scalar_multiply(router_logits, float(GEMMA4_RESIDUAL_DOWNSCALE))
+            routing_probs = g.softmax(router_logits, axis=-1)
+            topk_indices = _ensure_tensor_dtype(g, g.index(g.topk(router_logits, num_experts_per_tok), 0, axis=0), Graph.FP32)
+            normalize_default = True
+            activation = _activation_enum(node.attrs.get("activation"), default=0)
+
+        w3_start = w1_start + num_experts
+        w2_start = w3_start + num_experts
+        moe_kwargs = dict(
+            normalize_routing=bool(node.attrs.get("normalize_routing", normalize_default)),
+            epsilon=float(node.attrs.get("epsilon", 1.0e-6)),
+            routed_scaling_factor=float(node.attrs.get("routed_scaling_factor", 1.0)),
+        )
+        if activation is not None:
+            moe_kwargs["activation"] = activation
+        out = g.moe_layer_gated(
+            hidden,
+            routing_probs,
+            topk_indices,
+            [_tensor(env, v) for v in node.inputs[w1_start:w3_start]],
+            [_tensor(env, v) for v in node.inputs[w3_start:w2_start]],
+            [_tensor(env, v) for v in node.inputs[w2_start:]],
+            num_experts,
+            num_experts_per_tok,
+            **moe_kwargs,
+        )
+        output_value = ir.values.get(node.outputs[0])
+        output_dtype = _map_ir_dtype(output_value.dtype) if output_value is not None and output_value.dtype is not None else out.dtype
+        if out.dtype != output_dtype:
+            out = g.precision_cast(out, output_dtype)
+        return [out]
+
     if op == "addmm":
         bias = _tensor(env, node.inputs[0])
         lhs = _tensor(env, node.inputs[1])
@@ -1664,6 +1752,13 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) ->
         torch_dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype"))
         return [_materialize_constant_tensor(g, torch.ones(shape, dtype=torch_dtype))]
 
+    if op == "new_empty":
+        shape = tuple(int(v) for v in node.attrs.get("shape", ()))
+        if not shape:
+            raise NotImplementedError(f"new_empty requires a static shape, got {shape}")
+        torch_dtype = _materialize_constant_torch_dtype(node.attrs.get("dtype"))
+        return [_materialize_constant_tensor(g, torch.zeros(shape, dtype=torch_dtype))]
+
     if op == "tril":
         x = _tensor(env, node.inputs[0])
         shape = tuple(int(dim) for dim in x.shape)
@@ -1780,7 +1875,8 @@ def _lower_ir_node(g: Graph, node: IRNode, env: dict[str, Any], ir: IRGraph) ->
         embedding_tensor = _tensor(env, node.inputs[0])
         indices_tensor = _tensor(env, node.inputs[1])
         _debug_embedding_lowering(node, embedding_tensor, indices_tensor)
-        return [g.embedding_from_tensor(embedding_tensor, indices_tensor)]
+        out = g.embedding_from_tensor(embedding_tensor, indices_tensor)
+        return [out]
 
     if op == "conv_module":
         input_index = 0
diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py
index d934e80d0..a81090d82 100644
--- a/python/cactus/transpile/model_adapters.py
+++ b/python/cactus/transpile/model_adapters.py
@@ -269,6 +269,23 @@ def _module_device(module: torch.nn.Module) -> torch.device | None:
     return None
 
 
+def _module_has_meta_tensors(module: torch.nn.Module) -> bool:
+    return any(parameter.is_meta for parameter in module.parameters()) or any(
+        buffer.is_meta for buffer in module.buffers()
+    )
+
+
+def _to_meta_example_tensor(tensor: torch.Tensor | None) -> torch.Tensor | None:
+    if tensor is None or tensor.is_meta:
+        return tensor
+    return torch.empty_strided(
+        tuple(tensor.shape),
+        tuple(tensor.stride()),
+        dtype=tensor.dtype,
+        device="meta",
+    )
+
+
 def _unique_modules(*candidates: object) -> tuple[torch.nn.Module, ...]:
     modules: list[torch.nn.Module] = []
     seen: set[int] = set()
@@ -438,6 +455,656 @@ def _torch_is_compiling() -> bool:
     return False
 
 
+_CACTUS_TORCH_LIBRARIES: list[object] = []
+_LFM2_MOE_CUSTOM_OP_REGISTERED = False
+_LFM2_MOE_CUSTOM_OP = "cactus_transpile::lfm2_moe_layer_gated"
+_QWEN2_MOE_CUSTOM_OP_REGISTERED = False
+_QWEN2_MOE_CUSTOM_OP = "cactus_transpile::qwen2_moe_layer_gated"
+_GEMMA4_MOE_CUSTOM_OP_REGISTERED = False
+_GEMMA4_MOE_CUSTOM_OP = "cactus_transpile::gemma4_moe_layer_gated"
+
+
+def _ignore_duplicate_torch_registration(exc: RuntimeError) -> bool:
+    text = str(exc).lower()
+    return "already" in text or "duplicate" in text or "previously" in text
+
+
+def _moe_normalize_and_scale(routing_weights, normalize_routing, epsilon, routed_scaling_factor):
+    if normalize_routing:
+        routing_weights = routing_weights / (routing_weights.sum(dim=-1, keepdim=True) + float(epsilon))
+    return routing_weights * float(routed_scaling_factor)
+
+
+def _moe_scatter_experts(hidden, selected_experts, routing_weights, w1_weights, w3_weights, w2_weights, num_experts, activation):
+    output = torch.zeros_like(hidden)
+    for expert_idx in range(int(num_experts)):
+        selected = selected_experts == expert_idx
+        if not bool(selected.any()):
+            continue
+        token_idx, topk_pos = torch.where(selected)
+        current = hidden[token_idx]
+        gate = F.linear(current, w1_weights[expert_idx])
+        up = F.linear(current, w3_weights[expert_idx])
+        expert_out = F.linear(activation(gate) * up, w2_weights[expert_idx])
+        expert_out = expert_out * routing_weights[token_idx, topk_pos, None]
+        output.index_add_(0, token_idx, expert_out.to(dtype=output.dtype))
+    return output
+
+
+def _gelu_tanh(x):
+    return F.gelu(x, approximate="tanh")
+
+
+def _register_moe_custom_op(schema, op_name, impl, fake):
+    try:
+        library = torch.library.Library("cactus_transpile", "FRAGMENT")
+        library.define(schema)
+        _CACTUS_TORCH_LIBRARIES.append(library)
+    except RuntimeError as exc:
+        if not _ignore_duplicate_torch_registration(exc):
+            raise
+    try:
+        torch.library.impl(op_name, "CompositeExplicitAutograd")(impl)
+    except RuntimeError as exc:
+        if not _ignore_duplicate_torch_registration(exc):
+            raise
+    try:
+        torch.library.register_fake(op_name)(fake)
+    except RuntimeError as exc:
+        if not _ignore_duplicate_torch_registration(exc):
+            raise
+
+
+def _ensure_lfm2_moe_custom_op_registered() -> None:
+    global _LFM2_MOE_CUSTOM_OP_REGISTERED
+    if _LFM2_MOE_CUSTOM_OP_REGISTERED:
+        return
+
+    schema = (
+        "lfm2_moe_layer_gated("
+        "Tensor hidden, Tensor router_logits, Tensor expert_bias, "
+        "Tensor[] w1_weights, Tensor[] w3_weights, Tensor[] w2_weights, "
+        "int num_experts, int num_experts_per_tok, bool use_expert_bias, "
+        "bool normalize_routing, float epsilon, float routed_scaling_factor"
+        ") -> Tensor"
+    )
+
+    def _impl(hidden, router_logits, expert_bias, w1_weights, w3_weights, w2_weights,
+              num_experts, num_experts_per_tok, use_expert_bias, normalize_routing,
+              epsilon, routed_scaling_factor):
+        routing_probs = router_logits.sigmoid()
+        if use_expert_bias:
+            scores = routing_probs + expert_bias.to(device=routing_probs.device, dtype=routing_probs.dtype)
+            _, selected_experts = torch.topk(scores, k=int(num_experts_per_tok), dim=-1)
+            routing_weights = torch.gather(routing_probs, dim=1, index=selected_experts).type_as(router_logits)
+        else:
+            routing_weights, selected_experts = torch.topk(routing_probs, k=int(num_experts_per_tok), dim=-1)
+        routing_weights = _moe_normalize_and_scale(routing_weights, normalize_routing, epsilon, routed_scaling_factor)
+        return _moe_scatter_experts(hidden, selected_experts, routing_weights, w1_weights, w3_weights, w2_weights, num_experts, F.silu)
+
+    def _fake(hidden, router_logits, expert_bias, w1_weights, w3_weights, w2_weights,
+              num_experts, num_experts_per_tok, use_expert_bias, normalize_routing,
+              epsilon, routed_scaling_factor):
+        return hidden.new_empty(hidden.shape)
+
+    _register_moe_custom_op(schema, _LFM2_MOE_CUSTOM_OP, _impl, _fake)
+    _LFM2_MOE_CUSTOM_OP_REGISTERED = True
+
+
+def _ensure_qwen2_moe_custom_op_registered() -> None:
+    global _QWEN2_MOE_CUSTOM_OP_REGISTERED
+    if _QWEN2_MOE_CUSTOM_OP_REGISTERED:
+        return
+
+    schema = (
+        "qwen2_moe_layer_gated("
+        "Tensor hidden, Tensor router_logits, "
+        "Tensor[] w1_weights, Tensor[] w3_weights, Tensor[] w2_weights, "
+        "int num_experts, int num_experts_per_tok, bool normalize_routing, "
+        "float epsilon, float routed_scaling_factor"
+        ") -> Tensor"
+    )
+
+    def _impl(hidden, router_logits, w1_weights, w3_weights, w2_weights,
+              num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor):
+        routing_probs = F.softmax(router_logits, dim=-1, dtype=torch.float32).to(dtype=router_logits.dtype)
+        routing_weights, selected_experts = torch.topk(routing_probs, k=int(num_experts_per_tok), dim=-1)
+        routing_weights = _moe_normalize_and_scale(routing_weights, normalize_routing, epsilon, routed_scaling_factor)
+        return _moe_scatter_experts(hidden, selected_experts, routing_weights, w1_weights, w3_weights, w2_weights, num_experts, F.silu)
+
+    def _fake(hidden, router_logits, w1_weights, w3_weights, w2_weights,
+              num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor):
+        return hidden.new_empty(hidden.shape)
+
+    _register_moe_custom_op(schema, _QWEN2_MOE_CUSTOM_OP, _impl, _fake)
+    _QWEN2_MOE_CUSTOM_OP_REGISTERED = True
+
+
+def _ensure_gemma4_moe_custom_op_registered() -> None:
+    global _GEMMA4_MOE_CUSTOM_OP_REGISTERED
+    if _GEMMA4_MOE_CUSTOM_OP_REGISTERED:
+        return
+
+    schema = (
+        "gemma4_moe_layer_gated("
+        "Tensor hidden, Tensor router_logits, "
+        "Tensor[] w1_weights, Tensor[] w3_weights, Tensor[] w2_weights, "
+        "int num_experts, int num_experts_per_tok, bool normalize_routing, "
+        "float epsilon, float routed_scaling_factor"
+        ") -> Tensor"
+    )
+
+    def _impl(hidden, router_logits, w1_weights, w3_weights, w2_weights,
+              num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor):
+        routing_probs = F.softmax(router_logits, dim=-1, dtype=torch.float32).to(dtype=router_logits.dtype)
+        selected_experts = torch.topk(router_logits, k=int(num_experts_per_tok), dim=-1).indices
+        routing_weights = torch.gather(routing_probs, -1, selected_experts)
+        routing_weights = _moe_normalize_and_scale(routing_weights, normalize_routing, epsilon, routed_scaling_factor)
+        return _moe_scatter_experts(hidden, selected_experts, routing_weights, w1_weights, w3_weights, w2_weights, num_experts, _gelu_tanh)
+
+    def _fake(hidden, router_logits, w1_weights, w3_weights, w2_weights,
+              num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor):
+        return hidden.new_empty(hidden.shape)
+
+    _register_moe_custom_op(schema, _GEMMA4_MOE_CUSTOM_OP, _impl, _fake)
+    _GEMMA4_MOE_CUSTOM_OP_REGISTERED = True
+
+
+class _Lfm2MoeRuntimeMoeBlock(torch.nn.Module):
+
+    def __init__(self, block: torch.nn.Module, *, layer_index: int):
+        super().__init__()
+        _ensure_lfm2_moe_custom_op_registered()
+        experts = getattr(block, "experts", None)
+        gate_up = getattr(experts, "gate_up_proj", None)
+        down = getattr(experts, "down_proj", None)
+        gate = getattr(block, "gate", None)
+        if not isinstance(gate, torch.nn.Module) or not isinstance(gate_up, torch.Tensor) or not isinstance(down, torch.Tensor):
+            raise TypeError("LFM2-MoE runtime block requires gate, gate_up_proj, and down_proj")
+
+        self.layer_index = int(layer_index)
+        self.gate = gate
+        self.num_experts = int(getattr(block, "num_experts", int(gate_up.shape[0])) or int(gate_up.shape[0]))
+        self.top_k = int(getattr(block, "top_k", 0) or 0)
+        self.use_expert_bias = bool(getattr(block, "use_expert_bias", False))
+        self.normalize_routing = bool(getattr(block, "norm_topk_prob", False))
+        self.routed_scaling_factor = float(getattr(block, "routed_scaling_factor", 1.0) or 1.0)
+        self.epsilon = 1.0e-6
+        self.hidden_dim = int(gate_up.shape[2])
+        self.intermediate_dim = int(gate_up.shape[1]) // 2
+
+        if self.num_experts <= 0 or self.top_k <= 0:
+            raise ValueError("LFM2-MoE runtime block requires positive num_experts and top_k")
+        if int(gate_up.shape[0]) != self.num_experts or int(down.shape[0]) != self.num_experts:
+            raise ValueError("LFM2-MoE expert tensor count mismatch")
+        if int(gate_up.shape[1]) != 2 * self.intermediate_dim:
+            raise ValueError("LFM2-MoE gate_up_proj must have an even expert dimension")
+
+        self.w1_weights = torch.nn.ParameterList()
+        self.w3_weights = torch.nn.ParameterList()
+        self.w2_weights = torch.nn.ParameterList()
+        for expert_idx in range(self.num_experts):
+            self.w1_weights.append(torch.nn.Parameter(
+                gate_up[expert_idx, : self.intermediate_dim, :].detach().contiguous(),
+                requires_grad=False,
+            ))
+            self.w3_weights.append(torch.nn.Parameter(
+                gate_up[expert_idx, self.intermediate_dim :, :].detach().contiguous(),
+                requires_grad=False,
+            ))
+            self.w2_weights.append(torch.nn.Parameter(down[expert_idx].detach().contiguous(), requires_grad=False))
+
+        expert_bias = getattr(block, "expert_bias", None)
+        if self.use_expert_bias and isinstance(expert_bias, torch.Tensor):
+            bias_value = expert_bias.detach().to(dtype=torch.float32)
+        else:
+            bias_value = torch.zeros(self.num_experts, dtype=torch.float32, device=gate_up.device)
+        self.register_buffer("expert_bias", bias_value, persistent=True)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        batch_size, sequence_length, hidden_dim = hidden_states.shape
+        hidden_flat = hidden_states.reshape(-1, hidden_dim)
+        router_logits = self.gate(hidden_flat)
+        output = torch.ops.cactus_transpile.lfm2_moe_layer_gated(
+            hidden_flat,
+            router_logits,
+            self.expert_bias,
+            list(self.w1_weights),
+            list(self.w3_weights),
+            list(self.w2_weights),
+            self.num_experts,
+            self.top_k,
+            self.use_expert_bias,
+            self.normalize_routing,
+            self.epsilon,
+            self.routed_scaling_factor,
+        )
+        return output.reshape(batch_size, sequence_length, hidden_dim)
+
+
+def _is_lfm2_moe_sparse_block(block: object) -> bool:
+    experts = getattr(block, "experts", None)
+    return (
+        isinstance(block, torch.nn.Module)
+        and isinstance(getattr(block, "gate", None), torch.nn.Module)
+        and isinstance(getattr(experts, "gate_up_proj", None), torch.Tensor)
+        and isinstance(getattr(experts, "down_proj", None), torch.Tensor)
+    )
+
+
+def _is_lfm2_moe_model(model: torch.nn.Module) -> bool:
+    config = getattr(model, "config", None)
+    model_type = str(getattr(config, "model_type", "") or "").lower()
+    if model_type == "lfm2_moe":
+        return True
+    return type(model).__module__.startswith("transformers.models.lfm2_moe.")
+
+
+def _lfm2_backbone_from_model(model: torch.nn.Module) -> torch.nn.Module | None:
+    model_root = getattr(model, "model", None)
+    language_model = getattr(model_root, "language_model", None)
+    backbone = language_model if isinstance(language_model, torch.nn.Module) else model_root
+    return backbone if isinstance(backbone, torch.nn.Module) else None
+
+
+def _lfm2_position_embeddings(
+    backbone: torch.nn.Module,
+    hidden_states: torch.Tensor,
+    *,
+    position_ids: torch.Tensor,
+) -> object:
+    rotary_emb = getattr(backbone, "rotary_emb", None)
+    if callable(rotary_emb):
+        return rotary_emb(hidden_states, position_ids=position_ids)
+    pos_emb = getattr(backbone, "pos_emb", None)
+    if callable(pos_emb):
+        return pos_emb(hidden_states, position_ids=position_ids)
+    raise AttributeError(f"{type(backbone).__name__} has neither rotary_emb nor pos_emb")
+
+
+def _lfm2_causal_mask_for_capture(
+    create_causal_mask: Callable[..., object],
+    *,
+    backbone: torch.nn.Module,
+    inputs_embeds: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    position_ids: torch.Tensor,
+) -> object | None:
+    if inputs_embeds.is_meta:
+        return None
+    return create_causal_mask(
+        config=backbone.config,
+        inputs_embeds=inputs_embeds,
+        attention_mask=attention_mask,
+        past_key_values=None,
+        position_ids=position_ids,
+    )
+
+
+def _patch_lfm2_moe_feed_forwards(model: torch.nn.Module) -> None:
+    if not _is_lfm2_moe_model(model):
+        return
+    backbone = _lfm2_backbone_from_model(model)
+    layers = getattr(backbone, "layers", None)
+    if not isinstance(layers, torch.nn.ModuleList):
+        return
+    for layer_index, layer in enumerate(layers):
+        feed_forward = getattr(layer, "feed_forward", None)
+        if isinstance(feed_forward, _Lfm2MoeRuntimeMoeBlock):
+            continue
+        if _is_lfm2_moe_sparse_block(feed_forward):
+            setattr(layer, "feed_forward", _Lfm2MoeRuntimeMoeBlock(feed_forward, layer_index=layer_index))
+
+
+class _Qwen2MoeRuntimeMoeBlock(torch.nn.Module):
+
+    def __init__(self, block: torch.nn.Module, *, layer_index: int):
+        super().__init__()
+        _ensure_qwen2_moe_custom_op_registered()
+        experts = getattr(block, "experts", None)
+        gate_up = getattr(experts, "gate_up_proj", None)
+        down = getattr(experts, "down_proj", None)
+        gate = getattr(block, "gate", None)
+        gate_weight = getattr(gate, "weight", None)
+        shared_expert = getattr(block, "shared_expert", None)
+        shared_expert_gate = getattr(block, "shared_expert_gate", None)
+        if (
+            not isinstance(gate, torch.nn.Module)
+            or not isinstance(gate_weight, torch.Tensor)
+            or not isinstance(gate_up, torch.Tensor)
+            or not isinstance(down, torch.Tensor)
+            or not isinstance(shared_expert, torch.nn.Module)
+            or not isinstance(shared_expert_gate, torch.nn.Module)
+        ):
+            raise TypeError(
+                "Qwen2-MoE runtime block requires gate, packed experts, shared_expert, and shared_expert_gate"
+            )
+
+        self.layer_index = int(layer_index)
+        self.gate = gate
+        self.shared_expert = shared_expert
+        self.shared_expert_gate = shared_expert_gate
+        config = getattr(block, "config", None)
+        def _route_attr(*names, default=None):
+            for obj in (block, config):
+                if obj is None:
+                    continue
+                for name in names:
+                    val = getattr(obj, name, None)
+                    if val is not None:
+                        return val
+            return default
+        self.num_experts = int(_route_attr("num_experts", default=int(gate_up.shape[0])) or int(gate_up.shape[0]))
+        self.top_k = int(_route_attr("top_k", "num_experts_per_tok", default=0) or 0)
+        self.normalize_routing = bool(_route_attr("norm_topk_prob", default=False))
+        self.routed_scaling_factor = float(getattr(block, "routed_scaling_factor", 1.0) or 1.0)
+        self.epsilon = 1.0e-6
+        self.hidden_dim = int(gate_up.shape[2])
+        self.intermediate_dim = int(gate_up.shape[1]) // 2
+
+        if self.num_experts <= 0 or self.top_k <= 0:
+            raise ValueError("Qwen2-MoE runtime block requires positive num_experts and top_k")
+        if int(gate_up.shape[0]) != self.num_experts or int(down.shape[0]) != self.num_experts:
+            raise ValueError("Qwen2-MoE expert tensor count mismatch")
+        if int(gate_up.shape[1]) != 2 * self.intermediate_dim:
+            raise ValueError("Qwen2-MoE gate_up_proj must have an even expert dimension")
+
+        self.w1_weights = torch.nn.ParameterList()
+        self.w3_weights = torch.nn.ParameterList()
+        self.w2_weights = torch.nn.ParameterList()
+        for expert_idx in range(self.num_experts):
+            self.w1_weights.append(torch.nn.Parameter(
+                gate_up[expert_idx, : self.intermediate_dim, :].detach().contiguous(),
+                requires_grad=False,
+            ))
+            self.w3_weights.append(torch.nn.Parameter(
+                gate_up[expert_idx, self.intermediate_dim :, :].detach().contiguous(),
+                requires_grad=False,
+            ))
+            self.w2_weights.append(torch.nn.Parameter(down[expert_idx].detach().contiguous(), requires_grad=False))
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        batch_size, sequence_length, hidden_dim = hidden_states.shape
+        hidden_flat = hidden_states.reshape(-1, hidden_dim)
+        router_logits = F.linear(hidden_flat, self.gate.weight)
+        routed_output = torch.ops.cactus_transpile.qwen2_moe_layer_gated(
+            hidden_flat,
+            router_logits,
+            list(self.w1_weights),
+            list(self.w3_weights),
+            list(self.w2_weights),
+            self.num_experts,
+            self.top_k,
+            self.normalize_routing,
+            self.epsilon,
+            self.routed_scaling_factor,
+        )
+        shared_output = self.shared_expert(hidden_flat)
+        shared_output = torch.sigmoid(self.shared_expert_gate(hidden_flat)) * shared_output
+        output = routed_output + shared_output
+        return output.reshape(batch_size, sequence_length, hidden_dim)
+
+
+def _is_qwen2_moe_sparse_block(block: object) -> bool:
+    experts = getattr(block, "experts", None)
+    gate = getattr(block, "gate", None)
+    return (
+        isinstance(block, torch.nn.Module)
+        and isinstance(gate, torch.nn.Module)
+        and isinstance(getattr(gate, "weight", None), torch.Tensor)
+        and isinstance(getattr(experts, "gate_up_proj", None), torch.Tensor)
+        and isinstance(getattr(experts, "down_proj", None), torch.Tensor)
+        and isinstance(getattr(block, "shared_expert", None), torch.nn.Module)
+        and isinstance(getattr(block, "shared_expert_gate", None), torch.nn.Module)
+    )
+
+
+def _is_qwen2_moe_model(model: torch.nn.Module) -> bool:
+    config = getattr(model, "config", None)
+    model_type = str(getattr(config, "model_type", "") or "").lower()
+    if model_type == "qwen2_moe":
+        return True
+    return type(model).__module__.startswith("transformers.models.qwen2_moe.")
+
+
+def _qwen2_moe_backbone_from_model(model: torch.nn.Module) -> torch.nn.Module | None:
+    backbone = getattr(model, "model", None)
+    return backbone if isinstance(backbone, torch.nn.Module) else None
+
+
+def _patch_qwen2_moe_mlps(model: torch.nn.Module) -> None:
+    if not _is_qwen2_moe_model(model):
+        return
+    backbone = _qwen2_moe_backbone_from_model(model)
+    layers = getattr(backbone, "layers", None)
+    if not isinstance(layers, torch.nn.ModuleList):
+        return
+    for layer_index, layer in enumerate(layers):
+        mlp = getattr(layer, "mlp", None)
+        if isinstance(mlp, _Qwen2MoeRuntimeMoeBlock):
+            continue
+        if _is_qwen2_moe_sparse_block(mlp):
+            setattr(layer, "mlp", _Qwen2MoeRuntimeMoeBlock(mlp, layer_index=layer_index))
+
+
+class _Gemma4RuntimeMoeBlock(torch.nn.Module):
+
+    def __init__(
+        self,
+        block: torch.nn.Module,
+        *,
+        layer_index: int,
+        per_expert_scale: torch.Tensor | None = None,
+    ):
+        super().__init__()
+        _ensure_gemma4_moe_custom_op_registered()
+        gate_up = getattr(block, "gate_up_proj", None)
+        down = getattr(block, "down_proj", None)
+        if per_expert_scale is None:
+            per_expert_scale = getattr(block, "per_expert_scale", None)
+        if (
+            not isinstance(gate_up, torch.Tensor)
+            or not isinstance(down, torch.Tensor)
+            or not isinstance(per_expert_scale, torch.Tensor)
+        ):
+            raise TypeError("Gemma4 MoE runtime block requires gate_up_proj, down_proj, and per_expert_scale")
+
+        self.layer_index = int(layer_index)
+        self.num_experts = int(getattr(block, "num_experts", int(gate_up.shape[0])) or int(gate_up.shape[0]))
+        self.hidden_dim = int(gate_up.shape[2])
+        self.intermediate_dim = int(gate_up.shape[1]) // 2
+        if self.num_experts <= 0:
+            raise ValueError("Gemma4 MoE runtime block requires positive num_experts")
+        if int(gate_up.shape[0]) != self.num_experts or int(down.shape[0]) != self.num_experts:
+            raise ValueError("Gemma4 MoE expert tensor count mismatch")
+        if int(gate_up.shape[1]) != 2 * self.intermediate_dim:
+            raise ValueError("Gemma4 MoE gate_up_proj must have an even expert dimension")
+
+        self.w1_weights = torch.nn.ParameterList()
+        self.w3_weights = torch.nn.ParameterList()
+        self.w2_weights = torch.nn.ParameterList()
+        for expert_idx in range(self.num_experts):
+            self.w1_weights.append(torch.nn.Parameter(
+                gate_up[expert_idx, : self.intermediate_dim, :].detach().contiguous(),
+                requires_grad=False,
+            ))
+            self.w3_weights.append(torch.nn.Parameter(
+                gate_up[expert_idx, self.intermediate_dim :, :].detach().contiguous(),
+                requires_grad=False,
+            ))
+            scale = per_expert_scale[expert_idx].detach().to(device=down.device, dtype=down.dtype)
+            self.w2_weights.append(torch.nn.Parameter(
+                (down[expert_idx].detach() * scale).contiguous(),
+                requires_grad=False,
+            ))
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        router_logits: torch.Tensor,
+        *,
+        top_k: int,
+        normalize_routing: bool = True,
+        epsilon: float = 1.0e-6,
+    ) -> torch.Tensor:
+        return torch.ops.cactus_transpile.gemma4_moe_layer_gated(
+            hidden_states,
+            router_logits,
+            list(self.w1_weights),
+            list(self.w3_weights),
+            list(self.w2_weights),
+            self.num_experts,
+            int(top_k),
+            bool(normalize_routing),
+            float(epsilon),
+            1.0,
+        )
+
+
+class _Gemma4MoeRuntimeDecoderLayer(torch.nn.Module):
+    """Gemma4 decoder layer that preserves HF math while exporting MoE as one semantic op."""
+
+    def __init__(self, layer: torch.nn.Module, *, layer_index: int):
+        super().__init__()
+        self.config = layer.config
+        self.hidden_size = layer.hidden_size
+        self.layer_idx = layer.layer_idx
+        self.self_attn = layer.self_attn
+        config_layer_types = tuple(getattr(self.config, "layer_types", ()))
+        self.attention_type = getattr(layer, "attention_type", getattr(self.self_attn, "attention_type", None))
+        if self.attention_type is None:
+            self.attention_type = (
+                config_layer_types[layer_index]
+                if layer_index < len(config_layer_types)
+                else "full_attention"
+            )
+        self.mlp = layer.mlp
+        self.input_layernorm = layer.input_layernorm
+        self.post_attention_layernorm = layer.post_attention_layernorm
+        self.pre_feedforward_layernorm = layer.pre_feedforward_layernorm
+        self.post_feedforward_layernorm = layer.post_feedforward_layernorm
+        self.register_buffer("layer_scalar", layer.layer_scalar.detach().clone(), persistent=True)
+        self.hidden_size_per_layer_input = getattr(layer, "hidden_size_per_layer_input", 0)
+        if self.hidden_size_per_layer_input:
+            self.act_fn = layer.act_fn
+            self.per_layer_input_gate = layer.per_layer_input_gate
+            self.per_layer_projection = layer.per_layer_projection
+            self.post_per_layer_input_norm = layer.post_per_layer_input_norm
+        self.enable_moe_block = True
+        self.router = layer.router
+        self.moe = _Gemma4RuntimeMoeBlock(
+            layer.experts,
+            layer_index=layer_index,
+            per_expert_scale=layer.router.per_expert_scale,
+        )
+        self.post_feedforward_layernorm_1 = layer.post_feedforward_layernorm_1
+        self.post_feedforward_layernorm_2 = layer.post_feedforward_layernorm_2
+        self.pre_feedforward_layernorm_2 = layer.pre_feedforward_layernorm_2
+        self.top_k_experts = int(getattr(self.config, "top_k_experts", 0) or 0)
+        self.router_epsilon = float(getattr(self.config, "rms_norm_eps", 1.0e-6) or 1.0e-6)
+
+    def _router_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        router_states = self.router.norm(hidden_states)
+        scalar_root_size = getattr(self.router, "scalar_root_size", None)
+        if scalar_root_size is None:
+            root_size = getattr(self.router, "root_size", 1.0)
+            scalar_root_size = root_size.to(router_states.dtype) if isinstance(root_size, torch.Tensor) else float(root_size)
+        router_states = router_states * self.router.scale.to(router_states.dtype)
+        router_states = router_states * scalar_root_size
+        return self.router.proj(router_states)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: torch.Tensor = None,
+        per_layer_input: torch.Tensor = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: object | None = None,
+        use_cache: bool | None = False,
+        shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None,
+        **kwargs,
+    ) -> torch.Tensor:
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        hidden_states, _ = self.self_attn(
+            hidden_states=hidden_states,
+            position_embeddings=position_embeddings,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            shared_kv_states=shared_kv_states,
+            use_cache=use_cache,
+            **kwargs,
+        )
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states_1 = self.pre_feedforward_layernorm(hidden_states)
+        hidden_states_1 = self.mlp(hidden_states_1)
+        hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states_1)
+
+        hidden_states_flat = hidden_states.reshape(-1, hidden_states.shape[-1])
+        router_logits = self._router_logits(hidden_states_flat)
+        hidden_states_2 = self.pre_feedforward_layernorm_2(hidden_states_flat)
+        hidden_states_2 = self.moe(
+            hidden_states_2,
+            router_logits,
+            top_k=self.top_k_experts,
+            normalize_routing=True,
+            epsilon=self.router_epsilon,
+        )
+        hidden_states_2 = hidden_states_2.reshape(hidden_states.shape)
+        hidden_states_2 = self.post_feedforward_layernorm_2(hidden_states_2)
+
+        hidden_states = hidden_states_1 + hidden_states_2
+        hidden_states = self.post_feedforward_layernorm(hidden_states)
+        hidden_states = residual + hidden_states
+
+        if self.hidden_size_per_layer_input:
+            residual = hidden_states
+            hidden_states = self.per_layer_input_gate(hidden_states)
+            hidden_states = self.act_fn(hidden_states)
+            hidden_states = torch.multiply(hidden_states, per_layer_input)
+            hidden_states = self.per_layer_projection(hidden_states)
+            hidden_states = self.post_per_layer_input_norm(hidden_states)
+            hidden_states = residual + hidden_states
+
+        hidden_states = hidden_states * self.layer_scalar
+        return hidden_states
+
+
+def _is_gemma4_moe_decoder_layer(layer: object) -> bool:
+    return (
+        isinstance(layer, torch.nn.Module)
+        and bool(getattr(layer, "enable_moe_block", False))
+        and hasattr(layer, "router")
+        and hasattr(layer, "experts")
+        and not isinstance(layer, _Gemma4MoeRuntimeDecoderLayer)
+    )
+
+
+def _gemma4_backbone_from_model(model: torch.nn.Module) -> torch.nn.Module | None:
+    model_root = getattr(model, "model", None)
+    backbone = getattr(model_root, "language_model", model_root)
+    return backbone if isinstance(backbone, torch.nn.Module) else None
+
+
+def _patch_gemma4_moe_layers(model: torch.nn.Module) -> None:
+    if _family_key(model) != "gemma4":
+        return
+    backbone = _gemma4_backbone_from_model(model)
+    layers = getattr(backbone, "layers", None)
+    if not isinstance(layers, torch.nn.ModuleList):
+        return
+    for layer_index, layer in enumerate(layers):
+        if isinstance(layer, _Gemma4MoeRuntimeDecoderLayer):
+            continue
+        if _is_gemma4_moe_decoder_layer(layer):
+            layers[layer_index] = _Gemma4MoeRuntimeDecoderLayer(layer, layer_index=layer_index)
+
+
 def _gemma4_cpu_safe_text_mlp_enabled(layer: torch.nn.Module, hidden_states: torch.Tensor) -> bool:
     if hidden_states.device.type != "cpu" or hidden_states.dtype != torch.float16:
         return False
@@ -568,6 +1235,17 @@ def _gemma4_text_decoder_layer_forward(
     use_cache: bool = False,
     shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None,
 ) -> torch.Tensor:
+    if isinstance(layer, _Gemma4MoeRuntimeDecoderLayer):
+        return layer(
+            hidden_states,
+            position_embeddings=position_embeddings,
+            per_layer_input=per_layer_input,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            shared_kv_states=shared_kv_states,
+        )
     manual_attention_required = shared_kv_states is not None
     if not manual_attention_required and not _gemma4_cpu_safe_text_mlp_enabled(layer, hidden_states):
         return layer(
@@ -642,7 +1320,12 @@ def _gemma4_text_backbone_forward(
     }
     if shared_kv_states is _UNSET:
         num_shared_layers = int(getattr(backbone.config, "num_kv_shared_layers", 0) or 0)
-        shared_kv_states = {} if num_shared_layers > 0 else None
+        has_shared_kv_layers = num_shared_layers > 0 or any(
+            bool(getattr(getattr(layer, "self_attn", None), "is_kv_shared_layer", False))
+            or bool(getattr(getattr(layer, "self_attn", None), "store_full_length_kv", False))
+            for layer in getattr(backbone, "layers", ())
+        )
+        shared_kv_states = {} if has_shared_kv_layers else None
 
     config_layer_types = tuple(getattr(backbone.config, "layer_types", ()))
     end = backbone.config.num_hidden_layers if layer_end is None else min(int(layer_end), int(backbone.config.num_hidden_layers))
@@ -656,6 +1339,15 @@ def _gemma4_text_backbone_forward(
             "attention_type",
             config_layer_types[layer_index] if layer_index < len(config_layer_types) else "full_attention",
         )
+        layer_shared_kv_states = shared_kv_states
+        if shared_kv_states is not None:
+            layer_attn = getattr(decoder_layer, "self_attn", None)
+            layer_uses_shared_kv = (
+                bool(getattr(layer_attn, "is_kv_shared_layer", False))
+                or bool(getattr(layer_attn, "store_full_length_kv", False))
+            )
+            if not layer_uses_shared_kv:
+                layer_shared_kv_states = None
         hidden_states = _gemma4_text_decoder_layer_forward(
             decoder_layer,
             hidden_states,
@@ -665,7 +1357,7 @@ def _gemma4_text_backbone_forward(
             position_ids=position_ids,
             past_key_values=None,
             use_cache=False,
-            shared_kv_states=shared_kv_states,
+            shared_kv_states=layer_shared_kv_states,
         )
         if capture_layer_index is not None and layer_index == int(capture_layer_index):
             captured_hidden = hidden_states
@@ -894,11 +1586,14 @@ def _gemma4_vision_encoder_hidden_states(
 
     if _GEMMA4_CREATE_BIDIRECTIONAL_MASK is None:
         raise RuntimeError("Gemma4 bidirectional mask helper is unavailable in this transformers install")
-    attention_mask = _GEMMA4_CREATE_BIDIRECTIONAL_MASK(
-        config=config,
-        inputs_embeds=inputs_embeds,
-        attention_mask=attention_mask,
-    )
+    if inputs_embeds.is_meta:
+        attention_mask = (attention_mask.unsqueeze(-1) * attention_mask.unsqueeze(1)).unsqueeze(1)
+    else:
+        attention_mask = _GEMMA4_CREATE_BIDIRECTIONAL_MASK(
+            config=config,
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+        )
     position_embeddings = rotary_emb(hidden_states, pixel_position_ids)
     for decoder_layer in layers[:num_layers]:
         hidden_states = decoder_layer(
@@ -1398,9 +2093,14 @@ def _gemma4_build_standard_causal_mask_mapping(
     inputs_embeds: torch.Tensor,
     attention_mask: torch.Tensor | None,
     position_ids: torch.Tensor,
-) -> dict[str, torch.Tensor]:
+) -> dict[str, torch.Tensor | None]:
     if not callable(create_causal_mask) or not callable(create_sliding_window_causal_mask):
         raise RuntimeError("Gemma4 standard causal mask helpers are unavailable")
+    if inputs_embeds.is_meta or position_ids.is_meta or (attention_mask is not None and attention_mask.is_meta):
+        return {
+            "full_attention": None,
+            "sliding_attention": None,
+        }
     mask_kwargs = {
         "config": config,
         "inputs_embeds": inputs_embeds,
@@ -1544,12 +2244,14 @@ def get_transpile_metadata(self):
 class Lfm2CausalLMLogitsAdapter(torch.nn.Module):
     def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None):
         super().__init__()
+        _patch_lfm2_moe_feed_forwards(model)
         self.model = model
         model_root = getattr(model, "model", None)
         language_model = getattr(model_root, "language_model", None)
         self.backbone = language_model if isinstance(language_model, torch.nn.Module) else model_root
         self.lm_head = getattr(model, "lm_head", None)
         self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model)
+        self.adapter_family = _family_key(model)
         from transformers.models.lfm2.modeling_lfm2 import create_causal_mask  # type: ignore
 
         self._create_causal_mask = create_causal_mask
@@ -1567,16 +2269,16 @@ def forward(self, input_ids: torch.Tensor):
             else None
         )
         position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0)
-        causal_mask = self._create_causal_mask(
-            config=backbone.config,
+        causal_mask = _lfm2_causal_mask_for_capture(
+            self._create_causal_mask,
+            backbone=backbone,
             inputs_embeds=inputs_embeds,
             attention_mask=attention_mask,
-            past_key_values=None,
             position_ids=position_ids,
         )
 
         hidden_states = inputs_embeds
-        position_embeddings = backbone.rotary_emb(hidden_states, position_ids=position_ids)
+        position_embeddings = _lfm2_position_embeddings(backbone, hidden_states, position_ids=position_ids)
         layer_types = tuple(getattr(backbone.config, "layer_types", ()))
         linear_attention = attention_mask if inputs_embeds.shape[1] != 1 else None
 
@@ -1605,7 +2307,7 @@ def get_transpile_metadata(self):
             "graph": {
                 **_transpile_graph_meta(
                     self.model,
-                    adapter_family="lfm2",
+                    adapter_family=self.adapter_family,
                     adapter_type=type(self).__name__,
                     input_names=("input_ids",),
                 ),
@@ -1617,12 +2319,14 @@ def get_transpile_metadata(self):
 class Lfm2CausalLMStepAdapter(torch.nn.Module):
     def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None):
         super().__init__()
+        _patch_lfm2_moe_feed_forwards(model)
         self.model = model
         model_root = getattr(model, "model", None)
         language_model = getattr(model_root, "language_model", None)
         self.backbone = language_model if isinstance(language_model, torch.nn.Module) else model_root
         self.lm_head = getattr(model, "lm_head", None)
         self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model)
+        self.adapter_family = _family_key(model)
 
     def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor:
         backbone = self.backbone
@@ -1635,7 +2339,7 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch.
         causal_mask = None
 
         hidden_states = inputs_embeds
-        position_embeddings = backbone.rotary_emb(hidden_states, position_ids=text_position_ids)
+        position_embeddings = _lfm2_position_embeddings(backbone, hidden_states, position_ids=text_position_ids)
         layer_types = tuple(getattr(backbone.config, "layer_types", ()))
 
         for layer_index, decoder_layer in enumerate(backbone.layers[: backbone.config.num_hidden_layers]):
@@ -1659,7 +2363,7 @@ def get_transpile_metadata(self):
             "graph": {
                 **_transpile_graph_meta(
                     self.model,
-                    adapter_family="lfm2",
+                    adapter_family=self.adapter_family,
                     adapter_type=type(self).__name__,
                     input_names=("input_ids", "position_ids"),
                 ),
@@ -1677,6 +2381,7 @@ def _lfm2_vl_model_root(model: torch.nn.Module) -> torch.nn.Module:
 
 
 def _lfm2_language_backbone(model: torch.nn.Module) -> torch.nn.Module:
+    _patch_lfm2_moe_feed_forwards(model)
     root = getattr(model, "model", None)
     language_model = getattr(root, "language_model", None)
     backbone = language_model if isinstance(language_model, torch.nn.Module) else root
@@ -1880,15 +2585,15 @@ def forward(
         attention_mask: torch.Tensor,
         position_ids: torch.Tensor,
     ) -> torch.Tensor:
-        causal_mask = self._create_causal_mask(
-            config=self.backbone.config,
+        causal_mask = _lfm2_causal_mask_for_capture(
+            self._create_causal_mask,
+            backbone=self.backbone,
             inputs_embeds=inputs_embeds,
             attention_mask=attention_mask,
-            past_key_values=None,
             position_ids=position_ids,
         )
         hidden_states = inputs_embeds
-        position_embeddings = self.backbone.rotary_emb(hidden_states, position_ids=position_ids)
+        position_embeddings = _lfm2_position_embeddings(self.backbone, hidden_states, position_ids=position_ids)
         layer_types = tuple(getattr(self.backbone.config, "layer_types", ()))
         linear_attention = attention_mask if inputs_embeds.shape[1] != 1 else None
 
@@ -2186,6 +2891,7 @@ def _additive_mask_mapping(self, inputs_embeds: torch.Tensor) -> dict[str, torch
 class Gemma4CausalLMLogitsAdapter(torch.nn.Module):
     def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None):
         super().__init__()
+        _patch_gemma4_moe_layers(model)
         self.model = model
         model_backbone = model.model
         self.backbone = getattr(model_backbone, "language_model", model_backbone)
@@ -2366,6 +3072,7 @@ def _gemma4_apply_final_logit_softcapping(model: torch.nn.Module, logits: torch.
 
 class Gemma4MultimodalCausalLMLogitsAdapter(BoundInputAdapter):
     def __init__(self, model: torch.nn.Module, *, input_names: tuple[str, ...], weights_dir: str | None = None):
+        _patch_gemma4_moe_layers(model)
         super().__init__(
             model,
             input_names=input_names,
@@ -2844,6 +3551,7 @@ def __init__(
         native_image_pool_shapes: tuple[tuple[int, int, int], ...] | None = None,
     ):
         super().__init__()
+        _patch_gemma4_moe_layers(model)
         self.model = model
         self.input_names = input_names
         self.multimodal_backbone = model.model
@@ -3007,14 +3715,19 @@ def _prepare_decoder_inputs(
         attention_mask: torch.Tensor | None,
         token_type_ids: torch.Tensor,
         image_features: torch.Tensor,
-        audio_features: torch.Tensor,
+        audio_features: torch.Tensor | None = None,
     ) -> tuple[torch.Tensor, torch.Tensor, torch.LongTensor]:
         if self._native_merge_plan is not None:
+            audio_features_for_merge = audio_features
+            if audio_features_for_merge is None:
+                audio_features_for_merge = image_features.new_empty(
+                    (int(input_ids.shape[0]), 0, int(image_features.shape[-1]))
+                )
             inputs_embeds, per_layer_inputs_tokens = _gemma4_apply_native_merge_plan(
                 self.model,
                 input_ids=input_ids,
                 image_features=image_features,
-                audio_features=audio_features,
+                audio_features=audio_features_for_merge,
                 merge_plan=self._native_merge_plan,
                 pli_token_ids=self._native_merge_pli_token_ids,
             )
@@ -3044,10 +3757,11 @@ def _prepare_decoder_inputs(
                 image_mask.unsqueeze(-1).expand_as(inputs_embeds),
                 image_features.to(inputs_embeds.device, inputs_embeds.dtype),
             )
-            inputs_embeds = inputs_embeds.masked_scatter(
-                audio_mask.unsqueeze(-1).expand_as(inputs_embeds),
-                audio_features.to(inputs_embeds.device, inputs_embeds.dtype),
-            )
+            if audio_features is not None:
+                inputs_embeds = inputs_embeds.masked_scatter(
+                    audio_mask.unsqueeze(-1).expand_as(inputs_embeds),
+                    audio_features.to(inputs_embeds.device, inputs_embeds.dtype),
+                )
             per_layer_inputs_tokens = input_ids * text_mask.to(dtype=input_ids.dtype)
 
         per_layer_inputs: torch.Tensor
@@ -3154,10 +3868,15 @@ def __init__(
         *,
         weights_dir: str | None = None,
         native_merge_plan: _Gemma4NativeMergePlan | None = None,
+        include_audio: bool = True,
     ):
+        self.include_audio = bool(include_audio)
+        input_names = ("input_ids", "attention_mask", "token_type_ids", "image_features")
+        if self.include_audio:
+            input_names = (*input_names, "audio_features")
         super().__init__(
             model,
-            input_names=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"),
+            input_names=input_names,
             weights_dir=weights_dir,
             native_merge_plan=native_merge_plan,
         )
@@ -3168,7 +3887,7 @@ def forward(
         attention_mask: torch.Tensor | None,
         token_type_ids: torch.Tensor,
         image_features: torch.Tensor,
-        audio_features: torch.Tensor,
+        audio_features: torch.Tensor | None = None,
     ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.LongTensor]:
         return self._prepare_decoder_inputs(
             input_ids=input_ids,
@@ -3182,7 +3901,7 @@ def get_transpile_metadata(self):
         return {
             "graph": self._base_graph_meta(
                 adapter_type=type(self).__name__,
-                input_names=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"),
+                input_names=self.input_names,
             ),
         }
 
@@ -3377,7 +4096,7 @@ def forward(
                 shared_kv_states=shared_kv_states,
             )
             tail_masks = {
-                key: value[:, :, -1:, :] if value.ndim == 4 else value
+                key: value[:, :, -1:, :] if value is not None and value.ndim == 4 else value
                 for key, value in causal_mask_mapping.items()
             }
             tail_per_layer_inputs = (
@@ -3583,18 +4302,31 @@ def _build_gemma4_multimodal_component_specs(
     dynamic_batch: bool = False,
     max_slots: int = 1,
 ) -> list[ComponentModuleSpec]:
-    pixel_values = named_tensors["pixel_values"]
+    pixel_values = named_tensors.get("pixel_values")
     pixel_position_ids = named_tensors.get("pixel_position_ids")
-    input_features = named_tensors["input_features"]
-    input_features_mask = named_tensors["input_features_mask"]
+    input_features = named_tensors.get("input_features")
+    input_features_mask = named_tensors.get("input_features_mask")
     input_ids = named_tensors["input_ids"]
     attention_mask = named_tensors.get("attention_mask")
     token_type_ids = named_tensors["token_type_ids"]
-
-    requested_components = tuple(components or ("vision_encoder", "audio_encoder", "lm_encoder", "decoder"))
+    planning_input_ids = input_ids
+    low_memory_meta = _module_has_meta_tensors(model)
+    multimodal_backbone = getattr(model, "model", model)
+    model_has_audio = len(_gemma4_audio_modules(multimodal_backbone)) >= 2
+
+    has_vision_inputs = pixel_values is not None
+    has_audio_inputs = model_has_audio and input_features is not None and input_features_mask is not None
+    default_components = ["lm_encoder", "decoder"]
+    if has_vision_inputs:
+        default_components.insert(0, "vision_encoder")
+    if has_audio_inputs:
+        insert_at = 1 if has_vision_inputs else 0
+        default_components.insert(insert_at, "audio_encoder")
+    requested_components = tuple(components or tuple(default_components))
     requested_set = set(requested_components)
     if not requested_set:
         return []
+    include_audio = "audio_encoder" in requested_set or (components is None and has_audio_inputs)
 
     expanded_components: list[str] = []
 
@@ -3603,8 +4335,10 @@ def _require(component: str) -> None:
             expanded_components.append(component)
 
     if "decoder" in requested_set:
-        _require("vision_encoder")
-        _require("audio_encoder")
+        if has_vision_inputs or "vision_encoder" in requested_set:
+            _require("vision_encoder")
+        if include_audio:
+            _require("audio_encoder")
         _require("lm_encoder")
         _require("decoder_prefill_chunk")
         _require("decoder_embed_chunk")
@@ -3614,8 +4348,10 @@ def _require(component: str) -> None:
         _require("lm_encoder_media_step")
         _require("decoder_step")
     if "lm_encoder" in requested_set:
-        _require("vision_encoder")
-        _require("audio_encoder")
+        if has_vision_inputs or "vision_encoder" in requested_set:
+            _require("vision_encoder")
+        if include_audio:
+            _require("audio_encoder")
         _require("lm_encoder")
     if "decoder_prefill_chunk" in requested_set:
         _require("decoder_prefill_chunk")
@@ -3635,31 +4371,51 @@ def _require(component: str) -> None:
         _require("lm_encoder_step")
         _require("decoder_step")
 
-    vision_tower = getattr(getattr(model, "model", model), "vision_tower", None)
-    pooling_kernel_size = int(_module_or_config_attr(vision_tower, "pooling_kernel_size", 3) or 3)
-    native_image_soft_token_counts = _gemma4_static_image_soft_token_counts(
-        pixel_position_ids,
-        pooling_kernel_size=pooling_kernel_size,
-    )
-    native_image_pool_shapes = _gemma4_static_image_pool_shapes(
-        pixel_position_ids,
-        pooling_kernel_size=pooling_kernel_size,
-    )
+    if "vision_encoder" in expanded_components and pixel_values is None:
+        raise RuntimeError("Gemma4 vision component requested but pixel_values were not prepared")
+    if "audio_encoder" in expanded_components and (input_features is None or input_features_mask is None):
+        raise RuntimeError("Gemma4 audio component requested but input_features/input_features_mask were not prepared")
 
-    vision_encoder = Gemma4VisionEncoderAdapter(
-        model,
-        weights_dir=weights_dir,
-        native_image_soft_token_counts=native_image_soft_token_counts,
-        native_image_pool_shapes=native_image_pool_shapes,
-    ).eval()
-    vision_encoder_npu = Gemma4VisionEncoderAdapter(
-        model,
-        weights_dir=weights_dir,
-        native_image_soft_token_counts=native_image_soft_token_counts,
-        native_image_pool_shapes=None,
-    ).eval()
-    _gemma4_make_audio_mask_fp16_safe(model)
-    audio_encoder = Gemma4AudioEncoderAdapter(model, weights_dir=weights_dir).eval()
+    vision_encoder: Gemma4VisionEncoderAdapter | None = None
+    vision_encoder_npu: Gemma4VisionEncoderAdapter | None = None
+    audio_encoder: Gemma4AudioEncoderAdapter | None = None
+    native_image_soft_token_counts: tuple[int, ...] | None = None
+    native_image_pool_shapes: tuple[tuple[int, int, int], ...] | None = None
+    if "vision_encoder" in expanded_components:
+        vision_tower = getattr(getattr(model, "model", model), "vision_tower", None)
+        pooling_kernel_size = int(_module_or_config_attr(vision_tower, "pooling_kernel_size", 3) or 3)
+        native_image_soft_token_counts = _gemma4_static_image_soft_token_counts(
+            pixel_position_ids,
+            pooling_kernel_size=pooling_kernel_size,
+        )
+        native_image_pool_shapes = _gemma4_static_image_pool_shapes(
+            pixel_position_ids,
+            pooling_kernel_size=pooling_kernel_size,
+        )
+        vision_encoder = Gemma4VisionEncoderAdapter(
+            model,
+            weights_dir=weights_dir,
+            native_image_soft_token_counts=native_image_soft_token_counts,
+            native_image_pool_shapes=native_image_pool_shapes,
+        ).eval()
+        vision_encoder_npu = Gemma4VisionEncoderAdapter(
+            model,
+            weights_dir=weights_dir,
+            native_image_soft_token_counts=native_image_soft_token_counts,
+            native_image_pool_shapes=None,
+        ).eval()
+    if "audio_encoder" in expanded_components:
+        _gemma4_make_audio_mask_fp16_safe(model)
+        audio_encoder = Gemma4AudioEncoderAdapter(model, weights_dir=weights_dir).eval()
+
+    if low_memory_meta:
+        pixel_values = _to_meta_example_tensor(pixel_values)
+        pixel_position_ids = _to_meta_example_tensor(pixel_position_ids)
+        input_features = _to_meta_example_tensor(input_features)
+        input_features_mask = _to_meta_example_tensor(input_features_mask)
+        input_ids = _to_meta_example_tensor(input_ids)
+        attention_mask = _to_meta_example_tensor(attention_mask)
+        token_type_ids = _to_meta_example_tensor(token_type_ids)
 
     image_features: torch.Tensor | None = None
     audio_features: torch.Tensor | None = None
@@ -3676,27 +4432,33 @@ def _require(component: str) -> None:
             "lm_encoder" in expanded_components
             or "decoder" in expanded_components
         ):
+            if vision_encoder is None or pixel_values is None:
+                raise RuntimeError("Gemma4 vision features requested but vision encoder inputs are unavailable")
             image_features = vision_encoder(pixel_values, pixel_position_ids)
         if "audio_encoder" in expanded_components and (
             "lm_encoder" in expanded_components
             or "decoder" in expanded_components
         ):
+            if audio_encoder is None or input_features is None or input_features_mask is None:
+                raise RuntimeError("Gemma4 audio features requested but audio encoder inputs are unavailable")
             audio_features = audio_encoder(input_features, input_features_mask)
         if "lm_encoder" in expanded_components or "decoder" in expanded_components:
-            if image_features is None:
+            if image_features is None and vision_encoder is not None and pixel_values is not None:
                 image_features = vision_encoder(pixel_values, pixel_position_ids)
-            if audio_features is None:
+            if include_audio and audio_features is None and audio_encoder is not None and input_features is not None and input_features_mask is not None:
                 audio_features = audio_encoder(input_features, input_features_mask)
-            native_merge_plan = _gemma4_build_native_merge_plan(
-                getattr(model, "model", model),
-                input_ids,
-                image_feature_count=_gemma4_feature_token_count(image_features),
-                audio_feature_count=_gemma4_feature_token_count(audio_features),
-            )
+            if image_features is not None:
+                native_merge_plan = _gemma4_build_native_merge_plan(
+                    getattr(model, "model", model),
+                    planning_input_ids,
+                    image_feature_count=_gemma4_feature_token_count(image_features),
+                    audio_feature_count=_gemma4_feature_token_count(audio_features),
+                )
     lm_encoder = Gemma4LMEncoderAdapter(
         model,
         weights_dir=weights_dir,
         native_merge_plan=native_merge_plan,
+        include_audio=include_audio,
     ).eval()
     encoder_chunk_size = max(1, int(os.environ.get("CACTUS_GEMMA4_ENCODER_CHUNK", "128") or "128"))
     encoder_chunk_size = min(encoder_chunk_size, int(input_ids.shape[1]))
@@ -3711,13 +4473,27 @@ def _require(component: str) -> None:
 
     with torch.no_grad():
         if "decoder" in expanded_components:
-            decoder_inputs = lm_encoder(
-                input_ids,
-                attention_mask,
-                token_type_ids,
-                image_features,
-                audio_features,
-            )
+            if image_features is None:
+                raise RuntimeError("Gemma4 decoder spec requires precomputed image features")
+            if include_audio and audio_features is None:
+                raise RuntimeError("Gemma4 decoder spec requires precomputed audio features")
+            lm_encoder_inputs: tuple[torch.Tensor | None, ...]
+            if include_audio:
+                lm_encoder_inputs = (
+                    input_ids,
+                    attention_mask,
+                    token_type_ids,
+                    image_features,
+                    audio_features,
+                )
+            else:
+                lm_encoder_inputs = (
+                    input_ids,
+                    attention_mask,
+                    token_type_ids,
+                    image_features,
+                )
+            decoder_inputs = lm_encoder(*lm_encoder_inputs)
         if "lm_encoder_step" in expanded_components or "decoder_step" in expanded_components:
             step_input_ids = input_ids[:, -1:].contiguous()
             step_position_ids = torch.full(
@@ -3728,10 +4504,12 @@ def _require(component: str) -> None:
             )
             decoder_step_inputs = lm_encoder_step(step_input_ids, step_position_ids)
         if "lm_encoder_media_step" in expanded_components:
-            if audio_features is not None and int(audio_features.shape[1]) > 0:
-                media_step_embeds = audio_features[:, :1, :].contiguous()
-            elif image_features is not None and int(image_features.shape[1]) > 0:
-                media_step_embeds = image_features[:, :1, :].contiguous()
+            if audio_features is not None and _gemma4_feature_token_count(audio_features) > 0:
+                media_source = audio_features if audio_features.ndim == 3 else audio_features.unsqueeze(0)
+                media_step_embeds = media_source[:, :1, :].contiguous()
+            elif image_features is not None and _gemma4_feature_token_count(image_features) > 0:
+                media_source = image_features if image_features.ndim == 3 else image_features.unsqueeze(0)
+                media_step_embeds = media_source[:, :1, :].contiguous()
             else:
                 text_config = _gemma4_text_config(getattr(model, "model", model))
                 hidden_size = int(getattr(text_config, "hidden_size", 0) or 0)
@@ -3778,6 +4556,8 @@ def _require(component: str) -> None:
     }
     specs: list[ComponentModuleSpec] = []
     if "vision_encoder" in expanded_components:
+        if vision_encoder is None or vision_encoder_npu is None or pixel_values is None:
+            raise RuntimeError("Gemma4 vision_encoder spec requires prepared pixel_values")
         specs.append(ComponentModuleSpec(
             component="vision_encoder",
             module=vision_encoder,
@@ -3791,6 +4571,8 @@ def _require(component: str) -> None:
             npu_reparam=_gemma4_npu_projection_reparam,
         ))
     if "audio_encoder" in expanded_components:
+        if audio_encoder is None or input_features is None or input_features_mask is None:
+            raise RuntimeError("Gemma4 audio_encoder spec requires prepared audio features")
         specs.append(ComponentModuleSpec(
             component="audio_encoder",
             module=audio_encoder,
@@ -3802,13 +4584,22 @@ def _require(component: str) -> None:
             npu_reparam=_gemma4_npu_projection_reparam,
         ))
     if "lm_encoder" in expanded_components:
-        if image_features is None or audio_features is None:
-            raise RuntimeError("Gemma4 lm_encoder spec requires precomputed image/audio features")
+        if image_features is None:
+            raise RuntimeError("Gemma4 lm_encoder spec requires precomputed image features")
+        if include_audio and audio_features is None:
+            raise RuntimeError("Gemma4 lm_encoder spec requires precomputed audio features")
+        lm_encoder_example_inputs: tuple[torch.Tensor | None, ...]
+        lm_encoder_input_keys = ("input_ids", "attention_mask", "token_type_ids", "image_features")
+        if include_audio:
+            lm_encoder_example_inputs = (input_ids, attention_mask, token_type_ids, image_features, audio_features)
+            lm_encoder_input_keys = (*lm_encoder_input_keys, "audio_features")
+        else:
+            lm_encoder_example_inputs = (input_ids, attention_mask, token_type_ids, image_features)
         specs.append(ComponentModuleSpec(
             component="lm_encoder",
             module=lm_encoder,
-            example_inputs=(input_ids, attention_mask, token_type_ids, image_features, audio_features),
-            input_keys=("input_ids", "attention_mask", "token_type_ids", "image_features", "audio_features"),
+            example_inputs=lm_encoder_example_inputs,
+            input_keys=lm_encoder_input_keys,
             output_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS,
             graph_meta={**common_graph_meta, "component": "lm_encoder"},
             metadata={"family": "gemma4", "task": "multimodal_causal_lm_logits"},
@@ -4421,6 +5212,7 @@ def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None):
         self.model = model
         self.backbone = model.model
         self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model)
+        self.adapter_family = _family_key(model)
 
     def forward(self, input_ids: torch.Tensor):
         return self.debug_forward(input_ids)[0]
@@ -4482,7 +5274,7 @@ def get_transpile_metadata(self):
             "graph": {
                 **_transpile_graph_meta(
                     self.model,
-                    adapter_family="qwen3",
+                    adapter_family=self.adapter_family,
                     adapter_type=type(self).__name__,
                     input_names=("input_ids",),
                 ),
@@ -4499,6 +5291,7 @@ def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None):
         self.model = model
         self.backbone = model.model
         self.pad_token_id = pad_token_id if pad_token_id is not None else _resolve_model_pad_token_id(model)
+        self.adapter_family = _family_key(model)
 
     def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor:
         inputs_embeds = self.backbone.embed_tokens(input_ids)
@@ -4543,7 +5336,7 @@ def get_transpile_metadata(self):
             "graph": {
                 **_transpile_graph_meta(
                     self.model,
-                    adapter_family="qwen3",
+                    adapter_family=self.adapter_family,
                     adapter_type=type(self).__name__,
                     input_names=("input_ids", "position_ids"),
                 ),
@@ -4554,6 +5347,18 @@ def get_transpile_metadata(self):
         }
 
 
+class Qwen2MoeCausalLMLogitsAdapter(Qwen3CausalLMLogitsAdapter):
+    def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None):
+        _patch_qwen2_moe_mlps(model)
+        super().__init__(model, pad_token_id=pad_token_id)
+
+
+class Qwen2MoeCausalLMStepAdapter(Qwen3CausalLMStepAdapter):
+    def __init__(self, model: torch.nn.Module, *, pad_token_id: int | None = None):
+        _patch_qwen2_moe_mlps(model)
+        super().__init__(model, pad_token_id=pad_token_id)
+
+
 class Qwen3LMEncoderStepAdapter(torch.nn.Module):
     def __init__(self, model: torch.nn.Module):
         super().__init__()
@@ -4667,6 +5472,9 @@ def _build_qwen_causal_lm_component_specs(
     if family == "qwen3_5":
         decoder = Qwen35CausalLMLogitsAdapter(model, pad_token_id=pad_token_id).eval()
         decoder_step = Qwen35CausalLMStepAdapter(model, pad_token_id=pad_token_id).eval()
+    elif family == "qwen2_moe":
+        decoder = Qwen2MoeCausalLMLogitsAdapter(model, pad_token_id=pad_token_id).eval()
+        decoder_step = Qwen2MoeCausalLMStepAdapter(model, pad_token_id=pad_token_id).eval()
     elif family == "qwen3":
         decoder = Qwen3CausalLMLogitsAdapter(model, pad_token_id=pad_token_id).eval()
         decoder_step = Qwen3CausalLMStepAdapter(model, pad_token_id=pad_token_id).eval()
@@ -5400,6 +6208,8 @@ def _family_key(model: torch.nn.Module) -> str:
         return "qwen3_5"
     if module_name.startswith("transformers.models.qwen3_vl."):
         return "qwen3_5"
+    if module_name.startswith("transformers.models.qwen2_moe."):
+        return "qwen2_moe"
     if module_name.startswith("transformers.models.qwen3."):
         return "qwen3"
     if module_name.startswith("transformers.models.lfm2_vl."):
@@ -5411,6 +6221,8 @@ def _family_key(model: torch.nn.Module) -> str:
     model_type = str(getattr(getattr(model, "config", None), "model_type", "") or "").lower()
     if model_type == "needle":
         return "needle"
+    if model_type == "qwen2_moe":
+        return "qwen2_moe"
     if "nomic" in module_name.lower() or "nomic" in model_type:
         return "nomic"
     return "generic"
@@ -5759,12 +6571,13 @@ def _build_lfm2_causal_lm_component_specs(
         return None
     requested = tuple(components or ("decoder", "decoder_step"))
     requested_set = set(requested)
+    family = _family_key(model)
     common_graph_meta = {
         "weights_dir": weights_dir,
         "task": "causal_lm_logits",
-        "adapter_family": "lfm2",
+        "adapter_family": family,
     }
-    metadata = {"family": "lfm2", "task": "causal_lm_logits"}
+    metadata = {"family": family, "task": "causal_lm_logits"}
     chunk_components = {"lm_encoder_step", "lm_encoder_text_chunk", "decoder_prefill_chunk"}
     if chunk_components & requested_set:
         return _lfm2_chunked_pipeline_specs(
@@ -6133,7 +6946,7 @@ def build_component_module_specs(
             components=components,
             cache_context_length=cache_context_length,
         )
-    if family in {"qwen3", "qwen3_5"} and task == "causal_lm_logits":
+    if family in {"qwen2_moe", "qwen3", "qwen3_5"} and task == "causal_lm_logits":
         return _build_qwen_causal_lm_component_specs(
             model,
             named_tensors=named_tensors,
@@ -6167,7 +6980,7 @@ def build_component_module_specs(
             components=components,
             cache_context_length=cache_context_length,
         )
-    if family == "lfm2" and task == "causal_lm_logits":
+    if family in {"lfm2", "lfm2_moe"} and task == "causal_lm_logits":
         return _build_lfm2_causal_lm_component_specs(
             model,
             named_tensors=named_tensors,
@@ -6243,6 +7056,11 @@ def canonicalize_model_interface(
                 inner_model,
                 pad_token_id=padding_token_id_value,
             )
+        elif family == "qwen2_moe":
+            adapter_factory = lambda inner_model: Qwen2MoeCausalLMLogitsAdapter(  # type: ignore[assignment]
+                inner_model,
+                pad_token_id=padding_token_id_value,
+            )
         elif family == "qwen3":
             adapter_factory = lambda inner_model: Qwen3CausalLMLogitsAdapter(  # type: ignore[assignment]
                 inner_model,
diff --git a/python/cactus/transpile/model_profiles.py b/python/cactus/transpile/model_profiles.py
index 75580eb47..7d132182b 100644
--- a/python/cactus/transpile/model_profiles.py
+++ b/python/cactus/transpile/model_profiles.py
@@ -74,7 +74,7 @@ class ModelProfile:
 
 LFM2_PROFILE = ModelProfile(
     family="lfm2",
-    model_types=("lfm2_vl", "lfm2"),
+    model_types=("lfm2_vl", "lfm2", "lfm2_moe"),
     multimodal_context_tokens=256,
     input_combinations=((), ("image",)),
     model_id_aliases=(("lfm", "LiquidAI/LFM2-VL-450M"),),
@@ -94,6 +94,8 @@ class ModelProfile:
     text_component_plans=(
         ("lfm2forcausallm", ("decoder_step", "lm_encoder_step", "lm_encoder_text_chunk",
                              "decoder_prefill_chunk")),
+        ("lfm2moeforcausallm", ("decoder_step", "lm_encoder_step", "lm_encoder_text_chunk",
+                                "decoder_prefill_chunk")),
     ),
     needs_image=True,
     force_component_pipeline=True,
@@ -172,7 +174,7 @@ class ModelProfile:
 
 QWEN_PROFILE = ModelProfile(
     family="qwen",
-    model_types=("qwen", "qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"),
+    model_types=("qwen", "qwen2", "qwen2_moe", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"),
     multimodal_context_tokens=512,
     input_combinations=((), ("image",)),
     model_id_aliases=(
@@ -183,7 +185,7 @@ class ModelProfile:
         ("qwen3", "Qwen/Qwen3-1.7B"),
     ),
     model_id_markers=("qwen",),
-    family_aliases=("qwen2", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"),
+    family_aliases=("qwen2", "qwen2_moe", "qwen3", "qwen3_5", "qwen3.5", "qwen3_vl"),
     stop_tokens=("<|im_end|>",),
     avoid_native_loader=True,
     cached_step_components=("decoder_media_step",),
diff --git a/python/cactus/transpile/ops.py b/python/cactus/transpile/ops.py
index e1a870f8b..5e3ee5436 100644
--- a/python/cactus/transpile/ops.py
+++ b/python/cactus/transpile/ops.py
@@ -107,6 +107,43 @@ class OpSchema:
 
     # CUSTOM / FUSED OPS
     OpSchema("dense_mlp_tq_fused", num_inputs=4, attrs=("product_scale",), backend_op="dense_mlp_tq_fused"),
+    OpSchema(
+        "lfm2_moe_layer_gated",
+        num_inputs=None,
+        attrs=(
+            "num_experts",
+            "num_experts_per_tok",
+            "use_expert_bias",
+            "normalize_routing",
+            "epsilon",
+            "routed_scaling_factor",
+            "activation",
+        ),
+    ),
+    OpSchema(
+        "qwen2_moe_layer_gated",
+        num_inputs=None,
+        attrs=(
+            "num_experts",
+            "num_experts_per_tok",
+            "normalize_routing",
+            "epsilon",
+            "routed_scaling_factor",
+            "activation",
+        ),
+    ),
+    OpSchema(
+        "gemma4_moe_layer_gated",
+        num_inputs=None,
+        attrs=(
+            "num_experts",
+            "num_experts_per_tok",
+            "normalize_routing",
+            "epsilon",
+            "routed_scaling_factor",
+            "activation",
+        ),
+    ),
     OpSchema("rms_norm", num_inputs=2, attrs=("eps",), backend_op="rms_norm"),
     OpSchema("rope", num_inputs=1, attrs=("theta", "position_offset"), backend_op="rope"),
     OpSchema("rel_pos_bias", num_inputs=2, attrs=("scale",), backend_op="rel_pos_bias"),
diff --git a/python/cactus/transpile/optimize_graph.py b/python/cactus/transpile/optimize_graph.py
index ea2714d0d..e66d80881 100644
--- a/python/cactus/transpile/optimize_graph.py
+++ b/python/cactus/transpile/optimize_graph.py
@@ -394,7 +394,10 @@ def precompute_rope_tables(graph: IRGraph) -> bool:
             continue
         inv_freq_value_id, position_value_id = match
 
-        inv_freq = graph.constants[inv_freq_value_id].detach().cpu().to(torch.float64).reshape(-1)
+        inv_freq_constant = graph.constants[inv_freq_value_id]
+        if isinstance(inv_freq_constant, torch.Tensor) and inv_freq_constant.is_meta:
+            continue
+        inv_freq = inv_freq_constant.detach().cpu().to(torch.float64).reshape(-1)
         head_dim = int(inv_freq.numel()) * 2
         max_seq = _rope_table_max_seq(graph)
         positions = torch.arange(max_seq, dtype=torch.float64).reshape(max_seq, 1)
diff --git a/python/cactus/transpile/weight_binding.py b/python/cactus/transpile/weight_binding.py
index 68cd5992f..a82f14c6a 100644
--- a/python/cactus/transpile/weight_binding.py
+++ b/python/cactus/transpile/weight_binding.py
@@ -1,9 +1,11 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from functools import lru_cache
 import json
 import os
 from pathlib import Path
+import re
 from typing import Any
 
 
@@ -211,6 +213,31 @@ def _add_text_backbone_aliases(candidate: str) -> None:
         _add_text_backbone_aliases(candidate)
         _add_multimodal_backbone_aliases(candidate)
 
+    for candidate in tuple(aliases):
+        normalized = candidate
+        for prefix in (
+            "module.model.backbone.",
+            "adapter.model.backbone.",
+            "module.backbone.",
+            "adapter.backbone.",
+            "backbone.",
+            "module.model.",
+            "adapter.model.",
+            "module.",
+            "adapter.",
+        ):
+            if normalized.startswith(prefix):
+                normalized = normalized[len(prefix) :]
+                break
+        match = re.match(
+            r"^(?:model\.)?layers\.(\d+)\.feed_forward\.(w[123])_weights\.(\d+)$",
+            normalized,
+        )
+        if match is not None:
+            layer, weight_name, expert = match.groups()
+            _add_with_wrappers(f"model.layers.{layer}.feed_forward.experts.{expert}.{weight_name}.weight")
+            _add_with_wrappers(f"layers.{layer}.feed_forward.experts.{expert}.{weight_name}.weight")
+
     return tuple(aliases)
 
 
@@ -270,8 +297,15 @@ def _flatten_convert_manifest(root_manifest: dict[str, Any]) -> dict[str, object
     return flattened
 
 
-def _load_weights_manifest(root: Path) -> dict[str, object]:
-    manifest_path = root / "weights_manifest.json"
+@lru_cache(maxsize=16)
+def _load_weights_manifest_cached(
+    manifest_path_text: str,
+    *,
+    mtime_ns: int,
+    size: int,
+) -> dict[str, object]:
+    del mtime_ns, size
+    manifest_path = Path(manifest_path_text)
     if not manifest_path.exists():
         return {}
     try:
@@ -283,6 +317,18 @@ def _load_weights_manifest(root: Path) -> dict[str, object]:
     return _flatten_convert_manifest(loaded_manifest)
 
 
+def _load_weights_manifest(root: Path) -> dict[str, object]:
+    manifest_path = root / "weights_manifest.json"
+    if not manifest_path.exists():
+        return {}
+    stat = manifest_path.stat()
+    return _load_weights_manifest_cached(
+        str(manifest_path.resolve()),
+        mtime_ns=int(stat.st_mtime_ns),
+        size=int(stat.st_size),
+    )
+
+
 def _binding_from_manifest_entry(root: Path, source_name: str, entry: object) -> WeightBinding | None:
     if not isinstance(entry, dict):
         return None
diff --git a/python/cactus/transpile/weight_compat.py b/python/cactus/transpile/weight_compat.py
index dd9b02b65..61521890e 100644
--- a/python/cactus/transpile/weight_compat.py
+++ b/python/cactus/transpile/weight_compat.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import replace
+import json
 import shutil
 import struct
 from pathlib import Path
@@ -166,6 +167,8 @@ def _is_legacy_packed_int4_tensor(opened: _OpenedTensor) -> bool:
 def _can_materialize_compat_weight(source_tensor: object | None) -> bool:
     if source_tensor is None:
         return False
+    if getattr(source_tensor, "is_meta", False):
+        return False
     if isinstance(source_tensor, (str, bytes, bytearray, Path)):
         return False
     if np.isscalar(source_tensor):
@@ -174,6 +177,24 @@ def _can_materialize_compat_weight(source_tensor: object | None) -> bool:
 
 
 def _compat_weight_scale_factor(source_path: Path) -> float:
+    manifest_path = source_path.parent / "conversion_manifest.json"
+    if manifest_path.exists():
+        try:
+            rows = json.loads(manifest_path.read_text(encoding="utf-8"))
+        except (OSError, ValueError) as exc:
+            raise ValueError(f"unreadable conversion_manifest.json at {manifest_path}: {exc}") from exc
+        if isinstance(rows, list):
+            for row in rows:
+                if isinstance(row, dict) and row.get("output_file") == source_path.name:
+                    scale = row.get("scale_factor")
+                    if scale is None:
+                        return 1.0
+                    try:
+                        return float(scale)
+                    except (TypeError, ValueError) as exc:
+                        raise ValueError(
+                            f"non-numeric scale_factor {scale!r} for {source_path.name} in {manifest_path}"
+                        ) from exc
     parent_name = source_path.parent.name.lower()
     if "gemma-4" in parent_name or "gemma4" in parent_name:
         return gemma4_scale_factor(source_path.name)
diff --git a/python/tests/test_cli_transpile_defaults.py b/python/tests/test_cli_transpile_defaults.py
index 592e029fd..1a9f6c52f 100644
--- a/python/tests/test_cli_transpile_defaults.py
+++ b/python/tests/test_cli_transpile_defaults.py
@@ -144,6 +144,19 @@ def test_cli_convert_absorbs_graph_flags() -> None:
         parser.parse_args(["transpile", "google/gemma-4-E2B-it"])
 
 
+def test_cli_registers_low_memory_transpile_flag() -> None:
+    parser = cli.create_parser()
+    args = parser.parse_args([
+        "convert",
+        "Qwen/Qwen3-0.6B",
+        "--weights-dir",
+        "/tmp/x",
+        "--low-memory-load",
+    ])
+
+    assert args.low_memory_load is True
+
+
 def test_run_accepts_local_bundle_path() -> None:
     """`cactus run` accepts a HF id (org/model) OR a local path. Bare names
     like 'whisper-base' (no slash) are rejected."""
diff --git a/python/tests/test_component_plan.py b/python/tests/test_component_plan.py
new file mode 100644
index 000000000..4bd7bbb73
--- /dev/null
+++ b/python/tests/test_component_plan.py
@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from cactus.transpile.component_plan import infer_component_plan_from_config
+
+
+def test_gemma4_vision_only_config_does_not_request_audio_component() -> None:
+    config = {
+        "model_type": "gemma4",
+        "architectures": ["Gemma4ForConditionalGeneration"],
+        "vision_config": {"hidden_size": 1152},
+        "text_config": {"num_hidden_layers": 30},
+    }
+
+    plan = infer_component_plan_from_config(config, model_id="google/gemma-4-26B-A4B")
+
+    assert plan is not None
+    assert plan.task == "multimodal_causal_lm_logits"
+    assert plan.components == ("vision_encoder", "lm_encoder", "decoder")
+    assert plan.needs_image
+    assert not plan.needs_audio
+
+
+def test_gemma4_image_audio_config_keeps_audio_component() -> None:
+    config = {
+        "model_type": "gemma4",
+        "architectures": ["Gemma4ForConditionalGeneration"],
+        "vision_config": {"hidden_size": 1152},
+        "audio_config": {"hidden_size": 1024},
+        "text_config": {"num_hidden_layers": 30},
+    }
+
+    plan = infer_component_plan_from_config(config, model_id="google/gemma-4-E2B-it")
+
+    assert plan is not None
+    assert plan.components == ("vision_encoder", "audio_encoder", "lm_encoder", "decoder")
+    assert plan.needs_audio
diff --git a/python/tests/test_graph.py b/python/tests/test_graph.py
index c8b05f88e..59d41e414 100644
--- a/python/tests/test_graph.py
+++ b/python/tests/test_graph.py
@@ -112,6 +112,33 @@ def test_composed_inference_graph(self):
 
 class TestGraphTensorOps(unittest.TestCase):
 
+    def test_topk_from_fp16_outputs_fp32_indices_after_roundtrip(self):
+        g = Graph()
+        a = g.input((1, 5))
+        topk = g.topk(a, 3)
+        indices = g.index(topk, 0, axis=0)
+
+        data = np.array([[0.1, 4.0, 2.0, 5.0, 3.0]], dtype=np.float16)
+        g.set_input(a, data)
+        g.execute()
+
+        self.assertEqual(g.output_info(topk)["precision"], Graph.FP32)
+        self.assertEqual(g.output_info(indices)["precision"], Graph.FP32)
+        np.testing.assert_allclose(indices.numpy(), np.array([[3.0, 1.0, 4.0]], dtype=np.float32))
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = Path(tmpdir) / "topk_graph.cg"
+            g.save(path)
+
+            loaded = Graph.load(path)
+            loaded_a = Tensor(loaded, a.id, a.shape, a.dtype)
+            loaded_indices = Tensor(loaded, indices.id, indices.shape, indices.dtype)
+            loaded.set_input(loaded_a, data)
+            loaded.execute()
+
+            self.assertEqual(loaded.output_info(loaded_indices)["precision"], Graph.FP32)
+            np.testing.assert_allclose(loaded_indices.numpy(), np.array([[3.0, 1.0, 4.0]], dtype=np.float32))
+
     def test_view(self):
         g = Graph()
         a = g.input((2, 3))
diff --git a/python/tests/test_transpile_aten_ops.py b/python/tests/test_transpile_aten_ops.py
index b0f0ad7f4..d10c785bd 100644
--- a/python/tests/test_transpile_aten_ops.py
+++ b/python/tests/test_transpile_aten_ops.py
@@ -122,3 +122,151 @@ def test_dense_mlp_fusion_uses_topology_not_layer_names() -> None:
     assert fuse_dense_mlp_tq(graph) is True
     assert graph.nodes["down_linear"].op == "dense_mlp_tq_fused"
     assert graph.nodes["down_linear"].inputs == ["x", "pear_weight", "banana_weight", "plum_weight"]
+
+
+
+
+
+
+def _ensure_test_lfm2_moe_op_registered() -> None:
+    from cactus.transpile.model_adapters import _ensure_lfm2_moe_custom_op_registered
+    _ensure_lfm2_moe_custom_op_registered()
+
+
+def _ensure_test_qwen2_moe_op_registered() -> None:
+    from cactus.transpile.model_adapters import _ensure_qwen2_moe_custom_op_registered
+    _ensure_qwen2_moe_custom_op_registered()
+
+
+def _ensure_test_gemma4_moe_op_registered() -> None:
+    from cactus.transpile.model_adapters import _ensure_gemma4_moe_custom_op_registered
+    _ensure_gemma4_moe_custom_op_registered()
+
+
+class _FakeLfm2MoeBlock(torch.nn.Module):
+    def __init__(self) -> None:
+        super().__init__()
+        self.gate = torch.nn.Linear(4, 2, bias=False)
+        self.w1 = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(3, 4)) for _ in range(2)])
+        self.w3 = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(3, 4)) for _ in range(2)])
+        self.w2 = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(4, 3)) for _ in range(2)])
+        self.register_buffer("expert_bias", torch.zeros(2, dtype=torch.float32))
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_flat = hidden_states.reshape(-1, hidden_states.shape[-1])
+        return torch.ops.cactus_transpile.lfm2_moe_layer_gated(
+            hidden_flat,
+            self.gate(hidden_flat),
+            self.expert_bias,
+            list(self.w1),
+            list(self.w3),
+            list(self.w2),
+            2,
+            1,
+            True,
+            True,
+            1.0e-6,
+            1.0,
+        ).reshape(hidden_states.shape)
+
+
+class _FakeQwen2MoeBlock(torch.nn.Module):
+    def __init__(self) -> None:
+        super().__init__()
+        self.gate = torch.nn.Linear(4, 2, bias=False)
+        self.w1 = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(3, 4)) for _ in range(2)])
+        self.w3 = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(3, 4)) for _ in range(2)])
+        self.w2 = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(4, 3)) for _ in range(2)])
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_flat = hidden_states.reshape(-1, hidden_states.shape[-1])
+        return torch.ops.cactus_transpile.qwen2_moe_layer_gated(
+            hidden_flat,
+            self.gate(hidden_flat),
+            list(self.w1),
+            list(self.w3),
+            list(self.w2),
+            2,
+            1,
+            False,
+            1.0e-6,
+            1.0,
+        ).reshape(hidden_states.shape)
+
+
+class _FakeGemma4MoeBlock(torch.nn.Module):
+    def __init__(self) -> None:
+        super().__init__()
+        self.router = torch.nn.Linear(4, 2, bias=False)
+        self.w1 = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(3, 4)) for _ in range(2)])
+        self.w3 = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(3, 4)) for _ in range(2)])
+        self.w2 = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(4, 3)) for _ in range(2)])
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_flat = hidden_states.reshape(-1, hidden_states.shape[-1])
+        return torch.ops.cactus_transpile.gemma4_moe_layer_gated(
+            hidden_flat,
+            self.router(hidden_flat),
+            list(self.w1),
+            list(self.w3),
+            list(self.w2),
+            2,
+            1,
+            True,
+            1.0e-6,
+            1.0,
+        ).reshape(hidden_states.shape)
+
+
+def test_lfm2_moe_runtime_block_exports_single_semantic_op() -> None:
+    _ensure_test_lfm2_moe_op_registered()
+    block = _FakeLfm2MoeBlock().eval()
+    captured = capture_model(block, (torch.randn(1, 2, 4),), strict=False)
+
+    moe_nodes = [
+        captured.ir_graph.nodes[node_id]
+        for node_id in captured.ir_graph.order
+        if captured.ir_graph.nodes[node_id].op == "lfm2_moe_layer_gated"
+    ]
+
+    assert len(moe_nodes) == 1
+    assert moe_nodes[0].attrs["num_experts"] == 2
+    assert moe_nodes[0].attrs["num_experts_per_tok"] == 1
+    assert moe_nodes[0].attrs["use_expert_bias"] is True
+    assert len(moe_nodes[0].inputs) == 3 + 3 * 2
+
+
+def test_qwen2_moe_runtime_block_exports_single_semantic_op() -> None:
+    _ensure_test_qwen2_moe_op_registered()
+    block = _FakeQwen2MoeBlock().eval()
+    captured = capture_model(block, (torch.randn(1, 2, 4),), strict=False)
+
+    moe_nodes = [
+        captured.ir_graph.nodes[node_id]
+        for node_id in captured.ir_graph.order
+        if captured.ir_graph.nodes[node_id].op == "qwen2_moe_layer_gated"
+    ]
+
+    assert len(moe_nodes) == 1
+    assert moe_nodes[0].attrs["num_experts"] == 2
+    assert moe_nodes[0].attrs["num_experts_per_tok"] == 1
+    assert moe_nodes[0].attrs["normalize_routing"] is False
+    assert len(moe_nodes[0].inputs) == 2 + 3 * 2
+
+
+def test_gemma4_moe_runtime_block_exports_single_semantic_op() -> None:
+    _ensure_test_gemma4_moe_op_registered()
+    block = _FakeGemma4MoeBlock().eval()
+    captured = capture_model(block, (torch.randn(1, 2, 4),), strict=False)
+
+    moe_nodes = [
+        captured.ir_graph.nodes[node_id]
+        for node_id in captured.ir_graph.order
+        if captured.ir_graph.nodes[node_id].op == "gemma4_moe_layer_gated"
+    ]
+
+    assert len(moe_nodes) == 1
+    assert moe_nodes[0].attrs["num_experts"] == 2
+    assert moe_nodes[0].attrs["num_experts_per_tok"] == 1
+    assert moe_nodes[0].attrs["normalize_routing"] is True
+    assert len(moe_nodes[0].inputs) == 2 + 3 * 2
diff --git a/python/tests/test_transpile_importers.py b/python/tests/test_transpile_importers.py
new file mode 100644
index 000000000..cc3f53de9
--- /dev/null
+++ b/python/tests/test_transpile_importers.py
@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import torch
+
+from cactus.transpile.graph_ir import IRGraph
+from cactus.transpile.importers import ImportContext
+from cactus.transpile.importers import import_get_attr
+
+
+def test_import_get_attr_preserves_meta_tensor_constant() -> None:
+    ir = IRGraph(values={}, nodes={}, order=[], inputs=[], outputs=[])
+    ctx = ImportContext()
+    node = SimpleNamespace(name="p_weight", op="get_attr")
+    value = torch.empty((3, 4), device="meta", dtype=torch.float16)
+
+    import_get_attr(
+        ir,
+        node,
+        ctx,
+        value,
+        shape=(3, 4),
+        dtype="float16",
+        source_name="linear.weight",
+    )
+
+    constant = ir.constants["v_p_weight"]
+    assert isinstance(constant, torch.Tensor)
+    assert constant.is_meta
+    assert tuple(constant.shape) == (3, 4)
+    assert ir.values["v_p_weight"].meta["source_name"] == "linear.weight"
diff --git a/python/tests/test_transpile_weight_compat.py b/python/tests/test_transpile_weight_compat.py
index 29fc9c8f1..89235b988 100644
--- a/python/tests/test_transpile_weight_compat.py
+++ b/python/tests/test_transpile_weight_compat.py
@@ -259,6 +259,31 @@ def test_resolve_weight_binding_expands_language_model_backbone_aliases(tmp_path
     assert binding.path.endswith("layer_0_scalar.weights")
 
 
+def test_resolve_weight_binding_expands_lfm2_moe_runtime_expert_aliases(tmp_path: Path) -> None:
+    (tmp_path / "layer_2_moe_expert_0_w1.weights").write_bytes(b"")
+    (tmp_path / "weights_manifest.json").write_text(
+        json.dumps(
+            {
+                "weights": [
+                    {
+                        "source_name": "model.layers.2.feed_forward.experts.0.w1.weight",
+                        "output_name": "layer_2_moe_expert_0_w1.weights",
+                        "component": "language",
+                    }
+                ]
+            }
+        )
+    )
+
+    binding = resolve_weight_binding(
+        weights_dir=str(tmp_path),
+        source_name="module.backbone.layers.2.feed_forward.w1_weights.0",
+    )
+
+    assert binding is not None
+    assert binding.path.endswith("layer_2_moe_expert_0_w1.weights")
+
+
 def test_resolve_weight_binding_maps_module_backbone_to_model_manifest(tmp_path: Path) -> None:
     (tmp_path / "layer_0_q.weights").write_bytes(b"")
     (tmp_path / "weights_manifest.json").write_text(