diff --git a/.gitignore b/.gitignore index 8bf22b3ef..76327edd6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__ _site .jekyll-metadata *.csv +.claude/ diff --git a/prompt.txt b/prompt.txt new file mode 100644 index 000000000..794f6a77a --- /dev/null +++ b/prompt.txt @@ -0,0 +1,5 @@ +You are reviewing the design of a heterogeneous inference runtime that splits a large language model between a general purpose CPU and a spatial dataflow accelerator. The accelerator is organised as a two dimensional array of compute tiles, each with its own local memory and a pair of DMA channels, connected by a packet switched interconnect whose routes are configured once when a bitstream is loaded. Data reaches the array through shim tiles at the boundary, which translate buffer descriptors into bursts against external memory. Because the routes are fixed at configuration time, the shape of a computation is baked into the bitstream, and a design that works well for one sequence length can be badly matched to another. + +The model itself is a hybrid: three quarters of its decoder layers use a linear recurrent mixer with a fixed size state, and the remaining quarter use ordinary grouped query attention against a growing key value cache. The linear layers are cheap to extend token by token but expensive to snapshot, since their state must be copied in full before any speculative batch is attempted. The attention layers are the opposite: rolling back costs nothing more than moving a cursor, but every step must re-read the whole cache, so their cost grows with the length of the context rather than staying flat. + +Explain, in terms a systems engineer would find useful, why the number of dispatches to the accelerator can dominate the wall clock even when the arithmetic being dispatched is a small fraction of the total, and describe how you would measure whether a change that reduces dispatch count but moves arithmetic back onto the host is actually a net improvement. Be concrete about what you would instrument, what baseline you would compare against, and which measurements would tempt you toward a wrong conclusion. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 588149974..21e769f57 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -415,6 +415,7 @@ target_link_libraries(flm PUBLIC qwen3_5vl_npu qwen3_5_omni_npu qwen3_6_moe_npu + qwen3_8mtp_npu gemma_npu gemma_text_npu gemma4e_npu diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 4f794522e..1bf0270e4 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -305,6 +305,40 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ assert(this->last_token != -1); stop_reason_t reason = EOT_DETECTED; + + // Speculation is only sound under greedy sampling: the engine accepts a + // draft by comparing it against the base model's argmax, so the tokens it + // returns are argmax tokens. Honouring a temperature, top-p or a + // repetition penalty while taking that path would silently replace the + // user's sampler with greedy decoding -- the output stays fluent, so + // nothing would ever flag it. Anything but top_k == 1 stays on the + // ordinary loop. + // + // Note sample_greedy() still applies penalties when repeat_last_n != 0, + // and they reorder the logits before the argmax -- so top_k == 1 alone + // does NOT make the sampler's choice equal the model's argmax. All four + // conditions are load-bearing. + // + // supports_speculation() is defaulted to false on causal_lm, so every + // engine but qwen3_8mtp answers false here and keeps the ordinary loop. + // Evaluated once per generate(), not per token. + const bool spec_enabled = + this->lm_engine->supports_speculation() && this->sampler && + this->sampler->top_k == 1 && this->sampler->rep_penalty == 1.0f && + this->sampler->freq_penalty == 0.0f && this->sampler->pre_penalty == 0.0f; + // Draft depth. Named because it is also the hit-rate denominator: if the + // request and the accounting were two separate literals, tuning one would + // silently skew the metric that says whether the tuning helped. + // The engine clamps to its own MTP_STEPS and may draft fewer. + const int SPEC_MAX_DRAFT = 7; + // Announce BEFORE the first token is streamed. header_print writes to + // std::cout while tokens go to `os`; when a caller passes std::cout for + // both, a banner emitted once decoding is under way splits the output + // mid-sentence ("The[FLM] Speculative decoding enabled..."). + if (spec_enabled) { + header_print("FLM", "Speculative decoding enabled (MTP draft head)"); + } + int last_sampled_token = this->last_token; this->token_history.push_back(this->last_token); if (this->is_normal_token(last_sampled_token) && last_sampled_token != -1){ @@ -323,16 +357,91 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ reason = MAX_LENGTH_REACHED; return result; } + + // One accepted token, handled exactly as the single-token path handles a + // sampled one. Returns false when the loop must stop. + // + // Shared rather than duplicated on purpose: a speculative branch with its + // own copy of the streaming, history and eos checks is how a batch ends up + // emitting text past a stop token. + auto consume = [&](int token) -> bool { + this->total_tokens++; + last_sampled_token = token; + + this->profiler_list[TKOEN_DECODE_TIME].start(); + if (this->is_normal_token(token)){ // filter out special tokens + std::string token_str = this->tokenizer->run_time_decoder(token); + os << token_str << std::flush; + result += token_str; + } + this->profiler_list[TKOEN_DECODE_TIME].stop(1); + this->token_history.push_back(token); + if (this->is_eos(token)){ + meta_info.generated_tokens++; + if (this->forward_on_eos) { + this->lm_engine->forward(token); + } + return false; + } + meta_info.generated_tokens++; + if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ + reason = MAX_LENGTH_REACHED; + return false; + } + return this->total_tokens < this->MAX_L; + }; + while (this->total_tokens < this->MAX_L){ if (is_cancelled()) { reason = CANCEL_DETECTED; - // reset stream content + // reset stream content buffer_.clear(); current_mode_ = StreamEventType::CONTENT; tool_name_.clear(); is_in_tool_block_ = false; break; } + + if (spec_enabled) { + this->profiler_list[DECODING_TIME].start(); + std::vector accepted = + this->lm_engine->speculate(last_sampled_token, SPEC_MAX_DRAFT); + // Charge the cycle to however many tokens came out of it, so + // tok/s stays comparable with the non-speculative path. + this->profiler_list[DECODING_TIME].stop( + accepted.empty() ? 1 : (int)accepted.size()); + + // Cycle 1 opened by feeding the prompt window through the draft + // head to prime its KV cache -- prefill-shaped work that happened + // to run inside a speculate() call, so the decode clock above was + // running for it. Move it. Cycle 2 onward the head is already + // caught up and this is 0, which is why the transfer is driven by + // the engine rather than by a "first cycle" test here: after a + // context clear there is a new cycle 1, and only the engine knows. + // + // Time only, no token count: those rows are prompt positions the + // prefill counter has already been charged for once. + if (const uint64_t prime_us = + this->lm_engine->last_speculation_prime_us()) { + this->profiler_list[DECODING_TIME].add_time(-(int64_t)prime_us); + this->profiler_list[PREFILL_TIME].add_time((int64_t)prime_us); + } + + if (!accepted.empty()) { + // The engine has already committed these to its caches -- they + // must not be re-fed through forward(). An eos mid-batch stops + // here and the rest are dropped. + bool go_on = true; + for (int tok : accepted) { + if (!(go_on = consume(tok))) break; + } + if (!go_on) break; + continue; + } + // Empty means the engine declined this step (head not primed, no + // headroom under MAX_L). Fall through to the ordinary path. + } + this->profiler_list[DECODING_TIME].start(); buffer y = this->lm_engine->forward(last_sampled_token); this->profiler_list[DECODING_TIME].stop(1); @@ -341,29 +450,8 @@ std::string AutoModel::_shared_generate(chat_meta_info_t& meta_info, int length_ this->_apply_tool_choice_mask(y, meta_info); int sampled_token = this->sampler->sample(y); this->profiler_list[SAMPLING_TIME].stop(1); - this->total_tokens++; - last_sampled_token = sampled_token; - this->profiler_list[TKOEN_DECODE_TIME].start(); - if (this->is_normal_token(sampled_token)){ // filter out special tokens - std::string token_str = this->tokenizer->run_time_decoder(sampled_token); - os << token_str << std::flush; - result += token_str; - } - this->profiler_list[TKOEN_DECODE_TIME].stop(1); - this->token_history.push_back(sampled_token); - if (this->is_eos(sampled_token)){ - meta_info.generated_tokens++; - if (this->forward_on_eos) { - this->lm_engine->forward(last_sampled_token); - } - break; - } - meta_info.generated_tokens++; - if ((length_limit > 0) && (meta_info.generated_tokens >= length_limit)){ - reason = MAX_LENGTH_REACHED; - break; - } + if (!consume(sampled_token)) break; } meta_info.decoding_duration = (uint64_t)(time_utils::cast_to_us(this->profiler_list[DECODING_TIME].get_total_time()).first) * 1e3; meta_info.stop_reason = reason; @@ -564,6 +652,18 @@ std::string AutoModel::show_model_info() { /// \brief Show the profile /// \note The function will show the profile /// \note The function will return the profile +/// \note "Total time" is wall clock around insert() + generate(), while every +/// other row is a narrow window inside it, so the rows do not partition +/// the run. The "Untimed" row at the bottom is the remainder, and it is +/// printed precisely because that gap used to be invisible: Qwen3.8's +/// think preamble spent ~13 s of a 24 s run in four forward() calls +/// that _shared_generate()'s DECODING_TIME.reset() then discarded, and +/// nothing in this block said so. +/// \note The four narrow rows are only comparable with Total on a single-turn +/// run. DECODING_TIME is reset at the top of every _shared_generate() +/// while PREFILL_TIME and TOTAL_TIME accumulate across turns, so a +/// multi-turn session over-reports "Untimed" by the decode time of +/// every turn but the last. std::string AutoModel::show_profile() { std::stringstream ss; int total_tokens = this->lm_engine->get_current_context_length(); @@ -575,12 +675,29 @@ std::string AutoModel::show_profile() { ss << " Decoding time: " << time.first << " " << time.second << std::endl; time = this->profiler_list[PREFILL_TIME].get_total_time(); ss << " Prefill time: " << time.first << " " << time.second << std::endl; - // time = this->profiler_list[SAMPLING_TIME].get_total_time(); - // ss << " Sampling time: " << time.first << " " << time.second << std::endl; - // time = this->profiler_list[TKOEN_ENCODE_TIME].get_total_time(); - // ss << " Token encoding time: " << time.first << " " << time.second << std::endl; - // time = this->profiler_list[TKOEN_DECODE_TIME].get_total_time(); - // ss << " Token decoding time: " << time.first << " " << time.second << std::endl; + time = this->profiler_list[SAMPLING_TIME].get_total_time(); + ss << " Sampling time: " << time.first << " " << time.second << std::endl; + time = this->profiler_list[TKOEN_ENCODE_TIME].get_total_time(); + ss << " Token encoding time: " << time.first << " " << time.second << std::endl; + time = this->profiler_list[TKOEN_DECODE_TIME].get_total_time(); + ss << " Token decoding time: " << time.first << " " << time.second << std::endl; + // Same unit for all five before subtracting: get_total_time() re_unit()s + // each one independently, so their .first fields are not commensurable. + const float total_us = time_utils::cast_to_us(this->profiler_list[TOTAL_TIME].get_total_time()).first; + if (total_us > 0.0f) { + float timed_us = 0.0f; + for (profiler_type p : {DECODING_TIME, PREFILL_TIME, SAMPLING_TIME, + TKOEN_ENCODE_TIME, TKOEN_DECODE_TIME}) + timed_us += time_utils::cast_to_us(this->profiler_list[p].get_total_time()).first; + // re_unit() only scales upward, so a negative remainder would print as + // a seven-digit microsecond count. Scale the magnitude and put the + // sign back: negative is not an error to hide, it is the multi-turn + // case above announcing itself. + const float gap_us = total_us - timed_us; + time = time_utils::re_unit(std::make_pair(std::abs(gap_us), "us")); + ss << " Untimed: " << (gap_us < 0.0f ? -time.first : time.first) + << " " << time.second << std::endl; + } ss << " Average decoding speed: " << this->profiler_list[DECODING_TIME].get_average_speed() << " tokens/s" << std::endl; ss << " Average prefill speed: " << this->profiler_list[PREFILL_TIME].get_average_speed() << " tokens/s" << std::endl; // ss << " Average sampling speed: " << this->profiler_list[SAMPLING_TIME].get_average_speed() << " tokens/s" << std::endl; diff --git a/src/common/AutoModel/modeling_qwen3_8mtp.cpp b/src/common/AutoModel/modeling_qwen3_8mtp.cpp new file mode 100644 index 000000000..402ec6e52 --- /dev/null +++ b/src/common/AutoModel/modeling_qwen3_8mtp.cpp @@ -0,0 +1,853 @@ +/// \file modeling_qwen3_8mtp.cpp +/// \brief Qwen3_8MTP class +/// \author FastFlowLM Team +/// \date 2026-09-16 +/// \version 0.9.28 +/// \note AutoModel wrapper for Qwen3.8-27B. Multimodal when the checkpoint +/// ships vision_weight.q4nx; the image preprocessing itself lives in +/// modeling_qwen3_8mtp_image.cpp. +/// \note Speculative decode IS wired up: causal_lm carries the +/// supports_speculation()/speculate() hooks and AutoModel:: +/// _shared_generate() drives them under a greedy sampler. +/// \note generate() therefore delegates its decode loop to _shared_generate() +/// and keeps only the think-preamble replay. It previously carried a +/// private copy of the loop that called forward() + sample() directly, +/// which left speculation dead on the main chat path -- correct output, +/// no speedup, and a hit rate that never printed because no cycle ran. + +#include "AutoModel/modeling_qwen3_8mtp.hpp" + + +/************ Qwen3_8MTP family **************/ +Qwen3_8MTP::Qwen3_8MTP(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Qwen3_8MTP") {} + +void Qwen3_8MTP::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { + this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); + + this->q4nx = std::make_unique(this->model_path); + // lm_config->get("model_type", "") == qwen3_5 + this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); + + this->lm_engine->load_weights(*this->q4nx); + //free the q4nx + this->q4nx.reset(); + this->lm_engine->clear_context(); + this->setup_tokenizer(model_path); + this->sampler.reset(); + + this->enable_tool = true; + + // Qwen3.5-family recommended decoding defaults; the test harness overrides + // these with set_topk(1) when it wants a reproducible stream. + sampler_config config; + config.top_k = 20; + config.top_p = 0.8; + config.min_p = 0.0; + config.temperature = 0.7; + config.rep_penalty = 1.0; + config.freq_penalty = 1.0; + config.pre_penalty = 1.5f; + + this->set_sampler(config); + + // Image-processor geometry. patch_size / spatial_merge_size / + // temporal_patch_size are stated by config.json's vision_config and are + // read from it; the rest live in preprocessor_config.json, which the NPU2 + // checkpoint does not ship, so they fall back to the values that file + // carries upstream for Qwen3.8-27B. A wrong factor here does not fail -- + // it produces a grid the merger regroups across, which reads as a slightly + // confused caption. + { + const nlohmann::json& vc = this->lm_config->sub("vision_config"); + this->vision_patch_size = cfg_get(vc, "patch_size", 16); + this->vision_merge_size = cfg_get(vc, "spatial_merge_size", 2); + this->vision_temporal_patch_size = cfg_get(vc, "temporal_patch_size", 2); + this->vision_shortest_edge = cfg_get(vc, "shortest_edge", 65536); + this->vision_longest_edge = cfg_get(vc, "longest_edge", 16777216); + this->vision_rescale_factor = cfg_get(vc, "rescale_factor", 1.0f / 255.0f); + this->vision_image_mean = cfg_get(vc, "image_mean", 0.5f); + this->vision_image_std = cfg_get(vc, "image_std", 0.5f); + } + if (auto* eng = dynamic_cast(this->lm_engine.get())) { + if (eng->has_vision_tower()) + header_print("FLM", "vision backend: " << eng->vision_backend()); + } + + for (size_t i = 0; i < PROFILER_TYPE_NUM; i++) { + this->profiler_list[i].reset(); + } +} + +void Qwen3_8MTP::setup_tokenizer(std::string model_path) { + // tokenizer_config.json already lists eos_token_id as + // [248044 <|endoftext|>, 248046 <|im_end|>, 248048], and + // _shared_setup_tokenizer registers every entry of that array. config.json + // alone would only give 248044, which the chat template never emits -- + // that mismatch is the runaway-generation failure mode. + auto tokenizer_config = this->_shared_setup_tokenizer(model_path); +} + +std::string Qwen3_8MTP::apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools) { + minja::chat_template_inputs inputs; + inputs.add_generation_prompt = true; + inputs.messages = messages; + inputs.extra_context = this->extra_context; + inputs.extra_context["enable_thinking"] = this->enable_think; + if (!tools.empty() && this->enable_tool) + inputs.tools = tools; + return this->chat_tmpl->apply(inputs); +} + +bool Qwen3_8MTP::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { + // preprocess + this->profiler_list[TKOEN_ENCODE_TIME].start(); + std::string templated_text; + if (input.messages.empty() && input.prompt.empty()) { + header_print("WARNING", "No messages or prompt provided"); + return false; + } + + // <|image_pad|>, verified against tokenizer.json and against config.json's + // image_token_id. The engine reads the same id out of its own config, so a + // disagreement here surfaces as a span-count mismatch and a throw, not as + // a scrambled prompt. + constexpr int image_soft_token_id = 248056; + + qwen3_8mtp_npu* qwen3_8mtp_engine = dynamic_cast(this->lm_engine.get()); + + // A checkpoint without vision_weight.q4nx (or with FLM_Q38_VISION=0) has no + // tower, so an image would be templated into <|vision_start|><|image_pad|> + // ... placeholders that nothing ever fills -- the prompt would prefill + // garbage rather than fail. Drop them loudly in that case only. + const bool vision_ok = qwen3_8mtp_engine && qwen3_8mtp_engine->has_vision_tower(); + if (!vision_ok && !input.images.empty()) { + header_print("WARNING", "Qwen3.8-27B is loaded without a vision tower; ignoring " + << input.images.size() << " image(s)"); + input.images.clear(); + } + // Audio is not ported at all: the checkpoint carries no audio tower and + // there is no audio path in this engine. + if (!input.audios.empty()) { + header_print("WARNING", "Qwen3.8-27B has no audio path; ignoring " + << input.audios.size() << " audio clip(s)"); + input.audios.clear(); + } + + // ---------------------------------------------------------------------- + // Decode and preprocess every image BEFORE templating, so that an image + // that fails to load never gets a placeholder. If it did, the payload + // would fall out of step with the placeholders and the engine would + // splice image n's rows into image n+1's slots -- which every downstream + // stage accepts. + // + // `pixels` is one contiguous fp32 buffer for the whole prompt and it grows + // as images are appended, so the engine-facing pointers cannot be taken + // until every image is in. `pixel_offsets` records where each one landed. + // ---------------------------------------------------------------------- + std::vector host_images; + std::vector pixel_offsets; + std::vector pixels; + + auto stage_image = [&](qwen3_8mtp_host_image_t& image) -> bool { + if (image.width <= 0 || image.height <= 0) return false; + const size_t offset = pixels.size(); + this->preprocess_image(image, pixels); + if (image.grid_h <= 0 || image.grid_w <= 0) { + pixels.resize(offset); // undo a partial append + return false; + } + host_images.push_back(std::move(image)); + pixel_offsets.push_back(offset); + return true; + }; + + if (vision_ok) { + for (const auto& img_str : input.images) { + qwen3_8mtp_host_image_t image = this->load_image(img_str); + if (!stage_image(image)) + header_print("ERROR", "Skipping image that failed to load: " << img_str); + } + } + + if (!input.messages.empty()) { // already a formated messages, usually from REST API + json qwenvl_message = json::array(); + for (const auto& item : input.messages) { + json entry = item; + entry.erase("audios"); + if (!vision_ok || !item.contains("images")) { + entry.erase("images"); + qwenvl_message.push_back(entry); + continue; + } + + // Expand into the content-array form the chat template turns into + // <|vision_start|><|image_pad|><|vision_end|>, one image at a + // time, in message order. + json newContent = json::array(); + for (const auto& img : item["images"]) { + const std::string img_str = img.get(); + qwen3_8mtp_host_image_t image = this->load_image_base64(img_str); + if (!stage_image(image)) { + header_print("ERROR", "Skipping invalid base64 image; prefilling " + "language only for this item"); + continue; + } + newContent.push_back({ {"type", "image"}, {"image", img} }); + } + newContent.push_back({ {"type", "text"}, {"text", item["content"]} }); + qwenvl_message.push_back({ {"role", item["role"]}, {"content", newContent} }); + } + nlohmann::ordered_json ordered_messages = qwenvl_message; + templated_text = this->apply_chat_template(ordered_messages, input.tools); + } + else if (!input.prompt.empty()) { // a pure text, usually from the cli + nlohmann::ordered_json messages; + if (host_images.empty()) { + messages.push_back({ {"role", "user"}, {"content", input.prompt} }); + } + else { + nlohmann::ordered_json content; + content["role"] = "user"; + content["content"] = nlohmann::ordered_json::array(); + for (const auto& img_str : input.images) { + nlohmann::ordered_json image_obj; + image_obj["type"] = "image"; + image_obj["image"] = img_str; + content["content"].push_back(image_obj); + } + nlohmann::ordered_json text_obj; + text_obj["type"] = "text"; + text_obj["text"] = input.prompt; + content["content"].push_back(text_obj); + messages.push_back(content); + } + templated_text = this->apply_chat_template(messages); + } + + std::vector tokens_init = this->tokenizer->encode(templated_text); + + // The template emits ONE <|image_pad|> per image; the model wants one per + // MERGED patch, which is grid_h * grid_w / merge^2. Expand in place. + std::vector tokens; + if (host_images.empty()) { + tokens = std::move(tokens_init); + } + else { + const int merged = static_cast(this->vision_merge_size * + this->vision_merge_size); + size_t total_image_tokens = 0; + for (const auto& im : host_images) + total_image_tokens += static_cast(im.grid_h) * im.grid_w / merged; + tokens.reserve(tokens_init.size() + total_image_tokens); + + size_t image_counter = 0; + for (size_t i = 0; i < tokens_init.size(); i++) { + if (tokens_init[i] == image_soft_token_id && + image_counter < host_images.size()) { + const qwen3_8mtp_host_image_t& im = host_images[image_counter]; + const int n = im.grid_h * im.grid_w / merged; + tokens.insert(tokens.end(), static_cast(n), image_soft_token_id); + image_counter++; + } else { + tokens.push_back(tokens_init[i]); + } + } + if (image_counter != host_images.size()) { + // The template produced fewer placeholders than we staged images + // for, so some image's pixels have no slots to land in. Refusing is + // the only safe answer: prefilling would put image n's rows into + // image n+1's positions and read as a mildly confused caption. + header_print("ERROR", "templated " << image_counter << " image placeholder(s) " + "for " << host_images.size() << " image(s); refusing to prefill"); + return false; + } + header_print("FLM", "Total images: " << host_images.size() + << " (" << total_image_tokens << " image tokens)"); + } + + this->profiler_list[TKOEN_ENCODE_TIME].stop(tokens.size()); + + // ---------------------------------------------------------------------- + // Prompt-cache aware image alignment. + // + // AutoModel::_shared_insert prefix-matches `tokens` against + // `checkpoint_his` over the FULL length of checkpoint_his, and erases that + // prefix before prefilling only if every token of it matches. We must NOT + // erase `tokens` here -- _shared_insert needs the untrimmed sequence to + // run that very check. What does need fixing up locally is the payload, + // which holds pixels for the WHOLE prompt including images already in the + // cache from earlier turns: drop the fully-cached leading images so the + // survivors line up with the image tokens that survive the erase. + // ---------------------------------------------------------------------- + size_t prefix_skip_count = 0; + if (!host_images.empty()) { + const size_t idx = this->checkpoint_his.size(); + for (size_t i = 0; i < idx; i++) { + if (i < tokens.size() && tokens[i] == this->checkpoint_his[i]) prefix_skip_count++; + else break; + } + // Must match the entirety of checkpoint_his, otherwise _shared_insert + // clears the context and skips nothing. + if (prefix_skip_count != idx) prefix_skip_count = 0; + + if (prefix_skip_count > 0) { + const int merged = static_cast(this->vision_merge_size * + this->vision_merge_size); + size_t skipped_image_tokens = 0; + for (size_t i = 0; i < prefix_skip_count; i++) + if (tokens[i] == image_soft_token_id) skipped_image_tokens++; + + size_t images_to_drop = 0; + size_t consumed_image_tokens = 0; + for (const auto& im : host_images) { + const size_t img_tokens = + static_cast(im.grid_h) * im.grid_w / merged; + if (consumed_image_tokens + img_tokens > skipped_image_tokens) break; + consumed_image_tokens += img_tokens; + images_to_drop++; + } + + if (images_to_drop > 0) { + // The pixels themselves are NOT erased. pixel_offsets are + // absolute indices into `pixels`, so dropping the descriptors + // is enough and erasing the front of a 400 MB buffer to save + // nothing would be the only cost here. + host_images.erase(host_images.begin(), + host_images.begin() + images_to_drop); + pixel_offsets.erase(pixel_offsets.begin(), + pixel_offsets.begin() + images_to_drop); + header_print("FLM", "Prompt-cache hit: dropped " << images_to_drop + << " cached image(s) from payload"); + } + } + } + + // The last image token's index, expressed relative to the tokens that will + // SURVIVE _shared_insert's prefix erase. _chunked_insert hands the payload + // to chunk 0 only, so this is what grows chunk 0 to cover every image row; + // without it a second chunk would carry image tokens and no images, and + // the engine would throw. + int last_image_token_index = -1; + for (int i = static_cast(prefix_skip_count); i < (int)tokens.size(); i++) { + if (tokens[i] == image_soft_token_id) + last_image_token_index = i - static_cast(prefix_skip_count); + } + last_image_token_index++; // plus the end-of-image token + + // Engine-facing views. Built here, after every append to `pixels` is done, + // because the vector reallocates as it grows. + std::vector image_views(host_images.size()); + for (size_t i = 0; i < host_images.size(); i++) { + image_views[i].pixel_values = pixels.data() + pixel_offsets[i]; + image_views[i].grid_t = host_images[i].grid_t; + image_views[i].grid_h = host_images[i].grid_h; + image_views[i].grid_w = host_images[i].grid_w; + } + qwen3_8mtp_image_payload_t image_payload; + image_payload.images = image_views.data(); + image_payload.num_images = static_cast(image_views.size()); + + qwen3_8mtp_payload_t payload; + payload.images = &image_payload; + const bool has_images = image_payload.num_images > 0; + + // hardware + int restore_idx = -1; + + if (meta_info.restore_allowed) { + restore_idx = qwen3_8mtp_engine->restore(); + this->total_tokens = restore_idx; + this->token_history = checkpoint_his; // restore the token history to be consistent with the restored KV cache, which is crucial for correct functioning of _shared_insert's prefix-matching logic + } + + // The chat template's generation prompt ends with the think preamble: + // thinking on : "" "\n" -> 2 tokens + // thinking off: "" "\n\n" "" "\n\n" -> 4 tokens + // + // These used to be trimmed off here so generate() could re-feed them one + // at a time, which cost one full 64-layer weight stream per token -- + // forward() is _prefill_with_mm() on a one-row batch, and that batch's + // cost is the ~14 GB of packed weights, not the row. Measured at ~3.3 s + // each on this model, so the four-token thinking-off preamble was ~13 s + // of a 24 s short run, none of it visible in any printed timer. + // + // They are constants the tokenizer knew before the run started, so they + // ride in the prefill batch as 2-4 extra rows of a pass that was + // happening anyway. Two consequences beyond the time: + // - insert()'s own sample() now seeds `last_token` from the row after + // the last preamble token, which is the first answer token. It used + // to sample the row before the preamble and have generate() throw the + // result away. + // - `last_window_len` is left at the prompt length instead of 1, so the + // MTP head's step-0 catch-up is no longer clamped to a single row and + // the head starts the first cycle primed rather than cold. + // + // Thinking on still has to put "\n" on the screen -- it opens the + // reasoning block the user reads -- so record those ids for generate() to + // echo without forwarding. Thinking off streams nothing: its four tokens + // exist only to close a block that was never opened, and the old code + // decoded each one into a `token_str` it then dropped. + this->preamble_to_stream.clear(); + if (this->enable_think && tokens.size() >= 2) + this->preamble_to_stream.assign(tokens.end() - 2, tokens.end()); + + bool success = has_images + ? this->_shared_insert(meta_info, tokens, is_cancelled, &payload, last_image_token_index) + : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); + + checkpoint_his = token_history; + int checkpoint_idx = qwen3_8mtp_engine->checkpoint(); + return success; +} + +std::string Qwen3_8MTP::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { + std::string result; + assert(this->last_token != -1); + + // The think preamble is already in the KV cache and already in + // token_history: insert() prefilled it with the rest of the prompt + // instead of trimming it off for this function to re-feed a token at a + // time. All that is left here is the text, and only when thinking is on. + // + // No profiler is touched: there is no model work left to charge, and + // _shared_generate() resets DECODING_TIME and TKOEN_DECODE_TIME on entry + // anyway -- which is what used to silently discard the four forward() + // calls this block has replaced, so they never appeared in "Decoding + // time" and the 13 s they cost showed up only as a hole in "Total time". + for (int id : this->preamble_to_stream) { + std::string token_str = this->tokenizer->run_time_decoder(id); + result += token_str; + os << token_str << std::flush; + } + if (this->total_tokens >= this->MAX_L){ + header_print("WARNING", "Max length reached, stopping generation..."); + meta_info.stop_reason = MAX_LENGTH_REACHED; + return result; + } + + // Hand the decode loop to the base class rather than running a private + // copy of it. This function used to carry its own while() calling + // forward() + sample() directly, which meant the whole speculative path -- + // and with it the MTP hit rate -- was dead on the main chat path: only + // generate_with_prompt() and the milestone driver ever reached + // speculate(). The preamble above is the only part that is genuinely + // specific to this model, so it stays; the loop is not. + // + // _shared_generate() seeds from this->last_token, emits it, then feeds it + // to the model. That seed is insert()'s own sample of the prefill's last + // row -- which, now that the preamble is prefilled too, is the row that + // predicts the first answer token. Nothing to re-assign here. + result += this->_shared_generate(meta_info, length_limit, os, is_cancelled); + + // The engine counts every draft/verify cycle, so this prints whenever + // speculation actually ran, and prints nothing otherwise -- a checkpoint + // without an MTP head, or a non-greedy sampler, stays silent. + // + // Worth printing at all because a broken draft path is invisible in the + // text: verify overrides every rejected draft with the base model's own + // argmax, so bad drafting costs only speed. + // + // The newline is this function's own: header_print writes to std::cout + // while tokens stream to `os`, so without it the line would run on from + // the last token whenever log_raw_output is off (which is what supplies + // the break today -- not something to depend on from here). + if (auto* mtp = dynamic_cast(this->lm_engine.get())) { + if (mtp->speculation_cycles() > 0) { + if (!this->log_raw_output) std::cout << std::endl; + mtp->report_speculation_stats(); + } + } + + return result; +} + +/// \brief the shared profile, plus the MTP phase breakdown when it applies +/// \note "Decoding time" is a single number, but this model decodes in three +/// phases with unrelated cost structures: k serial one-layer draft steps, +/// one batched 64-layer verify over k+1 rows, and -- only on a rejection +/// -- a rollback and re-fold. Lumping them together hides the one thing +/// worth acting on, which is whether drafting is paying for itself. +/// \note The split cannot be made in AutoModel's profiler: from _shared_generate +/// a whole cycle is one speculate() call, so the phase boundary does not +/// exist there. The engine measures it and this appends the result. +/// \note One part of it does cross over. Step 0 of the first cycle after each +/// prefill is the draft head absorbing the prompt, which is prefill by +/// any honest reading, so the engine reports it through +/// last_speculation_prime_us() and _shared_generate moves those +/// microseconds from DECODING_TIME to PREFILL_TIME. The "Prime" row +/// below is therefore already OUT of the "Decoding time" above it -- +/// the only row in the breakdown of which that is true. +std::string Qwen3_8MTP::show_profile() { + // Base first, so the rows every model shares stay in one place and cannot + // drift from AutoModel's. + std::string ss = this->AutoModel::show_profile(); + + if (auto* mtp = dynamic_cast(this->lm_engine.get())) { + // Empty when speculation never ran -- a non-greedy sampler, or a + // checkpoint with no MTP head. Appending nothing then keeps those + // sessions byte-identical to what they printed before. + ss += mtp->speculation_timing(); + } + return ss; +} + +std::string Qwen3_8MTP::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os) { + if (!this->insert(meta_info, input)) { + return ""; + } + header_print("FLM", "Prompt inserted, starting generation..."); + if (this->enable_think) { + os << "\n" << std::flush; + } + return this->_shared_generate(meta_info, length_limit, os); +} + +// Non-stream +NonStreamResult Qwen3_8MTP::parse_nstream_content(const std::string response_text) { + NonStreamResult result; + + // Qwen3.5-style tool syntax: + // VALUE + std::string start_tag = ""; + std::string end_tag = ""; + std::string func_end_tag = ""; + std::string func_open = " fallback + size_t func_end_pos = response_text.find(func_end_tag, block_content_start); + if (func_end_pos != std::string::npos) { + block_end = func_end_pos + func_end_tag.length(); + } else { + block_end = response_text.length(); + } + search_from = block_end; + } + + std::string block = response_text.substr(block_content_start, block_end - block_content_start); + + std::string tool_name; + size_t func_start = block.find(func_open); + if (func_start != std::string::npos) { + func_start += func_open.length(); + size_t func_name_end = block.find(">", func_start); + if (func_name_end != std::string::npos) { + tool_name = block.substr(func_start, func_name_end - func_start); + } + } + + nlohmann::json args = nlohmann::json::object(); + size_t pos = 0; + + while (true) { + size_t param_start = block.find(param_open, pos); + if (param_start == std::string::npos) break; + + param_start += param_open.length(); + size_t param_name_end = block.find(">", param_start); + if (param_name_end == std::string::npos) break; + + std::string param_name = block.substr(param_start, param_name_end - param_start); + size_t value_start = param_name_end + 1; + size_t value_end = block.find(param_close, value_start); + + size_t next_param_pos = block.find(param_open, value_start); + size_t func_boundary_pos = block.find(func_end_tag, value_start); + + auto use_earlier_boundary = [&value_end](size_t boundary_pos) { + if (boundary_pos != std::string::npos && (value_end == std::string::npos || boundary_pos < value_end)) { + value_end = boundary_pos; + } + }; + + use_earlier_boundary(next_param_pos); + use_earlier_boundary(func_boundary_pos); + + if (value_end == std::string::npos) { + value_end = block.length(); + } + + std::string param_value = trim_tool_value(block.substr(value_start, value_end - value_start)); + + try { + args[param_name] = nlohmann::json::parse(param_value); + } + catch (...) { + args[param_name] = param_value; + } + + pos = value_end; + if (block.compare(value_end, param_close.length(), param_close) == 0) { + pos += param_close.length(); + } + } + + result.tool_calls_list.emplace_back(tool_name, args.dump()); + } + + if (result.tool_calls_list.empty()) { + size_t content_start = is_reasoning ? think_end_pos + think_end_tag.length() : 0; + result.content = response_text.substr(content_start); + } else { + // Populate legacy single-tool fields from the first call for backward compatibility + result.tool_name = result.tool_calls_list[0].first; + result.tool_args = result.tool_calls_list[0].second; + // Extract content before the first + size_t first_tool = response_text.find(start_tag); + size_t content_start = is_reasoning ? think_end_pos + think_end_tag.length() : 0; + if (first_tool != std::string::npos && first_tool > content_start) { + result.content = trim_tool_value(response_text.substr(content_start, first_tool - content_start)); + } + } + + return result; +} + +// Stream +StreamResult Qwen3_8MTP::parse_stream_content(const std::string content) { + return parse_stream_content_impl(content, false); +} + +StreamResult Qwen3_8MTP::parse_stream_content_final(const std::string content) { + return parse_stream_content_impl(content, true); +} + +StreamResult Qwen3_8MTP::parse_stream_content_impl(const std::string content, bool is_final) { + const std::string MARKER_THINK_START = ""; + const std::string MARKER_THINK_END = ""; + const std::string MARKER_TOOL_START = ""; + const std::string MARKER_TOOL_END = ""; + const std::string MARKER_FUNC_END = ""; + + + StreamResult result; + buffer_ += content; + + while (true) { + if (!is_in_tool_block_) { + size_t stray_end_pos = buffer_.find(MARKER_TOOL_END); + if (stray_end_pos != std::string::npos) { + buffer_.erase(stray_end_pos, MARKER_TOOL_END.length()); + } + } + + if (!is_in_tool_block_) { + size_t tool_start_pos = buffer_.find(MARKER_TOOL_START); + if (tool_start_pos != std::string::npos) { + if (tool_start_pos > 0) { + result.content = buffer_.substr(0, tool_start_pos); + result.type = current_mode_; + buffer_ = buffer_.substr(tool_start_pos); + return result; + } + + is_in_tool_block_ = true; + buffer_ = buffer_.substr(MARKER_TOOL_START.length()); + result.type = StreamEventType::WAITING; + return result; + } + } + + // tool calling process + if (is_in_tool_block_) { + size_t tool_end_pos = buffer_.find(MARKER_TOOL_END); + size_t func_end_pos = buffer_.find(MARKER_FUNC_END); + + if (tool_end_pos != std::string::npos || func_end_pos != std::string::npos || (is_final && !buffer_.empty())) { + size_t actual_end_pos = buffer_.size(); + size_t skip_length = 0; + + if (tool_end_pos != std::string::npos) { + actual_end_pos = tool_end_pos; + skip_length = MARKER_TOOL_END.length(); + } + else if (func_end_pos != std::string::npos) { + actual_end_pos = func_end_pos; + skip_length = MARKER_FUNC_END.length(); + } + + std::string block = buffer_.substr(0, actual_end_pos + skip_length); + buffer_ = buffer_.substr(actual_end_pos + skip_length); + is_in_tool_block_ = false; + + try { + result.type = StreamEventType::TOOL_DONE; + result.tool_id = "call_" + std::to_string(std::time(nullptr)); + + // parse function name + std::string func_open = "", func_start); + if (func_end != std::string::npos) { + result.tool_name = block.substr(func_start, func_end - func_start); + } + } + + // parse parameters + nlohmann::json args = nlohmann::json::object(); + std::string param_open = "", p_start); + if (p_name_end == std::string::npos) break; + std::string param_name = block.substr(p_start, p_name_end - p_start); + + size_t val_start = p_name_end + 1; + if (val_start < block.size() && block[val_start] == '\n') val_start++; + + size_t param_close_pos = block.find(param_close, val_start); + size_t val_end = param_close_pos; + + size_t next_param_pos = block.find(param_open, val_start); + size_t func_boundary_pos = block.find(MARKER_FUNC_END, val_start); + size_t tool_boundary_pos = block.find(MARKER_TOOL_END, val_start); + + auto use_earlier_boundary = [&val_end](size_t boundary_pos) { + if (boundary_pos != std::string::npos && (val_end == std::string::npos || boundary_pos < val_end)) { + val_end = boundary_pos; + } + }; + + use_earlier_boundary(next_param_pos); + use_earlier_boundary(func_boundary_pos); + use_earlier_boundary(tool_boundary_pos); + + if (val_end == std::string::npos && is_final) { + val_end = block.size(); + } + if (val_end == std::string::npos) break; + + std::string param_value = block.substr(val_start, val_end - val_start); + + // Enhanced trim: handle multiple newlines or spaces that the model may generate after a parameter + while(!param_value.empty() && (param_value.back() == '\n' || param_value.back() == '\r' || param_value.back() == ' ')) { + param_value.pop_back(); + } + + try { + // Try to parse as native JSON type (Integer, Float, Boolean, Array, Object) + args[param_name] = nlohmann::json::parse(param_value); + } + catch (...) { + args[param_name] = param_value; + } + + search_pos = param_close_pos != std::string::npos && val_end == param_close_pos + ? val_end + param_close.length() + : val_end; + } + result.tool_args_str = args.dump(); + return result; + } + catch (...) { + result.type = StreamEventType::CONTENT; + result.content = "[Error parsing tool call]"; + return result; + } + } + else { + result.type = StreamEventType::WAITING; + return result; + } + } + + if (current_mode_ == StreamEventType::CONTENT) { + size_t think_start_pos = buffer_.find(MARKER_THINK_START); + if (think_start_pos != std::string::npos) { + if (think_start_pos > 0) { + result.content = buffer_.substr(0, think_start_pos); + result.type = StreamEventType::CONTENT; + buffer_ = buffer_.substr(think_start_pos); + return result; + } + buffer_ = buffer_.substr(MARKER_THINK_START.length()); + current_mode_ = StreamEventType::REASONING; + continue; + } + } + else if (current_mode_ == StreamEventType::REASONING) { + size_t think_end_pos = buffer_.find(MARKER_THINK_END); + if (think_end_pos != std::string::npos) { + if (think_end_pos > 0) { + result.content = buffer_.substr(0, think_end_pos); + result.type = StreamEventType::REASONING; + buffer_ = buffer_.substr(think_end_pos); + return result; + } + buffer_ = buffer_.substr(MARKER_THINK_END.length()); + current_mode_ = StreamEventType::CONTENT; + continue; + } + } + + if (!buffer_.empty()) { + size_t last_lt = buffer_.rfind('<'); + // If '<' appears at the end (possibly an incomplete or tag) + if (last_lt != std::string::npos && (buffer_.length() - last_lt) <= 15) { + if (last_lt > 0) { + // Only output the content before '<' + result.content = buffer_.substr(0, last_lt); + result.type = current_mode_; + buffer_ = buffer_.substr(last_lt); + return result; + } else { + // If '<' is the first character in the buffer, directly wait for the next chunk + result.type = StreamEventType::WAITING; + return result; + } + } + + result.content = buffer_; + result.type = current_mode_; + buffer_.clear(); + return result; + } + + break; + } + + result.type = current_mode_; + return result; +} diff --git a/src/common/AutoModel/modeling_qwen3_8mtp_image.cpp b/src/common/AutoModel/modeling_qwen3_8mtp_image.cpp new file mode 100644 index 000000000..4b570cc4e --- /dev/null +++ b/src/common/AutoModel/modeling_qwen3_8mtp_image.cpp @@ -0,0 +1,273 @@ +/// \file modeling_qwen3_8mtp_image.cpp +/// \brief Qwen3.8-27B image preprocessing: file/base64 -> the engine's patches +/// \author FastFlowLM Team +/// \date 2026-09-22 +/// \version 0.9.28 +/// +/// This is the host half of the vision path. It reproduces what the HF +/// `Qwen3_5ImageProcessor` does to a PIL image, so that what reaches +/// qwen3_8mtp_npu::encode_image() is byte-for-byte the `pixel_values` the +/// reference dump was captured with: +/// +/// decode -> (optional pre-resize) -> HWC to CHW +/// -> smart_resize to a multiple of patch*merge +/// -> bicubic antialias resize +/// -> rescale by 1/255, normalise by mean/std +/// -> replicate the frame temporal_patch_size times +/// -> reorder into MERGE-BLOCK order +/// +/// Modelled on modeling_qwen3_5vl_image.cpp; the differences are all in +/// preprocess_image() and are commented where they occur. + +#include "AutoModel/modeling_qwen3_8mtp.hpp" + +#include +#include +#include + + +// --------------------------------------------------------------------------- +// decode +// --------------------------------------------------------------------------- + +/// The pre-resize ladder, shared by both loaders. Factored out rather than +/// copied because the two Qwen3_5VL copies of it have already drifted -- one +/// logs with a trailing newline and the other does not. +static int _pre_resize_max_height(int level, int decoded_height) { + switch (level) { + case 1: return 480; + case 2: return 720; + case 3: return 1080; + case 4: return 1440; + case 5: return 2160; + case 6: return 2880; + case 7: return 3240; + case 8: return 4320; + default: return decoded_height; // no resizing + } +} + +void Qwen3_8MTP::_apply_pre_resize(image_data_t& decoded) { + if (this->image_pre_resize <= 0) return; + + const int max_height = _pre_resize_max_height(this->image_pre_resize, + decoded.height); + if (decoded.height <= max_height) return; + + image_data_t resized_image; + const float ratio = static_cast(max_height) / + static_cast(decoded.height); + const int target_width = static_cast(static_cast(decoded.width) * ratio); + const int target_height = max_height; + header_print_r("FLM", "Qwen3.8 resizing image from (" + + std::to_string(decoded.width) + ", " + + std::to_string(decoded.height) + ") to (" + + std::to_string(target_width) + ", " + + std::to_string(target_height) + ")\n"); + if (image_reader_.resize_image(decoded, target_width, target_height, resized_image)) { + image_reader_.recycle(decoded); + decoded = std::move(resized_image); + } +} + +/// Shared tail of both loaders: CHW reorder and hand-off into the host struct. +qwen3_8mtp_host_image_t Qwen3_8MTP::_finish_load(image_data_t& decoded) { + qwen3_8mtp_host_image_t empty_result; + image_data_t reordered; + + this->_apply_pre_resize(decoded); + + if (!image_reader_.reorder_hwc_to_chw(decoded, reordered)) { + image_reader_.recycle(decoded); + return empty_result; + } + image_reader_.recycle(decoded); + + qwen3_8mtp_host_image_t result; + result.width = reordered.width; + result.height = reordered.height; + result._data = std::move(reordered.pixels); + image_reader_.recycle(reordered); + return result; +} + +qwen3_8mtp_host_image_t Qwen3_8MTP::load_image(const std::string& filename) { + image_data_t decoded; + if (!image_reader_.load_image(filename, decoded)) + return qwen3_8mtp_host_image_t{}; + return this->_finish_load(decoded); +} + +qwen3_8mtp_host_image_t Qwen3_8MTP::load_image_base64(const std::string& base64_string) { + image_data_t decoded; + if (!image_reader_.load_image_base64(base64_string, decoded)) + return qwen3_8mtp_host_image_t{}; + return this->_finish_load(decoded); +} + + +// --------------------------------------------------------------------------- +// smart_resize +// --------------------------------------------------------------------------- + +/// Port of transformers' `smart_resize`. `factor` is patch_size * merge_size, +/// which for this checkpoint is 16 * 2 = 32: the grid must be a whole number +/// of patches AND a whole number of 2x2 merge blocks, or the merger's regroup +/// reads across an image boundary. +void Qwen3_8MTP::smart_resize( + int height, + int width, + int& h_bar, + int& w_bar, + int factor, + int min_pixels, + int max_pixels +) { + const double aspect_ratio = static_cast(std::max(height, width)) / + static_cast(std::min(height, width)); + if (aspect_ratio > 200.0) { + header_print("WARNING", "absolute aspect ratio must be smaller than 200, got " + << aspect_ratio); + } + + h_bar = static_cast(std::round(static_cast(height) / factor)) * factor; + w_bar = static_cast(std::round(static_cast(width) / factor)) * factor; + + const long long total_pixels = static_cast(h_bar) * w_bar; + + if (total_pixels > max_pixels) { + const double beta = std::sqrt((static_cast(height) * width) / max_pixels); + h_bar = std::max(factor, + static_cast(std::floor(height / beta / factor)) * factor); + w_bar = std::max(factor, + static_cast(std::floor(width / beta / factor)) * factor); + } else if (total_pixels < min_pixels) { + const double beta = std::sqrt(static_cast(min_pixels) / + (static_cast(height) * width)); + h_bar = static_cast(std::ceil(height * beta / factor)) * factor; + w_bar = static_cast(std::ceil(width * beta / factor)) * factor; + } +} + + +// --------------------------------------------------------------------------- +// preprocess +// --------------------------------------------------------------------------- + +/// \brief turn one decoded CHW image into the engine's `pixel_values` rows +/// \param image decoded uint8 (3, H, W); its pixels are freed on the way out +/// \param pixel_values appended to, fp32, [patches][3 * temporal * patch^2] +/// +/// Two differences from the Qwen3_5VL version: +/// +/// 1. The output is fp32, because qwen3_8mtp_npu::encode_image() takes +/// `const float*`. The vision tower's ops carry activations across the +/// CPU/NPU seam as fp32 host buffers so that the two backends are drop-in +/// for each other, and pixel_values is just the first such buffer. +/// +/// 2. The geometry constants come from config.json's vision_config, with +/// Qwen3.8-27B's preprocessor_config.json values as the fallback. The +/// Qwen3_5VL path reads converter-injected QWEN3_5_* keys off the engine; +/// this checkpoint carries none of them. +/// +/// The reorder itself is staged through bf16, because +/// imgproc::reorder_patches_inplace only writes bf16 and it is not worth a +/// second 90-line copy of that AVX-512 permutation to avoid the round trip. +/// In the tower's shipping mode that round trip is exactly inert: cpu_mm_op +/// rounds its `x` to bf16 before the first multiply anyway, so the patch +/// embedding consumes the same bits either way. It is NOT inert under +/// FLM_Q38_VISION_FP32=1, which is a debug control and not a shipping path. +void Qwen3_8MTP::preprocess_image(qwen3_8mtp_host_image_t& image, + std::vector& pixel_values) { + const int width = image.width; + const int height = image.height; + const int channels = 3; // RGB; vision_config.in_channels + + const unsigned patch_size = this->vision_patch_size; + const unsigned merge_size = this->vision_merge_size; + const unsigned temporal_size = this->vision_temporal_patch_size; + + int resized_height = 0; + int resized_width = 0; + smart_resize(height, width, + resized_height, resized_width, + static_cast(patch_size * merge_size), + static_cast(this->vision_shortest_edge), + static_cast(this->vision_longest_edge)); + + const size_t single_frame_size = static_cast(resized_height) * + resized_width * channels; + const size_t total_patch_size = single_frame_size * temporal_size; + const unsigned grid_h = static_cast(resized_height) / patch_size; + const unsigned grid_w = static_cast(resized_width) / patch_size; + + auto resize_image = imgproc::avx512::resize_bicubic_antialias_rgb_planar_avx512( + image._data.data(), width, height, resized_width, resized_height, true); + + // Reused across calls: a multi-image prompt would otherwise allocate and + // free two buffers of this size per image. + static thread_local std::vector patch_vector_scratch; + static thread_local std::vector reorder_scratch; + if (patch_vector_scratch.size() < total_patch_size) + patch_vector_scratch.resize(total_patch_size); + if (reorder_scratch.size() < total_patch_size) + reorder_scratch.resize(total_patch_size); + + imgproc::avx512::rescale_and_normalize_avx512( + resize_image.data(), patch_vector_scratch.data(), + resized_width, resized_height, channels, + true, this->vision_rescale_factor, + true, this->vision_image_mean, this->vision_image_std); + + // A still image has one frame, which the processor replicates to fill the + // temporal patch. grid_t therefore stays 1 while the per-patch row is + // 3 * temporal_patch_size * patch^2 wide. + for (unsigned l = 1; l < temporal_size; l++) { + std::memcpy(patch_vector_scratch.data() + l * single_frame_size, + patch_vector_scratch.data(), + single_frame_size * sizeof(float)); + } + + imgproc::reorder_patches_inplace( + patch_vector_scratch.data(), + reorder_scratch.data(), + 1, 1, // batch_size, grid_t -- one still image + static_cast(temporal_size), + channels, + static_cast(grid_h), static_cast(grid_w), + static_cast(merge_size), + static_cast(patch_size)); + + const size_t prev = pixel_values.size(); + pixel_values.resize(prev + total_patch_size); + float* dst = pixel_values.data() + prev; + for (size_t i = 0; i < total_patch_size; i++) + dst[i] = static_cast(reorder_scratch[i]); + + image.width_resized = resized_width; + image.height_resized = resized_height; + image.grid_t = 1; + image.grid_h = static_cast(grid_h); + image.grid_w = static_cast(grid_w); + image._data.free(); // the uint8 source is dead from here on +} + + +bool Qwen3_8MTP::preprocess_image_file(const std::string& path, + std::vector& pixel_values, + int& grid_t, int& grid_h, int& grid_w) { + grid_t = grid_h = grid_w = 0; + qwen3_8mtp_host_image_t image = this->load_image(path); + if (image.width <= 0 || image.height <= 0) return false; + + const size_t offset = pixel_values.size(); + this->preprocess_image(image, pixel_values); + if (image.grid_h <= 0 || image.grid_w <= 0) { + pixel_values.resize(offset); + return false; + } + grid_t = image.grid_t; + grid_h = image.grid_h; + grid_w = image.grid_w; + return true; +} diff --git a/src/include/AutoModel/all_models.hpp b/src/include/AutoModel/all_models.hpp index 71aee93c0..32a621a2e 100644 --- a/src/include/AutoModel/all_models.hpp +++ b/src/include/AutoModel/all_models.hpp @@ -20,6 +20,7 @@ #include "modeling_qwen3_5vl.hpp" #include "modeling_qwen3_5_omni.hpp" #include "modeling_qwen3_6_moe.hpp" +#include "modeling_qwen3_8mtp.hpp" #include "modeling_nanbeige.hpp" #include "modeling_gemma4e.hpp" #include "modeling_hunyuan.hpp" @@ -41,6 +42,7 @@ typedef enum { qwen3_5, qwen3_5_omni, qwen3_6_moe, + qwen3_8mtp, gemma3, gemma3_text, gemma4e, @@ -72,6 +74,7 @@ inline std::pair> get_auto_model(const s {"qwen3.5", SupportedModelFamily::qwen3_5}, {"qwen3.5-omni", SupportedModelFamily::qwen3_5_omni}, {"qwen3.6-moe", SupportedModelFamily::qwen3_6_moe}, + {"qwen3.8-mtp", SupportedModelFamily::qwen3_8mtp}, {"gemma3", SupportedModelFamily::gemma3}, {"gemma3-text", SupportedModelFamily::gemma3_text}, {"gemma4e", SupportedModelFamily::gemma4e}, @@ -155,6 +158,9 @@ inline std::pair> get_auto_model(const s case SupportedModelFamily::qwen3_6_moe: auto_chat_engine = std::make_unique(npu_device_inst); break; + case SupportedModelFamily::qwen3_8mtp: + auto_chat_engine = std::make_unique(npu_device_inst); + break; case SupportedModelFamily::lfm2: auto_chat_engine = std::make_unique(npu_device_inst); break; diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index 5b54ec3ba..d13d6ba71 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -27,6 +27,7 @@ #include "models/qwen3vl_flash/qwen3vl_flash.hpp" #include "models/qwen3_5vl/qwen3_5vl_npu.hpp" #include "models/qwen3_6_moe/qwen3_6_moe_npu.hpp" +#include "models/qwen3_8mtp/qwen3_8mtp_npu.hpp" #include "models/gemma/gemma_npu.hpp" #include "models/gemma_text/gemma_text_npu.hpp" #include "models/gemma4e/gemma4e_npu.hpp" diff --git a/src/include/AutoModel/modeling_qwen3_8mtp.hpp b/src/include/AutoModel/modeling_qwen3_8mtp.hpp new file mode 100644 index 000000000..f40c6b631 --- /dev/null +++ b/src/include/AutoModel/modeling_qwen3_8mtp.hpp @@ -0,0 +1,241 @@ +/// \file modeling_qwen3_8mtp.hpp +/// \brief Qwen3_8MTP class +/// \author FastFlowLM Team +/// \date 2026-09-16 +/// \version 0.9.28 +/// \note AutoModel wrapper for Qwen3.8-27B (model_type qwen3_5): 64 layers, +/// 48 GatedDeltaNet + 16 full attention, plus a 1-layer MTP draft head. +/// \note Multimodal when the checkpoint ships vision_weight.q4nx alongside +/// model.q4nx; text-only otherwise. Images are dropped loudly rather +/// than templated into placeholders nothing fills -- that failure mode +/// prefills garbage and still produces fluent text, so it never flags +/// itself. Ask the engine with has_vision_tower(), do not assume. +/// \note The think ids are 248068/248069, NOT the 151667/151668 that +/// modeling_qwen3.hpp hardcodes. Copying those gives a model whose +/// reasoning block never closes. + +#pragma once +#include "AutoModel/automodel.hpp" +#include "image/image_reader.hpp" +#include "image_process_utils/imageproc.hpp" +#include "image_process_utils/imageprocAVX512.hpp" +#include "base64.hpp" + + +/// \brief one decoded image, host side, on its way to the engine +/// \note Deliberately NOT qwen3_8mtp_image_t: that struct is the engine's ABI +/// and carries only a pointer and a grid. Everything here -- the uint8 +/// CHW pixels, the pre- and post-resize extents -- is preprocessing +/// state the engine must never see, and keeping it out of the shared +/// header is what lets the engine struct stay a POD across the .so +/// boundary. +struct qwen3_8mtp_host_image_t { + int width = 0; + int height = 0; + int width_resized = 0; ///< assigned by preprocess_image + int height_resized = 0; + int grid_t = 1; ///< a still image is one temporal frame + int grid_h = 0; ///< PATCHES, not pixels + int grid_w = 0; + + bytes _data; ///< uint8 (3, H, W); freed by preprocess_image +}; + + +/************ Qwen3_8MTP **************/ +class Qwen3_8MTP : public AutoModel { +private: + + bool enable_think = false; + bool enable_tool = true; + + /// \note Qwen3.8 vocabulary, verified against tokenizer.json: + /// 248068, 248069. The chat template emits + /// "\n" when thinking is on and "\n\n\n\n" + /// when it is off, exactly as Qwen3.5 does. + int think_start_id = 248068; + int think_end_id = 248069; + + /// \brief the generation prompt's trailing ids that generate() must echo + /// \note insert() prefills the whole templated prompt, think preamble + /// included, so by the time generate() runs those tokens are already + /// in the cache and in token_history. What is not done is putting + /// them on screen, and only thinking-on has anything to show: it + /// opens a reasoning block the user reads. Thinking-off leaves this + /// empty. + /// \note Recorded at insert() time rather than rebuilt from + /// think_start_id/198 in generate(), so a caller that flips + /// enable_think between the two calls echoes what was actually + /// prefilled instead of what the flag says now. + std::vector preamble_to_stream; + + void setup_tokenizer(std::string model_path); + + // ---- image pipeline --------------------------------------------------- + // + // Modelled on Qwen3_5VL's, with one deliberate difference: this engine's + // encode_image() takes `const float*`, not bf16. See preprocess_image(). + + ImageReader image_reader_; + + /// \brief cap the decoded height before preprocessing; 0 = no cap + /// \note Purely a cost knob. smart_resize below still runs and still has + /// the final say over the grid, so this only changes how many + /// patches the tower is asked to encode -- which at ~70 ms per + /// merged token on the CPU backend is the whole encode time. + int image_pre_resize = 0; + + /// Image-processor geometry. Read from config.json's vision_config in + /// load_model() where the checkpoint states it, and defaulted to + /// Qwen3.8-27B's preprocessor_config.json where it does not -- the NPU2 + /// checkpoint ships no preprocessor_config.json, and the converter does + /// not inject the QWEN3_5_* keys that Qwen3_5VL reads. + /// + /// image_mean and image_std are 0.5, NOT the OPENAI_CLIP constants. Using + /// CLIP's costs no error anyone would notice at the tower's output and + /// every bit of it at the model's, which is why it is written down. + unsigned int vision_patch_size = 16; + unsigned int vision_merge_size = 2; + unsigned int vision_temporal_patch_size = 2; + unsigned int vision_shortest_edge = 65536; + unsigned int vision_longest_edge = 16777216; + float vision_rescale_factor = 1.0f / 255.0f; + float vision_image_mean = 0.5f; + float vision_image_std = 0.5f; + + qwen3_8mtp_host_image_t load_image(const std::string& filename); + qwen3_8mtp_host_image_t load_image_base64(const std::string& base64_string); + + /// Shared tails of the two loaders. Factored out because the Qwen3_5VL + /// pair they are modelled on is a verbatim copy that has already drifted. + void _apply_pre_resize(image_data_t& decoded); + qwen3_8mtp_host_image_t _finish_load(image_data_t& decoded); + + void smart_resize( + int height, int width, + int& h_bar, int& w_bar, + int factor, + int min_pixels, + int max_pixels); + + /// \brief resize, normalise and reorder one image; appends to `pixel_values` + /// \note Appends in MERGE-BLOCK order, which is what the engine's + /// qwen3_8mtp_image_t contract requires. Raster order is not a + /// crash -- the tower runs and the image is silently scrambled. + void preprocess_image(qwen3_8mtp_host_image_t& image, + std::vector& pixel_values); + +public: + Qwen3_8MTP(flm_rt::device* npu_device_inst); + + /// \brief decode + preprocess one image file into the engine's pixel_values + /// \return false if the image could not be decoded or preprocessed + /// + /// The whole image pipeline with no model behind it. It exists so the host + /// preprocessing can be scored against a captured `pixel_values` without + /// loading 17 GB of weights first -- and it needs scoring separately, + /// because the tower's own verification feeds it a reference pixel_values + /// and therefore says nothing about how this code produces one. A wrong + /// patch order here passes every tower check and still scrambles the image. + /// + /// Safe to call on a default-constructed Qwen3_8MTP: it touches only + /// image_reader_ and the vision_* geometry, which is defaulted to this + /// checkpoint's values and only refined by load_model(). + bool preprocess_image_file(const std::string& path, + std::vector& pixel_values, + int& grid_t, int& grid_h, int& grid_w); + + void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; + bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; + std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; + std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; + std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + NonStreamResult parse_nstream_content(const std::string response_text); + StreamResult parse_stream_content(const std::string content); + StreamResult parse_stream_content_final(const std::string content) override; + + /// \brief the base profile plus the MTP draft/verify/replay breakdown + /// \note Overridden because "Decoding time" is one number for a model with + /// three distinct decode phases. The runtime profiler cannot split + /// them -- from its side a cycle is one opaque speculate() call -- + /// so the breakdown is measured in the engine and appended here. + /// \note Calls the base implementation rather than reproducing its rows, + /// so the shared statistics cannot drift out of step with it. + /// \note Appends nothing when speculation never ran, which keeps the + /// non-greedy and no-MTP-head paths byte-identical to before. + std::string show_profile() override; + +private: + StreamResult parse_stream_content_impl(const std::string content, bool is_final); + +public: + + /// \brief Configure a parameter with type-erased value + /// \param parameter_name the name of the parameter + /// \param value the value to set (can be any type) + /// \return true if the parameter was configured successfully, false otherwise + bool configure_parameter(std::string parameter_name, const std::any& value) override { + if (parameter_name == "enable_think") { + try { + this->enable_think = std::any_cast(value); + return true; + } catch (const std::bad_any_cast&) { + return false; + } + } + else if (parameter_name == "reasoning_effort") { + std::string reasoning_effort; + try { + reasoning_effort = std::any_cast(value); + if (reasoning_effort == "high" || reasoning_effort == "medium" || reasoning_effort == "low") + this->enable_think = true; + else if (reasoning_effort == "none") + this->enable_think = false; + else + header_print("WARNING", "Reasoning effort must be 'none', 'low', 'medium' or 'high'!"); + return true; + } catch (const std::bad_any_cast&) { + return false; + } + } + else if (parameter_name == "toggle_think") { + this->enable_think = !this->enable_think; + return true; + } + else if (parameter_name == "system_prompt") { + try { + this->user_system_prompt = std::any_cast(value); + this->extra_context["user_system_prompt"] = this->user_system_prompt; + return true; + } catch (const std::bad_any_cast&) { + return false; + } + } + else if (parameter_name == "img_pre_resize") { + // Same ladder and the same key name as Qwen3_5VL, so a caller that + // drives both models does not need to special-case this one. + try { + this->image_pre_resize = std::any_cast(value); + int target_size; + if (this->image_pre_resize <= 0) target_size = 0; + else if (this->image_pre_resize == 1) target_size = 480; + else if (this->image_pre_resize <= 2) target_size = 720; + else if (this->image_pre_resize <= 3) target_size = 1080; + else if (this->image_pre_resize <= 4) target_size = 1440; + else if (this->image_pre_resize <= 5) target_size = 2160; + else if (this->image_pre_resize <= 6) target_size = 2880; + else if (this->image_pre_resize <= 7) target_size = 3240; + else if (this->image_pre_resize <= 8) target_size = 4320; + else { this->image_pre_resize = 0; target_size = 0; } + if (this->image_pre_resize > 0) + header_print_r("FLM", "Qwen3.8 pre-resize image height to " + + std::to_string(target_size) + + " pixels if larger than that"); + return true; + } catch (const std::bad_any_cast&) { + return false; + } + } + return AutoModel::configure_parameter(parameter_name, value); + } +}; diff --git a/src/include/causal_lm.hpp b/src/include/causal_lm.hpp index d4682a95a..9c096bb31 100644 --- a/src/include/causal_lm.hpp +++ b/src/include/causal_lm.hpp @@ -6,6 +6,8 @@ /// \note This class is a virtual class for causal language models /// \note All other models should inherit from this class so that they can be used in the same way. #pragma once +#include + #include "tensor_utils/q4_npu_eXpress.hpp" #include "tensor_2d.hpp" #include "utils/utils.hpp" @@ -61,4 +63,63 @@ class causal_lm { virtual int checkpoint() = 0; virtual int restore() = 0; + + // ---- speculative decoding --------------------------------------------- + // Defaulted, not pure: only engines with a draft head override these, and + // a pure virtual here would fail to compile all 16 other subclasses. + // + // ABI WARNING -- adding a virtual to this class is a lockstep change. + // Every engine .so emits its own vtable for its causal_lm subclass and flm + // loads those .so files prebuilt; flm emits no vtable of its own. Appending + // here renumbers the vtable, so any .so not rebuilt keeps one that is short + // by the number of methods added, and calling a new method reads past its + // end. Measured: these two took the vtable from 120 to 136 bytes, and every + // stale .so segfaulted on its first decode step -- after a clean prefill, + // with no diagnostic. A link check cannot catch it; the vtable symbol still + // resolves, it is merely the wrong size. + // + // So: every .so in src/lib/xrt must be rebuilt and re-copied in the same + // change as any edit to this class. Verify with + // readelf -sW .so | grep _ZTV # all engine vtables must agree + // not with "it compiled" and not with "it linked". + + /// \brief whether speculate() can currently run + /// \return false unless the engine has a draft head and loaded weights for it + /// \note Default false: an engine that does not override this is never + /// asked to speculate and keeps the single-token path exactly. + virtual bool supports_speculation() const { return false; } + + /// \brief propose several tokens at once and return the ones the model agrees with + /// \param last_token the most recently committed token -- the one the caller + /// would otherwise pass to forward() + /// \param max_draft how many tokens to draft; the engine may draft fewer + /// \return the committed tokens, in order, at least one on success; + /// empty means "could not speculate this step" + /// \note An empty return is not an error and not end-of-stream. It is how + /// the engine declines -- no draft head, not enough context, no room + /// under MAX_L -- and the caller must fall back to forward() + + /// sample() for that step. + /// \note PRECONDITION: greedy sampling. Acceptance is an exact integer + /// compare against the base model's argmax, so the tokens returned + /// are argmax tokens. Calling this with top_k != 1, a temperature, or + /// repetition penalties active silently substitutes greedy decoding + /// for the sampler the user configured -- the output stays fluent, so + /// nothing downstream would ever flag it. The caller must gate on its + /// own sampler; the engine cannot see it. + /// \note The engine has already committed these tokens to its caches. The + /// caller must not re-feed them through forward(). + virtual std::vector speculate(int last_token, int max_draft) { return {}; } + + /// \brief how much of the last speculate() was prompt-phase, in microseconds + /// \return 0 by default, and 0 on any cycle that did no prompt-phase work + /// \note A draft head keeps its own KV cache, and on the first cycle after + /// a prefill that cache is empty: the cycle opens by re-absorbing the + /// prompt window before it drafts anything. That pass costs what a + /// prefill costs and scales with the prompt, but it happens inside a + /// speculate() call, where the caller's decode timer is running. + /// Engines that can tell the two apart report the split here so the + /// caller can move it to its prefill bucket; engines that cannot + /// return 0 and lose nothing, since the time is already counted. + /// \note Valid only until the next speculate(). + virtual uint64_t last_speculation_prime_us() const { return 0; } }; \ No newline at end of file diff --git a/src/include/minja/minja.hpp b/src/include/minja/minja.hpp index 58cdb4176..35757e916 100644 --- a/src/include/minja/minja.hpp +++ b/src/include/minja/minja.hpp @@ -1351,6 +1351,14 @@ class BinaryOpExpr : public Expression { if (name == "iterable") return l.is_iterable(); if (name == "sequence") return l.is_array(); if (name == "defined") return !l.is_null(); + // Upstream minja (commit 3e4c61c) implements `is defined` but not + // its negation, and throws on the test NAME before ever looking at + // the value -- so `x is undefined` fails even when x is set. + // VariableExpr::do_evaluate returns a null Value for a missing + // name, so this is the exact mirror of the line above. + // Qwen3.8-27B's chat template opens with + // `{%- if enable_thinking is undefined or enable_thinking is true %}`. + if (name == "undefined") return l.is_null(); if (name == "true") return l.to_bool(); if (name == "false") return !l.to_bool(); throw std::runtime_error("Unknown type for 'is' operator: " + name); diff --git a/src/include/models/qwen3_8mtp/qwen3_8mtp_npu.hpp b/src/include/models/qwen3_8mtp/qwen3_8mtp_npu.hpp new file mode 100644 index 000000000..f45c14221 --- /dev/null +++ b/src/include/models/qwen3_8mtp/qwen3_8mtp_npu.hpp @@ -0,0 +1,302 @@ +/// \file qwen3_8mtp_npu.hpp +/// \brief qwen3_8mtp_npu class +/// \author FastFlowLM Team +/// \date 2026-09-16 +/// \version 0.9.28 +/// \note Qwen3.8-27B: 64 layers, 48 GatedDeltaNet + 16 full attention in the +/// pattern [lin,lin,lin,full] x 16, plus a 1-layer MTP draft head. +/// \note Phase 1 is CPU-only -- no xclbin, no NPU sequence header. The +/// constructor still takes an npu_xclbin_manager* so the signature +/// matches every other engine and Phase 2 needs no call-site change. +/// \note Nothing from detail/ may be included here: the runtime's +/// automodel.hpp pulls this header in. +#pragma once +#include "lm_config.hpp" +#include "npu_utils/npu_utils.hpp" +#include "tensor_utils/q4_npu_eXpress.hpp" +#include "tensor_2d.hpp" +#include "utils/utils.hpp" +#include "causal_lm.hpp" +#if USEAVX2 +#include // For AVX intrinsics +#endif +#include + + +/// \brief one image's patches, as the HF image processor emits them +/// \note `pixel_values` is [grid_t*grid_h*grid_w][in_channels * temporal_patch * +/// patch^2] fp32, already in MERGE-BLOCK order (the processor's +/// reshape/permute), which is what makes the merger's 2x2 regroup a free +/// reinterpretation downstream. Feeding raster order is not a crash: the +/// tower runs and the image is silently scrambled. +/// \note grid_h and grid_w count PATCHES, not pixels, and must both be +/// multiples of spatial_merge_size. +struct qwen3_8mtp_image_t { + const float* pixel_values = nullptr; + int grid_t = 1; + int grid_h = 0; + int grid_w = 0; +}; + +/// \brief the images a prefill window refers to, in prompt order +/// \note The engine matches these against the window's image_token_id rows and +/// throws if the counts disagree -- a mismatch would splice one image's +/// rows into another's slots, which every downstream stage accepts. +struct qwen3_8mtp_image_payload_t { + const qwen3_8mtp_image_t* images = nullptr; + int num_images = 0; +}; + +/// \brief optional payload for prefill() +/// \note This is the same `void* payload` seam qwen3_5vl_npu uses for images. +/// Phase 1 needs it for one thing the reference forces: a multimodal +/// window's position ids are NOT 0..L-1. In the dumped prompt 221 tokens +/// occupy only 165 positions, because a 16x16 image block consumes 64 +/// token slots while advancing position by 8. Anything that reproduces a +/// captured window must therefore supply positions explicitly. +/// \note Both pointers are borrowed and only read during the call. +struct qwen3_8mtp_payload_t { + /// [3][L] m-rope position ids, channels t,h,w. Null = pure text, in which + /// case the engine emits ctx..ctx+L-1 on all three channels. + const int32_t* position_ids = nullptr; + /// [L][hidden_size] fp32 hidden states to use instead of the embedding + /// lookup. Null = look every id up in model.embed_tokens. Phase 2's vision + /// encoder writes image rows here; the milestone driver uses it to feed a + /// captured inputs_embeds. + const float* embeds = nullptr; + /// Raw images for the engine's own vision tower to encode. Appended last so + /// existing brace-initialised call sites keep compiling. + /// \note `embeds` wins if both are set: a caller supplying hidden states has + /// already resolved its image rows, and running the tower again would + /// be work whose result is thrown away. + const qwen3_8mtp_image_payload_t* images = nullptr; +}; + +/// \note This is the only engine that overrides causal_lm's speculation hooks. +/// They are defaulted in the base, so the other 15 engines keep the +/// single-token path unchanged -- but their .so files must still be +/// rebuilt in lockstep, because the added virtuals renumber the shared +/// vtable. See the ABI warning in causal_lm.hpp. +class qwen3_8mtp_npu : public causal_lm{ +public: + /// \brief initialize the qwen3_8mtp_npu + /// \param config the configuration + /// \param npu_instance the npu instance (unused in phase 1, CPU only) + /// \param MAX_L the maximum context length + qwen3_8mtp_npu(LM_Config config, npu_xclbin_manager *npu_instance, int MAX_L = 4096); + ~qwen3_8mtp_npu(); + + /// \brief forward one token + /// \param ids the token id + /// \return the last-token logits + buffer forward(int ids) override; + + /// \brief prefill a window of tokens + /// \param ids the token ids + /// \param payload reserved for the phase-2 vision path + /// \return the last-token logits + /// \note Appends to the KV/SSM state; never resets. _chunked_insert calls + /// this repeatedly for one prompt. + buffer prefill(std::vector& ids, void* payload = nullptr) override; + + /// \brief set the context length + /// \param L the context length + void set_context_length(int L) override; + + /// \brief load the weights + /// \param q4nx the q4nx + void load_weights(Q4NX& q4nx) override; + + /// \brief clear the context + void clear_context() override; + + /// \brief get the k cache + /// \note dead ceremony; nothing in the runtime calls it + buffer get_k_cache(int layer_idx, int idx) override; + + /// \brief get the v cache + /// \note dead ceremony; nothing in the runtime calls it + buffer get_v_cache(int layer_idx, int idx) override; + + /// \brief update the max length + /// \param MAX_L the max length + void update_max_length(uint32_t MAX_L) override; + + /// \brief get the current context length + /// \return the current context length + int get_current_context_length() override; + int checkpoint() override; + int restore() override; + + /// \brief true once a checkpoint carrying the mtp.* tensors is loaded + /// \note Answers false before load_weights(), and false forever for a + /// checkpoint without an MTP head -- that is still a valid base + /// model, it just decodes one token at a time. + bool supports_speculation() const override; + + /// \brief draft with the MTP head, verify against the full stack, commit + /// \param last_token the token the caller would otherwise pass to forward() + /// \param max_draft draft depth, clamped to MTP_STEPS (7) + /// \return the committed tokens (1..max_draft+1), or {} to decline + /// \note The returned tokens are already in the caches. Feeding them back + /// through forward() would double-append them. + /// \note Greedy only -- see the precondition on causal_lm::speculate. + std::vector speculate(int last_token, int max_draft) override; + + // ---- speculation statistics ------------------------------------------- + // All non-virtual: they add no vtable slot, so unlike the two speculation + // hooks on causal_lm these cost no lockstep rebuild of the other engines. + // + // Worth reporting at all because a broken draft path is SILENT. Verify + // overrides every rejected draft with the base model's own argmax, so bad + // drafting produces correct text and merely burns time -- the hit rate is + // the only outward symptom. Without it a regression here reads as "the + // machine feels slow today". + + /// \brief mean drafts accepted per speculative cycle since load + double mean_accepted_length() const; + + /// \brief fraction of offered drafts the base model agreed with, in [0,1] + /// \note Denominator is drafts actually offered, which is not + /// cycles x MTP_STEPS: k is clamped by max_draft and by the MAX_L + /// headroom check, so short cycles would otherwise be charged full. + /// \note The free token committed at the first mismatch is NOT counted as + /// a hit -- it is the base model's argmax, not a correct draft. + /// Counting it would floor this at 1/k for a head that never once + /// guessed right. + double draft_hit_rate() const; + + /// \brief speculative cycles run since load; 0 means speculation never ran + /// \note Check this before reading a rate: with no cycles the rates are + /// 0.0, which is indistinguishable from "ran and missed everything". + uint64_t speculation_cycles() const; + + /// \brief one-line summary, or "" if no cycle ever ran + std::string speculation_stats() const; + + /// \brief print that summary; prints nothing when no cycle ran + /// \note Engine-side rather than in the runtime decode loop on purpose: + /// this model's AutoModel wrapper has its own generate() that does + /// not go through _shared_generate, so a runtime-side counter would + /// stay silent on the main chat path. + void report_speculation_stats() const; + + /// \brief multi-line draft/verify/replay time breakdown, or "" if no cycle + /// \note Lives here rather than in the runtime's profiler because from + /// automodel.cpp one cycle IS one speculate() call -- the phase + /// boundary does not exist at that level, so DECODING_TIME can only + /// ever charge draft, verify and replay to a single number. + /// \note Indented two levels so it sits under show_profile()'s + /// "Statistics:" block. Ends with a newline. + std::string speculation_timing() const; + + /// \brief (draft + verify + replay) / cycle time, in [0,1]; 0 if no cycle + /// \note Denominator is the measured cycle, not the sum of the parts, so a + /// phase that is double-counted pushes this ABOVE 1.0 instead of + /// normalising itself away. That is the whole point of exposing it: + /// a breakdown whose parts do not add up to the whole misleads more + /// than no breakdown at all, and only this ratio catches it. + double speculation_phase_fraction() const; + + /// \brief microseconds of the LAST speculate() that were prompt-phase work + /// \return 0 on every cycle but the first one after each prefill() + /// \note Step 0 of that first cycle is the draft head absorbing the prompt: + /// its catch-up window is the whole last prefill chunk, run once, + /// through the prefill bitstream. It is prefill, not decode, and it + /// scales with the PROMPT -- so leaving it inside DECODING_TIME makes + /// decode tok/s a function of prompt length. The caller moves this + /// many microseconds from DECODING_TIME to PREFILL_TIME. + /// \note Read it immediately after speculate(); the next call overwrites it. + uint64_t last_speculation_prime_us() const override; + + /// \brief zero the counters, e.g. between benchmark runs + /// \note Counters are cumulative since load and survive clear_context(), + /// so a per-turn rate needs this at the start of each turn. + /// \note Zeroes the timers too -- a rate and a duration measured over + /// different windows do not compose. + void reset_speculation_stats(); + + // ---- phase 2 and 3: MTP draft + verify ------------------------------- + // Not part of causal_lm. The runtime cannot drive speculation yet, so + // these are the engine-side half only, reachable from the milestone + // driver and from a future speculate() override. + + /// \brief the raw hidden states the last prefill()/forward() produced + /// \return [L][hidden_size] fp32, layer 63's output -- BEFORE model.norm. + /// Valid until the next prefill()/forward(). + /// \note This is the layer tap, for scoring against prefill.layer_63_out. + /// It is NOT what the MTP head consumes -- see last_normed_states(). + const float* last_hidden_states(int& L, int& D) const; + + /// \brief the same states with model.norm applied + /// \return [L][hidden_size] fp32, equal to the reference's + /// prefill.final_norm_out. Valid until the next call. + /// \note This is what the MTP head's fusion takes as `prev_hidden`, and + /// the distinction is load-bearing: the dump's mtp.step0.prev_hidden + /// matches final_norm_out exactly (cos 1.0) and the pre-norm tap + /// only to cos 0.94. Feeding the raw tap makes the head fuse + /// unnormalised states -- it still drafts plausible tokens, so the + /// failure shows up only as a quietly poor acceptance rate. + const float* last_normed_states(int& L, int& D); + + /// \brief draft up to k continuations with the MTP head + /// \param ids the window's token ids (k=1 in steady state) + /// \param prev_hidden [T][hidden_size], the states that predicted them + /// \param T window length + /// \param k how many tokens to draft + /// \return the drafted tokens, greedy argmax; empty if no MTP head loaded + /// \note Greedy is not a default, it is the contract: acceptance is an + /// exact integer compare, so a sampled draft would match only by + /// coincidence and the speedup would vanish. + std::vector mtp_draft(const std::vector& ids, + const float* prev_hidden, int T, int k); + + /// \brief run the k+1 verify tokens through all 64 layers and accept + /// \param verify_ids the k+1 tokens: the committed token then the k drafts + /// \param drafts the k drafted tokens, to compare against + /// \param out_argmax receives the base model's argmax at all k+1 positions + /// \return how many drafts were accepted; `accepted + 1` tokens commit + /// \note Snapshots the stack first and rolls back on a partial accept -- + /// the delta-net fold has no inverse, so there is no other way back. + int prefill_verify(const std::vector& verify_ids, + const std::vector& drafts, + std::vector& out_argmax); + + /// \brief whether the checkpoint carried an MTP head + bool has_mtp_head() const; + + // ---- module 4: the vision tower --------------------------------------- + // Not part of causal_lm. prefill() drives the tower itself when the payload + // carries images; these expose it directly so the milestone driver can + // score the encoder against a dump without a prompt around it. + + /// \brief true once a checkpoint with vision_weight.q4nx is loaded + /// \note False before load_weights(), and false for a text-only checkpoint. + /// Passing images to prefill() then throws rather than silently + /// dropping them. + bool has_vision_tower() const; + + /// \brief language tokens one grid produces: t*h*w / spatial_merge_size^2 + /// \note The prompt must carry exactly this many image_token_id rows. + int image_tokens_for(int grid_t, int grid_h, int grid_w) const; + + /// \brief run one image through the tower + /// \param out [image_tokens_for(...)][hidden_size] fp32 + /// \throws if no vision tower is loaded + void encode_image(const float* pixel_values, int grid_t, int grid_h, + int grid_w, float* out); + + /// \brief which backend the tower's matmuls and attention are running on + /// \return e.g. "mm=cpu attn=cpu bf16", or "" with no tower loaded + std::string vision_backend() const; + + /// \brief capture the tower's per-stage intermediates on the next encode + /// \param fn called as (stage_name, [rows][cols] fp32, rows, cols); the + /// pointer dies when the call returns. Pass {} to stop capturing. + /// \note Costs nothing when unset, so this is the only instrumentation the + /// tower carries -- there is no debug build of it. + void set_vision_tap(std::function fn); +private: + struct Impl; + Impl* _impl; +}; diff --git a/src/include/utils/profiler.hpp b/src/include/utils/profiler.hpp index 6e6802200..1b2f811e6 100644 --- a/src/include/utils/profiler.hpp +++ b/src/include/utils/profiler.hpp @@ -39,6 +39,18 @@ class profiler{ return end_time; } + /// \brief add (or, when negative, remove) time without running the clock + /// \param us microseconds to fold into the total + /// \note For moving a measured interval between buckets: an interval that + /// was timed inside one phase but belongs to another cannot be + /// re-measured, only transferred, and start()/stop() cannot express + /// that. Clamped at zero, so a transfer that overshoots empties the + /// bucket instead of wrapping the unsigned total into nonsense. + void add_time(int64_t us){ + double t = (double)this->total_time.first + (double)us; + this->total_time.first = t > 0.0 ? (float)t : 0.0f; + } + /// \brief reset the profiler void reset(){ this->start_time = time_utils::now(); diff --git a/src/lib/xrt/dequant.dll b/src/lib/xrt/dequant.dll index 6bb0e3b72..9da491c19 100644 Binary files a/src/lib/xrt/dequant.dll and b/src/lib/xrt/dequant.dll differ diff --git a/src/lib/xrt/gemm.dll b/src/lib/xrt/gemm.dll index e3ff852fc..ef65ee5d9 100644 Binary files a/src/lib/xrt/gemm.dll and b/src/lib/xrt/gemm.dll differ diff --git a/src/lib/xrt/gemma4_12b_npu.dll b/src/lib/xrt/gemma4_12b_npu.dll index 33a555700..4cde37571 100644 Binary files a/src/lib/xrt/gemma4_12b_npu.dll and b/src/lib/xrt/gemma4_12b_npu.dll differ diff --git a/src/lib/xrt/gemma4_12b_npu.lib b/src/lib/xrt/gemma4_12b_npu.lib index 471386f8a..1595a3c94 100644 Binary files a/src/lib/xrt/gemma4_12b_npu.lib and b/src/lib/xrt/gemma4_12b_npu.lib differ diff --git a/src/lib/xrt/gemma4e_flash.dll b/src/lib/xrt/gemma4e_flash.dll index 14753888e..7ec270b96 100644 Binary files a/src/lib/xrt/gemma4e_flash.dll and b/src/lib/xrt/gemma4e_flash.dll differ diff --git a/src/lib/xrt/gemma4e_flash.lib b/src/lib/xrt/gemma4e_flash.lib index 3a74dd7ed..9044da5bb 100644 Binary files a/src/lib/xrt/gemma4e_flash.lib and b/src/lib/xrt/gemma4e_flash.lib differ diff --git a/src/lib/xrt/gemma4e_npu.dll b/src/lib/xrt/gemma4e_npu.dll index 022db6f44..c511810bf 100644 Binary files a/src/lib/xrt/gemma4e_npu.dll and b/src/lib/xrt/gemma4e_npu.dll differ diff --git a/src/lib/xrt/gemma4e_npu.lib b/src/lib/xrt/gemma4e_npu.lib index 2404e56f7..18b9246b9 100644 Binary files a/src/lib/xrt/gemma4e_npu.lib and b/src/lib/xrt/gemma4e_npu.lib differ diff --git a/src/lib/xrt/gemma_embedding.dll b/src/lib/xrt/gemma_embedding.dll index d651aa7aa..1bf1a17c1 100644 Binary files a/src/lib/xrt/gemma_embedding.dll and b/src/lib/xrt/gemma_embedding.dll differ diff --git a/src/lib/xrt/gemma_npu.dll b/src/lib/xrt/gemma_npu.dll index febcf58ab..7f2b2a562 100644 Binary files a/src/lib/xrt/gemma_npu.dll and b/src/lib/xrt/gemma_npu.dll differ diff --git a/src/lib/xrt/gemma_npu.lib b/src/lib/xrt/gemma_npu.lib index de68e7c7f..2a46f3253 100644 Binary files a/src/lib/xrt/gemma_npu.lib and b/src/lib/xrt/gemma_npu.lib differ diff --git a/src/lib/xrt/gemma_text_npu.dll b/src/lib/xrt/gemma_text_npu.dll index 71988f276..3c386ac80 100644 Binary files a/src/lib/xrt/gemma_text_npu.dll and b/src/lib/xrt/gemma_text_npu.dll differ diff --git a/src/lib/xrt/gemma_text_npu.lib b/src/lib/xrt/gemma_text_npu.lib index 4ae1c758c..9dcccab2e 100644 Binary files a/src/lib/xrt/gemma_text_npu.lib and b/src/lib/xrt/gemma_text_npu.lib differ diff --git a/src/lib/xrt/gpt_oss_npu.dll b/src/lib/xrt/gpt_oss_npu.dll index 5c7d93fce..abfefa8fe 100644 Binary files a/src/lib/xrt/gpt_oss_npu.dll and b/src/lib/xrt/gpt_oss_npu.dll differ diff --git a/src/lib/xrt/gpt_oss_npu.lib b/src/lib/xrt/gpt_oss_npu.lib index 4bbe16c9e..8a8863163 100644 Binary files a/src/lib/xrt/gpt_oss_npu.lib and b/src/lib/xrt/gpt_oss_npu.lib differ diff --git a/src/lib/xrt/hunyuan_npu.dll b/src/lib/xrt/hunyuan_npu.dll index 701a34215..30809606a 100644 Binary files a/src/lib/xrt/hunyuan_npu.dll and b/src/lib/xrt/hunyuan_npu.dll differ diff --git a/src/lib/xrt/hunyuan_npu.lib b/src/lib/xrt/hunyuan_npu.lib index 7cd519743..82fde8348 100644 Binary files a/src/lib/xrt/hunyuan_npu.lib and b/src/lib/xrt/hunyuan_npu.lib differ diff --git a/src/lib/xrt/lfm2_npu.dll b/src/lib/xrt/lfm2_npu.dll index 9f10dfa71..4bbed0c44 100644 Binary files a/src/lib/xrt/lfm2_npu.dll and b/src/lib/xrt/lfm2_npu.dll differ diff --git a/src/lib/xrt/lfm2_npu.lib b/src/lib/xrt/lfm2_npu.lib index d47859694..7b4c6fc12 100644 Binary files a/src/lib/xrt/lfm2_npu.lib and b/src/lib/xrt/lfm2_npu.lib differ diff --git a/src/lib/xrt/libqwen3_8mtp_npu.so b/src/lib/xrt/libqwen3_8mtp_npu.so new file mode 100755 index 000000000..b522198df Binary files /dev/null and b/src/lib/xrt/libqwen3_8mtp_npu.so differ diff --git a/src/lib/xrt/llama_npu.dll b/src/lib/xrt/llama_npu.dll index 578afa7de..c08673521 100644 Binary files a/src/lib/xrt/llama_npu.dll and b/src/lib/xrt/llama_npu.dll differ diff --git a/src/lib/xrt/llama_npu.lib b/src/lib/xrt/llama_npu.lib index 3429bad25..5d66b77c9 100644 Binary files a/src/lib/xrt/llama_npu.lib and b/src/lib/xrt/llama_npu.lib differ diff --git a/src/lib/xrt/lm_head.dll b/src/lib/xrt/lm_head.dll index ce2e25b6d..2a217d9e8 100644 Binary files a/src/lib/xrt/lm_head.dll and b/src/lib/xrt/lm_head.dll differ diff --git a/src/lib/xrt/mha.dll b/src/lib/xrt/mha.dll index 71463e6da..2169360f1 100644 Binary files a/src/lib/xrt/mha.dll and b/src/lib/xrt/mha.dll differ diff --git a/src/lib/xrt/nanbeige_npu.dll b/src/lib/xrt/nanbeige_npu.dll index f1e832cc0..a54ac85bb 100644 Binary files a/src/lib/xrt/nanbeige_npu.dll and b/src/lib/xrt/nanbeige_npu.dll differ diff --git a/src/lib/xrt/nanbeige_npu.lib b/src/lib/xrt/nanbeige_npu.lib index b9f0fc970..ee71e476c 100644 Binary files a/src/lib/xrt/nanbeige_npu.lib and b/src/lib/xrt/nanbeige_npu.lib differ diff --git a/src/lib/xrt/phi4_npu.dll b/src/lib/xrt/phi4_npu.dll index a7478c83e..eb3c548ef 100644 Binary files a/src/lib/xrt/phi4_npu.dll and b/src/lib/xrt/phi4_npu.dll differ diff --git a/src/lib/xrt/phi4_npu.lib b/src/lib/xrt/phi4_npu.lib index 9a086d34e..675902da5 100644 Binary files a/src/lib/xrt/phi4_npu.lib and b/src/lib/xrt/phi4_npu.lib differ diff --git a/src/lib/xrt/q4_npu_eXpress.dll b/src/lib/xrt/q4_npu_eXpress.dll index 5b565df90..9bbe87671 100644 Binary files a/src/lib/xrt/q4_npu_eXpress.dll and b/src/lib/xrt/q4_npu_eXpress.dll differ diff --git a/src/lib/xrt/qwen2_npu.dll b/src/lib/xrt/qwen2_npu.dll index 4db755b06..149203b11 100644 Binary files a/src/lib/xrt/qwen2_npu.dll and b/src/lib/xrt/qwen2_npu.dll differ diff --git a/src/lib/xrt/qwen2_npu.lib b/src/lib/xrt/qwen2_npu.lib index 20b9a8a31..37a3ba968 100644 Binary files a/src/lib/xrt/qwen2_npu.lib and b/src/lib/xrt/qwen2_npu.lib differ diff --git a/src/lib/xrt/qwen2vl_npu.dll b/src/lib/xrt/qwen2vl_npu.dll index 9d3221f03..34d5be9a0 100644 Binary files a/src/lib/xrt/qwen2vl_npu.dll and b/src/lib/xrt/qwen2vl_npu.dll differ diff --git a/src/lib/xrt/qwen2vl_npu.lib b/src/lib/xrt/qwen2vl_npu.lib index 4d6d2bd13..ed8fb9ebd 100644 Binary files a/src/lib/xrt/qwen2vl_npu.lib and b/src/lib/xrt/qwen2vl_npu.lib differ diff --git a/src/lib/xrt/qwen3_5_omni_npu.dll b/src/lib/xrt/qwen3_5_omni_npu.dll index c2cb57454..85591473d 100644 Binary files a/src/lib/xrt/qwen3_5_omni_npu.dll and b/src/lib/xrt/qwen3_5_omni_npu.dll differ diff --git a/src/lib/xrt/qwen3_5vl_npu.dll b/src/lib/xrt/qwen3_5vl_npu.dll index 3f2c256a5..8ff5db5a2 100644 Binary files a/src/lib/xrt/qwen3_5vl_npu.dll and b/src/lib/xrt/qwen3_5vl_npu.dll differ diff --git a/src/lib/xrt/qwen3_5vl_npu.lib b/src/lib/xrt/qwen3_5vl_npu.lib index d389a4331..adfbe0c98 100644 Binary files a/src/lib/xrt/qwen3_5vl_npu.lib and b/src/lib/xrt/qwen3_5vl_npu.lib differ diff --git a/src/lib/xrt/qwen3_6_moe_npu.dll b/src/lib/xrt/qwen3_6_moe_npu.dll index 4bd12b994..d21dca15e 100644 Binary files a/src/lib/xrt/qwen3_6_moe_npu.dll and b/src/lib/xrt/qwen3_6_moe_npu.dll differ diff --git a/src/lib/xrt/qwen3_6_moe_npu.lib b/src/lib/xrt/qwen3_6_moe_npu.lib index 1c683fbe7..8c36e2639 100644 Binary files a/src/lib/xrt/qwen3_6_moe_npu.lib and b/src/lib/xrt/qwen3_6_moe_npu.lib differ diff --git a/src/lib/xrt/qwen3_8mtp_npu.dll b/src/lib/xrt/qwen3_8mtp_npu.dll new file mode 100644 index 000000000..6486fcaea Binary files /dev/null and b/src/lib/xrt/qwen3_8mtp_npu.dll differ diff --git a/src/lib/xrt/qwen3_8mtp_npu.lib b/src/lib/xrt/qwen3_8mtp_npu.lib new file mode 100644 index 000000000..335db3b86 Binary files /dev/null and b/src/lib/xrt/qwen3_8mtp_npu.lib differ diff --git a/src/lib/xrt/qwen3_npu.dll b/src/lib/xrt/qwen3_npu.dll index 660f543ba..564a065c5 100644 Binary files a/src/lib/xrt/qwen3_npu.dll and b/src/lib/xrt/qwen3_npu.dll differ diff --git a/src/lib/xrt/qwen3_npu.lib b/src/lib/xrt/qwen3_npu.lib index d1ff0764e..63e10b66d 100644 Binary files a/src/lib/xrt/qwen3_npu.lib and b/src/lib/xrt/qwen3_npu.lib differ diff --git a/src/lib/xrt/qwen3vl_flash.dll b/src/lib/xrt/qwen3vl_flash.dll index 323371fe4..096283c25 100644 Binary files a/src/lib/xrt/qwen3vl_flash.dll and b/src/lib/xrt/qwen3vl_flash.dll differ diff --git a/src/lib/xrt/qwen3vl_flash.lib b/src/lib/xrt/qwen3vl_flash.lib index dfdf9f672..42660be35 100644 Binary files a/src/lib/xrt/qwen3vl_flash.lib and b/src/lib/xrt/qwen3vl_flash.lib differ diff --git a/src/lib/xrt/qwen3vl_npu.dll b/src/lib/xrt/qwen3vl_npu.dll index 132923064..dbf45f5c3 100644 Binary files a/src/lib/xrt/qwen3vl_npu.dll and b/src/lib/xrt/qwen3vl_npu.dll differ diff --git a/src/lib/xrt/qwen3vl_npu.lib b/src/lib/xrt/qwen3vl_npu.lib index fbd369bbf..bdc2c6fbe 100644 Binary files a/src/lib/xrt/qwen3vl_npu.lib and b/src/lib/xrt/qwen3vl_npu.lib differ diff --git a/src/lib/xrt/whisper_npu.dll b/src/lib/xrt/whisper_npu.dll index cb575944b..f16363303 100644 Binary files a/src/lib/xrt/whisper_npu.dll and b/src/lib/xrt/whisper_npu.dll differ diff --git a/src/model_list.json b/src/model_list.json index cd40dfc7a..10eba7b9a 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -475,6 +475,38 @@ "footprint": 22.1 } }, + "qwen3.8-mtp": { + "27b": { + "name": "Qwen3.8-27B-NPU2", + "url": "https://huggingface.co/FastFlowLM/Qwen3.8-27B-NPU2", + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.8-27B-NPU2/tree/main", + "ms_url": "https://modelscope.cn/models/amd/Qwen3.8-27B-NPU2", + "size": 27000000000, + "flm_min_version": "1.0.3", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "chat_template.jinja" + ], + "vlm": false, + "default_context_length": 4096, + "max_prefill_len": 256, + "details": { + "format": "NPU2", + "family": "qwen3.8-mtp", + "think": true, + "parameter_size": "27B", + "quantization_level": "Q4_K" + }, + "label": [ + "reasoning", + "tool-calling" + ], + "footprint": 17.1 + } + }, "lfm2": { "1.2b": { "name": "LFM2-1.2B-NPU2", diff --git a/src/test/qwen3_8mtp_npu/CMakeLists.txt b/src/test/qwen3_8mtp_npu/CMakeLists.txt new file mode 100644 index 000000000..e2927e9f7 --- /dev/null +++ b/src/test/qwen3_8mtp_npu/CMakeLists.txt @@ -0,0 +1,47 @@ +cmake_minimum_required(VERSION 3.22) +project(qwen3_8mtp_npu VERSION 1.0.0 LANGUAGES CXX) + +include(${CMAKE_CURRENT_LIST_DIR}/../CMakeLists.txt) +npu_test_setup() + +# Find and enable OpenMP for multi-threading +find_package(OpenMP) +if(OpenMP_CXX_FOUND) + message(STATUS "OpenMP found - enabling multi-threading support for test") +else() + message(WARNING "OpenMP not found - some optimizations will run single-threaded") +endif() + +add_npu_test( + test_qwen3_8mtp_npu + test/qwen3_8mtp_npu + USE_AUTOMODEL + USE_TOKENIZER + USE_SAMPLER + SOURCES + "${CMAKE_SOURCE_DIR}/../../common/AutoModel/modeling_qwen3_8mtp.cpp" +) + +# Add OpenMP compiler flags if available +if(OpenMP_CXX_FOUND) + target_compile_options(test_qwen3_8mtp_npu PUBLIC + $<$:/openmp> + $<$>:-fopenmp> + ) +endif() + +target_link_libraries(test_qwen3_8mtp_npu PUBLIC + qwen3_8mtp_npu + xrt_coreutil +) + +# Link OpenMP if available +if(OpenMP_CXX_FOUND) + target_link_libraries(test_qwen3_8mtp_npu PUBLIC OpenMP::OpenMP_CXX) +endif() + +# Add test target +add_custom_target(test_qwen3_8mtp_npu_target + DEPENDS test_qwen3_8mtp_npu + COMMENT "Building test_qwen3_8mtp_npu executable" +) diff --git a/src/test/qwen3_8mtp_npu/Makefile b/src/test/qwen3_8mtp_npu/Makefile new file mode 100644 index 000000000..e53a45d17 --- /dev/null +++ b/src/test/qwen3_8mtp_npu/Makefile @@ -0,0 +1,132 @@ +# ============================================================================= +# Qwen3.8-27B (MTP) NPU Test Makefile +# ============================================================================= +# +# This Makefile builds the standalone test harness for Qwen3.8-27B. +# +# Usage: +# make - Build the test executable +# make clean - Remove all built files +# make test - Build and run with a short prompt +# +# ============================================================================= +-include ../common.mk + + +SOURCES += test.cpp +SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += ../../common/AutoModel/modeling_qwen3_8mtp.cpp +SOURCES += ../../common/AutoModel/modeling_qwen3_8mtp_image.cpp +SOURCES += ../../common/image_process_utils/imageproc.cpp +SOURCES += ../../common/image_process_utils/imageprocAVX512.cpp +SOURCES += ../../common/image/image_reader.cpp +SOURCES += ../../common/tokenizer/tokenizer.cpp +SOURCES += ../../common/modules/sampler.cpp + +HEADERS += ../../include/models/qwen3_8mtp/qwen3_8mtp_npu.hpp +HEADERS += ../../include/AutoModel/modeling_qwen3_8mtp.hpp + + +ifeq ($(WSL), 0) + +LDFLAGS += -lqwen3_8mtp_npu +# image_reader.cpp decodes through ffmpeg, same as the Qwen3_5VL test. +LDFLAGS += -lavformat -lavcodec -lavutil -lswscale -lswresample + +CPP_SOURCES := $(filter %.cpp,$(SOURCES)) +OBJECTS := $(addprefix $(BUILD_DIR)/,$(notdir $(CPP_SOURCES:.cpp=.o))) +DEPS := $(OBJECTS:.o=.d) +VPATH := $(sort $(dir $(CPP_SOURCES))) + +all: $(BUILD_DIR)/test_qwen3_8mtp_npu + +$(BUILD_DIR): + mkdir -p $(BUILD_DIR) + +$(BUILD_DIR)/%.o: %.cpp | $(BUILD_DIR) + $(CXX) $(CXX_FLAGS) -c $< -o $@ + +$(BUILD_DIR)/test_qwen3_8mtp_npu: $(OBJECTS) + $(CXX) $(CXX_FLAGS) -o $@ $^ $(LDFLAGS) $(DEPENDENCY_LDFLAGS) + +# Scores the HOST image preprocessing against a captured pixel_values. Separate +# from the test binary because it loads no model and touches no device -- see +# the header comment in verify_preprocess.cpp for why it is not covered by +# tools/verify_n14_vision.cpp. +VERIFY_SOURCES := $(filter-out test.cpp,$(CPP_SOURCES)) verify_preprocess.cpp +VERIFY_OBJECTS := $(addprefix $(BUILD_DIR)/,$(notdir $(VERIFY_SOURCES:.cpp=.o))) +DEPS += $(VERIFY_OBJECTS:.o=.d) + +verify_preprocess: $(BUILD_DIR)/verify_preprocess + +$(BUILD_DIR)/verify_preprocess: $(VERIFY_OBJECTS) + $(CXX) $(CXX_FLAGS) -o $@ $^ $(LDFLAGS) $(DEPENDENCY_LDFLAGS) + +REF_IMAGE ?= /scratch/michyu/Projects/FastFlowLM_IRON/models_simple/test_input/observation_images_image.png +REF_DUMP ?= /scratch/michyu/models/Qwen3.8-27B-NPU2/vision_ref.safetensors + +verify: $(BUILD_DIR)/verify_preprocess + cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib/xrt \ + && ./verify_preprocess $(REF_IMAGE) $(REF_DUMP) + +clean: + rm -rf $(BUILD_DIR) + +# Decode is ~1-2 s/token here, so -n stays small. +# +# The draft head now runs on hardware by default. MTP_CPU=1 used to be forced +# because MTP_layer.xclbin timed out its second dispatch (npu_mtp_layer: +# dispatch state 8 at cache row 1); that is repaired, so the default is 0 and +# the variable is only kept as an A/B knob. Speculation is output-neutral -- +# speculate() verifies every draft against the real stack -- so either setting +# produces the same text, and only throughput moves. +# +# MTP_CPU=0 draft head on the NPU: MTP_layer.xclbin for the six one-token +# steps, attention_prefill.xclbin for step 0's length-L window +# MTP_CPU=1 draft head on the CPU (attention stack stays on the NPU) +# +# MTP_PREFILL=0 keeps the head on the NPU but forces step 0 back onto the +# per-row MTP_layer.xclbin path -- the A/B for the step-0 split alone. +# PROMPT=0 reads ../../../../prompt.txt (346 tokens) instead of the 8-token +# smoke prompt; use it when measuring, not when just checking it runs. +MTP_CPU ?= 0 +MTP_PREFILL ?= 1 +PROMPT ?= 1 +NTOK ?= 512 + +test: $(BUILD_DIR)/test_qwen3_8mtp_npu + cp ../../model_list.json $(BUILD_DIR)/model_list.json + cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib/xrt \ + && export FLM_Q38_MTP_CPU=$(MTP_CPU) \ + && export FLM_Q38_MTP_PREFILL=$(MTP_PREFILL) \ + && ./test_qwen3_8mtp_npu --model qwen3.8-mtp:27b -s $(PROMPT) -n $(NTOK) + +-include $(DEPS) +.PHONY: all clean test verify verify_preprocess + +else + +# WSL build environment +# Use CMake to invoke the Visual Studio +PWSH := powershell.exe + +all: directories test + +directories: + mkdir -p $(BUILD_DIR) + +$(BUILD_DIR)/test_qwen3_8mtp_npu.exe: $(SOURCES) + cd $(BUILD_DIR) && $(PWSH) -Command "cmake ../../../test/qwen3_8mtp_npu" + cd $(BUILD_DIR) && $(PWSH) -Command "cmake --build . --config Release --target test_qwen3_8mtp_npu" + +clean: + rm -rf $(BUILD_DIR) + + +test: directories $(BUILD_DIR)/test_qwen3_8mtp_npu.exe + cp ../../model_list.json $(BUILD_DIR)/model_list.json + cd $(BUILD_DIR) && ${PWSH} -Command "\$$env:PATH = '..\..\..\lib;' + \$$env:PATH; .\test_qwen3_8mtp_npu.exe -m qwen3.8-mtp:27b -s 1 -p 0 -n 16" + +.PHONY: all clean test directories + +endif diff --git a/src/test/qwen3_8mtp_npu/activate.sh b/src/test/qwen3_8mtp_npu/activate.sh new file mode 100755 index 000000000..e69412e0b --- /dev/null +++ b/src/test/qwen3_8mtp_npu/activate.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +# Model weights live at $FLM_MODEL_PATH/models/Qwen3.8-27B-NPU2. +# model_list.json's "model_path" is "models", so this must point at the PARENT +# of that directory, not at it. +export FLM_MODEL_PATH="/scratch/$USER" + +# copy src to dst only if the contents differ; dst may be a directory or a +# destination filename (used to rename layer.xclbin -> attention_layer.xclbin) +copy_if_different() { + local src="$1" dst="$2" + [ -d "$dst" ] && dst="$dst/$(basename "$src")" + if cmp -s "$src" "$dst"; then + echo "unchanged: $dst" + else + cp "$src" "$dst" && echo "updated: $dst" + fi +} + +# copy lib +copy_if_different /scratch/$USER/Projects/FastFlowLM_IRON/FLM_DLL/build/lib/libqwen3_8mtp_npu.so ../../lib/xrt + +# Phase 1 is CPU-only -- no xclbins to stage. The NPU kernel binaries land here +# in phase 2 (dequant_mm / lm_head / layer / GateDeltaNet_prefill), at which +# point add the matching copy_if_different lines for +# ../../xclbins/Qwen3.8-27B-NPU2/. +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/MTP_decoding/build/QWEN3_8_27B/xclbins/MTP_layer.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/ +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/gate_delta_net_verify/build/QWEN3_8_27B/xclbins/deltanet_layer.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/ +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/gated_attention_verify/build/QWEN3_8_27B/xclbins/layer.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/attention_layer.xclbin +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/gated_attention_prefill/build/xclbins/attn.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/attention_prefill.xclbin +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/lm_head_npu_bin/build/QWEN3_8_27B/xclbins/lm_head.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/ +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/lm_head_8/build/QWEN3_8_27B/xclbins/lm_head.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/lm_head_8.xclbin +# The fused Q4_K dequant + GEMM the MTP head's step-0 projections run on. Its +# design is not named layer.xclbin, so it stages under its own name unrenamed. +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/dequant_mm/build/xclbins/dequant_mm.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/ + +# N10's two: the depthwise conv1d (+ SiLU + the q/k l2norms) and the gated +# delta-rule fold, which together are the 48 linear layers' whole recurrence. +# Both keep their own names; nothing else stages a conv.xclbin. +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/conv1d_prefill/build/xclbins/conv.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/ +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/gate_delta_net_prefill/build/xclbins/GateDeltaNet_prefill.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/ + +# The vision tower's attention (N15). Its source is also attn.xclbin, same as +# gated_attention_prefill above, so it MUST be renamed on the way in or the two +# designs land on the same file and the tower silently drives the language +# model's causal core. The engine looks for it under this name and falls back +# to the host tower if it is absent. +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/vision_attn/build/xclbins/attn.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/vision_attn.xclbin + +# The vision tower's matmuls (N16), the other ~7 s of a 1200-patch image. Two +# designs: A (m=64,k=384,n=48) serves every shape, B (m=32,k=384,n=128) exists +# for merger.fc2 alone, whose N of 5120 is not a multiple of A's tile_N of 384. +# BOTH sources are build//xclbins/mm.xclbin, so both must be renamed or +# the second copy overwrites the first and the tower drives the wrong tile shape. +# The engine wants both or it leaves the matmuls on the host; it never runs one. +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/vision_mm/build/VISION_MM_A/xclbins/mm.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/vision_mm_a.xclbin +copy_if_different /scratch/michyu/Projects/FastFlowLM_IRON/FLM_Xclbin/Qwen3_8/vision_mm/build/VISION_MM_B/xclbins/mm.xclbin /scratch/michyu/FastFlowLM/src/xclbins/Qwen3.8-27B-NPU2/vision_mm_b.xclbin diff --git a/src/test/qwen3_8mtp_npu/test.cpp b/src/test/qwen3_8mtp_npu/test.cpp new file mode 100644 index 000000000..bfbd39819 --- /dev/null +++ b/src/test/qwen3_8mtp_npu/test.cpp @@ -0,0 +1,161 @@ +#include +#include +#include "utils/utils.hpp" +#include "utils/vm_args.hpp" +#include "AutoModel/modeling_qwen3_8mtp.hpp" +#include "model_list.hpp" + +flm_rt::device npu_device_global; + +int main(int argc, char* argv[]) { + #ifdef __WINDOWS__ + SetConsoleOutputCP(CP_UTF8); + SetConsoleCP(CP_UTF8); + // Set thread priority to low + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_LOWEST); + #endif + + arg_utils::po::options_description desc("Allowed options"); + arg_utils::po::variables_map vm; + desc.add_options()("model,m", arg_utils::po::value()->required(), "Model file"); + desc.add_options()("Short,s", arg_utils::po::value()->default_value(true), "Short Prompt"); + desc.add_options()("Preemption,p", arg_utils::po::value()->default_value(false), "Preemption"); + desc.add_options()("Think,t", arg_utils::po::value()->default_value(false), "Enable thinking"); + desc.add_options()("Length,n", arg_utils::po::value()->default_value(32), "Max generated tokens"); + desc.add_options()("Image,i", arg_utils::po::value()->default_value(""), "Image file to attach; runs one multimodal turn and ignores -s"); + arg_utils::po::store(arg_utils::po::parse_command_line(argc, argv, desc), vm); + + std::string tag = vm["model"].as(); + bool short_prompt = vm["Short"].as(); + bool preemption = vm["Preemption"].as(); + bool enable_think = vm["Think"].as(); + int length_limit = vm["Length"].as(); + std::string image_path = vm["Image"].as(); + std::cout << "Model: " << tag << std::endl; + std::string exe_dir = utils::get_executable_directory(); + std::string model_dir = utils::get_models_directory(); + std::string model_list_path = exe_dir + "/model_list.json"; + model_list model_list(model_list_path, model_dir); + + header_print("info", "Initializing chat model..."); + std::string model_path = model_list.get_model_path(tag); + std::pair model_info_pair = model_list.get_model_info(tag); + nlohmann::json model_info = model_info_pair.second; + std::cout << "Model path: " << model_path << std::endl; + + std::unique_ptr chat = std::make_unique(&npu_device_global); + npu_device_global = flm_rt::device(0); + + chat->load_model(model_path, model_info, -1, preemption); + header_print("info", "Model loaded"); + chat_meta_info_t meta_info; + lm_uniform_input_t uniformed_input; + // Greedy. Phase 1 runs the 64-layer stack on the CPU at roughly 1-2 s/token, + // so a deterministic stream is what makes a short run worth comparing + // against the milestone driver at all. + // + // set_topk(1) alone is NOT enough to get greedy decoding here, and it is + // not enough to enable speculation. load_model() installs the Qwen3.5 + // recommended defaults, which include freq_penalty 1.0 and pre_penalty + // 1.5, and sample_greedy() still applies penalties when repeat_last_n != 0 + // -- they reorder the logits before the argmax. So the sampler's argmax is + // not the model's argmax, MTP acceptance is an exact compare against the + // model's argmax, and _shared_generate's gate correctly refuses to + // speculate. Zeroing them is what actually makes this run greedy. + sampler_config greedy; + greedy.top_k = 1; + greedy.rep_penalty = 1.0f; // 1.0 == disabled + greedy.freq_penalty = 0.0f; + greedy.pre_penalty = 0.0f; + chat->set_sampler(greedy); + chat->configure_parameter("enable_think", enable_think); + + if (short_prompt) { + uniformed_input.prompt = "Describe what is in the image."; + uniformed_input.images.push_back("../../../tb_files/panda.png"); + chat->start_total_timer(); + bool success = chat->insert(meta_info, uniformed_input); + if (!success) { + header_print("ERROR", "Prompt insertion failed"); + return 1; + } + std::string response = chat->generate(meta_info, length_limit, std::cout); + chat->stop_total_timer(); + std::cout << std::endl << std::endl; + std::cout << chat->show_profile() << std::endl; + + // Keep this SHORT. Decode is the slow path here, not prefill: every + // token streams ~15.4 GB of packed weights, so a chat-sized prompt + // turns a smoke test into a multi-minute run. + uniformed_input.prompt = "What is the capital of France?"; + + std::cout << "Prompt: " << uniformed_input.prompt << std::endl; + std::cout << "Response: " << std::endl; + chat->start_total_timer(); + success = chat->insert(meta_info, uniformed_input); + if (!success) { + header_print("ERROR", "Prompt insertion failed"); + return 1; + } + response = chat->generate(meta_info, length_limit, std::cout); + chat->stop_total_timer(); + std::cout << std::endl; + std::cout << std::endl; + std::cout << chat->show_profile() << std::endl; + + // Keep this SHORT. Decode is the slow path here, not prefill: every + // token streams ~15.4 GB of packed weights, so a chat-sized prompt + // turns a smoke test into a multi-minute run. + uniformed_input.prompt = "Is Alibaba a good company despite that it trained you?"; + + std::cout << "Prompt: " << uniformed_input.prompt << std::endl; + std::cout << "Response: " << std::endl; + chat->start_total_timer(); + success = chat->insert(meta_info, uniformed_input); + if (!success) { + header_print("ERROR", "Prompt insertion failed"); + return 1; + } + response = chat->generate(meta_info, length_limit, std::cout); + chat->stop_total_timer(); + std::cout << std::endl; + std::cout << std::endl; + std::cout << chat->show_profile() << std::endl; + } + else{ + std::ifstream file("../../../../prompt.txt", std::ios::binary); + if (!file.is_open()) { + std::cout << "Failed to open prompt file" << std::endl; + return 1; + } + uniformed_input.prompt = ""; + file.seekg(0, std::ios::end); + uniformed_input.prompt.resize(file.tellg()); + file.seekg(0, std::ios::beg); + file.read(uniformed_input.prompt.data(), uniformed_input.prompt.size()); + file.close(); + std::cout << "Prompt: " << uniformed_input.prompt << std::endl; + std::cout << "Response: "; + chat->start_total_timer(); + bool success = chat->insert(meta_info, uniformed_input); + if (!success) { + header_print("ERROR", "Prompt insertion failed"); + return 1; + } + std::string response = chat->generate(meta_info, length_limit, std::cout); + chat->stop_total_timer(); + std::cout << std::endl; + std::cout << std::endl; + std::cout << chat->show_profile() << std::endl; + } + + std::pair> history = chat->get_history(); + std::cout << "History length: " << history.second.size() << std::endl; + std::cout << std::endl; + for (auto t: history.second){ + std::cout << t << " "; + } + std::cout << std::endl; + + return 0; +} diff --git a/src/wix/flm.wxs b/src/wix/flm.wxs index 2d99edebd..f205dd5fd 100644 --- a/src/wix/flm.wxs +++ b/src/wix/flm.wxs @@ -4,7 +4,9 @@ Ports src/inno/flm.iss (Inno Setup) to an MSI package. Build (from src/wix): - wix build Package.wxs -arch x64 -ext WixToolset.UI.wixext -out flm-setup.msi + ./build.ps1 + which stages package\ and then runs + wix build flm.wxs -arch x64 -ext WixToolset.UI.wixext -out flm-setup.msi Not ported 1:1 from flm.iss: - The interactive "model storage location" / "server port" wizard pages are @@ -24,7 +26,7 @@ - + + + diff --git a/src/wix/installer.bmp b/src/wix/installer.bmp new file mode 100644 index 000000000..bd7c5eaff Binary files /dev/null and b/src/wix/installer.bmp differ diff --git a/src/wix/top.bmp b/src/wix/top.bmp new file mode 100644 index 000000000..e7789de7d Binary files /dev/null and b/src/wix/top.bmp differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/GateDeltaNet_prefill.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/GateDeltaNet_prefill.xclbin new file mode 100644 index 000000000..6e29b6179 Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/GateDeltaNet_prefill.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/MTP_layer.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/MTP_layer.xclbin new file mode 100644 index 000000000..b5e81d99b Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/MTP_layer.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/attention_layer.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/attention_layer.xclbin new file mode 100644 index 000000000..bffb9e8ed Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/attention_layer.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/attention_prefill.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/attention_prefill.xclbin new file mode 100644 index 000000000..e91d19c2b Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/attention_prefill.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/conv.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/conv.xclbin new file mode 100644 index 000000000..311d97423 Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/conv.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/deltanet_layer.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/deltanet_layer.xclbin new file mode 100644 index 000000000..e0d4b2987 Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/deltanet_layer.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/dequant_mm.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/dequant_mm.xclbin new file mode 100644 index 000000000..5e1f14afc Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/dequant_mm.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/lm_head.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/lm_head.xclbin new file mode 100644 index 000000000..d37f9d496 Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/lm_head.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/lm_head_8.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/lm_head_8.xclbin new file mode 100644 index 000000000..4a38b6708 Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/lm_head_8.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/vision_attn.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/vision_attn.xclbin new file mode 100644 index 000000000..0c369d79d Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/vision_attn.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/vision_mm_a.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/vision_mm_a.xclbin new file mode 100644 index 000000000..319459e14 Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/vision_mm_a.xclbin differ diff --git a/src/xclbins/Qwen3.8-27B-NPU2/vision_mm_b.xclbin b/src/xclbins/Qwen3.8-27B-NPU2/vision_mm_b.xclbin new file mode 100644 index 000000000..d5f91afb7 Binary files /dev/null and b/src/xclbins/Qwen3.8-27B-NPU2/vision_mm_b.xclbin differ