From 2f4717a45f672a35521574a92b6c6593feda99bd Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Sun, 28 Jun 2026 23:48:31 +0000 Subject: [PATCH 01/13] Add NVIDIA Nemotron-3.5-ASR streaming multilingual backend Adds a new eddy::nemotron backend for NVIDIA's nemotron-3.5-asr-streaming-0.6b: a cache-aware streaming FastConformer-RNNT with prompt-conditioned multilingual decoding (40+ languages). This is distinct from the Parakeet path (stateless overlapping-chunk encoder + TDT duration decoding) and needs its own module: cache-aware streaming encoder loop, integer prompt_id language conditioning, plain RNNT greedy decode, and a dedicated tokenizer (Nemotron emits standalone U+2581 word-boundary tokens, so text is rebuilt by concatenating raw pieces then mapping U+2581 -> space). - include/eddy/models/nemotron/nemotron.hpp, src/models/nemotron/ nemotron_openvino.cpp: OpenVINO backend (preprocessor/encoder/ decoder/joint IR + vocab + metadata). - examples/cpp/nemotron_cli.cpp: CLI (--device, --lang, --model-dir). - model_configs.hpp: nemotron-streaming entry pointing at FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov. Adds ModelConfig::repo_subdir so files download from the repo's fp16/ subfolder (default precision) while staying flat in cache. - ensure_models.cpp: honor repo_subdir in the download URL. Accuracy (FLEURS full test splits, FP32 IR, CPU; matches FP16): en_us WER 11.78, es_419 WER 6.99, fr_fr WER 12.92, cmn_hans_cn CER 21.05, ja_jp CER 15.12 (weighted 13.70) -- at or below the FluidAudio CoreML reference on every language. RTFx ~3.66x single-CPU. Closes #8 --- CMakeLists.txt | 3 + examples/cpp/CMakeLists.txt | 5 +- examples/cpp/nemotron_cli.cpp | 110 ++++++ include/eddy/core/model_configs.hpp | 27 +- include/eddy/models/nemotron/nemotron.hpp | 79 ++++ src/models/nemotron/nemotron_openvino.cpp | 423 ++++++++++++++++++++++ src/utils/ensure_models.cpp | 7 +- 7 files changed, 650 insertions(+), 4 deletions(-) create mode 100644 examples/cpp/nemotron_cli.cpp create mode 100644 include/eddy/models/nemotron/nemotron.hpp create mode 100644 src/models/nemotron/nemotron_openvino.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c55c96..c7428da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,9 @@ target_sources(eddy src/models/parakeet-v2/parakeet_decoder.cpp src/models/parakeet-v2/parakeet_chunking.cpp src/models/parakeet-v2/tokenizer.cpp + + # Nemotron cache-aware streaming implementation + src/models/nemotron/nemotron_openvino.cpp ) # Link dependencies diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 4d8c615..c345eb1 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -7,10 +7,13 @@ endif() add_executable(parakeet_cli parakeet_cli.cpp) target_link_libraries(parakeet_cli PRIVATE eddy) +add_executable(nemotron_cli nemotron_cli.cpp) +target_link_libraries(nemotron_cli PRIVATE eddy) + add_executable(hf_fetch_models hf_fetch_models.cpp) target_link_libraries(hf_fetch_models PRIVATE eddy) add_executable(benchmark_fleurs benchmark_fleurs.cpp) target_link_libraries(benchmark_fleurs PRIVATE eddy) -install(TARGETS parakeet_cli hf_fetch_models benchmark_fleurs DESTINATION bin) +install(TARGETS parakeet_cli nemotron_cli hf_fetch_models benchmark_fleurs DESTINATION bin) diff --git a/examples/cpp/nemotron_cli.cpp b/examples/cpp/nemotron_cli.cpp new file mode 100644 index 0000000..d1abb1f --- /dev/null +++ b/examples/cpp/nemotron_cli.cpp @@ -0,0 +1,110 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// CLI for the NVIDIA Nemotron-3.5-ASR-Streaming-Multilingual 0.6B backend. + +#include "eddy/backends/openvino_backend.hpp" +#include "eddy/core/app_dir.hpp" +#include "eddy/models/nemotron/nemotron.hpp" +#include "eddy/utils/audio_utils.hpp" + +#include +#include +#include +#include +#include + +void print_usage(const char* prog) { + std::cout << "Usage: " << prog << " [options]\n\n"; + std::cout << "Options:\n"; + std::cout << " --device OpenVINO device (default: CPU). CPU, AUTO, NPU\n"; + std::cout << " --lang Language: en-US, zh-CN, ... or auto (default: auto)\n"; + std::cout << " --model-dir Directory with nemotron_*.xml/bin + metadata.json\n"; + std::cout << " (default: per-user model cache for 'nemotron-streaming')\n"; + std::cout << " --help Show this help\n"; +} + +int main(int argc, char* argv[]) { + std::cout.setf(std::ios::unitbuf); + if (argc < 2) { + print_usage(argv[0]); + return 1; + } + + std::string audio_file, device = "CPU", lang = "auto", model_dir_arg; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + if (a == "--help" || a == "-h") { + print_usage(argv[0]); + return 0; + } else if (a == "--device" && i + 1 < argc) { + device = argv[++i]; + } else if (a == "--lang" && i + 1 < argc) { + lang = argv[++i]; + } else if (a == "--model-dir" && i + 1 < argc) { + model_dir_arg = argv[++i]; + } else { + audio_file = a; + } + } + if (audio_file.empty()) { + std::cerr << "Error: no audio file specified\n\n"; + print_usage(argv[0]); + return 1; + } + + std::cout << "=== Nemotron 3.5 ASR Streaming CLI ===\n\n"; + + try { + auto pcm = eddy::audio::read_wav(audio_file); + const float audio_seconds = static_cast(pcm.size()) / 16000.0f; + std::cout << "Audio: " << audio_file << " (" << std::fixed << std::setprecision(2) + << audio_seconds << "s)\n"; + + std::filesystem::path model_dir = + model_dir_arg.empty() ? eddy::get_model_assets_dir("nemotron-streaming") + : std::filesystem::path(model_dir_arg); + std::cout << "Models: " << model_dir.string() << "\n"; + + eddy::OpenVINOOptions ov_opts; + ov_opts.device = device; + ov_opts.cache_dir = eddy::get_model_dir("nemotron-streaming").string(); + auto backend = std::make_shared(ov_opts); + + eddy::nemotron::ModelPaths paths{ + .preprocessor = (model_dir / "nemotron_preprocessor.xml").string(), + .encoder = (model_dir / "nemotron_encoder.xml").string(), + .decoder = (model_dir / "nemotron_decoder.xml").string(), + .joint = (model_dir / "nemotron_joint.xml").string(), + .vocab_json = (model_dir / "nemotron_vocab.json").string(), + .metadata_json = (model_dir / "metadata.json").string(), + }; + eddy::nemotron::Config cfg; + cfg.device = device; + cfg.language = lang; + + eddy::nemotron::OpenVINONemotron model(backend, paths, cfg); + std::cout << "Compiling + warming up (" << device << ") ... "; + model.warmup(); + std::cout << "[OK]\n\n"; + + std::cout << std::string(70, '=') << "\nTRANSCRIBING...\n" << std::string(70, '=') << "\n\n"; + const auto result = model.transcribe(pcm); + + const float rtfx = result.latency_ms > 0.0 + ? audio_seconds / static_cast(result.latency_ms / 1000.0) + : 0.0f; + + std::cout << "Result:\n" << std::string(70, '-') << "\n"; + std::cout << result.text << "\n" << std::string(70, '-') << "\n\n"; + std::cout << "prompt_id_used: " << result.prompt_id_used << "\n"; + std::cout << "detected_lang: " << (result.detected_language.empty() ? "(none)" : result.detected_language) << "\n"; + std::cout << "tokens: " << result.token_ids.size() << "\n"; + std::cout << "processing time: " << std::fixed << std::setprecision(0) << result.latency_ms << " ms\n"; + std::cout << "real-time factor:" << std::fixed << std::setprecision(1) << rtfx << "x\n"; + return 0; + } catch (const std::exception& e) { + std::cerr << "\n[ERROR] " << e.what() << "\n"; + return 1; + } +} diff --git a/include/eddy/core/model_configs.hpp b/include/eddy/core/model_configs.hpp index 129cba6..6a31a9e 100644 --- a/include/eddy/core/model_configs.hpp +++ b/include/eddy/core/model_configs.hpp @@ -14,6 +14,9 @@ struct ModelConfig { std::string repo_id; // HuggingFace repository ID (e.g., "org/model-name") std::vector required_files; // List of required model files (xml, bin, json) std::string cache_subdir; // Subdirectory name in cache (e.g., "parakeet-v2") + std::string repo_subdir; // Optional subfolder within the repo (e.g., "fp16"); + // files download from /resolve/main// + // but are stored flat in the cache. Empty => repo root. }; // Available model configurations (similar to FluidAudio's ModelNames.swift) @@ -41,10 +44,32 @@ namespace model_configs { .cache_subdir = "parakeet-v3" }; + // NVIDIA Nemotron-3.5-ASR-Streaming-Multilingual 0.6B (cache-aware + // streaming FastConformer-RNNT, prompt-conditioned multilingual). + // Distinct file set + metadata.json (cache shapes, prompt_dictionary, + // lang_tag_token_ids) consumed by the eddy::nemotron backend. + inline const std::vector NEMOTRON_FILES = { + "nemotron_encoder.xml", "nemotron_encoder.bin", + "nemotron_decoder.xml", "nemotron_decoder.bin", + "nemotron_joint.xml", "nemotron_joint.bin", + "nemotron_preprocessor.xml", "nemotron_preprocessor.bin", + "nemotron_vocab.json", "metadata.json" + }; + + inline const ModelConfig NEMOTRON_STREAMING = { + .repo_id = "FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov", + .required_files = NEMOTRON_FILES, + .cache_subdir = "nemotron-streaming", + // FP16 IR (identical transcripts to FP32, ~half the size, NPU-friendly). + // FP32 also available under the "fp32" subfolder of the same repo. + .repo_subdir = "fp16" + }; + // Model name lookup map inline const std::map MODEL_MAP = { {"parakeet-v2", PARAKEET_V2}, - {"parakeet-v3", PARAKEET_V3} + {"parakeet-v3", PARAKEET_V3}, + {"nemotron-streaming", NEMOTRON_STREAMING} }; // Default model diff --git a/include/eddy/models/nemotron/nemotron.hpp b/include/eddy/models/nemotron/nemotron.hpp new file mode 100644 index 0000000..68523ec --- /dev/null +++ b/include/eddy/models/nemotron/nemotron.hpp @@ -0,0 +1,79 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// NVIDIA Nemotron-3.5-ASR-Streaming-Multilingual 0.6B backend. +// +// Unlike the Parakeet TDT path (stateless encoder + overlapping-chunk +// dedup + token/duration heads), Nemotron is a *cache-aware streaming* +// FastConformer-RNNT: the encoder carries cache tensors across chunks and +// takes an int `prompt_id` for language conditioning. Decoding is plain +// RNNT (no duration head). This warrants a separate module rather than +// overloading the Parakeet pipeline. + +#pragma once + +#include +#include +#include + +namespace eddy { +class OpenVINOBackend; +} + +namespace eddy::nemotron { + +/// Paths to the exported OpenVINO IR plus tokenizer/metadata. +/// File layout matches export_openvino.py / the HF model repo. +struct ModelPaths { + std::string preprocessor; // nemotron_preprocessor.xml (audio -> mel) + std::string encoder; // nemotron_encoder.xml (mel + caches + prompt_id -> encoded + caches) + std::string decoder; // nemotron_decoder.xml (token + lstm state -> dec_out + state) + std::string joint; // nemotron_joint.xml (enc_step + dec_step -> logits) + std::string vocab_json; // nemotron_vocab.json (id -> piece) + std::string metadata_json; // metadata.json (shapes, blank_idx, prompt_dictionary, ...) +}; + +struct Config { + std::string device = "AUTO"; // OpenVINO device for encoder/decoder/joint (preprocessor always CPU) + /// Language for prompt conditioning. Accepts dictionary keys ("en-US"), + /// 2-letter codes ("en" -> first "en-*"), or "auto" (model self-detects). + std::string language = "auto"; + size_t max_symbols_per_frame = 10; // RNNT inner-loop safety cap +}; + +struct TranscriptionResult { + std::string text; // lang-tag tokens stripped + std::string detected_language; // first tag emitted, if any (empty otherwise) + int prompt_id_used = 0; + std::vector token_ids; // raw emitted token ids (pre-strip) + double latency_ms = 0.0; +}; + +/// Streaming Nemotron ASR over OpenVINO. Construct, then transcribe whole +/// PCM buffers (internally chunked with cache-aware state continuity). +class OpenVINONemotron { +public: + OpenVINONemotron(std::shared_ptr backend, ModelPaths paths, Config config); + ~OpenVINONemotron(); + + OpenVINONemotron(const OpenVINONemotron&) = delete; + OpenVINONemotron& operator=(const OpenVINONemotron&) = delete; + + /// Compile models + load tokenizer/metadata (lazy; called by transcribe()). + void warmup(); + + /// Transcribe 16 kHz mono float32 PCM in [-1, 1]. + TranscriptionResult transcribe(const std::vector& pcm_16k_mono); + + /// Resolve a language string to its integer prompt id using the model's + /// prompt_dictionary (falls back to "auto"). + [[nodiscard]] int resolve_prompt_id(const std::string& language) const; + + struct Impl; + +private: + void ensure_compiled(); + std::unique_ptr impl_; +}; + +} // namespace eddy::nemotron diff --git a/src/models/nemotron/nemotron_openvino.cpp b/src/models/nemotron/nemotron_openvino.cpp new file mode 100644 index 0000000..0c39ee4 --- /dev/null +++ b/src/models/nemotron/nemotron_openvino.cpp @@ -0,0 +1,423 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// Cache-aware streaming inference for NVIDIA Nemotron-3.5-ASR-Streaming +// Multilingual 0.6B. Port of the validated Python reference +// (nemotron-ov-export/transcribe_ov.py), which mirrors mobius's CoreML +// streaming loop: chunk raw audio -> preprocessor -> cache-aware encoder +// (+ prompt_id) -> greedy RNNT decode, carrying encoder caches and LSTM +// state across chunks. + +#include "eddy/models/nemotron/nemotron.hpp" + +#include "eddy/backends/openvino_backend.hpp" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eddy::nemotron { + +namespace { + +// Mel feature buffer in [bins, frames] row-major (bin-major, matching the +// [1, bins, T] tensor layout the encoder expects). +struct MelBuf { + std::vector data; // size = bins * frames + size_t bins = 0; + size_t frames = 0; +}; + +ov::Tensor make_i32(int value) { + ov::Tensor t(ov::element::i32, ov::Shape{1}); + t.data()[0] = value; + return t; +} + +// SentencePiece word boundary marker (U+2581 "▁"). Unlike Parakeet, +// Nemotron's multilingual tokenizer emits standalone ▁ tokens, so the +// faithful decode is "concatenate pieces, then replace ▁ with space" +// (matches the validated Python reference), not per-piece prefix logic. +constexpr std::string_view kWordBoundary = "\xE2\x96\x81"; + +std::string finalize_text(std::string s) { + // Replace every ▁ with a space. + std::string out; + out.reserve(s.size()); + for (size_t i = 0; i < s.size();) { + if (s.compare(i, kWordBoundary.size(), kWordBoundary) == 0) { + out.push_back(' '); + i += kWordBoundary.size(); + } else { + out.push_back(s[i]); + ++i; + } + } + // Trim leading/trailing whitespace. + const auto b = out.find_first_not_of(" \t\n\r"); + const auto e = out.find_last_not_of(" \t\n\r"); + if (b == std::string::npos) return ""; + return out.substr(b, e - b + 1); +} + +} // namespace + +struct OpenVINONemotron::Impl { + std::shared_ptr backend; + ModelPaths paths; + Config config; + + std::vector vocab; // id -> piece (raw, ▁-marked) + + ov::CompiledModel preproc, encoder, decoder, joint; + ov::InferRequest preproc_req, encoder_req, decoder_req, joint_req; + + // Metadata + int sample_rate = 16000; + int mel_features = 128; + int chunk_mel_frames = 112; + int pre_encode_cache = 9; + int total_mel_frames = 121; + int blank_idx = 13087; + int vocab_size = 13087; + int decoder_hidden = 640; + int decoder_layers = 2; + int default_prompt_id = 101; + ov::Shape cache_channel_shape; + ov::Shape cache_time_shape; + std::map prompt_dictionary; + std::set lang_tag_token_ids; + + ov::element::Type token_et = ov::element::i32; + + std::once_flag compile_once; + std::mutex infer_guard; + + size_t chunk_samples() const { + return static_cast(static_cast(chunk_mel_frames) * 0.01 * sample_rate); + } +}; + +OpenVINONemotron::OpenVINONemotron(std::shared_ptr backend, + ModelPaths paths, Config config) + : impl_(std::make_unique()) { + if (!backend) { + throw std::invalid_argument("OpenVINO backend is null"); + } + impl_->backend = std::move(backend); + impl_->paths = std::move(paths); + impl_->config = std::move(config); +} + +OpenVINONemotron::~OpenVINONemotron() = default; + +void OpenVINONemotron::warmup() { ensure_compiled(); } + +int OpenVINONemotron::resolve_prompt_id(const std::string& language) const { + const auto& dict = impl_->prompt_dictionary; + auto it = dict.find(language); + if (it != dict.end()) { + return it->second; + } + if (language.size() == 2) { + const std::string prefix = language + "-"; + for (const auto& [k, v] : dict) { + if (k.size() >= prefix.size() && + std::equal(prefix.begin(), prefix.end(), k.begin(), + [](char a, char b) { return std::tolower(a) == std::tolower(b); })) { + return v; + } + } + } + return impl_->default_prompt_id; +} + +void OpenVINONemotron::ensure_compiled() { + std::call_once(impl_->compile_once, [this]() { + auto& core = impl_->backend->core(); + const std::string device = impl_->config.device.empty() ? "AUTO" : impl_->config.device; + + // --- Load metadata.json --- + { + std::ifstream f(impl_->paths.metadata_json); + if (!f.good()) { + throw std::runtime_error("Failed to open Nemotron metadata: " + impl_->paths.metadata_json); + } + nlohmann::json m; + f >> m; + impl_->sample_rate = m.value("sample_rate", 16000); + impl_->mel_features = m.value("mel_features", 128); + impl_->chunk_mel_frames = m.value("chunk_mel_frames", 112); + impl_->pre_encode_cache = m.value("pre_encode_cache", 9); + impl_->total_mel_frames = m.value("total_mel_frames", 121); + impl_->blank_idx = m.value("blank_idx", 13087); + impl_->vocab_size = m.value("vocab_size", 13087); + impl_->decoder_hidden = m.value("decoder_hidden", 640); + impl_->decoder_layers = m.value("decoder_layers", 2); + impl_->default_prompt_id = m.value("default_prompt_id", 101); + + auto to_shape = [](const nlohmann::json& arr) { + ov::Shape s; + for (const auto& d : arr) s.push_back(d.get()); + return s; + }; + impl_->cache_channel_shape = to_shape(m.at("cache_channel_shape")); + impl_->cache_time_shape = to_shape(m.at("cache_time_shape")); + + if (m.contains("prompt_dictionary")) { + for (auto& [k, v] : m["prompt_dictionary"].items()) { + impl_->prompt_dictionary[k] = v.get(); + } + } + if (m.contains("lang_tag_token_ids")) { + for (const auto& id : m["lang_tag_token_ids"]) { + impl_->lang_tag_token_ids.insert(id.get()); + } + } + } + + // --- Compile models (preprocessor on CPU; rest on chosen device) --- + std::string preproc_device = "CPU"; + if (const char* env = std::getenv("EDDY_PREPROC_DEVICE")) { + if (*env) preproc_device = env; + } + impl_->preproc = core.compile_model(impl_->paths.preprocessor, preproc_device); + impl_->encoder = core.compile_model(impl_->paths.encoder, device); + impl_->decoder = core.compile_model(impl_->paths.decoder, device); + impl_->joint = core.compile_model(impl_->paths.joint, device); + + impl_->preproc_req = impl_->preproc.create_infer_request(); + impl_->encoder_req = impl_->encoder.create_infer_request(); + impl_->decoder_req = impl_->decoder.create_infer_request(); + impl_->joint_req = impl_->joint.create_infer_request(); + + impl_->token_et = impl_->decoder.input("token").get_element_type(); + + // --- Vocab (id -> piece). Flat {"0":"piece", ...} format. --- + { + std::ifstream f(impl_->paths.vocab_json); + if (!f.good()) { + throw std::runtime_error("Failed to open Nemotron vocab: " + impl_->paths.vocab_json); + } + nlohmann::json v; + f >> v; + size_t max_id = 0; + for (auto& [k, _] : v.items()) { + max_id = std::max(max_id, static_cast(std::stoul(k))); + } + impl_->vocab.assign(max_id + 1, std::string{}); + for (auto& [k, val] : v.items()) { + impl_->vocab[std::stoul(k)] = val.get(); + } + } + }); +} + +TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) { + ensure_compiled(); + std::lock_guard lock(impl_->infer_guard); + + const auto t_start = std::chrono::steady_clock::now(); + + auto& I = *impl_; + const size_t bins = static_cast(I.mel_features); + const size_t total = static_cast(I.total_mel_frames); + const size_t pre_cache = static_cast(I.pre_encode_cache); + const size_t chunk_samples = I.chunk_samples(); + + const int prompt_id = resolve_prompt_id(I.config.language); + + // Persistent encoder caches (carried across chunks). + ov::Tensor cache_channel(ov::element::f32, I.cache_channel_shape); + ov::Tensor cache_time(ov::element::f32, I.cache_time_shape); + std::memset(cache_channel.data(), 0, cache_channel.get_byte_size()); + std::memset(cache_time.data(), 0, cache_time.get_byte_size()); + ov::Tensor cache_len = make_i32(0); + + // Persistent LSTM state. + const ov::Shape lstm_shape{static_cast(I.decoder_layers), 1, + static_cast(I.decoder_hidden)}; + ov::Tensor h(ov::element::f32, lstm_shape); + ov::Tensor c(ov::element::f32, lstm_shape); + std::memset(h.data(), 0, h.get_byte_size()); + std::memset(c.data(), 0, c.get_byte_size()); + + int last_token = I.blank_idx; + std::vector all_tokens; + + MelBuf mel_cache; // last pre_encode_cache frames of previous chunk's mel + + const ov::Tensor prompt_tensor = make_i32(prompt_id); + + size_t off = 0; + while (off < pcm.size()) { + const size_t end = std::min(off + chunk_samples, pcm.size()); + + // Raw audio chunk, padded to chunk_samples. + ov::Tensor audio(ov::element::f32, ov::Shape{1, chunk_samples}); + float* adst = audio.data(); + std::memset(adst, 0, audio.get_byte_size()); + std::copy(pcm.begin() + static_cast(off), pcm.begin() + static_cast(end), adst); + + // Preprocessor: audio -> mel [1, bins, T_mel] + I.preproc_req.set_tensor("audio", audio); + I.preproc_req.set_tensor("audio_length", make_i32(static_cast(chunk_samples))); + I.preproc_req.infer(); + const ov::Tensor mel_out = I.preproc_req.get_tensor("mel"); + const ov::Shape mel_shape = mel_out.get_shape(); // [1, bins, T_mel] + const size_t t_mel = mel_shape[2]; + const float* mel_src = mel_out.data(); + + // Build encoder mel input [1, bins, total]: prepend cache (or zero + // pre_encode_cache on first chunk), then pad/trim to total. + ov::Tensor mel_in(ov::element::f32, ov::Shape{1, bins, total}); + float* mdst = mel_in.data(); + std::memset(mdst, 0, mel_in.get_byte_size()); + + const size_t cache_frames = mel_cache.frames; // 0 on first chunk + const size_t lead = (cache_frames > 0) ? cache_frames : pre_cache; // zero-pad lead on first chunk + for (size_t bin = 0; bin < bins; ++bin) { + float* row = mdst + bin * total; + size_t col = 0; + // leading cache frames + for (size_t t = 0; t < lead && col < total; ++t, ++col) { + if (cache_frames > 0) { + row[col] = mel_cache.data[bin * cache_frames + t]; + } // else zero (already memset) + } + // current chunk mel frames + for (size_t t = 0; t < t_mel && col < total; ++t, ++col) { + row[col] = mel_src[bin * t_mel + t]; + } + } + + // Update mel_cache = last pre_encode_cache frames of current chunk mel. + const size_t keep = std::min(pre_cache, t_mel); + mel_cache.bins = bins; + mel_cache.frames = keep; + mel_cache.data.assign(bins * keep, 0.0f); + for (size_t bin = 0; bin < bins; ++bin) { + for (size_t t = 0; t < keep; ++t) { + mel_cache.data[bin * keep + t] = mel_src[bin * t_mel + (t_mel - keep + t)]; + } + } + + // Encoder: mel + caches + prompt_id -> encoded + caches + I.encoder_req.set_tensor("mel", mel_in); + I.encoder_req.set_tensor("mel_length", make_i32(static_cast(total))); + I.encoder_req.set_tensor("cache_channel", cache_channel); + I.encoder_req.set_tensor("cache_time", cache_time); + I.encoder_req.set_tensor("cache_len", cache_len); + I.encoder_req.set_tensor("prompt_id", prompt_tensor); + I.encoder_req.infer(); + + const ov::Tensor encoded = I.encoder_req.get_tensor("encoded"); // [1, D, T_enc] + // Persist updated caches (copy out before next infer overwrites them). + { + const ov::Tensor cc = I.encoder_req.get_tensor("cache_channel_out"); + const ov::Tensor ctt = I.encoder_req.get_tensor("cache_time_out"); + const ov::Tensor cl = I.encoder_req.get_tensor("cache_len_out"); + std::memcpy(cache_channel.data(), cc.data(), cache_channel.get_byte_size()); + std::memcpy(cache_time.data(), ctt.data(), cache_time.get_byte_size()); + cache_len.data()[0] = cl.data()[0]; + } + + const ov::Shape enc_shape = encoded.get_shape(); // [1, D, T_enc] + const size_t enc_d = enc_shape[1]; + const size_t t_enc = enc_shape[2]; + const float* enc_data = encoded.data(); + + // Greedy RNNT decode over encoder frames. + ov::Tensor enc_step(ov::element::f32, ov::Shape{1, enc_d, 1}); + for (size_t t = 0; t < t_enc; ++t) { + float* es = enc_step.data(); + for (size_t ch = 0; ch < enc_d; ++ch) { + es[ch] = enc_data[ch * t_enc + t]; + } + + for (size_t sym = 0; sym < I.config.max_symbols_per_frame; ++sym) { + // Decoder + ov::Tensor token(I.token_et, ov::Shape{1, 1}); + if (I.token_et == ov::element::i64) { + token.data()[0] = last_token; + } else { + token.data()[0] = last_token; + } + I.decoder_req.set_tensor("token", token); + I.decoder_req.set_tensor("token_length", make_i32(1)); + I.decoder_req.set_tensor("h_in", h); + I.decoder_req.set_tensor("c_in", c); + I.decoder_req.infer(); + const ov::Tensor dec_out = I.decoder_req.get_tensor("decoder_out"); // [1, H, 1] + + // Joint + I.joint_req.set_tensor("encoder", enc_step); + I.joint_req.set_tensor("decoder", dec_out); + I.joint_req.infer(); + const ov::Tensor logits = I.joint_req.get_tensor("logits"); // [1,1,1,V] + const float* lg = logits.data(); + const size_t vsz = logits.get_size(); + + int best = 0; + float best_score = lg[0]; + for (size_t i = 1; i < vsz; ++i) { + if (lg[i] > best_score) { + best_score = lg[i]; + best = static_cast(i); + } + } + + if (best == I.blank_idx) { + break; + } + all_tokens.push_back(best); + last_token = best; + // Advance LSTM state on emission. + std::memcpy(h.data(), I.decoder_req.get_tensor("h_out").data(), h.get_byte_size()); + std::memcpy(c.data(), I.decoder_req.get_tensor("c_out").data(), c.get_byte_size()); + } + } + + off += chunk_samples; + } + + // Strip blank / out-of-range / language-tag tokens; concatenate pieces. + auto piece = [&](int tok) -> const std::string& { + static const std::string empty; + return (tok >= 0 && tok < static_cast(I.vocab.size())) ? I.vocab[tok] : empty; + }; + + TranscriptionResult result; + result.prompt_id_used = prompt_id; + result.token_ids = all_tokens; + std::string body; + for (int tok : all_tokens) { + if (tok == I.blank_idx || tok >= I.vocab_size) continue; + if (I.lang_tag_token_ids.count(tok)) { + if (result.detected_language.empty()) { + result.detected_language = finalize_text(piece(tok)); + } + continue; + } + body += piece(tok); + } + result.text = finalize_text(body); + + const auto t_end = std::chrono::steady_clock::now(); + result.latency_ms = std::chrono::duration(t_end - t_start).count(); + return result; +} + +} // namespace eddy::nemotron diff --git a/src/utils/ensure_models.cpp b/src/utils/ensure_models.cpp index 125ffc5..924459b 100644 --- a/src/utils/ensure_models.cpp +++ b/src/utils/ensure_models.cpp @@ -108,8 +108,11 @@ bool download_models(const eddy::ModelConfig& config, continue; } - // Construct HuggingFace URL - const std::string url = "https://huggingface.co/" + config.repo_id + "/resolve/main/" + filename; + // Construct HuggingFace URL. When repo_subdir is set, files live in a + // subfolder of the repo (e.g. "fp16/") but are still stored flat locally. + const std::string remote_rel = + config.repo_subdir.empty() ? filename : config.repo_subdir + "/" + filename; + const std::string url = "https://huggingface.co/" + config.repo_id + "/resolve/main/" + remote_rel; // Notify progress if (progress_callback) { From 16da4dd60929da1504f3be1ea10e8df053461efc Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 03:10:12 +0000 Subject: [PATCH 02/13] Address PR #10 code review: cache assert, prompt-id lazy compile, hardening - nemotron_openvino.cpp: assert cache_*_out byte sizes match the pre-allocated input caches before memcpy (mismatched IR export now trips here instead of silently over-/under-reading); resolve_prompt_id lazily calls the call_once-guarded ensure_compiled() so it is safe to call before transcribe()/warmup(); drop unused , add (std::tolower) + ; comment the blank_idx == vocab_size filter. - ensure_models.cpp: reject shell-unsafe characters in the curl URL and output path before std::system (repo_subdir widened the interpolation surface; today all fields are constants but ModelConfig is caller-supplied). - hf_fetch_models.cpp: list available models dynamically from MODEL_MAP instead of the stale "parakeet-v2" literal. Not changed: the audio_length finding. The CLI passes the full padded chunk_samples to match the WER-validated Python reference (transcribe_ov.py pads the final chunk then passes chunk.shape[1]); switching to the unpadded length would diverge from the validated path. --- examples/cpp/hf_fetch_models.cpp | 4 +++- src/models/nemotron/nemotron_openvino.cpp | 16 +++++++++++++++- src/utils/ensure_models.cpp | 23 +++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/examples/cpp/hf_fetch_models.cpp b/examples/cpp/hf_fetch_models.cpp index 108142c..63ccdaf 100644 --- a/examples/cpp/hf_fetch_models.cpp +++ b/examples/cpp/hf_fetch_models.cpp @@ -53,7 +53,9 @@ int main(int argc, char** argv) { auto it = MODEL_MAP.find(model_name); if (it == MODEL_MAP.end()) { std::cerr << "ERROR: Unknown model: " << model_name << "\n"; - std::cerr << "Available models: parakeet-v2\n"; + std::cerr << "Available models:"; + for (const auto& [k, _] : MODEL_MAP) std::cerr << " " << k; + std::cerr << "\n"; return 1; } diff --git a/src/models/nemotron/nemotron_openvino.cpp b/src/models/nemotron/nemotron_openvino.cpp index 0c39ee4..f1c1ba2 100644 --- a/src/models/nemotron/nemotron_openvino.cpp +++ b/src/models/nemotron/nemotron_openvino.cpp @@ -18,8 +18,9 @@ #include #include +#include +#include #include -#include #include #include #include @@ -126,6 +127,10 @@ OpenVINONemotron::~OpenVINONemotron() = default; void OpenVINONemotron::warmup() { ensure_compiled(); } int OpenVINONemotron::resolve_prompt_id(const std::string& language) const { + // prompt_dictionary is populated by ensure_compiled(); make this safe to call + // standalone (before transcribe()/warmup()). ensure_compiled() is std::call_once + // guarded, so this is a cheap no-op once compiled. + const_cast(this)->ensure_compiled(); const auto& dict = impl_->prompt_dictionary; auto it = dict.find(language); if (it != dict.end()) { @@ -329,6 +334,12 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) const ov::Tensor cc = I.encoder_req.get_tensor("cache_channel_out"); const ov::Tensor ctt = I.encoder_req.get_tensor("cache_time_out"); const ov::Tensor cl = I.encoder_req.get_tensor("cache_len_out"); + // Cache-aware streaming: the *_out caches are the same fixed shape as the + // input caches (the ring buffer is re-filled in place), so we copy back + // into the pre-allocated input tensors. Assert the byte sizes agree so a + // mismatched IR export trips here instead of silently over-/under-reading. + assert(cc.get_byte_size() == cache_channel.get_byte_size()); + assert(ctt.get_byte_size() == cache_time.get_byte_size()); std::memcpy(cache_channel.data(), cc.data(), cache_channel.get_byte_size()); std::memcpy(cache_time.data(), ctt.data(), cache_time.get_byte_size()); cache_len.data()[0] = cl.data()[0]; @@ -404,6 +415,9 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) result.token_ids = all_tokens; std::string body; for (int tok : all_tokens) { + // blank intentionally sits at index vocab_size (== blank_idx), so the + // explicit blank check is redundant with `tok >= vocab_size`; kept for + // clarity since the two are configured independently from metadata.json. if (tok == I.blank_idx || tok >= I.vocab_size) continue; if (I.lang_tag_token_ids.count(tok)) { if (result.detected_language.empty()) { diff --git a/src/utils/ensure_models.cpp b/src/utils/ensure_models.cpp index 924459b..b7ed4bf 100644 --- a/src/utils/ensure_models.cpp +++ b/src/utils/ensure_models.cpp @@ -2,6 +2,7 @@ #include "eddy/utils/ensure_models.hpp" +#include #include #include #include @@ -15,9 +16,31 @@ static bool file_nonempty(const std::filesystem::path& p) { std::filesystem::file_size(p, ec) > 0; } +// The download shells out via std::system, so any interpolated component must +// be free of characters that could break out of the double-quoted argument. +// Today every field is a compile-time constant, but ModelConfig is caller- +// supplied, so reject anything outside a conservative path/URL charset rather +// than risk command injection. +static bool is_shell_safe(const std::string& s) { + for (const unsigned char c : s) { + const bool ok = std::isalnum(c) || c == '.' || c == '_' || c == '-' || + c == '/' || c == ':' || c == '~'; + if (!ok) return false; + } + return true; +} + static bool download_single_file(const std::string& url, const std::filesystem::path& output_path, std::string* error_msg = nullptr) { + // Refuse to build a shell command from unsafe components. + if (!is_shell_safe(url) || !is_shell_safe(output_path.string())) { + if (error_msg) { + *error_msg = "Refusing to download: unsafe characters in URL or path: " + url; + } + return false; + } + // Create parent directory std::error_code ec; std::filesystem::create_directories(output_path.parent_path(), ec); From e1afdfaa5785e0f946859a98cafa69d7638db4af Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 04:20:43 +0000 Subject: [PATCH 03/13] Add nemotron-streaming-int8 model config (weight-only INT8 encoder) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit INT8 weight-only encoder (per-channel symmetric; Conformer relative-pos projections kept FP16 to satisfy the OV CPU plugin) + FP16 decoder/joint/ preprocessor, served from the "int8" subfolder of the same HF repo. WER matches FP16/FP32 (en_us 10.99 vs 11.78); ~half the RAM of FP16 (2.1GB vs 3.9GB peak) and ~half the disk (749MB vs 1.3GB). No CPU speed gain (weights decompress to float on x86) — the win is memory footprint, chiefly for Intel NPU / memory-constrained deployments. --- include/eddy/core/model_configs.hpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/include/eddy/core/model_configs.hpp b/include/eddy/core/model_configs.hpp index 6a31a9e..7117715 100644 --- a/include/eddy/core/model_configs.hpp +++ b/include/eddy/core/model_configs.hpp @@ -65,11 +65,25 @@ namespace model_configs { .repo_subdir = "fp16" }; + // INT8 weight-only encoder (per-channel symmetric; Conformer relative-pos + // projections kept FP16) + FP16 decoder/joint/preprocessor. WER matches + // FP16/FP32 (en_us 10.99 vs 11.78); ~half the RAM of FP16 (2.1GB vs 3.9GB) + // and ~half the disk. No CPU speed gain (weights decompress to float on + // x86); the win is memory footprint — chiefly for Intel NPU / constrained + // deployments. Same repo, "int8" subfolder. + inline const ModelConfig NEMOTRON_STREAMING_INT8 = { + .repo_id = "FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov", + .required_files = NEMOTRON_FILES, + .cache_subdir = "nemotron-streaming-int8", + .repo_subdir = "int8" + }; + // Model name lookup map inline const std::map MODEL_MAP = { {"parakeet-v2", PARAKEET_V2}, {"parakeet-v3", PARAKEET_V3}, - {"nemotron-streaming", NEMOTRON_STREAMING} + {"nemotron-streaming", NEMOTRON_STREAMING}, + {"nemotron-streaming-int8", NEMOTRON_STREAMING_INT8} }; // Default model From aa04da24e3974981ef5fad40ce1e9b59b824e176 Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 12:57:05 +0000 Subject: [PATCH 04/13] Add Nemotron C-API + rename shared model-download util namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the Nemotron streaming backend through the eddy_c C API (previously CLI-only), mirroring the Parakeet C API: - EddyNemotronModel handle + EddyNemotronConfig (device/model_dir/language) - EddyNemotronResult (text, detected_language, token_ids, prompt_id_used, latency_ms) with eddy_nemotron_free_result - eddy_nemotron_create / _destroy / _infer_file / _infer_buffer model_dir defaults to the "nemotron-streaming" Eddy cache; language is fixed at creation (recreate to change). Verified end-to-end against the FP16 IR: transcript matches the CLI/Python reference word-for-word. Rename eddy::parakeet -> eddy::model_utils for the model-download helper (ensure_models.{hpp,cpp}): it is model-agnostic (operates on any eddy::ModelConfig) and already serves Nemotron downloads, so the model-specific namespace was misleading. Updated all call sites (parakeet_cli, benchmark_fleurs, hf_fetch_models, eddy_c). openvino_utils stays under eddy::parakeet — it is coupled to the Parakeet ModelFile type and unused by Nemotron. Addresses the follow-up noted in the PR #10 review. --- examples/cpp/benchmark_fleurs.cpp | 2 +- examples/cpp/hf_fetch_models.cpp | 2 +- examples/cpp/parakeet_cli.cpp | 2 +- include/eddy/eddy_c.h | 50 +++++++++++ include/eddy/utils/ensure_models.hpp | 7 +- src/eddy_c.cpp | 130 ++++++++++++++++++++++++++- src/utils/ensure_models.cpp | 6 +- 7 files changed, 187 insertions(+), 12 deletions(-) diff --git a/examples/cpp/benchmark_fleurs.cpp b/examples/cpp/benchmark_fleurs.cpp index dc51b0c..fa68192 100644 --- a/examples/cpp/benchmark_fleurs.cpp +++ b/examples/cpp/benchmark_fleurs.cpp @@ -507,7 +507,7 @@ int main(int argc, char* argv[]) { // Load Parakeet v3 models auto cache_model_dir = eddy::get_model_assets_dir("parakeet-v3"); std::string fetch_err; - if (!eddy::parakeet::check_models_available(cache_model_dir, &fetch_err)) { + if (!eddy::model_utils::check_models_available(cache_model_dir, &fetch_err)) { if (!fetch_err.empty()) std::cout << "[INFO] " << fetch_err << "\n"; } diff --git a/examples/cpp/hf_fetch_models.cpp b/examples/cpp/hf_fetch_models.cpp index 63ccdaf..d16b851 100644 --- a/examples/cpp/hf_fetch_models.cpp +++ b/examples/cpp/hf_fetch_models.cpp @@ -87,7 +87,7 @@ int main(int argc, char** argv) { // Download models using library function std::string error_msg; - bool success = eddy::parakeet::download_models( + bool success = eddy::model_utils::download_models( config, fs::path(target_dir), &error_msg, diff --git a/examples/cpp/parakeet_cli.cpp b/examples/cpp/parakeet_cli.cpp index 70cd10e..021c5cb 100644 --- a/examples/cpp/parakeet_cli.cpp +++ b/examples/cpp/parakeet_cli.cpp @@ -109,7 +109,7 @@ int main(int argc, char* argv[]) { auto cache_model_dir = eddy::get_model_assets_dir(model_name); std::filesystem::path model_dir; std::string fetch_err; - if (!eddy::parakeet::check_models_available(cache_model_dir, &fetch_err)) { + if (!eddy::model_utils::check_models_available(cache_model_dir, &fetch_err)) { if (!fetch_err.empty()) std::cout << "[INFO] " << fetch_err << "\n"; } diff --git a/include/eddy/eddy_c.h b/include/eddy/eddy_c.h index dba5d6a..16df676 100644 --- a/include/eddy/eddy_c.h +++ b/include/eddy/eddy_c.h @@ -226,6 +226,56 @@ EDDY_API EddyError eddy_parakeet_infer_buffer(EddyParakeetModel model, const flo EDDY_API char* eddy_parakeet_decode_tokens(EddyParakeetModel model, const int* token_ids, size_t count); EDDY_API void eddy_parakeet_free_result(EddyParakeetResult* result); +// ----------------------------- +// Nemotron streaming (OpenVINO) C API +// ----------------------------- + +// Opaque handle for a Nemotron streaming model. +typedef void* EddyNemotronModel; + +typedef struct { + const char* device; // "CPU", "NPU", or "AUTO" (default "CPU") + const char* model_dir; // Dir with nemotron_*.xml/bin + metadata.json. + // NULL or "cache" => Eddy cache for "nemotron-streaming". + const char* language; // "en-US", "zh-CN", ..., or "auto" (default "auto") +} EddyNemotronConfig; + +typedef struct { + char* text; // full transcript (lang-tag tokens stripped); free with eddy_nemotron_free_result + char* detected_language; // first tag emitted, or "" ; freed with the result + int* token_ids; // raw emitted token ids (pre-strip); freed with the result + size_t num_tokens; + int prompt_id_used; // integer prompt id selected for conditioning + double latency_ms; +} EddyNemotronResult; + +/** + * @brief Create a Nemotron streaming model. + * @param config Device / model_dir / language. Language conditions decoding and + * is fixed at creation; recreate the handle to change it. + * @param error_message Out param for error (can be NULL). Free with eddy_free_string. + * @return Model handle or NULL on failure. + */ +EDDY_API EddyNemotronModel eddy_nemotron_create(EddyNemotronConfig config, char** error_message); + +/** @brief Destroy a Nemotron model handle. */ +EDDY_API void eddy_nemotron_destroy(EddyNemotronModel model); + +/** + * @brief Transcribe a 16 kHz mono WAV file. + * @param result Out param; free with eddy_nemotron_free_result. + */ +EDDY_API EddyError eddy_nemotron_infer_file(EddyNemotronModel model, const char* wav_path, EddyNemotronResult* result, char** error_message); + +/** + * @brief Transcribe a raw float32 PCM buffer (16 kHz mono, normalized [-1, 1]). + * @param result Out param; free with eddy_nemotron_free_result. + */ +EDDY_API EddyError eddy_nemotron_infer_buffer(EddyNemotronModel model, const float* pcm, size_t length, int sample_rate, EddyNemotronResult* result, char** error_message); + +/** @brief Free an EddyNemotronResult (text, detected_language, token_ids). */ +EDDY_API void eddy_nemotron_free_result(EddyNemotronResult* result); + // Utility /** diff --git a/include/eddy/utils/ensure_models.hpp b/include/eddy/utils/ensure_models.hpp index 92651a6..664bbda 100644 --- a/include/eddy/utils/ensure_models.hpp +++ b/include/eddy/utils/ensure_models.hpp @@ -1,4 +1,5 @@ -// Centralized helper to check and download Parakeet OpenVINO model files. +// Centralized helper to check and download OpenVINO model files. +// Model-agnostic: operates on any eddy::ModelConfig (Parakeet, Nemotron, ...). #pragma once @@ -8,7 +9,7 @@ #include #include -namespace eddy::parakeet { +namespace eddy::model_utils { // Checks if all required model files exist in target_dir. // Returns true if all files are present, false otherwise. @@ -39,5 +40,5 @@ using DownloadProgressCallback = std::functionsecond; // Create progress callback wrapper - eddy::parakeet::DownloadProgressCallback cpp_callback = nullptr; + eddy::model_utils::DownloadProgressCallback cpp_callback = nullptr; if (progress_callback) { cpp_callback = [progress_callback, user_data](const std::string& filename, int current, int total) { progress_callback(filename.c_str(), current, total, user_data); @@ -74,7 +75,7 @@ EddyError eddy_download_parakeet_models( // Download models std::string last_error; - bool success = eddy::parakeet::download_models( + bool success = eddy::model_utils::download_models( config, std::filesystem::path(target_dir), &last_error, @@ -397,7 +398,7 @@ EDDY_API EddyParakeetModel eddy_parakeet_create(EddyParakeetConfig config, char* } std::string err; - (void)eddy::parakeet::check_models_available(model_dir, &err); + (void)eddy::model_utils::check_models_available(model_dir, &err); #if defined(_WIN32) if (!std::filesystem::exists(model_dir)) { auto legacy = eddy::get_app_data_dir() / "cache" / "models" / "parakeet-v2" / "files"; @@ -510,4 +511,127 @@ EDDY_API char* eddy_parakeet_decode_tokens(EddyParakeetModel handle, const int* return copy_string(txt); } +// ----------------------------- +// Nemotron streaming C API +// ----------------------------- + +static constexpr const char* kNemotronModelName = "nemotron-streaming"; + +struct CNemotron { + std::unique_ptr model; +}; + +EDDY_API void eddy_nemotron_free_result(EddyNemotronResult* result) { + if (!result) return; + if (result->text) { delete[] result->text; result->text = nullptr; } + if (result->detected_language) { delete[] result->detected_language; result->detected_language = nullptr; } + if (result->token_ids) { delete[] result->token_ids; result->token_ids = nullptr; } + result->num_tokens = 0; +} + +EDDY_API EddyNemotronModel eddy_nemotron_create(EddyNemotronConfig config, char** error_message) { + try { + const std::string device = config.device ? config.device : "CPU"; + const std::string language = config.language ? config.language : "auto"; + + // Resolve model directory: explicit dir, else (NULL/"cache") the Eddy cache. + std::filesystem::path model_dir; + if (config.model_dir && std::string(config.model_dir).size() > 0 && + std::string(config.model_dir) != "cache") { + model_dir = config.model_dir; + } else { + model_dir = eddy::get_model_assets_dir(kNemotronModelName); + } + + auto backend = std::make_shared( + eddy::OpenVINOOptions{ .device = device, + .cache_dir = eddy::get_model_dir(kNemotronModelName).string() } + ); + + eddy::nemotron::ModelPaths paths{ + .preprocessor = (model_dir / "nemotron_preprocessor.xml").string(), + .encoder = (model_dir / "nemotron_encoder.xml").string(), + .decoder = (model_dir / "nemotron_decoder.xml").string(), + .joint = (model_dir / "nemotron_joint.xml").string(), + .vocab_json = (model_dir / "nemotron_vocab.json").string(), + .metadata_json = (model_dir / "metadata.json").string(), + }; + + eddy::nemotron::Config cfg; + cfg.device = device; + cfg.language = language; + + auto handle = std::make_unique(); + handle->model = std::make_unique(backend, paths, cfg); + return static_cast(handle.release()); + } catch (const std::exception& e) { + if (error_message) *error_message = capture_exception(e); + return nullptr; + } catch (...) { + if (error_message) *error_message = copy_string("[Eddy Error] Unknown exception in nemotron create"); + return nullptr; + } +} + +EDDY_API void eddy_nemotron_destroy(EddyNemotronModel handle) { + if (!handle) return; + delete static_cast(handle); +} + +static EddyError nemotron_fill_result(const eddy::nemotron::TranscriptionResult& res, EddyNemotronResult* out) { + out->text = copy_string(res.text); + out->detected_language = copy_string(res.detected_language); + out->prompt_id_used = res.prompt_id_used; + out->latency_ms = res.latency_ms; + out->num_tokens = res.token_ids.size(); + if (out->num_tokens > 0) { + out->token_ids = new int[out->num_tokens]; + for (size_t i = 0; i < out->num_tokens; ++i) out->token_ids[i] = res.token_ids[i]; + } else { + out->token_ids = nullptr; + } + return EDDY_OK; +} + +EDDY_API EddyError eddy_nemotron_infer_buffer(EddyNemotronModel handle, const float* pcm, size_t length, + int sample_rate, EddyNemotronResult* out, char** err) { + if (!handle || !pcm || !out) { + if (err) *err = copy_string("[Eddy Error] Invalid argument: null pointer"); + return EDDY_ERROR_INVALID_ARGUMENT; + } + if (sample_rate != 16000) { + if (err) *err = copy_string("[Eddy Error] Nemotron expects 16kHz mono audio"); + return EDDY_ERROR_INVALID_ARGUMENT; + } + try { + auto* h = static_cast(handle); + std::vector samples(pcm, pcm + length); + return nemotron_fill_result(h->model->transcribe(samples), out); + } catch (const std::exception& e) { + if (err) *err = capture_exception(e); + return EDDY_ERROR_INFERENCE_FAILED; + } catch (...) { + if (err) *err = copy_string("[Eddy Error] Unknown exception during nemotron inference"); + return EDDY_ERROR_UNKNOWN; + } +} + +EDDY_API EddyError eddy_nemotron_infer_file(EddyNemotronModel handle, const char* wav_path, + EddyNemotronResult* out, char** err) { + if (!handle || !wav_path || !out) { + if (err) *err = copy_string("[Eddy Error] Invalid argument: null pointer"); + return EDDY_ERROR_INVALID_ARGUMENT; + } + try { + auto pcm = eddy::audio::read_wav(wav_path); + return eddy_nemotron_infer_buffer(handle, pcm.data(), pcm.size(), 16000, out, err); + } catch (const std::exception& e) { + if (err) *err = capture_exception(e); + return EDDY_ERROR_FILE_NOT_FOUND; + } catch (...) { + if (err) *err = copy_string("[Eddy Error] Unknown exception in nemotron infer_file"); + return EDDY_ERROR_UNKNOWN; + } +} + } // extern "C" diff --git a/src/utils/ensure_models.cpp b/src/utils/ensure_models.cpp index b7ed4bf..d6dd8a3 100644 --- a/src/utils/ensure_models.cpp +++ b/src/utils/ensure_models.cpp @@ -1,4 +1,4 @@ -// Centralized check and download for Parakeet model files. +// Centralized check and download for OpenVINO model files (model-agnostic). #include "eddy/utils/ensure_models.hpp" @@ -7,7 +7,7 @@ #include #include -namespace eddy::parakeet { +namespace eddy::model_utils { static bool file_nonempty(const std::filesystem::path& p) { std::error_code ec; @@ -168,4 +168,4 @@ bool download_models(const eddy::ModelConfig& config, return true; } -} // namespace eddy::parakeet +} // namespace eddy::model_utils From df37fa3b15a2a5a8553bac9616a4655a76a35188 Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:22:51 +0000 Subject: [PATCH 05/13] Address PR #10 review: Release-safe cache guards, hoist RNNT allocs, C-API hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encoder/decoder (nemotron_openvino.cpp): - Replace assert() cache-size checks with runtime `throw` (Release builds define NDEBUG, so the asserts were compiled out); add the same guard for the decoder h_out/c_out LSTM state copies. - Hoist the per-chunk (audio, mel_in) and inner-loop (token, token_length, mel_length, audio_length) tensors above the loops — they have fixed shape/value for the whole call; the inner token/token_length allocs were the hottest path. Drop now-unused . - Make ensure_compiled() const so resolve_prompt_id() (const) can lazily compile without the const_cast UB; document why it's safe (only mutates *impl_ through the unique_ptr). - Keep audio_length = full padded chunk_samples (matches the WER-validated transcribe_ov.py), now with a comment explaining why end-off would diverge. C API (eddy_c.cpp / eddy_c.h): - eddy_nemotron_infer_file: only a genuinely missing file returns FILE_NOT_FOUND; read_wav format/decode errors now return INFERENCE_FAILED. - eddy_nemotron_infer_buffer: zero-init *out and free on catch so a throw mid-fill can't leak the already-allocated strings. - Hoist the doubly-constructed model_dir string; annotate token_ids ownership. CLI (nemotron_cli.cpp): - Add --model to pick the cache variant (e.g. nemotron-streaming-int8) without --model-dir wrangling. - Error (not silently swallow as the audio path) when a known flag is given without a value, and reject unknown -options. Align nemotron::Config default device to "CPU" to match the C API and CLI. Builds clean; FP16/INT8 transcripts match the reference word-for-word. --- examples/cpp/nemotron_cli.cpp | 40 +++++++++---- include/eddy/eddy_c.h | 2 +- include/eddy/models/nemotron/nemotron.hpp | 9 ++- src/eddy_c.cpp | 25 +++++--- src/models/nemotron/nemotron_openvino.cpp | 72 ++++++++++++++++------- 5 files changed, 106 insertions(+), 42 deletions(-) diff --git a/examples/cpp/nemotron_cli.cpp b/examples/cpp/nemotron_cli.cpp index d1abb1f..7c88e14 100644 --- a/examples/cpp/nemotron_cli.cpp +++ b/examples/cpp/nemotron_cli.cpp @@ -19,8 +19,10 @@ void print_usage(const char* prog) { std::cout << "Options:\n"; std::cout << " --device OpenVINO device (default: CPU). CPU, AUTO, NPU\n"; std::cout << " --lang Language: en-US, zh-CN, ... or auto (default: auto)\n"; + std::cout << " --model Model variant: nemotron-streaming (FP16, default) or\n"; + std::cout << " nemotron-streaming-int8. Selects the cache dir.\n"; std::cout << " --model-dir Directory with nemotron_*.xml/bin + metadata.json\n"; - std::cout << " (default: per-user model cache for 'nemotron-streaming')\n"; + std::cout << " (overrides --model; default: cache for the --model variant)\n"; std::cout << " --help Show this help\n"; } @@ -31,18 +33,35 @@ int main(int argc, char* argv[]) { return 1; } - std::string audio_file, device = "CPU", lang = "auto", model_dir_arg; + std::string audio_file, device = "CPU", lang = "auto", model_dir_arg, + model_name = "nemotron-streaming"; for (int i = 1; i < argc; ++i) { std::string a = argv[i]; + // A flag that needs a value but is the last arg must error, not fall through + // to the positional branch (which would swallow the flag as the audio path). + auto take_value = [&](const char* flag, std::string& dst) -> bool { + if (i + 1 >= argc) { + std::cerr << "Error: " << flag << " requires an argument\n"; + return false; + } + dst = argv[++i]; + return true; + }; if (a == "--help" || a == "-h") { print_usage(argv[0]); return 0; - } else if (a == "--device" && i + 1 < argc) { - device = argv[++i]; - } else if (a == "--lang" && i + 1 < argc) { - lang = argv[++i]; - } else if (a == "--model-dir" && i + 1 < argc) { - model_dir_arg = argv[++i]; + } else if (a == "--device") { + if (!take_value("--device", device)) return 1; + } else if (a == "--lang") { + if (!take_value("--lang", lang)) return 1; + } else if (a == "--model") { + if (!take_value("--model", model_name)) return 1; + } else if (a == "--model-dir") { + if (!take_value("--model-dir", model_dir_arg)) return 1; + } else if (!a.empty() && a[0] == '-') { + std::cerr << "Error: unknown option " << a << "\n\n"; + print_usage(argv[0]); + return 1; } else { audio_file = a; } @@ -62,13 +81,14 @@ int main(int argc, char* argv[]) { << audio_seconds << "s)\n"; std::filesystem::path model_dir = - model_dir_arg.empty() ? eddy::get_model_assets_dir("nemotron-streaming") + model_dir_arg.empty() ? eddy::get_model_assets_dir(model_name) : std::filesystem::path(model_dir_arg); + std::cout << "Model: " << model_name << "\n"; std::cout << "Models: " << model_dir.string() << "\n"; eddy::OpenVINOOptions ov_opts; ov_opts.device = device; - ov_opts.cache_dir = eddy::get_model_dir("nemotron-streaming").string(); + ov_opts.cache_dir = eddy::get_model_dir(model_name).string(); auto backend = std::make_shared(ov_opts); eddy::nemotron::ModelPaths paths{ diff --git a/include/eddy/eddy_c.h b/include/eddy/eddy_c.h index 16df676..aec82da 100644 --- a/include/eddy/eddy_c.h +++ b/include/eddy/eddy_c.h @@ -243,7 +243,7 @@ typedef struct { typedef struct { char* text; // full transcript (lang-tag tokens stripped); free with eddy_nemotron_free_result char* detected_language; // first tag emitted, or "" ; freed with the result - int* token_ids; // raw emitted token ids (pre-strip); freed with the result + int* token_ids; // raw emitted token ids (pre-strip); must be freed with eddy_nemotron_free_result size_t num_tokens; int prompt_id_used; // integer prompt id selected for conditioning double latency_ms; diff --git a/include/eddy/models/nemotron/nemotron.hpp b/include/eddy/models/nemotron/nemotron.hpp index 68523ec..b5892c4 100644 --- a/include/eddy/models/nemotron/nemotron.hpp +++ b/include/eddy/models/nemotron/nemotron.hpp @@ -34,7 +34,9 @@ struct ModelPaths { }; struct Config { - std::string device = "AUTO"; // OpenVINO device for encoder/decoder/joint (preprocessor always CPU) + // CPU is the default (safe, tested path; matches the eddy_c C API default and + // the CLI). Set "AUTO"/"NPU"/"GPU" to target other OpenVINO devices. + std::string device = "CPU"; // OpenVINO device for encoder/decoder/joint (preprocessor always CPU) /// Language for prompt conditioning. Accepts dictionary keys ("en-US"), /// 2-letter codes ("en" -> first "en-*"), or "auto" (model self-detects). std::string language = "auto"; @@ -72,7 +74,10 @@ class OpenVINONemotron { struct Impl; private: - void ensure_compiled(); + // const: only mutates *impl_ (reachable through the unique_ptr in a const + // method) and is std::call_once-guarded, so resolve_prompt_id() (const) can + // lazily compile without a const_cast. + void ensure_compiled() const; std::unique_ptr impl_; }; diff --git a/src/eddy_c.cpp b/src/eddy_c.cpp index a3918a3..b40d2df 100644 --- a/src/eddy_c.cpp +++ b/src/eddy_c.cpp @@ -535,13 +535,10 @@ EDDY_API EddyNemotronModel eddy_nemotron_create(EddyNemotronConfig config, char* const std::string language = config.language ? config.language : "auto"; // Resolve model directory: explicit dir, else (NULL/"cache") the Eddy cache. - std::filesystem::path model_dir; - if (config.model_dir && std::string(config.model_dir).size() > 0 && - std::string(config.model_dir) != "cache") { - model_dir = config.model_dir; - } else { - model_dir = eddy::get_model_assets_dir(kNemotronModelName); - } + const std::string md = config.model_dir ? config.model_dir : ""; + std::filesystem::path model_dir = + (!md.empty() && md != "cache") ? std::filesystem::path(md) + : eddy::get_model_assets_dir(kNemotronModelName); auto backend = std::make_shared( eddy::OpenVINOOptions{ .device = device, @@ -603,14 +600,19 @@ EDDY_API EddyError eddy_nemotron_infer_buffer(EddyNemotronModel handle, const fl if (err) *err = copy_string("[Eddy Error] Nemotron expects 16kHz mono audio"); return EDDY_ERROR_INVALID_ARGUMENT; } + // Zero-init so a throw mid-fill (e.g. copy_string OOM after text is set) is + // cleaned up by eddy_nemotron_free_result in the catch instead of leaking. + *out = EddyNemotronResult{}; try { auto* h = static_cast(handle); std::vector samples(pcm, pcm + length); return nemotron_fill_result(h->model->transcribe(samples), out); } catch (const std::exception& e) { + eddy_nemotron_free_result(out); if (err) *err = capture_exception(e); return EDDY_ERROR_INFERENCE_FAILED; } catch (...) { + eddy_nemotron_free_result(out); if (err) *err = copy_string("[Eddy Error] Unknown exception during nemotron inference"); return EDDY_ERROR_UNKNOWN; } @@ -622,12 +624,19 @@ EDDY_API EddyError eddy_nemotron_infer_file(EddyNemotronModel handle, const char if (err) *err = copy_string("[Eddy Error] Invalid argument: null pointer"); return EDDY_ERROR_INVALID_ARGUMENT; } + // Only a genuinely missing file is FILE_NOT_FOUND; read_wav also throws for + // format/channel/sample-rate/decode errors, which are not filesystem issues. + std::error_code ec; + if (!std::filesystem::exists(wav_path, ec)) { + if (err) *err = copy_string("[Eddy Error] WAV file not found: " + std::string(wav_path)); + return EDDY_ERROR_FILE_NOT_FOUND; + } try { auto pcm = eddy::audio::read_wav(wav_path); return eddy_nemotron_infer_buffer(handle, pcm.data(), pcm.size(), 16000, out, err); } catch (const std::exception& e) { if (err) *err = capture_exception(e); - return EDDY_ERROR_FILE_NOT_FOUND; + return EDDY_ERROR_INFERENCE_FAILED; } catch (...) { if (err) *err = copy_string("[Eddy Error] Unknown exception in nemotron infer_file"); return EDDY_ERROR_UNKNOWN; diff --git a/src/models/nemotron/nemotron_openvino.cpp b/src/models/nemotron/nemotron_openvino.cpp index f1c1ba2..b079439 100644 --- a/src/models/nemotron/nemotron_openvino.cpp +++ b/src/models/nemotron/nemotron_openvino.cpp @@ -18,7 +18,6 @@ #include #include -#include #include #include #include @@ -128,9 +127,10 @@ void OpenVINONemotron::warmup() { ensure_compiled(); } int OpenVINONemotron::resolve_prompt_id(const std::string& language) const { // prompt_dictionary is populated by ensure_compiled(); make this safe to call - // standalone (before transcribe()/warmup()). ensure_compiled() is std::call_once - // guarded, so this is a cheap no-op once compiled. - const_cast(this)->ensure_compiled(); + // standalone (before transcribe()/warmup()). ensure_compiled() is const (it only + // mutates *impl_, reachable through the unique_ptr in a const method) and + // std::call_once guarded, so this is a cheap no-op once compiled. + ensure_compiled(); const auto& dict = impl_->prompt_dictionary; auto it = dict.find(language); if (it != dict.end()) { @@ -149,7 +149,7 @@ int OpenVINONemotron::resolve_prompt_id(const std::string& language) const { return impl_->default_prompt_id; } -void OpenVINONemotron::ensure_compiled() { +void OpenVINONemotron::ensure_compiled() const { std::call_once(impl_->compile_once, [this]() { auto& core = impl_->backend->core(); const std::string device = impl_->config.device.empty() ? "AUTO" : impl_->config.device; @@ -266,19 +266,34 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) const ov::Tensor prompt_tensor = make_i32(prompt_id); + // Hot-path tensors whose shapes/values are fixed for the whole call — allocate + // once and reuse across chunks and the inner RNNT loop instead of re-allocating + // every iteration (the inner `token`/`token_length` allocs dominate otherwise). + ov::Tensor audio(ov::element::f32, ov::Shape{1, chunk_samples}); + ov::Tensor mel_in(ov::element::f32, ov::Shape{1, bins, total}); + ov::Tensor token(I.token_et, ov::Shape{1, 1}); + const ov::Tensor token_length = make_i32(1); + const ov::Tensor mel_length = make_i32(static_cast(total)); + // audio_length is intentionally the FULL padded chunk size on every chunk, + // including the final short chunk. This mirrors the WER-validated Python + // reference (transcribe_ov.py: the chunk is np.pad'd to chunk_samples and + // chunk.shape[1] == chunk_samples is passed). The preprocessor IR does not gate + // mel output on this value, so passing the real (end-off) count would diverge + // from the validated path without changing output. + const ov::Tensor audio_length = make_i32(static_cast(chunk_samples)); + size_t off = 0; while (off < pcm.size()) { const size_t end = std::min(off + chunk_samples, pcm.size()); - // Raw audio chunk, padded to chunk_samples. - ov::Tensor audio(ov::element::f32, ov::Shape{1, chunk_samples}); + // Raw audio chunk, padded to chunk_samples (reusing the hoisted tensor). float* adst = audio.data(); std::memset(adst, 0, audio.get_byte_size()); std::copy(pcm.begin() + static_cast(off), pcm.begin() + static_cast(end), adst); // Preprocessor: audio -> mel [1, bins, T_mel] I.preproc_req.set_tensor("audio", audio); - I.preproc_req.set_tensor("audio_length", make_i32(static_cast(chunk_samples))); + I.preproc_req.set_tensor("audio_length", audio_length); I.preproc_req.infer(); const ov::Tensor mel_out = I.preproc_req.get_tensor("mel"); const ov::Shape mel_shape = mel_out.get_shape(); // [1, bins, T_mel] @@ -286,8 +301,8 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) const float* mel_src = mel_out.data(); // Build encoder mel input [1, bins, total]: prepend cache (or zero - // pre_encode_cache on first chunk), then pad/trim to total. - ov::Tensor mel_in(ov::element::f32, ov::Shape{1, bins, total}); + // pre_encode_cache on first chunk), then pad/trim to total. (reusing the + // hoisted tensor; fully overwritten via memset + fills below.) float* mdst = mel_in.data(); std::memset(mdst, 0, mel_in.get_byte_size()); @@ -321,7 +336,7 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) // Encoder: mel + caches + prompt_id -> encoded + caches I.encoder_req.set_tensor("mel", mel_in); - I.encoder_req.set_tensor("mel_length", make_i32(static_cast(total))); + I.encoder_req.set_tensor("mel_length", mel_length); I.encoder_req.set_tensor("cache_channel", cache_channel); I.encoder_req.set_tensor("cache_time", cache_time); I.encoder_req.set_tensor("cache_len", cache_len); @@ -336,10 +351,15 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) const ov::Tensor cl = I.encoder_req.get_tensor("cache_len_out"); // Cache-aware streaming: the *_out caches are the same fixed shape as the // input caches (the ring buffer is re-filled in place), so we copy back - // into the pre-allocated input tensors. Assert the byte sizes agree so a - // mismatched IR export trips here instead of silently over-/under-reading. - assert(cc.get_byte_size() == cache_channel.get_byte_size()); - assert(ctt.get_byte_size() == cache_time.get_byte_size()); + // into the pre-allocated input tensors. Check the byte sizes agree so a + // mismatched IR export throws here instead of silently over-/under-reading. + // Runtime check (not assert): Release builds define NDEBUG. + if (cc.get_byte_size() != cache_channel.get_byte_size() || + ctt.get_byte_size() != cache_time.get_byte_size()) { + throw std::runtime_error( + "Nemotron encoder cache_*_out byte size differs from the pre-allocated " + "input cache; the model IR does not match metadata.json cache shapes."); + } std::memcpy(cache_channel.data(), cc.data(), cache_channel.get_byte_size()); std::memcpy(cache_time.data(), ctt.data(), cache_time.get_byte_size()); cache_len.data()[0] = cl.data()[0]; @@ -359,15 +379,15 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) } for (size_t sym = 0; sym < I.config.max_symbols_per_frame; ++sym) { - // Decoder - ov::Tensor token(I.token_et, ov::Shape{1, 1}); + // Decoder (token/token_length tensors hoisted above the loops; just + // overwrite the scalar token value each iteration). if (I.token_et == ov::element::i64) { token.data()[0] = last_token; } else { token.data()[0] = last_token; } I.decoder_req.set_tensor("token", token); - I.decoder_req.set_tensor("token_length", make_i32(1)); + I.decoder_req.set_tensor("token_length", token_length); I.decoder_req.set_tensor("h_in", h); I.decoder_req.set_tensor("c_in", c); I.decoder_req.infer(); @@ -395,9 +415,19 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) } all_tokens.push_back(best); last_token = best; - // Advance LSTM state on emission. - std::memcpy(h.data(), I.decoder_req.get_tensor("h_out").data(), h.get_byte_size()); - std::memcpy(c.data(), I.decoder_req.get_tensor("c_out").data(), c.get_byte_size()); + // Advance LSTM state on emission. h_out/c_out are the same fixed shape as + // h_in/c_in by construction; guard so a mismatched decoder IR throws + // instead of corrupting the fixed-size state tensors. + const ov::Tensor h_out = I.decoder_req.get_tensor("h_out"); + const ov::Tensor c_out = I.decoder_req.get_tensor("c_out"); + if (h_out.get_byte_size() != h.get_byte_size() || + c_out.get_byte_size() != c.get_byte_size()) { + throw std::runtime_error( + "Nemotron decoder h_out/c_out byte size differs from the LSTM state " + "tensors; the decoder IR does not match the expected layer/hidden dims."); + } + std::memcpy(h.data(), h_out.data(), h.get_byte_size()); + std::memcpy(c.data(), c_out.data(), c.get_byte_size()); } } From eca8b1d8e65fbb875b1048ea56d0253c01c6dc7b Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:46:44 -0400 Subject: [PATCH 06/13] Address PR #10 review: C-API result init, fail-fast create, metadata/cache hardening - eddy_nemotron_infer_file: zero-init *out before FILE_NOT_FOUND return so a caller-then-free_result does not delete[] uninitialized pointers. - eddy_nemotron_create: fail fast via check_models_available(NEMOTRON_FILES), matching the eddy_parakeet_create contract instead of deferring to first infer. - nemotron metadata: require cache_channel_shape/cache_time_shape explicitly with a path-aware error rather than letting nlohmann's bare key-not-found propagate. - mel_cache: resize instead of assign (every element is overwritten; drop the redundant per-chunk zero-fill). - chunk_samples(): integer frames*rate/100, avoiding the 0.01 FP rounding hazard. - ensure_models: split is_shell_safe into tight is_url_safe / is_path_safe; URL charset drops '~', path charset adds native '\' and space (fixes Windows paths). - CLAUDE.md: document both Parakeet/batch and Nemotron/streaming as supported. Co-Authored-By: Claude Opus 4.8 (1M context) --- claude.md | 9 +++++-- src/eddy_c.cpp | 17 +++++++++++++ src/models/nemotron/nemotron_openvino.cpp | 17 +++++++++++-- src/utils/ensure_models.cpp | 29 ++++++++++++++++++----- 4 files changed, 62 insertions(+), 10 deletions(-) diff --git a/claude.md b/claude.md index 1c5a245..1b05d8a 100644 --- a/claude.md +++ b/claude.md @@ -28,12 +28,17 @@ Match FluidAudio's Parakeet v2 (Swift/CoreML) implementation in C++/OpenVINO: ## Architecture -**Batch Chunking** (current focus): +Two first-class ASR paths are supported: + +**Batch (Parakeet)** — `eddy::parakeet`, stateless overlapping-chunk encoder: - 10s chunks with 3s overlap - 2D search deduplication at boundaries - LSTM state continuity across chunks -**NOT building streaming yet** - focus is on batch processing of complete audio files. +**Streaming (Nemotron)** — `eddy::nemotron`, cache-aware streaming FastConformer-RNNT: +- Carries `cache_channel`/`cache_time`/`cache_len` across chunks (`att_context=[56,0]`) +- Integer `prompt_id` per-chunk language conditioning (40+ languages) +- Plain RNNT greedy decode (no TDT duration bins) ## Testing diff --git a/src/eddy_c.cpp b/src/eddy_c.cpp index b40d2df..4ce09b2 100644 --- a/src/eddy_c.cpp +++ b/src/eddy_c.cpp @@ -545,6 +545,19 @@ EDDY_API EddyNemotronModel eddy_nemotron_create(EddyNemotronConfig config, char* .cache_dir = eddy::get_model_dir(kNemotronModelName).string() } ); + // Fail fast at create time if the model files are missing, matching the + // eddy_parakeet_create contract. Otherwise the first failure only surfaces + // deep inside ensure_compiled() on the initial transcribe()/warmup() call. + { + std::string check_err; + if (!eddy::model_utils::check_models_available( + model_dir, &check_err, eddy::model_configs::NEMOTRON_FILES)) { + throw std::runtime_error( + "Nemotron model files not available in '" + model_dir.string() + + "': " + check_err); + } + } + eddy::nemotron::ModelPaths paths{ .preprocessor = (model_dir / "nemotron_preprocessor.xml").string(), .encoder = (model_dir / "nemotron_encoder.xml").string(), @@ -628,6 +641,10 @@ EDDY_API EddyError eddy_nemotron_infer_file(EddyNemotronModel handle, const char // format/channel/sample-rate/decode errors, which are not filesystem issues. std::error_code ec; if (!std::filesystem::exists(wav_path, ec)) { + // Zero-init before the early return so a caller that calls + // eddy_nemotron_free_result on *out after FILE_NOT_FOUND does not + // delete[] uninitialized/garbage pointers. + *out = EddyNemotronResult{}; if (err) *err = copy_string("[Eddy Error] WAV file not found: " + std::string(wav_path)); return EDDY_ERROR_FILE_NOT_FOUND; } diff --git a/src/models/nemotron/nemotron_openvino.cpp b/src/models/nemotron/nemotron_openvino.cpp index b079439..db8ef9a 100644 --- a/src/models/nemotron/nemotron_openvino.cpp +++ b/src/models/nemotron/nemotron_openvino.cpp @@ -106,7 +106,9 @@ struct OpenVINONemotron::Impl { std::mutex infer_guard; size_t chunk_samples() const { - return static_cast(static_cast(chunk_mel_frames) * 0.01 * sample_rate); + // mel hop is 10 ms => frames * sample_rate / 100. Pure integer math avoids + // the rounding hazard of multiplying by the non-representable 0.01. + return static_cast(chunk_mel_frames) * static_cast(sample_rate) / 100; } }; @@ -178,6 +180,14 @@ void OpenVINONemotron::ensure_compiled() const { for (const auto& d : arr) s.push_back(d.get()); return s; }; + // These two keys have no sensible default (they size the encoder caches), + // so require them explicitly with a path-aware message rather than letting + // nlohmann's bare "key not found" propagate from a truncated metadata.json. + if (!m.contains("cache_channel_shape") || !m.contains("cache_time_shape")) { + throw std::runtime_error( + "Nemotron metadata missing required cache shape keys " + "(cache_channel_shape / cache_time_shape): " + impl_->paths.metadata_json); + } impl_->cache_channel_shape = to_shape(m.at("cache_channel_shape")); impl_->cache_time_shape = to_shape(m.at("cache_time_shape")); @@ -327,7 +337,10 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) const size_t keep = std::min(pre_cache, t_mel); mel_cache.bins = bins; mel_cache.frames = keep; - mel_cache.data.assign(bins * keep, 0.0f); + // resize, not assign: every element is unconditionally overwritten by the + // loop below (bin*keep + t is a bijection over [0, bins*keep)), so the + // zero-fill assign() would do is pure waste on this per-chunk hot buffer. + mel_cache.data.resize(bins * keep); for (size_t bin = 0; bin < bins; ++bin) { for (size_t t = 0; t < keep; ++t) { mel_cache.data[bin * keep + t] = mel_src[bin * t_mel + (t_mel - keep + t)]; diff --git a/src/utils/ensure_models.cpp b/src/utils/ensure_models.cpp index d6dd8a3..58db799 100644 --- a/src/utils/ensure_models.cpp +++ b/src/utils/ensure_models.cpp @@ -19,22 +19,39 @@ static bool file_nonempty(const std::filesystem::path& p) { // The download shells out via std::system, so any interpolated component must // be free of characters that could break out of the double-quoted argument. // Today every field is a compile-time constant, but ModelConfig is caller- -// supplied, so reject anything outside a conservative path/URL charset rather -// than risk command injection. -static bool is_shell_safe(const std::string& s) { +// supplied, so reject anything outside a conservative charset rather than risk +// command injection. The long-term fix is to drop std::system for a direct +// libcurl call and eliminate this class of concern entirely. +// +// The URL and the local output path get *separate* allowlists: the URL is the +// real injection surface (it's assembled from caller-supplied ModelConfig +// fields) so it stays tight — only the characters an https HuggingFace URL +// needs, and notably no '~'. The output path is application-controlled (the +// Eddy cache dir) but must also tolerate native Windows paths, so it +// additionally allows the native separator '\\' and spaces (the drive-letter +// ':' is already covered by the shared charset). +static bool charset_ok(const std::string& s, bool allow_path_chars) { for (const unsigned char c : s) { - const bool ok = std::isalnum(c) || c == '.' || c == '_' || c == '-' || - c == '/' || c == ':' || c == '~'; + bool ok = std::isalnum(c) || c == '.' || c == '_' || c == '-' || + c == '/' || c == ':'; + if (!ok && allow_path_chars) { + // Native Windows paths: backslash separators and spaces inside the + // double-quoted argument. None of these can break out of the quotes. + ok = (c == '\\' || c == ' '); + } if (!ok) return false; } return true; } +static bool is_url_safe(const std::string& s) { return charset_ok(s, /*allow_path_chars=*/false); } +static bool is_path_safe(const std::string& s) { return charset_ok(s, /*allow_path_chars=*/true); } + static bool download_single_file(const std::string& url, const std::filesystem::path& output_path, std::string* error_msg = nullptr) { // Refuse to build a shell command from unsafe components. - if (!is_shell_safe(url) || !is_shell_safe(output_path.string())) { + if (!is_url_safe(url) || !is_path_safe(output_path.string())) { if (error_msg) { *error_msg = "Refusing to download: unsafe characters in URL or path: " + url; } From 7e29a800cc5575c6923484a7d7a5c18af3ca53d4 Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 15:42:01 -0400 Subject: [PATCH 07/13] Nemotron: native C++ mel preprocessor + NPU encoder fix Preprocessor: replace the dynamic-shape nemotron_preprocessor.xml with a native C++ log-mel featurizer (MelFeaturizer) reproducing NeMo's AudioToMelSpectrogramPreprocessor exactly (preemph 0.97 -> center-pad -> Hann-windowed 512-pt STFT via a self-contained radix-2 FFT -> power -> slaney mel -> log(x+2^-24)). Reverse-engineered from the IR (validated to ~1e-5 on window/filterbank constants and <=0.14% WER end-to-end over 1050 FLEURS files). A startup guard verifies the featurizer's frame geometry matches the model's chunk_mel_frames. nemotron_preprocessor.xml/.bin dropped from NEMOTRON_FILES. NPU: the OpenVINO NPU plugin miscompiles BitwiseNot on a boolean (it does an integer complement, so ~0=-1 and ~1=-2 are both 'true'), making the FastConformer attention mask all-true -> uniform softmax -> encoder output collapses to ~0 -> empty transcripts. Rewrite BitwiseNot(bool) -> LogicalNot in the encoder IR before compiling (no-op on CPU/GPU). Nemotron now runs correctly on NPU at WER/CER parity with CPU and ~2x the throughput (RTFx ~22x vs ~11x). Adds benchmark_nemotron_fleurs (FLEURS WER/CER/RTFx harness, UTF-8/CJK-aware CER) used to validate the above across en/es/fr/zh/ja on CPU and NPU. Co-Authored-By: Claude Opus 4.8 (1M context) --- CMakeLists.txt | 1 + examples/cpp/CMakeLists.txt | 5 +- examples/cpp/benchmark_nemotron_fleurs.cpp | 341 ++++++++++++++++++ include/eddy/core/model_configs.hpp | 5 +- .../models/nemotron/nemotron_featurizer.hpp | 59 +++ src/models/nemotron/nemotron_featurizer.cpp | 193 ++++++++++ src/models/nemotron/nemotron_openvino.cpp | 100 +++-- 7 files changed, 677 insertions(+), 27 deletions(-) create mode 100644 examples/cpp/benchmark_nemotron_fleurs.cpp create mode 100644 include/eddy/models/nemotron/nemotron_featurizer.hpp create mode 100644 src/models/nemotron/nemotron_featurizer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c7428da..85614bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,7 @@ target_sources(eddy # Nemotron cache-aware streaming implementation src/models/nemotron/nemotron_openvino.cpp + src/models/nemotron/nemotron_featurizer.cpp ) # Link dependencies diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index c345eb1..a7cd1fa 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -16,4 +16,7 @@ target_link_libraries(hf_fetch_models PRIVATE eddy) add_executable(benchmark_fleurs benchmark_fleurs.cpp) target_link_libraries(benchmark_fleurs PRIVATE eddy) -install(TARGETS parakeet_cli nemotron_cli hf_fetch_models benchmark_fleurs DESTINATION bin) +add_executable(benchmark_nemotron_fleurs benchmark_nemotron_fleurs.cpp) +target_link_libraries(benchmark_nemotron_fleurs PRIVATE eddy) + +install(TARGETS parakeet_cli nemotron_cli hf_fetch_models benchmark_fleurs benchmark_nemotron_fleurs DESTINATION bin) diff --git a/examples/cpp/benchmark_nemotron_fleurs.cpp b/examples/cpp/benchmark_nemotron_fleurs.cpp new file mode 100644 index 0000000..9074300 --- /dev/null +++ b/examples/cpp/benchmark_nemotron_fleurs.cpp @@ -0,0 +1,341 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// FLEURS Multilingual ASR Benchmark for the Nemotron streaming backend. +// +// Mirrors benchmark_fleurs.cpp (Parakeet) but drives eddy::nemotron:: +// OpenVINONemotron with per-language prompt conditioning. Loads the model once +// per language (language is fixed at handle construction), loops the FLEURS +// split, and reports WER / CER / RTFx. +// +// Usage: +// benchmark_nemotron_fleurs.exe --languages en_us,es_419,fr_fr \ +// --samples 0 --device NPU --model-dir --output results.json + +#include "eddy/backends/openvino_backend.hpp" +#include "eddy/core/app_dir.hpp" +#include "eddy/models/nemotron/nemotron.hpp" +#include "eddy/utils/audio_utils.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +// FLEURS code -> human name (subset; only used for display). +const std::map LANG_NAMES = { + {"en_us", "English (US)"}, {"es_419", "Spanish (LatAm)"}, {"fr_fr", "French (France)"}, + {"de_de", "German (Germany)"}, {"it_it", "Italian (Italy)"}, {"ru_ru", "Russian (Russia)"}, + {"nl_nl", "Dutch"}, {"pl_pl", "Polish"}, {"uk_ua", "Ukrainian"}, + {"sk_sk", "Slovak"}, {"cs_cz", "Czech"}, {"bg_bg", "Bulgarian"}, + {"hr_hr", "Croatian"}, {"ro_ro", "Romanian"}, {"fi_fi", "Finnish"}, + {"hu_hu", "Hungarian"}, {"sv_se", "Swedish"}, {"et_ee", "Estonian"}, + {"da_dk", "Danish"}, {"lt_lt", "Lithuanian"}, {"el_gr", "Greek"}, + {"mt_mt", "Maltese"}, {"lv_lv", "Latvian"}, {"sl_si", "Slovenian"}, + {"cmn_hans_cn", "Chinese (Mandarin)"}, {"ja_jp", "Japanese"}, +}; + +// Map a FLEURS code ("en_us") to a Nemotron prompt-dictionary tag ("en-US"). +// es_419 (Latin-American Spanish) is special-cased to es-US; everything else is +// the generic "xx_yy" -> "xx-YY". Unknown tags fall back inside the backend +// (2-letter, then "auto"). +std::string fleurs_to_nemotron_lang(const std::string& code) { + if (code == "es_419") return "es-US"; + if (code == "cmn_hans_cn") return "zh-CN"; // FLEURS Mandarin -> Nemotron zh-CN + if (code == "ja_jp") return "ja-JP"; + auto us = code.find('_'); + if (us == std::string::npos) return code; + std::string lang = code.substr(0, us); + std::string region = code.substr(us + 1); + for (char& c : region) c = static_cast(std::toupper(static_cast(c))); + return lang + "-" + region; +} + +// Split a UTF-8 string into codepoint substrings (1-4 bytes each). +std::vector utf8_chars(const std::string& s) { + std::vector cps; + size_t i = 0; + while (i < s.size()) { + unsigned char c = static_cast(s[i]); + size_t len = (c < 0x80) ? 1 : ((c >> 5) == 0x6) ? 2 : ((c >> 4) == 0xE) ? 3 + : ((c >> 3) == 0x1E) ? 4 : 1; + if (i + len > s.size()) len = 1; + cps.push_back(s.substr(i, len)); + i += len; + } + return cps; +} + +// Normalize: lowercase ASCII alphanumerics, collapse other ASCII to single +// spaces, and KEEP non-ASCII codepoints verbatim (so CJK/accented characters +// survive for CER). UTF-8 aware, unlike the byte-wise version in +// benchmark_fleurs.cpp which would strip all CJK. +std::string normalize_text(const std::string& text) { + std::string result; + result.reserve(text.size()); + bool last_space = false; + for (const std::string& cp : utf8_chars(text)) { + if (cp.size() == 1) { + unsigned char c = static_cast(cp[0]); + if (std::isalnum(c)) { + result += static_cast(std::tolower(c)); + last_space = false; + } else if (!last_space && !result.empty()) { + result += ' '; + last_space = true; + } + } else { + result += cp; // non-ASCII (CJK etc.) kept as-is + last_space = false; + } + } + if (!result.empty() && result.back() == ' ') result.pop_back(); + return result; +} + +int levenshtein(const std::vector& a, const std::vector& b) { + const size_t m = a.size(), n = b.size(); + std::vector prev(n + 1), cur(n + 1); + for (size_t j = 0; j <= n; ++j) prev[j] = static_cast(j); + for (size_t i = 1; i <= m; ++i) { + cur[0] = static_cast(i); + for (size_t j = 1; j <= n; ++j) { + if (a[i - 1] == b[j - 1]) cur[j] = prev[j - 1]; + else cur[j] = 1 + std::min({prev[j], cur[j - 1], prev[j - 1]}); + } + std::swap(prev, cur); + } + return prev[n]; +} + +std::vector words(const std::string& s) { + std::vector w; + std::istringstream iss(s); + std::string t; + while (iss >> t) w.push_back(t); + return w; +} + +double wer(const std::string& ref, const std::string& hyp) { + auto r = words(normalize_text(ref)), h = words(normalize_text(hyp)); + if (r.empty()) return h.empty() ? 0.0 : 1.0; + return static_cast(levenshtein(r, h)) / r.size(); +} + +double cer(const std::string& ref, const std::string& hyp) { + // Character error rate over UTF-8 codepoints (spaces dropped). Codepoint- + // level, so CJK characters count as one token each. + std::vector r, h; + for (const auto& cp : utf8_chars(normalize_text(ref))) if (cp != " ") r.push_back(cp); + for (const auto& cp : utf8_chars(normalize_text(hyp))) if (cp != " ") h.push_back(cp); + if (r.empty()) return h.empty() ? 0.0 : 1.0; + return static_cast(levenshtein(r, h)) / r.size(); +} + +struct Sample { std::string id, audio_path, transcription; }; + +std::vector load_samples(const std::string& cache_dir, const std::string& lang, int max_samples) { + std::vector samples; + fs::path dir = fs::path(cache_dir) / lang; + if (!fs::exists(dir)) { + std::cerr << "Warning: language dir not found: " << dir << "\n"; + return samples; + } + std::map trans; + fs::path tf = dir / (lang + ".trans.txt"); + if (fs::exists(tf)) { + std::ifstream f(tf); + std::string line; + while (std::getline(f, line)) { + size_t sp = line.find(' '); + if (sp != std::string::npos) trans[line.substr(0, sp)] = line.substr(sp + 1); + } + } + std::vector wavs; + for (const auto& e : fs::directory_iterator(dir)) + if (e.path().extension() == ".wav") wavs.push_back(e.path()); + std::sort(wavs.begin(), wavs.end()); + if (max_samples > 0 && wavs.size() > static_cast(max_samples)) + wavs.resize(max_samples); + for (const auto& w : wavs) { + std::string id = w.stem().string(); + Sample s{id, w.string(), trans.count(id) ? trans[id] : ""}; + samples.push_back(s); + } + return samples; +} + +struct LangResult { + std::string lang, name; + double wer = 0, cer = 0, rtfx = 0, total_audio = 0, total_proc = 0; + int processed = 0, skipped = 0; +}; + +void print_usage(const char* p) { + std::cout << "Nemotron FLEURS Benchmark\n\nUsage: " << p << " [options]\n" + << " --languages Comma-separated FLEURS codes (default: en_us,es_419,fr_fr)\n" + << " --samples Max samples per language (0 = all; default 0)\n" + << " --device OpenVINO device CPU/NPU/GPU/AUTO (default CPU)\n" + << " --model-dir Dir with nemotron_*.xml/bin + metadata.json\n" + << " --output Output JSON (default nemotron_fleurs_results.json)\n" + << " --debug Per-file hypothesis/reference\n"; +} + +int main(int argc, char* argv[]) { + std::cout.setf(std::ios::unitbuf); + if (argc < 2) { print_usage(argv[0]); return 1; } + + std::string cache_dir = argv[1]; + std::vector langs = {"en_us", "es_419", "fr_fr"}; + int max_samples = 0; + std::string device = "CPU", model_dir, output = "nemotron_fleurs_results.json"; + bool debug = false; + + for (int i = 2; i < argc; ++i) { + std::string a = argv[i]; + auto next = [&](std::string& dst) { if (i + 1 < argc) dst = argv[++i]; }; + if (a == "--help" || a == "-h") { print_usage(argv[0]); return 0; } + else if (a == "--languages") { std::string v; next(v); langs.clear(); + std::istringstream ss(v); std::string t; while (std::getline(ss, t, ',')) langs.push_back(t); } + else if (a == "--samples") { std::string v; next(v); max_samples = std::stoi(v); } + else if (a == "--device") next(device); + else if (a == "--model-dir") next(model_dir); + else if (a == "--output") next(output); + else if (a == "--debug") debug = true; + } + + // Default model dir: %LOCALAPPDATA%/eddy/models/nemotron-streaming-int8/files + if (model_dir.empty()) { + const char* lad = std::getenv("LOCALAPPDATA"); + if (lad) model_dir = (fs::path(lad) / "eddy" / "models" / "nemotron-streaming-int8" / "files").string(); + } + + std::cout << "=== Nemotron FLEURS Benchmark ===\n"; + std::cout << "Cache: " << cache_dir << "\n"; + std::cout << "Model dir: " << model_dir << "\n"; + std::cout << "Device: " << device << "\n"; + std::cout << "Samples: " << (max_samples == 0 ? "all" : std::to_string(max_samples)) << " per language\n\n"; + + eddy::nemotron::ModelPaths paths{ + .preprocessor = (fs::path(model_dir) / "nemotron_preprocessor.xml").string(), + .encoder = (fs::path(model_dir) / "nemotron_encoder.xml").string(), + .decoder = (fs::path(model_dir) / "nemotron_decoder.xml").string(), + .joint = (fs::path(model_dir) / "nemotron_joint.xml").string(), + .vocab_json = (fs::path(model_dir) / "nemotron_vocab.json").string(), + .metadata_json = (fs::path(model_dir) / "metadata.json").string(), + }; + + eddy::OpenVINOOptions ov_opts; + ov_opts.device = device; + ov_opts.cache_dir = eddy::get_model_dir("nemotron-streaming-int8").string(); + auto backend = std::make_shared(ov_opts); + + std::vector results; + for (const auto& lang : langs) { + std::string tag = fleurs_to_nemotron_lang(lang); + std::cout << "Processing " << lang << " (prompt lang=" << tag << ")...\n"; + + auto samples = load_samples(cache_dir, lang, max_samples); + if (samples.empty()) { std::cerr << " no samples; skipping\n\n"; continue; } + std::cout << " Loaded " << samples.size() << " samples\n"; + + eddy::nemotron::Config cfg; + cfg.device = device; + cfg.language = tag; + + std::shared_ptr model; + try { + model = std::make_shared(backend, paths, cfg); + std::cout << " Compiling + warming up (" << device << ") ... "; + model->warmup(); + std::cout << "[OK]\n"; + } catch (const std::exception& e) { + std::cerr << " [ERROR] model init failed: " << e.what() << "\n\n"; + continue; + } + + LangResult lr; + lr.lang = lang; + lr.name = LANG_NAMES.count(lang) ? LANG_NAMES.at(lang) : lang; + double sum_wer = 0, sum_cer = 0; + + for (const auto& s : samples) { + try { + auto pcm = eddy::audio::read_wav(s.audio_path); + double audio_sec = pcm.size() / 16000.0; + auto t0 = std::chrono::high_resolution_clock::now(); + auto res = model->transcribe(pcm); + auto t1 = std::chrono::high_resolution_clock::now(); + double proc_sec = std::chrono::duration_cast(t1 - t0).count() / 1000.0; + + if (!s.transcription.empty()) { + double w = wer(s.transcription, res.text); + double c = cer(s.transcription, res.text); + sum_wer += w; sum_cer += c; + if (debug) { + std::cout << " [" << s.id << "] WER=" << std::fixed << std::setprecision(1) << (w * 100) << "%\n" + << " hyp: " << res.text << "\n ref: " << s.transcription << "\n"; + } + } + lr.total_audio += audio_sec; + lr.total_proc += proc_sec; + lr.processed++; + } catch (const std::exception& e) { + std::cerr << " Warning: " << s.id << ": " << e.what() << "\n"; + lr.skipped++; + } + } + + if (lr.processed > 0) { + lr.wer = sum_wer / lr.processed; + lr.cer = sum_cer / lr.processed; + lr.rtfx = lr.total_proc > 0 ? lr.total_audio / lr.total_proc : 0; + } + results.push_back(lr); + std::cout << " " << lang << ": WER=" << std::fixed << std::setprecision(2) << (lr.wer * 100) + << "% CER=" << (lr.cer * 100) << "% RTFx=" << std::setprecision(2) << lr.rtfx + << "x (" << lr.processed << " processed, " << lr.skipped << " skipped)\n\n"; + } + + // JSON output + { + std::ofstream f(output); + f << "{\n \"benchmark\": \"FLEURS Nemotron streaming\",\n \"device\": \"" << device + << "\",\n \"model_dir\": \""; + for (char c : model_dir) { if (c == '\\') f << "\\\\"; else f << c; } + f << "\",\n \"results\": [\n"; + for (size_t i = 0; i < results.size(); ++i) { + const auto& r = results[i]; + f << " {\"language\": \"" << r.lang << "\", \"wer\": " << (r.wer * 100) + << ", \"cer\": " << (r.cer * 100) << ", \"rtfx\": " << r.rtfx + << ", \"processed\": " << r.processed << ", \"skipped\": " << r.skipped + << ", \"audioSec\": " << r.total_audio << ", \"procSec\": " << r.total_proc << "}"; + f << (i + 1 < results.size() ? ",\n" : "\n"); + } + f << " ]\n}\n"; + } + + std::cout << std::string(72, '=') << "\nSUMMARY (device=" << device << ")\n" << std::string(72, '=') << "\n"; + double tot_audio = 0, tot_proc = 0; + for (const auto& r : results) { + std::cout << std::left << std::setw(18) << r.name << " | WER=" << std::right << std::fixed + << std::setprecision(2) << std::setw(6) << (r.wer * 100) << "% CER=" << std::setw(6) + << (r.cer * 100) << "% RTFx=" << std::setw(5) << std::setprecision(2) << r.rtfx + << "x n=" << r.processed << "\n"; + tot_audio += r.total_audio; tot_proc += r.total_proc; + } + std::cout << std::string(72, '-') << "\nAudio-weighted RTFx: " << std::fixed << std::setprecision(2) + << (tot_proc > 0 ? tot_audio / tot_proc : 0) << "x (total audio " + << std::setprecision(0) << tot_audio << "s / proc " << tot_proc << "s)\n"; + std::cout << "Results saved to: " << output << "\n"; + return 0; +} diff --git a/include/eddy/core/model_configs.hpp b/include/eddy/core/model_configs.hpp index 7117715..0860453 100644 --- a/include/eddy/core/model_configs.hpp +++ b/include/eddy/core/model_configs.hpp @@ -47,12 +47,13 @@ namespace model_configs { // NVIDIA Nemotron-3.5-ASR-Streaming-Multilingual 0.6B (cache-aware // streaming FastConformer-RNNT, prompt-conditioned multilingual). // Distinct file set + metadata.json (cache shapes, prompt_dictionary, - // lang_tag_token_ids) consumed by the eddy::nemotron backend. + // lang_tag_token_ids) consumed by the eddy::nemotron backend. The mel + // preprocessor is computed natively in C++ (eddy::nemotron::MelFeaturizer), + // so nemotron_preprocessor.xml/.bin are intentionally NOT required. inline const std::vector NEMOTRON_FILES = { "nemotron_encoder.xml", "nemotron_encoder.bin", "nemotron_decoder.xml", "nemotron_decoder.bin", "nemotron_joint.xml", "nemotron_joint.bin", - "nemotron_preprocessor.xml", "nemotron_preprocessor.bin", "nemotron_vocab.json", "metadata.json" }; diff --git a/include/eddy/models/nemotron/nemotron_featurizer.hpp b/include/eddy/models/nemotron/nemotron_featurizer.hpp new file mode 100644 index 0000000..2fef11a --- /dev/null +++ b/include/eddy/models/nemotron/nemotron_featurizer.hpp @@ -0,0 +1,59 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 +// +// Native C++ log-mel featurizer for the Nemotron streaming model — a drop-in +// replacement for the `nemotron_preprocessor.xml` OpenVINO IR. +// +// It reproduces NeMo's AudioToMelSpectrogramPreprocessor exactly (verified +// against the IR to fp16-storage precision): preemphasis 0.97 -> center pad +// n_fft/2 zeros -> framed STFT (Hann(win_length) centred in n_fft, hop) -> +// power spectrum -> slaney mel filterbank -> log(x + guard). The IR carries no +// per-feature normalisation; frames past the valid audio length are zeroed +// (matching the IR's length mask). + +#pragma once + +#include +#include + +namespace eddy::nemotron { + +class MelFeaturizer { + public: + // Nemotron defaults: 16 kHz, 128 mels, 25 ms Hann window (400 samples), + // 10 ms hop (160), 512-pt FFT, preemphasis 0.97, log guard 2^-24. + explicit MelFeaturizer(int sample_rate = 16000, int n_mels = 128); + + // Compute log-mel for `n` samples of 16 kHz mono float PCM. `valid_samples` + // is the number of non-padding samples (frames whose index >= valid_samples/ + // hop are zeroed, mirroring the OV preprocessor). Fills `out_mel` with + // [n_mels * frames] in bin-major layout (out_mel[bin*frames + t]) and sets + // `out_frames`. + void compute(const float* audio, std::size_t n, int valid_samples, + std::vector& out_mel, std::size_t& out_frames) const; + + int n_mels() const { return n_mels_; } + int sample_rate() const { return sample_rate_; } + + private: + int sample_rate_; + int n_mels_; + int n_fft_; + int hop_; + int win_length_; + int n_freq_; // n_fft_/2 + 1 + float preemph_; + float log_guard_; + + std::vector window_; // [n_fft_]: Hann(win_length_) centred, 0 elsewhere + std::vector mel_fb_; // [n_mels_ * n_freq_], row-major (slaney) + + // Radix-2 FFT precomputed tables (size n_fft_). + std::vector bitrev_; + std::vector tw_cos_; // [n_fft_/2] + std::vector tw_sin_; // [n_fft_/2] + + void fft(std::vector& re, std::vector& im) const; +}; + +} // namespace eddy::nemotron diff --git a/src/models/nemotron/nemotron_featurizer.cpp b/src/models/nemotron/nemotron_featurizer.cpp new file mode 100644 index 0000000..fd8aad6 --- /dev/null +++ b/src/models/nemotron/nemotron_featurizer.cpp @@ -0,0 +1,193 @@ +// Copyright (C) 2025 Eddy SDK +// SPDX-License-Identifier: Apache-2.0 + +#include "eddy/models/nemotron/nemotron_featurizer.hpp" + +#define _USE_MATH_DEFINES +#include + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +#include +#include +#include + +namespace eddy::nemotron { + +namespace { + +// librosa slaney hz<->mel (htk=False). +inline double hz_to_mel(double hz) { + const double f_sp = 200.0 / 3.0; + const double min_log_hz = 1000.0; + const double min_log_mel = min_log_hz / f_sp; + const double logstep = std::log(6.4) / 27.0; + if (hz < min_log_hz) return hz / f_sp; + return min_log_mel + std::log(hz / min_log_hz) / logstep; +} + +inline double mel_to_hz(double mel) { + const double f_sp = 200.0 / 3.0; + const double min_log_hz = 1000.0; + const double min_log_mel = min_log_hz / f_sp; + const double logstep = std::log(6.4) / 27.0; + if (mel < min_log_mel) return f_sp * mel; + return min_log_hz * std::exp(logstep * (mel - min_log_mel)); +} + +} // namespace + +MelFeaturizer::MelFeaturizer(int sample_rate, int n_mels) + : sample_rate_(sample_rate), + n_mels_(n_mels), + n_fft_(512), + hop_(static_cast(sample_rate * 0.01 + 0.5)), // 10 ms -> 160 + win_length_(static_cast(sample_rate * 0.025 + 0.5)), // 25 ms -> 400 + n_freq_(512 / 2 + 1), + preemph_(0.97f), + log_guard_(6e-8f) { + // The featurizer is calibrated for NeMo's 16 kHz Nemotron config (25 ms / + // 10 ms framing -> win 400 / hop 160, 512-pt FFT). A 512 FFT only fits the + // window if win_length <= n_fft; guard so an unexpected sample rate fails + // loudly instead of silently producing garbage mel. + if (win_length_ > n_fft_) { + throw std::runtime_error( + "MelFeaturizer: window length " + std::to_string(win_length_) + + " exceeds n_fft " + std::to_string(n_fft_) + + " (sample_rate " + std::to_string(sample_rate_) + + " unsupported by the 512-pt featurizer)."); + } + + // Hann window (periodic=False) of win_length_, centred in the n_fft_ frame + // (torch pads (n_fft - win_length)/2 on the left). Zeros elsewhere. + window_.assign(n_fft_, 0.0f); + const int off = (n_fft_ - win_length_) / 2; + for (int i = 0; i < win_length_; ++i) { + const double w = 0.5 - 0.5 * std::cos(2.0 * M_PI * i / (win_length_ - 1)); + window_[off + i] = static_cast(w); + } + + // Slaney mel filterbank [n_mels_, n_freq_], norm='slaney', fmin=0, fmax=sr/2. + const double fmin = 0.0; + const double fmax = sample_rate_ / 2.0; + std::vector f_pts(n_mels_ + 2); + { + const double mmin = hz_to_mel(fmin); + const double mmax = hz_to_mel(fmax); + for (int i = 0; i < n_mels_ + 2; ++i) { + const double mel = mmin + (mmax - mmin) * i / (n_mels_ + 1); + f_pts[i] = mel_to_hz(mel); + } + } + std::vector fft_freqs(n_freq_); + for (int k = 0; k < n_freq_; ++k) { + fft_freqs[k] = (sample_rate_ / 2.0) * k / (n_freq_ - 1); + } + mel_fb_.assign(static_cast(n_mels_) * n_freq_, 0.0f); + for (int i = 0; i < n_mels_; ++i) { + const double lo = f_pts[i], ce = f_pts[i + 1], hi = f_pts[i + 2]; + const double enorm = 2.0 / (hi - lo); // slaney normalization + for (int k = 0; k < n_freq_; ++k) { + const double left = (fft_freqs[k] - lo) / (ce - lo); + const double right = (hi - fft_freqs[k]) / (hi - ce); + double v = std::min(left, right); + if (v < 0.0) v = 0.0; + mel_fb_[static_cast(i) * n_freq_ + k] = static_cast(v * enorm); + } + } + + // Radix-2 FFT tables: bit-reversal permutation + twiddle factors. + bitrev_.resize(n_fft_); + int log2n = 0; + while ((1 << log2n) < n_fft_) ++log2n; + for (int i = 0; i < n_fft_; ++i) { + int r = 0; + for (int b = 0; b < log2n; ++b) + if (i & (1 << b)) r |= 1 << (log2n - 1 - b); + bitrev_[i] = r; + } + tw_cos_.resize(n_fft_ / 2); + tw_sin_.resize(n_fft_ / 2); + for (int i = 0; i < n_fft_ / 2; ++i) { + const double ang = -2.0 * M_PI * i / n_fft_; + tw_cos_[i] = static_cast(std::cos(ang)); + tw_sin_[i] = static_cast(std::sin(ang)); + } +} + +// In-place iterative radix-2 Cooley-Tukey FFT, size n_fft_. +void MelFeaturizer::fft(std::vector& re, std::vector& im) const { + const int n = n_fft_; + for (int i = 0; i < n; ++i) { + const int j = bitrev_[i]; + if (j > i) { + std::swap(re[i], re[j]); + std::swap(im[i], im[j]); + } + } + for (int len = 2; len <= n; len <<= 1) { + const int half = len >> 1; + const int step = n / len; // twiddle stride + for (int base = 0; base < n; base += len) { + for (int k = 0; k < half; ++k) { + const float wc = tw_cos_[k * step]; + const float ws = tw_sin_[k * step]; + const int a = base + k; + const int b = base + k + half; + const float br = re[b] * wc - im[b] * ws; + const float bi = re[b] * ws + im[b] * wc; + re[b] = re[a] - br; + im[b] = im[a] - bi; + re[a] += br; + im[a] += bi; + } + } + } +} + +void MelFeaturizer::compute(const float* audio, std::size_t n, int valid_samples, + std::vector& out_mel, std::size_t& out_frames) const { + // Preemphasis: y[0]=x[0]; y[i]=x[i]-0.97*x[i-1]. + std::vector y(n); + if (n > 0) y[0] = audio[0]; + for (std::size_t i = 1; i < n; ++i) y[i] = audio[i] - preemph_ * audio[i - 1]; + + // Center pad n_fft/2 zeros each side, then frame with hop. The padded length + // is n + n_fft; #frames = 1 + (padded - n_fft)/hop = 1 + n/hop. + const int pad = n_fft_ / 2; + const std::size_t frames = 1 + n / static_cast(hop_); + out_frames = frames; + out_mel.assign(static_cast(n_mels_) * frames, 0.0f); + + const std::size_t valid_frames = + static_cast(valid_samples) / static_cast(hop_); + + std::vector re(n_fft_), im(n_fft_), power(n_freq_); + for (std::size_t f = 0; f < frames; ++f) { + if (f >= valid_frames) continue; // length mask: zero (already zeroed) + + // Frame covers padded[f*hop : f*hop+n_fft]; padded index j maps to y[j-pad]. + const long start = static_cast(f * static_cast(hop_)) - pad; + for (int t = 0; t < n_fft_; ++t) { + const long src = start + t; + const float s = (src >= 0 && src < static_cast(n)) ? y[static_cast(src)] : 0.0f; + re[t] = s * window_[t]; + im[t] = 0.0f; + } + + fft(re, im); + for (int k = 0; k < n_freq_; ++k) power[k] = re[k] * re[k] + im[k] * im[k]; + + // mel = mel_fb (n_mels x n_freq) @ power; then log(mel + guard). + for (int mbin = 0; mbin < n_mels_; ++mbin) { + const float* row = &mel_fb_[static_cast(mbin) * n_freq_]; + float acc = 0.0f; + for (int k = 0; k < n_freq_; ++k) acc += row[k] * power[k]; + out_mel[static_cast(mbin) * frames + f] = std::log(acc + log_guard_); + } + } +} + +} // namespace eddy::nemotron diff --git a/src/models/nemotron/nemotron_openvino.cpp b/src/models/nemotron/nemotron_openvino.cpp index db8ef9a..193f91c 100644 --- a/src/models/nemotron/nemotron_openvino.cpp +++ b/src/models/nemotron/nemotron_openvino.cpp @@ -11,8 +11,14 @@ #include "eddy/models/nemotron/nemotron.hpp" #include "eddy/backends/openvino_backend.hpp" +#include "eddy/models/nemotron/nemotron_featurizer.hpp" + +#include #include +#include +#include +#include #include #include @@ -40,6 +46,30 @@ struct MelBuf { size_t frames = 0; }; +// The OpenVINO NPU plugin miscompiles BitwiseNot on a boolean tensor: it does +// an integer bitwise complement, so ~0 = -1 and ~1 = -2 are *both* nonzero +// ("true"). The FastConformer attention mask is built with a `~` over a bool, +// so on NPU the mask becomes all-true -> every key is masked -> uniform softmax +// -> the encoder output collapses to ~0 and every transcript is empty. Replace +// BitwiseNot(bool) with the semantically identical LogicalNot, which the NPU +// compiles correctly; it is a no-op on CPU/GPU. Returns the (possibly rewritten) +// model ready to compile. +std::shared_ptr load_npu_safe(ov::Core& core, const std::string& xml) { + auto model = core.read_model(xml); + bool changed = false; + for (const auto& node : model->get_ordered_ops()) { + if (ov::as_type_ptr(node)) { + auto repl = std::make_shared(node->input_value(0)); + repl->set_friendly_name(node->get_friendly_name()); + ov::copy_runtime_info(node, repl); + ov::replace_node(node, repl); + changed = true; + } + } + if (changed) model->validate_nodes_and_infer_types(); + return model; +} + ov::Tensor make_i32(int value) { ov::Tensor t(ov::element::i32, ov::Shape{1}); t.data()[0] = value; @@ -81,8 +111,11 @@ struct OpenVINONemotron::Impl { std::vector vocab; // id -> piece (raw, ▁-marked) - ov::CompiledModel preproc, encoder, decoder, joint; - ov::InferRequest preproc_req, encoder_req, decoder_req, joint_req; + ov::CompiledModel encoder, decoder, joint; + ov::InferRequest encoder_req, decoder_req, joint_req; + + // Native C++ log-mel featurizer replacing the nemotron_preprocessor.xml IR. + std::unique_ptr featurizer; // Metadata int sample_rate = 16000; @@ -203,17 +236,41 @@ void OpenVINONemotron::ensure_compiled() const { } } - // --- Compile models (preprocessor on CPU; rest on chosen device) --- - std::string preproc_device = "CPU"; - if (const char* env = std::getenv("EDDY_PREPROC_DEVICE")) { - if (*env) preproc_device = env; + // --- Mel featurizer (native C++, replaces nemotron_preprocessor.xml) --- + // The IR preprocessor is dynamic-shaped (NPU-incompatible) and adds an + // OV inference per chunk; the native featurizer reproduces it exactly + // (validated to fp16-storage precision). paths.preprocessor is unused. + impl_->featurizer = std::make_unique(impl_->sample_rate, impl_->mel_features); + // Guard: the featurizer's framing must match the model's expected geometry. + // A full chunk (chunk_mel_frames * sample_rate/100 samples) must yield + // chunk_mel_frames + 1 mel frames; otherwise metadata (sample_rate / + // chunk_mel_frames) disagrees with the hardcoded 10 ms hop and the mel would + // silently misalign with the encoder. + { + const size_t cs = impl_->chunk_samples(); + std::vector probe(cs, 0.0f); + std::vector mel_probe; + size_t probe_frames = 0; + impl_->featurizer->compute(probe.data(), cs, static_cast(cs), mel_probe, probe_frames); + const size_t expected = static_cast(impl_->chunk_mel_frames) + 1; + if (probe_frames != expected) { + throw std::runtime_error( + "Nemotron C++ featurizer geometry mismatch: produced " + + std::to_string(probe_frames) + " frames for a chunk, expected " + + std::to_string(expected) + " (chunk_mel_frames=" + + std::to_string(impl_->chunk_mel_frames) + ", sample_rate=" + + std::to_string(impl_->sample_rate) + "). The model's featurizer " + "config differs from the hardcoded 25 ms/10 ms framing."); + } } - impl_->preproc = core.compile_model(impl_->paths.preprocessor, preproc_device); - impl_->encoder = core.compile_model(impl_->paths.encoder, device); + + // --- Compile models on the chosen device --- + // The encoder carries the attention-mask BitwiseNot that the NPU plugin + // miscompiles, so load it through the rewrite (no-op on CPU/GPU). + impl_->encoder = core.compile_model(load_npu_safe(core, impl_->paths.encoder), device); impl_->decoder = core.compile_model(impl_->paths.decoder, device); impl_->joint = core.compile_model(impl_->paths.joint, device); - impl_->preproc_req = impl_->preproc.create_infer_request(); impl_->encoder_req = impl_->encoder.create_infer_request(); impl_->decoder_req = impl_->decoder.create_infer_request(); impl_->joint_req = impl_->joint.create_infer_request(); @@ -273,6 +330,7 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) std::vector all_tokens; MelBuf mel_cache; // last pre_encode_cache frames of previous chunk's mel + std::vector mel_scratch; // per-chunk featurizer output [bins * t_mel] const ov::Tensor prompt_tensor = make_i32(prompt_id); @@ -284,13 +342,6 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) ov::Tensor token(I.token_et, ov::Shape{1, 1}); const ov::Tensor token_length = make_i32(1); const ov::Tensor mel_length = make_i32(static_cast(total)); - // audio_length is intentionally the FULL padded chunk size on every chunk, - // including the final short chunk. This mirrors the WER-validated Python - // reference (transcribe_ov.py: the chunk is np.pad'd to chunk_samples and - // chunk.shape[1] == chunk_samples is passed). The preprocessor IR does not gate - // mel output on this value, so passing the real (end-off) count would diverge - // from the validated path without changing output. - const ov::Tensor audio_length = make_i32(static_cast(chunk_samples)); size_t off = 0; while (off < pcm.size()) { @@ -301,14 +352,15 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) std::memset(adst, 0, audio.get_byte_size()); std::copy(pcm.begin() + static_cast(off), pcm.begin() + static_cast(end), adst); - // Preprocessor: audio -> mel [1, bins, T_mel] - I.preproc_req.set_tensor("audio", audio); - I.preproc_req.set_tensor("audio_length", audio_length); - I.preproc_req.infer(); - const ov::Tensor mel_out = I.preproc_req.get_tensor("mel"); - const ov::Shape mel_shape = mel_out.get_shape(); // [1, bins, T_mel] - const size_t t_mel = mel_shape[2]; - const float* mel_src = mel_out.data(); + // Preprocessor: audio -> mel [bins, T_mel] (native C++ featurizer). + // audio_length is intentionally the FULL padded chunk size on every chunk + // (mirrors the WER-validated reference: the chunk is zero-padded to + // chunk_samples and that full length is passed), so the length mask zeros + // only the trailing frame, which the assembly below trims anyway. + size_t t_mel = 0; + I.featurizer->compute(adst, chunk_samples, static_cast(chunk_samples), + mel_scratch, t_mel); + const float* mel_src = mel_scratch.data(); // bin-major [bins * t_mel] // Build encoder mel input [1, bins, total]: prepend cache (or zero // pre_encode_cache on first chunk), then pad/trim to total. (reusing the From 4fdd486d18d79f5678070437878dc837fa6dca0e Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:28:54 -0400 Subject: [PATCH 08/13] benchmark_nemotron_fleurs: FluidAudio-matched scoring (CJK char-level + Unicode punct strip) Match FluidAudio's NemotronMultilingualFleursBenchmark / WERCalculator: route CJK languages (ja/ko/zh/cmn/yue/th/lo) through character-level CER (whitespace WER is meaningless without word segmentation; report WER=CER), and strip Unicode Mark/Symbol/Punctuation (approx category M/S/P over the blocks that occur in FLEURS) to a space in normalization instead of keeping non-ASCII punctuation. Brings CJK CER onto the published reference: ja 19.0 -> 15.46 (ref 15.15), zh 21.78 -> 20.18 (ref 20.94). en/es/fr unaffected (ASCII) or marginally improved (European punctuation now stripped). Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/cpp/benchmark_nemotron_fleurs.cpp | 64 +++++++++++++++++----- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/examples/cpp/benchmark_nemotron_fleurs.cpp b/examples/cpp/benchmark_nemotron_fleurs.cpp index 9074300..97c6437 100644 --- a/examples/cpp/benchmark_nemotron_fleurs.cpp +++ b/examples/cpp/benchmark_nemotron_fleurs.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -75,26 +76,52 @@ std::vector utf8_chars(const std::string& s) { return cps; } -// Normalize: lowercase ASCII alphanumerics, collapse other ASCII to single -// spaces, and KEEP non-ASCII codepoints verbatim (so CJK/accented characters -// survive for CER). UTF-8 aware, unlike the byte-wise version in -// benchmark_fleurs.cpp which would strip all CJK. +// Decode a 1-4 byte UTF-8 codepoint string to its Unicode scalar value. +uint32_t cp_scalar(const std::string& cp) { + unsigned char c0 = static_cast(cp[0]); + if (cp.size() == 1) return c0; + if (cp.size() == 2) return ((c0 & 0x1F) << 6) | (static_cast(cp[1]) & 0x3F); + if (cp.size() == 3) + return ((c0 & 0x0F) << 12) | ((static_cast(cp[1]) & 0x3F) << 6) | + (static_cast(cp[2]) & 0x3F); + return ((c0 & 0x07) << 18) | ((static_cast(cp[1]) & 0x3F) << 12) | + ((static_cast(cp[2]) & 0x3F) << 6) | (static_cast(cp[3]) & 0x3F); +} + +// Approximate the Whisper/FluidAudio normalizer's "replace every Mark/Symbol/ +// Punctuation (Unicode category M/S/P) with a space" step over the codepoint +// blocks that actually occur in FLEURS refs/hyps. Keeps letters (incl. CJK, +// kana, hangul, accented Latin, Greek, Cyrillic) and digits. +bool is_punct_or_symbol(uint32_t c) { + return (c >= 0x00A1 && c <= 0x00BF) || c == 0x00D7 || c == 0x00F7 || // Latin-1 punct/symbols, × ÷ + (c >= 0x2000 && c <= 0x206F) || // general punctuation – — ' ' " " … + (c >= 0x2070 && c <= 0x20CF) || // super/subscripts, currency symbols + (c >= 0x2100 && c <= 0x2BFF) || // letterlike/number forms, arrows, math, misc symbols + (c >= 0x3000 && c <= 0x303F) || // CJK symbols and punctuation 。、「」（） + (c >= 0xFF01 && c <= 0xFF0F) || // fullwidth ！＂＃…／ + (c >= 0xFF1A && c <= 0xFF20) || // fullwidth ：；＜＝＞？＠ + (c >= 0xFF3B && c <= 0xFF40) || // fullwidth ［＼］＾＿｀ + (c >= 0xFF5B && c <= 0xFF65); // fullwidth ｛｜｝、。etc +} + +// Normalize ~ FluidAudio's basicNormalize: lowercase ASCII, replace Unicode +// punctuation/symbols (M/S/P) with single spaces, keep letters/digits and +// diacritics/CJK, collapse whitespace. (Whisper's English number-word folding +// is intentionally omitted — "similar enough" per the multilingual path.) std::string normalize_text(const std::string& text) { std::string result; result.reserve(text.size()); bool last_space = false; + auto sep = [&]() { if (!last_space && !result.empty()) { result += ' '; last_space = true; } }; for (const std::string& cp : utf8_chars(text)) { if (cp.size() == 1) { unsigned char c = static_cast(cp[0]); - if (std::isalnum(c)) { - result += static_cast(std::tolower(c)); - last_space = false; - } else if (!last_space && !result.empty()) { - result += ' '; - last_space = true; - } + if (std::isalnum(c)) { result += static_cast(std::tolower(c)); last_space = false; } + else sep(); + } else if (is_punct_or_symbol(cp_scalar(cp))) { + sep(); } else { - result += cp; // non-ASCII (CJK etc.) kept as-is + result += cp; // letter / CJK / diacritic — keep last_space = false; } } @@ -102,6 +129,14 @@ std::string normalize_text(const std::string& text) { return result; } +// CJK / no-space scripts: word-level WER over whitespace tokens is meaningless, +// so FluidAudio routes these through character-level scoring (matches Whisper / +// ESPnet). FLEURS code prefixes. +bool is_cjk_lang(const std::string& code) { + auto p = [&](const char* s) { return code.rfind(s, 0) == 0; }; + return p("ja") || p("ko") || p("zh") || p("cmn") || p("yue") || p("th") || p("lo"); +} + int levenshtein(const std::vector& a, const std::vector& b) { const size_t m = a.size(), n = b.size(); std::vector prev(n + 1), cur(n + 1); @@ -266,6 +301,7 @@ int main(int argc, char* argv[]) { LangResult lr; lr.lang = lang; lr.name = LANG_NAMES.count(lang) ? LANG_NAMES.at(lang) : lang; + const bool cjk = is_cjk_lang(lang); double sum_wer = 0, sum_cer = 0; for (const auto& s : samples) { @@ -278,8 +314,10 @@ int main(int argc, char* argv[]) { double proc_sec = std::chrono::duration_cast(t1 - t0).count() / 1000.0; if (!s.transcription.empty()) { - double w = wer(s.transcription, res.text); + // CJK: character-level rate reported in both WER and CER + // (FluidAudio convention — whitespace WER is meaningless). double c = cer(s.transcription, res.text); + double w = cjk ? c : wer(s.transcription, res.text); sum_wer += w; sum_cer += c; if (debug) { std::cout << " [" << s.id << "] WER=" << std::fixed << std::setprecision(1) << (w * 100) << "%\n" From 1045024e43b04bf14c5fb9d43ef35354bbb18417 Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:33:01 -0400 Subject: [PATCH 09/13] docs(benchmarks): add Nemotron streaming multilingual NPU results (FLEURS) 5-language FLEURS results (en/es/fr WER, zh/ja CER) for nemotron-streaming-int8 on Intel NPU at ~22-24x RTFx, scored with the FluidAudio methodology. Matches the OpenVINO FP32 reference and beats the FluidAudio CoreML reference on es/fr/zh/ja. Documents the NPU BitwiseNot->LogicalNot enablement fix and the native C++ mel preprocessor. Co-Authored-By: Claude Opus 4.8 (1M context) --- BENCHMARK_RESULTS.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/BENCHMARK_RESULTS.md b/BENCHMARK_RESULTS.md index 3f02d5e..1d7db65 100644 --- a/BENCHMARK_RESULTS.md +++ b/BENCHMARK_RESULTS.md @@ -97,6 +97,36 @@ Comprehensive benchmark results for eddy ASR on LibriSpeech test-clean and FLEUR --- +## Nemotron Streaming Multilingual 0.6B (FLEURS) + +**Model**: `nemotron-streaming-int8` (weight-only INT8 encoder, FP16 decoder/joint) +**Device**: Intel NPU · **Software**: OpenVINO 2025.0 · **Decoding**: greedy, forced language +**Preprocessor**: native C++ log-mel featurizer (replaces the dynamic-shape OV preprocessor) +**Scoring**: FluidAudio methodology — WER for spaced languages, character-level CER for CJK +(`ja`/`zh`), Whisper-style punctuation/symbol stripping + +| Language | Metric | eddy NPU (int8) | OpenVINO FP32 ref | FluidAudio CoreML ref | RTFx | Samples | +|----------|--------|----------------:|------------------:|----------------------:|-----:|--------:| +| English (US) | WER | 12.48 | 11.78 | 12.09 | 22.3× | 350 | +| Spanish (LatAm) | WER | 7.03 | 6.99 | 9.01 | 22.5× | 350 | +| French (France) | WER | 13.35 | 12.92 | 15.18 | 21.7× | 350 | +| Chinese (Mandarin) | CER | 20.18 | 21.05 | 24.54 | 23.7× | 945 | +| Japanese | CER | 15.46 | 15.12 | 16.86 | 23.3× | 650 | + +**Audio-weighted RTFx**: ~22–24× on Intel NPU (≈2× the CPU figure of ~11×). + +**Notes**: +- INT8-on-NPU accuracy matches the OpenVINO FP32 reference within noise and beats the + FluidAudio CoreML reference on es/fr/zh/ja. +- **NPU enablement**: the OpenVINO NPU plugin miscompiles `BitwiseNot` on a boolean + (integer complement → mask all-true → encoder collapses to ~0 → empty transcripts). + eddy rewrites `BitwiseNot → LogicalNot` in the encoder IR before compiling (no-op on + CPU/GPU); without it the NPU produces empty output for this model. +- Unlike Parakeet (overlapping-chunk + 2D dedup), Nemotron is cache-aware streaming RNNT + with per-chunk `prompt_id` language conditioning. + +--- + ## Performance Notes ### Best Performing Languages (WER < 10%) From 31428c80bc23bdebe049cf4c475f40e890b8c750 Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:48:58 -0400 Subject: [PATCH 10/13] nemotron: support both multilingual and English speech-streaming variants Auto-detect prompt conditioning from the encoder's input ports: the multilingual encoder has a prompt_id input, the English nemotron-speech- streaming encoder does not. When absent, skip prompt_id resolution and don't feed the tensor. One eddy::nemotron backend now serves both models; everything else (C++ mel featurizer, NPU BitwiseNot fix, cache-aware decode loop) is shared. Adds nemotron-speech-streaming{,-int8} model configs (English, FluidInference/ nemotron-speech-streaming-en-0.6b-ov). Multilingual path regression-checked (en_us NPU unchanged at WER 12.2 on the smoke set). Co-Authored-By: Claude Opus 4.8 (1M context) --- include/eddy/core/model_configs.hpp | 24 ++++++++++++++++++++++- src/models/nemotron/nemotron_openvino.cpp | 21 +++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/include/eddy/core/model_configs.hpp b/include/eddy/core/model_configs.hpp index 0860453..9d29d99 100644 --- a/include/eddy/core/model_configs.hpp +++ b/include/eddy/core/model_configs.hpp @@ -79,12 +79,34 @@ namespace model_configs { .repo_subdir = "int8" }; + // NVIDIA nemotron-speech-streaming-en-0.6b: the monolingual (English) sibling + // of the multilingual model. Same FastConformer cache-aware RNNT, but no + // prompt/language conditioning (the eddy backend auto-detects the absent + // encoder prompt_id input). Same flat file set as NEMOTRON_FILES. + // NOTE: the OpenVINO IR repo is produced by the mobius export pipeline; until + // it is published this config resolves but downloads will 404. + inline const ModelConfig NEMOTRON_SPEECH = { + .repo_id = "FluidInference/nemotron-speech-streaming-en-0.6b-ov", + .required_files = NEMOTRON_FILES, + .cache_subdir = "nemotron-speech-streaming", + .repo_subdir = "fp16" + }; + + inline const ModelConfig NEMOTRON_SPEECH_INT8 = { + .repo_id = "FluidInference/nemotron-speech-streaming-en-0.6b-ov", + .required_files = NEMOTRON_FILES, + .cache_subdir = "nemotron-speech-streaming-int8", + .repo_subdir = "int8" + }; + // Model name lookup map inline const std::map MODEL_MAP = { {"parakeet-v2", PARAKEET_V2}, {"parakeet-v3", PARAKEET_V3}, {"nemotron-streaming", NEMOTRON_STREAMING}, - {"nemotron-streaming-int8", NEMOTRON_STREAMING_INT8} + {"nemotron-streaming-int8", NEMOTRON_STREAMING_INT8}, + {"nemotron-speech-streaming", NEMOTRON_SPEECH}, + {"nemotron-speech-streaming-int8", NEMOTRON_SPEECH_INT8} }; // Default model diff --git a/src/models/nemotron/nemotron_openvino.cpp b/src/models/nemotron/nemotron_openvino.cpp index 193f91c..3cfc3b3 100644 --- a/src/models/nemotron/nemotron_openvino.cpp +++ b/src/models/nemotron/nemotron_openvino.cpp @@ -133,6 +133,12 @@ struct OpenVINONemotron::Impl { std::map prompt_dictionary; std::set lang_tag_token_ids; + // Whether the encoder takes a `prompt_id` input. True for the multilingual + // model (per-chunk language conditioning); false for the monolingual English + // speech-streaming model. Auto-detected from the encoder's input ports so one + // backend serves both variants. + bool has_prompt = false; + ov::element::Type token_et = ov::element::i32; std::once_flag compile_once; @@ -275,6 +281,14 @@ void OpenVINONemotron::ensure_compiled() const { impl_->decoder_req = impl_->decoder.create_infer_request(); impl_->joint_req = impl_->joint.create_infer_request(); + // Detect prompt conditioning from the encoder's input ports: the + // multilingual encoder has a "prompt_id" input; the English speech-streaming + // encoder does not. Drives whether transcribe() feeds a prompt_id tensor. + impl_->has_prompt = false; + for (const auto& in : impl_->encoder.inputs()) { + if (in.get_names().count("prompt_id")) { impl_->has_prompt = true; break; } + } + impl_->token_et = impl_->decoder.input("token").get_element_type(); // --- Vocab (id -> piece). Flat {"0":"piece", ...} format. --- @@ -309,7 +323,8 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) const size_t pre_cache = static_cast(I.pre_encode_cache); const size_t chunk_samples = I.chunk_samples(); - const int prompt_id = resolve_prompt_id(I.config.language); + // prompt_id only applies to the multilingual (prompt-conditioned) encoder. + const int prompt_id = I.has_prompt ? resolve_prompt_id(I.config.language) : 0; // Persistent encoder caches (carried across chunks). ov::Tensor cache_channel(ov::element::f32, I.cache_channel_shape); @@ -332,7 +347,7 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) MelBuf mel_cache; // last pre_encode_cache frames of previous chunk's mel std::vector mel_scratch; // per-chunk featurizer output [bins * t_mel] - const ov::Tensor prompt_tensor = make_i32(prompt_id); + const ov::Tensor prompt_tensor = make_i32(prompt_id); // unused when !has_prompt // Hot-path tensors whose shapes/values are fixed for the whole call — allocate // once and reuse across chunks and the inner RNNT loop instead of re-allocating @@ -405,7 +420,7 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) I.encoder_req.set_tensor("cache_channel", cache_channel); I.encoder_req.set_tensor("cache_time", cache_time); I.encoder_req.set_tensor("cache_len", cache_len); - I.encoder_req.set_tensor("prompt_id", prompt_tensor); + if (I.has_prompt) I.encoder_req.set_tensor("prompt_id", prompt_tensor); I.encoder_req.infer(); const ov::Tensor encoded = I.encoder_req.get_tensor("encoded"); // [1, D, T_enc] From 1b229918d07e6637c2d3b02b6bcd64a5646a75b3 Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:18:09 -0400 Subject: [PATCH 11/13] nemotron-speech configs: reuse the multilingual HF repo (en/ subfolders) Point nemotron-speech-streaming{,-int8} at the existing FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov repo under en/fp16 and en/int8 instead of a separate HF space. Co-Authored-By: Claude Opus 4.8 (1M context) --- include/eddy/core/model_configs.hpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/include/eddy/core/model_configs.hpp b/include/eddy/core/model_configs.hpp index 9d29d99..09eedd7 100644 --- a/include/eddy/core/model_configs.hpp +++ b/include/eddy/core/model_configs.hpp @@ -83,20 +83,19 @@ namespace model_configs { // of the multilingual model. Same FastConformer cache-aware RNNT, but no // prompt/language conditioning (the eddy backend auto-detects the absent // encoder prompt_id input). Same flat file set as NEMOTRON_FILES. - // NOTE: the OpenVINO IR repo is produced by the mobius export pipeline; until - // it is published this config resolves but downloads will 404. + // Shares the multilingual HF repo (no separate space) under "en/" subfolders. inline const ModelConfig NEMOTRON_SPEECH = { - .repo_id = "FluidInference/nemotron-speech-streaming-en-0.6b-ov", + .repo_id = "FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov", .required_files = NEMOTRON_FILES, .cache_subdir = "nemotron-speech-streaming", - .repo_subdir = "fp16" + .repo_subdir = "en/fp16" }; inline const ModelConfig NEMOTRON_SPEECH_INT8 = { - .repo_id = "FluidInference/nemotron-speech-streaming-en-0.6b-ov", + .repo_id = "FluidInference/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b-ov", .required_files = NEMOTRON_FILES, .cache_subdir = "nemotron-speech-streaming-int8", - .repo_subdir = "int8" + .repo_subdir = "en/int8" }; // Model name lookup map From 28f9f083730b460126a59a8a19a76eb03b7eee77 Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:32:46 -0400 Subject: [PATCH 12/13] nemotron: CLI + comments cover both multilingual and English variants nemotron_cli --model help lists nemotron-streaming[-int8] (multilingual) and nemotron-speech-streaming[-int8] (English); banner/header no longer hardcode '3.5 Multilingual'. Backend file header documents that one path serves both variants (prompt auto-detected). Comment-only/help-text changes. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/cpp/nemotron_cli.cpp | 15 ++++++++++----- src/models/nemotron/nemotron_openvino.cpp | 14 ++++++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/examples/cpp/nemotron_cli.cpp b/examples/cpp/nemotron_cli.cpp index 7c88e14..4935964 100644 --- a/examples/cpp/nemotron_cli.cpp +++ b/examples/cpp/nemotron_cli.cpp @@ -1,7 +1,9 @@ // Copyright (C) 2025 Eddy SDK // SPDX-License-Identifier: Apache-2.0 // -// CLI for the NVIDIA Nemotron-3.5-ASR-Streaming-Multilingual 0.6B backend. +// CLI for the NVIDIA Nemotron cache-aware streaming ASR backend: the +// 3.5-ASR-Streaming-Multilingual 0.6B model and the English speech-streaming +// 0.6B model (selected via --model). #include "eddy/backends/openvino_backend.hpp" #include "eddy/core/app_dir.hpp" @@ -18,9 +20,12 @@ void print_usage(const char* prog) { std::cout << "Usage: " << prog << " [options]\n\n"; std::cout << "Options:\n"; std::cout << " --device OpenVINO device (default: CPU). CPU, AUTO, NPU\n"; - std::cout << " --lang Language: en-US, zh-CN, ... or auto (default: auto)\n"; - std::cout << " --model Model variant: nemotron-streaming (FP16, default) or\n"; - std::cout << " nemotron-streaming-int8. Selects the cache dir.\n"; + std::cout << " --lang Language: en-US, zh-CN, ... or auto (default: auto).\n"; + std::cout << " Ignored by the English speech-streaming model.\n"; + std::cout << " --model Model variant (selects the cache dir):\n"; + std::cout << " nemotron-streaming[-int8] multilingual (40+ langs)\n"; + std::cout << " nemotron-speech-streaming[-int8] English, no language prompt\n"; + std::cout << " Default: nemotron-streaming (FP16).\n"; std::cout << " --model-dir Directory with nemotron_*.xml/bin + metadata.json\n"; std::cout << " (overrides --model; default: cache for the --model variant)\n"; std::cout << " --help Show this help\n"; @@ -72,7 +77,7 @@ int main(int argc, char* argv[]) { return 1; } - std::cout << "=== Nemotron 3.5 ASR Streaming CLI ===\n\n"; + std::cout << "=== Nemotron ASR Streaming CLI ===\n\n"; try { auto pcm = eddy::audio::read_wav(audio_file); diff --git a/src/models/nemotron/nemotron_openvino.cpp b/src/models/nemotron/nemotron_openvino.cpp index 3cfc3b3..520392e 100644 --- a/src/models/nemotron/nemotron_openvino.cpp +++ b/src/models/nemotron/nemotron_openvino.cpp @@ -1,12 +1,14 @@ // Copyright (C) 2025 Eddy SDK // SPDX-License-Identifier: Apache-2.0 // -// Cache-aware streaming inference for NVIDIA Nemotron-3.5-ASR-Streaming -// Multilingual 0.6B. Port of the validated Python reference -// (nemotron-ov-export/transcribe_ov.py), which mirrors mobius's CoreML -// streaming loop: chunk raw audio -> preprocessor -> cache-aware encoder -// (+ prompt_id) -> greedy RNNT decode, carrying encoder caches and LSTM -// state across chunks. +// Cache-aware streaming inference for NVIDIA Nemotron FastConformer-RNNT ASR. +// Serves both the 3.5-ASR-Streaming-Multilingual 0.6B model (per-chunk prompt_id +// language conditioning) and the English speech-streaming 0.6B model (no prompt, +// auto-detected from the encoder's inputs). +// +// Pipeline per chunk: native C++ mel featurizer -> cache-aware encoder +// (+ prompt_id when present) -> greedy RNNT decode, carrying the encoder caches +// and decoder LSTM state across chunks. #include "eddy/models/nemotron/nemotron.hpp" From 19e0ec679dc35747dad7a32757227a0a644d8e59 Mon Sep 17 00:00:00 2001 From: Alex-Wengg <36247722+Alex-Wengg@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:47:21 -0400 Subject: [PATCH 13/13] Address code review: C-API result init, download cleanup, NPU rewrite guard - eddy_c: zero-init *out on ALL early/error returns (sample_rate mismatch, malformed-WAV INFERENCE_FAILED), not just FILE_NOT_FOUND, so callers that free after any error don't delete[] uninitialized pointers. - ensure_models: delete partial files on download failure (a truncated file would otherwise pass file_nonempty and be skipped, loading a corrupt model); reject '..' path components (charset allows '.'/'/'). - nemotron encoder: guard the BitwiseNot->LogicalNot rewrite on boolean input type (only valid for bool); verify cache_len_out is i32 for symmetry with the cache_*_out byte-size checks. - featurizer: ptrdiff_t frame indices (long is 32-bit on Win64); clarify the log-guard constant (6e-8, the IR's stored value) and the length-mask vs OV. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../eddy/models/nemotron/nemotron_featurizer.hpp | 3 ++- src/eddy_c.cpp | 15 ++++++++------- src/models/nemotron/nemotron_featurizer.cpp | 11 ++++++++--- src/models/nemotron/nemotron_openvino.cpp | 11 ++++++++--- src/utils/ensure_models.cpp | 13 +++++++++++++ 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/include/eddy/models/nemotron/nemotron_featurizer.hpp b/include/eddy/models/nemotron/nemotron_featurizer.hpp index 2fef11a..0a4333e 100644 --- a/include/eddy/models/nemotron/nemotron_featurizer.hpp +++ b/include/eddy/models/nemotron/nemotron_featurizer.hpp @@ -21,7 +21,8 @@ namespace eddy::nemotron { class MelFeaturizer { public: // Nemotron defaults: 16 kHz, 128 mels, 25 ms Hann window (400 samples), - // 10 ms hop (160), 512-pt FFT, preemphasis 0.97, log guard 2^-24. + // 10 ms hop (160), 512-pt FFT, preemphasis 0.97, log guard 6e-8 (the value + // stored in the exported IR; NeMo's 2^-24 rounds to this at fp16). explicit MelFeaturizer(int sample_rate = 16000, int n_mels = 128); // Compute log-mel for `n` samples of 16 kHz mono float PCM. `valid_samples` diff --git a/src/eddy_c.cpp b/src/eddy_c.cpp index 4ce09b2..092d350 100644 --- a/src/eddy_c.cpp +++ b/src/eddy_c.cpp @@ -609,13 +609,14 @@ EDDY_API EddyError eddy_nemotron_infer_buffer(EddyNemotronModel handle, const fl if (err) *err = copy_string("[Eddy Error] Invalid argument: null pointer"); return EDDY_ERROR_INVALID_ARGUMENT; } + // Zero-init before any other early return so callers that follow the + // "always safe to eddy_nemotron_free_result" contract never delete[] + // uninitialized pointers (and so a throw mid-fill is cleaned up in catch). + *out = EddyNemotronResult{}; if (sample_rate != 16000) { if (err) *err = copy_string("[Eddy Error] Nemotron expects 16kHz mono audio"); return EDDY_ERROR_INVALID_ARGUMENT; } - // Zero-init so a throw mid-fill (e.g. copy_string OOM after text is set) is - // cleaned up by eddy_nemotron_free_result in the catch instead of leaking. - *out = EddyNemotronResult{}; try { auto* h = static_cast(handle); std::vector samples(pcm, pcm + length); @@ -637,14 +638,14 @@ EDDY_API EddyError eddy_nemotron_infer_file(EddyNemotronModel handle, const char if (err) *err = copy_string("[Eddy Error] Invalid argument: null pointer"); return EDDY_ERROR_INVALID_ARGUMENT; } + // Zero-init on every path (FILE_NOT_FOUND, a read_wav throw on a malformed + // file, ...) so a caller that frees *out after any error never delete[]s + // uninitialized pointers. + *out = EddyNemotronResult{}; // Only a genuinely missing file is FILE_NOT_FOUND; read_wav also throws for // format/channel/sample-rate/decode errors, which are not filesystem issues. std::error_code ec; if (!std::filesystem::exists(wav_path, ec)) { - // Zero-init before the early return so a caller that calls - // eddy_nemotron_free_result on *out after FILE_NOT_FOUND does not - // delete[] uninitialized/garbage pointers. - *out = EddyNemotronResult{}; if (err) *err = copy_string("[Eddy Error] WAV file not found: " + std::string(wav_path)); return EDDY_ERROR_FILE_NOT_FOUND; } diff --git a/src/models/nemotron/nemotron_featurizer.cpp b/src/models/nemotron/nemotron_featurizer.cpp index fd8aad6..d894968 100644 --- a/src/models/nemotron/nemotron_featurizer.cpp +++ b/src/models/nemotron/nemotron_featurizer.cpp @@ -161,6 +161,10 @@ void MelFeaturizer::compute(const float* audio, std::size_t n, int valid_samples out_frames = frames; out_mel.assign(static_cast(n_mels_) * frames, 0.0f); + // Frames at index >= valid_samples/hop are zeroed. This intentionally has no + // "+1" (unlike `frames` above): it mirrors the OV preprocessor's length mask, + // whose mel_length = audio_length/hop (verified — for a full chunk it zeroes + // exactly the trailing frame, which the encoder-input assembly then trims). const std::size_t valid_frames = static_cast(valid_samples) / static_cast(hop_); @@ -169,10 +173,11 @@ void MelFeaturizer::compute(const float* audio, std::size_t n, int valid_samples if (f >= valid_frames) continue; // length mask: zero (already zeroed) // Frame covers padded[f*hop : f*hop+n_fft]; padded index j maps to y[j-pad]. - const long start = static_cast(f * static_cast(hop_)) - pad; + // ptrdiff_t (not long, which is 32-bit on Win64) to avoid overflow on long audio. + const std::ptrdiff_t start = static_cast(f * static_cast(hop_)) - pad; for (int t = 0; t < n_fft_; ++t) { - const long src = start + t; - const float s = (src >= 0 && src < static_cast(n)) ? y[static_cast(src)] : 0.0f; + const std::ptrdiff_t src = start + t; + const float s = (src >= 0 && src < static_cast(n)) ? y[static_cast(src)] : 0.0f; re[t] = s * window_[t]; im[t] = 0.0f; } diff --git a/src/models/nemotron/nemotron_openvino.cpp b/src/models/nemotron/nemotron_openvino.cpp index 520392e..4b87c3b 100644 --- a/src/models/nemotron/nemotron_openvino.cpp +++ b/src/models/nemotron/nemotron_openvino.cpp @@ -60,7 +60,11 @@ std::shared_ptr load_npu_safe(ov::Core& core, const std::string& xml) auto model = core.read_model(xml); bool changed = false; for (const auto& node : model->get_ordered_ops()) { - if (ov::as_type_ptr(node)) { + // Only a BitwiseNot over a boolean is equivalent to LogicalNot. Guard on the + // input element type so a future IR with an integer BitwiseNot isn't silently + // miscompiled (LogicalNot would change both semantics and output dtype). + if (ov::as_type_ptr(node) && + node->get_input_element_type(0) == ov::element::boolean) { auto repl = std::make_shared(node->input_value(0)); repl->set_friendly_name(node->get_friendly_name()); ov::copy_runtime_info(node, repl); @@ -437,9 +441,10 @@ TranscriptionResult OpenVINONemotron::transcribe(const std::vector& pcm) // mismatched IR export throws here instead of silently over-/under-reading. // Runtime check (not assert): Release builds define NDEBUG. if (cc.get_byte_size() != cache_channel.get_byte_size() || - ctt.get_byte_size() != cache_time.get_byte_size()) { + ctt.get_byte_size() != cache_time.get_byte_size() || + cl.get_element_type() != ov::element::i32) { throw std::runtime_error( - "Nemotron encoder cache_*_out byte size differs from the pre-allocated " + "Nemotron encoder cache_*_out shape/type differs from the pre-allocated " "input cache; the model IR does not match metadata.json cache shapes."); } std::memcpy(cache_channel.data(), cc.data(), cache_channel.get_byte_size()); diff --git a/src/utils/ensure_models.cpp b/src/utils/ensure_models.cpp index 58db799..f6af35d 100644 --- a/src/utils/ensure_models.cpp +++ b/src/utils/ensure_models.cpp @@ -57,6 +57,14 @@ static bool download_single_file(const std::string& url, } return false; } + // The charset permits '.' and '/', so reject ".." components explicitly to + // stop a caller-supplied filename from writing outside the target directory. + for (const auto& part : output_path) { + if (part == "..") { + if (error_msg) *error_msg = "Refusing to download: path traversal in " + output_path.string(); + return false; + } + } // Create parent directory std::error_code ec; @@ -75,6 +83,10 @@ static bool download_single_file(const std::string& url, // Execute download int ret = std::system(curl_cmd.c_str()); if (ret != 0) { + // Remove any partial file: an interrupted transfer leaves a truncated file + // that file_nonempty() would later accept, silently skipping a re-download + // and loading a corrupt model. + std::filesystem::remove(output_path, ec); if (error_msg) { *error_msg = "curl failed with exit code " + std::to_string(ret) + " for URL: " + url; } @@ -84,6 +96,7 @@ static bool download_single_file(const std::string& url, // Verify downloaded file auto size = std::filesystem::file_size(output_path, ec); if (ec || size == 0) { + std::filesystem::remove(output_path, ec); if (error_msg) { *error_msg = "Downloaded file is missing or empty: " + output_path.string(); }