Skip to content

Add Nemotron ASR transcription support #741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ncylich wants to merge 11 commits into
base: main
Choose a base branch
from
Open

Add Nemotron ASR transcription support #741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
714b6f1
Add Nemotron ASR transcription support
ncylich Jun 19, 2026
3b60d8f
Add Nemotron ASR NPU audio encoder support
ncylich Jun 19, 2026
3045158
Use CPU/GPU Nemotron ASR encoder window
ncylich Jun 19, 2026
0a71b86
Support one-minute Nemotron ASR encoder variants
ncylich Jun 19, 2026
9969b9f
Tighten Nemotron ASR streaming integration
ncylich Jun 19, 2026
716ccb6
Tighten Nemotron ASR reference parity
ncylich Jun 20, 2026
cb693cd
Keep Parakeet conformer conv weights stable
ncylich Jun 20, 2026
b2ec433
Attach INT8 mmap weight scales consistently
ncylich Jun 20, 2026
c998153
Fix causal ASR conv parity
ncylich Jun 20, 2026
4a61ddb
Warm Core ML audio encoders before transcription
ncylich Jun 20, 2026
7f5486e
Restore causal depthwise SIMD path
ncylich Jun 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion cactus-engine/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,14 @@ set(_CACTUS_ENGINE_ARCHIVES
set(_CACTUS_ENGINE_DEPS cactus_engine cactus_graph cactus_kernels)

if(APPLE)
find_program(CACTUS_APPLE_LIBTOOL
NAMES libtool
PATHS /usr/bin
NO_DEFAULT_PATH
REQUIRED
)
add_custom_target(cactus_engine_bundle ALL
COMMAND libtool -static -o ${_CACTUS_ENGINE_BUNDLE} ${_CACTUS_ENGINE_ARCHIVES}
COMMAND ${CACTUS_APPLE_LIBTOOL} -static -o ${_CACTUS_ENGINE_BUNDLE} ${_CACTUS_ENGINE_ARCHIVES}
DEPENDS ${_CACTUS_ENGINE_DEPS}
BYPRODUCTS ${_CACTUS_ENGINE_BUNDLE}
)
Expand Down
30 changes: 29 additions & 1 deletion cactus-engine/src/engine.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,13 +147,19 @@ struct Config {
float rope_scaling_factor = 1.0f;
float rope_mscale_all_dim = 0.0f;

enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, NEEDLE = 18};
enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, NEEDLE = 18, NEMOTRON_ASR = 19};
uint32_t predictor_hidden_dim = 0;
uint32_t predictor_num_layers = 0;
uint32_t tdt_joint_dim = 0;
uint32_t tdt_num_durations = 0;
uint32_t tdt_blank_id = 0;
std::vector<uint32_t> tdt_durations;
uint32_t rnnt_joint_dim = 0;
uint32_t rnnt_blank_id = 0;
uint32_t prompt_dim = 0;
uint32_t default_prompt_id = 0;
uint32_t max_symbols_per_step = 10;
std::unordered_map<std::string, uint32_t> prompt_dictionary;

ModelType model_type = ModelType::GEMMA4;

Expand Down Expand Up @@ -672,11 +678,28 @@ class Model {
double raw_decode_ms = 0.0;
};

struct RnntStreamState {
bool initialized = false;
uint32_t last_token = 0;
size_t time_index = 0;
std::vector<std::vector<uint8_t>> dec_state;
std::vector<uint32_t> pending;
float confirmed_sec = 0.0f;
size_t decoded_tokens = 0;
double raw_decode_ms = 0.0;
};

std::vector<uint32_t> transcribe_parakeet_tdt(const std::vector<float>& audio_features,
ParakeetTdtStreamState* stream = nullptr,
bool is_final = true,
size_t end_frame = 0,
const std::atomic<bool>* should_stop = nullptr);
std::vector<uint32_t> transcribe_nemotron_asr(const std::vector<float>& audio_features,
const std::string& target_lang = "auto",
RnntStreamState* stream = nullptr,
bool is_final = true,
size_t end_frame = 0,
const std::atomic<bool>* should_stop = nullptr);
std::vector<uint32_t> transcribe_whisper_seq2seq(const std::vector<float>& audio_features,
const std::vector<uint32_t>& decoder_prompt_tokens,
size_t max_tokens,
Expand Down Expand Up @@ -724,7 +747,9 @@ class Model {
size_t last_prefill_tail_padding_tokens() const { return last_prefill_tail_padding_tokens_; }

bool load_npu_audio_encoder(const std::string& model_path, const std::string& compute_units = "");
bool load_npu_audio_encoders(const std::vector<std::string>& model_paths, const std::string& compute_units = "");
bool has_npu_audio_encoder() const { return npu_audio_encoder_ != nullptr; }
npu::NPUEncoder* select_npu_audio_encoder(size_t feature_dim, size_t frame_count) const;

bool load_npu_vision_encoder(const std::string& model_path);
bool has_npu_vision_encoder() const { return npu_vision_encoder_ != nullptr; }
Expand Down Expand Up @@ -884,11 +909,14 @@ class Model {

std::string family_;
std::string npu_audio_encoder_mlpackage_;
std::vector<std::string> npu_audio_encoder_mlpackages_;
std::string npu_audio_compute_units_;
size_t npu_audio_prewarm_frames_ = 0;
std::string npu_vision_encoder_mlpackage_;
std::string npu_source_encoder_mlpackage_;

std::unique_ptr<npu::NPUEncoder> npu_audio_encoder_;
std::vector<std::unique_ptr<npu::NPUEncoder>> npu_audio_encoders_;
std::unique_ptr<npu::NPUEncoder> npu_vision_encoder_;
std::unique_ptr<npu::NPUEncoder> npu_source_encoder_;

Expand Down
Loading