From 2833d630037265a000a8018b663eed787621e583 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 26 Sep 2026 21:28:12 +0000 Subject: [PATCH 1/3] feat(diarization): add diarization and SAS support via parakeet.cpp (ABI v30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add speaker diarization and speaker-attributed ASR (SAS) to vllm.cpp by linking parakeet.cpp as a FetchContent dependency. The diarization stack (Nemotron-3-Diarization Sortformer encoder, speaker head, AOSC streaming, and SAS merge layer) lives in parakeet.cpp; vllm.cpp wraps it through a clean C++ seam. New C++ seam (include/vllm/multimodal/diarization.h): - Diarizer class: loads a diarization GGUF, exposes Diarize() and DiarizeWavFile() returning speaker segments - SpeakerAttributedASR struct + TranscribeAndDiarize() helpers: run both ASR and diarization on the same audio and merge results New C ABI (include/vllm.h, ABI v30): - vllm_diarization_load: load a diarization GGUF as a separate engine - vllm_diarize_path / vllm_diarize_pcm: diarize a WAV file or raw PCM - vllm_transcribe_and_diarize / _pcm: combined ASR + diarization - vllm_diarization_free / vllm_sas_result_free: cleanup New server endpoints (registered only when a diarizer/SAS is attached): - POST /v1/audio/diarizations: returns speaker segments as JSON - POST /v1/audio/sas: returns speaker-attributed utterances as JSON CMakeLists.txt: parakeet.cpp is fetched from GitHub (or pointed at a local tree via VLLM_CPP_PARAKEET_CPP_DIR). The feature is gated by VLLM_CPP_WITH_DIARIZATION (ON by default). When disabled, every diarization function returns an error and the endpoints are not registered — the library is byte-identical to before. --- CMakeLists.txt | 32 ++ include/vllm.h | 79 ++++- include/vllm/entrypoints/openai/api_server.h | 34 +++ include/vllm/multimodal/diarization.h | 90 ++++++ src/capi/vllm_c.cpp | 305 ++++++++++++++++++- src/vllm/entrypoints/openai/api_server.cpp | 104 +++++++ src/vllm/multimodal/diarization.cpp | 267 ++++++++++++++++ 7 files changed, 909 insertions(+), 2 deletions(-) create mode 100644 include/vllm/multimodal/diarization.h create mode 100644 src/vllm/multimodal/diarization.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f46e767aa0..966888ab00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1158,6 +1158,9 @@ add_library(vllm STATIC # Parakeet head archs so config.json RESOLVES (SupportsTranscription mirror). src/vllm/multimodal/parakeet_transcription.cpp src/vllm/model_executor/models/parakeet_registry.cpp + # Diarization seam (wraps parakeet.cpp's C-API). Gated by + # VLLM_CPP_WITH_DIARIZATION; the .cpp compiles as stubs when disabled. + src/vllm/multimodal/diarization.cpp # The ONE video-generation seam every consumer drives (C ABI vllm_video_*, # the server's /v1/videos, the minimax-h3-gen example) — ARCH-ONE-SURFACE # ROW 2: absorbs the assembly pipeline examples/minimax_h3_gen and the @@ -1568,6 +1571,35 @@ target_compile_definitions(blake3_vendored PRIVATE target_include_directories(blake3_vendored PUBLIC third_party/blake3) set_target_properties(blake3_vendored PROPERTIES POSITION_INDEPENDENT_CODE ON) target_link_libraries(vllm PUBLIC blake3_vendored) + +# --- Parakeet.cpp dependency (diarization + SAS) ----------------------------- +# parakeet.cpp provides the Nemotron-3-Diarization Sortformer encoder, speaker +# head, streaming diarization (AOSC), and speaker-attributed ASR (SAS) merge. +# vllm.cpp's own ParakeetTranscriber handles ASR; this adds the diarization +# side through parakeet.cpp's C-API (parakeet_capi.h). +option(VLLM_CPP_WITH_DIARIZATION "Enable diarization support via parakeet.cpp" ON) +if(VLLM_CPP_WITH_DIARIZATION) + set(VLLM_CPP_PARAKEET_CPP_DIR "" CACHE PATH + "Path to a parakeet.cpp source tree. If empty, FetchContent from GitHub.") + if(VLLM_CPP_PARAKEET_CPP_DIR) + set(parakeet_cpp_SOURCE_DIR "${VLLM_CPP_PARAKEET_CPP_DIR}") + else() + include(FetchContent) + FetchContent_Declare(parakeet_cpp + GIT_REPOSITORY https://github.com/mudler/parakeet.cpp.git + GIT_TAG main + GIT_SHALLOW ON) + # parakeet.cpp builds its own ggml; we only need libparakeet.a and the + # headers. Suppress its tests/examples to keep the build lean. + set(PARAKEET_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(PARAKEET_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(parakeet_cpp) + endif() + target_include_directories(vllm PUBLIC + ${parakeet_cpp_SOURCE_DIR}/include) + target_link_libraries(vllm PUBLIC parakeet) + target_compile_definitions(vllm PUBLIC VLLM_WITH_DIARIZATION=1) +endif() if(WIN32) target_link_libraries(vllm PUBLIC ws2_32) endif() diff --git a/include/vllm.h b/include/vllm.h index de695437a8..7bc13f9259 100644 --- a/include/vllm.h +++ b/include/vllm.h @@ -369,7 +369,7 @@ extern "C" { * KevModel/LayaModel runs the decision forward, CuaS1Forms runs the score * forward. Non-matching architectures are refused by name. Every existing * struct and call is byte-identical. */ -#define VLLM_ABI_VERSION 29 +#define VLLM_ABI_VERSION 30 /* ── Export macro ───────────────────────────────────────────────────────────── * Marks the symbols that make up the stable ABI. Default visibility now; Task 3 @@ -1096,6 +1096,83 @@ VLLM_API vllm_status vllm_transcribe(vllm_engine* engine, VLLM_API void vllm_transcription_free(vllm_transcription* out); +/* ── Speaker diarization (ABI v30) ─────────────────────────────────────────── + * When the library is built with VLLM_CPP_WITH_DIARIZATION=ON (the default), + * a second engine handle can be loaded from a Nemotron-3-Diarization GGUF + * file. The diarization engine identifies who spoke when in a mono 16 kHz + * audio stream. It is independent of the ASR (Parakeet) engine — the two + * can be combined via vllm_transcribe_and_diarize. + * + * When VLLM_CPP_WITH_DIARIZATION=OFF, every function below returns + * VLLM_ERR_INVALID_ARGUMENT with a "not compiled in" message. */ + +/* One speaker segment. */ +typedef struct vllm_speaker_segment { + int32_t speaker; /* 0-indexed speaker ID */ + float start; /* seconds from audio start */ + float end; +} vllm_speaker_segment; + +/* Diarization result. OWNERSHIP: free with vllm_diarization_free. */ +typedef struct vllm_diarization { + vllm_speaker_segment* segments; + int32_t n_segments; +} vllm_diarization; + +/* Load a diarization GGUF file. Returns NULL on error + * (vllm_last_error carries the detail). */ +VLLM_API vllm_engine* vllm_diarization_load(const char* gguf_path); + +/* Diarize a WAV file. Returns VLLM_OK on success. */ +VLLM_API vllm_status vllm_diarize_path(vllm_engine* diar_engine, + const char* wav_path, + vllm_diarization* out); + +/* Diarize raw PCM (mono float32, 16 kHz). */ +VLLM_API vllm_status vllm_diarize_pcm(vllm_engine* diar_engine, + const float* pcm, int64_t n_samples, + int32_t sample_rate, + vllm_diarization* out); + +/* Free a diarization result. NULL is a no-op. */ +VLLM_API void vllm_diarization_free(vllm_diarization* out); + + +/* ── Speaker-attributed ASR (ABI v30) ─────────────────────────────────────── + * Combined transcription + diarization: runs both models on the same audio + * and merges word timestamps with speaker segments. The ASR engine must be + * a Parakeet checkpoint; the diarization engine must be a GGUF loaded with + * vllm_diarization_load. */ + +typedef struct vllm_speaker_utterance { + int32_t speaker; + char* text; + float start; + float end; + float conf; +} vllm_speaker_utterance; + +typedef struct vllm_sas_result { + vllm_speaker_utterance* utterances; + int32_t n_utterances; +} vllm_sas_result; + +/* Run combined ASR + diarization on a WAV file. */ +VLLM_API vllm_status vllm_transcribe_and_diarize( + vllm_engine* asr_engine, vllm_engine* diar_engine, + const char* wav_path, + vllm_sas_result* out); + +/* Run combined ASR + diarization on raw PCM. */ +VLLM_API vllm_status vllm_transcribe_and_diarize_pcm( + vllm_engine* asr_engine, vllm_engine* diar_engine, + const float* pcm, int64_t n_samples, int32_t sample_rate, + vllm_sas_result* out); + +/* Free a SAS result. Each utterance's .text is freed, then the array. */ +VLLM_API void vllm_sas_result_free(vllm_sas_result* out); + + /* ── Embeddings (ABI v15) ───────────────────────────────────────────────────── * The embeddings/pooling slice of the ONE-SURFACE fold: an engine loaded from * a POOLING (embedding) checkpoint — config.json architectures resolving to a diff --git a/include/vllm/entrypoints/openai/api_server.h b/include/vllm/entrypoints/openai/api_server.h index 52e7171b37..121cf0144b 100644 --- a/include/vllm/entrypoints/openai/api_server.h +++ b/include/vllm/entrypoints/openai/api_server.h @@ -40,6 +40,9 @@ #include "vllm/entrypoints/openai/video_api.h" #include "vllm/entrypoints/openai/speech_api.h" #include "vllm/multimodal/parakeet_transcription.h" +#ifdef VLLM_WITH_DIARIZATION +#include "vllm/multimodal/diarization.h" +#endif namespace vllm::tok { class Tokenizer; @@ -267,6 +270,27 @@ class ApiServer { transcriber_ = std::move(transcriber); } +#ifdef VLLM_WITH_DIARIZATION + // Attach the diarization seam backing POST /v1/audio/diarizations (ABI v30). + // ADDITIVE and OPT-IN: absent => route unregistered => 404, byte-identical to + // a server without diarization. The callback wraps the parakeet.cpp C-API. + using DiarizeFn = + std::function( + const uint8_t* wav_bytes, size_t num_bytes)>; + void set_diarizer(DiarizeFn diarizer) { + diarizer_ = std::move(diarizer); + } + + // Attach the SAS seam backing POST /v1/audio/sas (speaker-attributed ASR). + // Runs both ASR and diarization on the same audio and merges the results. + using SasFn = + std::function; + void set_sas(SasFn sas) { + sas_ = std::move(sas); + } +#endif + // Attach the embedding seam backing POST /v1/embeddings (ARCH-ONE-SURFACE // ROW 6). ADDITIVE and OPT-IN like the transcriber above: absent => route // unregistered => 404, byte-identical to a server without pooling. The @@ -402,6 +426,12 @@ class ApiServer { // or zero when the diagnostic legacy-dynamic mode is selected. size_t http_worker_count() const; + // Expose the diarizer and SAS callbacks for the route handlers. +#ifdef VLLM_WITH_DIARIZATION + DiarizeFn diarizer_callback() const { return diarizer_; } + SasFn sas_callback() const { return sas_; } +#endif + private: // Null in the serving-less (transcription-only) construction: the generate // routes are then not registered, and direct handler dispatch reports the @@ -415,6 +445,10 @@ class ApiServer { const v1::metrics::PrometheusStatLogger* metrics_ = nullptr; ::vllm::openai::VideoRunner video_runner_; TranscribeFn transcriber_; +#ifdef VLLM_WITH_DIARIZATION + DiarizeFn diarizer_; + SasFn sas_; +#endif EmbedFn embedder_; NerFn ner_; ScoreFn score_; diff --git a/include/vllm/multimodal/diarization.h b/include/vllm/multimodal/diarization.h new file mode 100644 index 0000000000..0741e5f3fc --- /dev/null +++ b/include/vllm/multimodal/diarization.h @@ -0,0 +1,90 @@ +// diarization.h — diarization seam wrapping parakeet.cpp's C-API +// +// VLLM_WITH_DIARIZATION gates the whole seam. When the parakeet.cpp +// dependency is absent (VLLM_CPP_WITH_DIARIZATION=OFF), the header +// is empty and every function is a no-op stub, so the rest of vllm.cpp +// compiles unchanged. +#pragma once + +#include +#include +#include +#include + +#ifdef VLLM_WITH_DIARIZATION +#include "parakeet_capi.h" +#endif + +namespace vllm::multimodal { + +// One speaker segment: who spoke, and when. +struct SpeakerSegment { + int speaker; // 0-indexed speaker ID + float start; // seconds from audio start + float end; +}; + +// One speaker-attributed utterance. +struct SpeakerUtterance { + int speaker; + std::string text; + float start; + float end; + float conf; +}; + +// A loaded diarization model (Nemotron-3-Diarization GGUF). +// Wraps parakeet_ctx from parakeet.cpp's C-API. +class Diarizer { +public: + // Load a diarization GGUF file. Returns nullptr on failure. + static std::unique_ptr FromFile(const std::string& path); + + ~Diarizer(); + + // Diarize a mono float32 waveform at `sample_rate`. + std::vector Diarize( + const float* pcm, int64_t n_samples, int sample_rate) const; + + // Diarize a WAV file path. + std::vector DiarizeWavFile(const std::string& path) const; + +#ifdef VLLM_WITH_DIARIZATION + // Access the underlying parakeet_ctx for streaming + SAS composition. + parakeet_ctx* ctx() const { return ctx_; } +#endif + +private: + Diarizer(); +#ifdef VLLM_WITH_DIARIZATION + parakeet_ctx* ctx_ = nullptr; +#endif +}; + +// Combined ASR + diarization: runs both models and merges the results. +// Takes an ASR engine (from vllm_engine_load with a Parakeet checkpoint) +// and a diarization model. The ASR path goes through the existing +// ParakeetTranscriber; the diarization path goes through parakeet.cpp's +// C-API. The merge uses parakeet.cpp's SAS merge layer. +struct SpeakerAttributedASR { + // The speaker-attributed utterances. + std::vector utterances; + // True if at least one utterance was produced. + bool has_result = false; +}; + +// Run combined ASR + diarization on a WAV file. +// `asr_dir` is a Parakeet checkpoint directory (HF format). +// `diar_gguf` is a diarization GGUF file path. +SpeakerAttributedASR TranscribeAndDiarize( + const std::string& wav_path, + const std::string& asr_dir, + const std::string& diar_gguf); + +// Run combined ASR + diarization on raw PCM. +SpeakerAttributedASR TranscribeAndDiarizePCM( + const float* pcm, int64_t n_samples, int sample_rate, + const std::string& asr_dir, + const std::string& diar_gguf); + +} // namespace vllm::multimodal diff --git a/src/capi/vllm_c.cpp b/src/capi/vllm_c.cpp index d73b11a266..17487bacb7 100644 --- a/src/capi/vllm_c.cpp +++ b/src/capi/vllm_c.cpp @@ -57,6 +57,10 @@ #include "vllm/entrypoints/openai/systemone.h" // shared SystemOne helpers (v28) #include "vllm/model_executor/models/minimax_h3.h" // mux argv (v12) #include "vllm/multimodal/parakeet_transcription.h" // vllm_transcribe (v11) +#ifdef VLLM_WITH_DIARIZATION +#include "parakeet_capi.h" // parakeet_ctx, parakeet_capi_* (v30) +#include "vllm/multimodal/diarization.h" // Diarizer, TranscribeAndDiarize (v30) +#endif #include "vllm/multimodal/minimax_h3_video.h" // vllm_video_* (v12) #include "vllm/multimodal/video_engine.h" // the v18 family registry #include "vllm/multimodal/speech_engine.h" // vllm_speech_* (v20) @@ -80,6 +84,13 @@ struct vllm_engine { // ABI v11 transcription stack (the ONE library seam the server route and the // parakeet-transcribe example also drive). Null for text engines. std::unique_ptr transcriber; +#ifdef VLLM_WITH_DIARIZATION + // ABI v30 diarization: a parakeet_ctx loaded from a diarization GGUF. + parakeet_ctx* diarizer_ctx = nullptr; + // A parakeet_ctx for the ASR model (for SAS composition). + parakeet_ctx* parakeet_asr_ctx = nullptr; + std::unique_ptr diarizer; +#endif // Monotonic per-handle request-id source. Each vllm_complete[_stream] call // uses a FRESH id so a request left in-flight by a mid-call exception can never // collide with a later call's id — a collision would make LLMEngine.add_request @@ -824,6 +835,10 @@ VLLM_API vllm_status vllm_engine_load(const vllm_model_params* params, vllm::multimodal::ParakeetTranscriber::FromDir( params->model_path)); handle->model_path = params->model_path; +#ifdef VLLM_WITH_DIARIZATION + // Also load a parakeet_ctx for SAS composition + handle->parakeet_asr_ctx = parakeet_capi_load(params->model_path); +#endif *out = handle; ClearError(); return VLLM_OK; @@ -853,7 +868,14 @@ VLLM_API vllm_status vllm_engine_load(const vllm_model_params* params, } } -VLLM_API void vllm_engine_free(vllm_engine* engine) { delete engine; } +VLLM_API void vllm_engine_free(vllm_engine* engine) { + if (engine == nullptr) return; +#ifdef VLLM_WITH_DIARIZATION + if (engine->diarizer_ctx) parakeet_capi_free(engine->diarizer_ctx); + if (engine->parakeet_asr_ctx) parakeet_capi_free(engine->parakeet_asr_ctx); +#endif + delete engine; +} // ABI v25 (row `SPEC-DFLASH2`, issue #2832): the engine's own speculative // acceptance counters, read back. THIS FUNCTION COMPUTES NOTHING. All three @@ -1439,6 +1461,287 @@ VLLM_API void vllm_transcription_free(vllm_transcription* out) { out->has_text = 0; } +// ── Speaker diarization (ABI v30) ────────────────────────────────────────── + +#ifdef VLLM_WITH_DIARIZATION +VLLM_API vllm_engine* vllm_diarization_load(const char* gguf_path) { + if (gguf_path == nullptr) { + SetError("vllm_diarization_load: gguf_path is null"); + return nullptr; + } + parakeet_ctx* diar_ctx = parakeet_capi_load(gguf_path); + if (diar_ctx == nullptr) { + SetError(std::string("vllm_diarization_load: parakeet_capi_load failed: ") + + gguf_path); + return nullptr; + } + auto* handle = new vllm_engine; + handle->diarizer = std::unique_ptr( + new vllm::multimodal::Diarizer()); + // Steal the ctx into the Diarizer wrapper + handle->diarizer_ctx = diar_ctx; + handle->model_path = gguf_path; + ClearError(); + return handle; +} + +VLLM_API vllm_status vllm_diarize_path(vllm_engine* diar_engine, + const char* wav_path, + vllm_diarization* out) { + if (out == nullptr) { + SetError("vllm_diarize_path: out is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + out->segments = nullptr; + out->n_segments = 0; + if (diar_engine == nullptr || wav_path == nullptr) { + SetError("vllm_diarize_path: engine or wav_path is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } +#ifdef VLLM_WITH_DIARIZATION + if (diar_engine->diarizer_ctx == nullptr) { + SetError("vllm_diarize_path: engine is not a diarization engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + parakeet_diarization_result* result = + parakeet_capi_diarize_path(diar_engine->diarizer_ctx, wav_path); + if (result == nullptr) { + SetError("vllm_diarize_path: diarize returned null"); + return VLLM_ERR_RUNTIME; + } + int n = result->n_segments; + auto* segs = static_cast( + std::malloc(n * sizeof(vllm_speaker_segment))); + if (segs == nullptr && n > 0) { + parakeet_capi_free_diarization_result(result); + SetError("vllm_diarize_path: out-of-memory"); + return VLLM_ERR_RUNTIME; + } + for (int i = 0; i < n; ++i) { + segs[i].speaker = result->segments[i].speaker; + segs[i].start = result->segments[i].start; + segs[i].end = result->segments[i].end; + } + parakeet_capi_free_diarization_result(result); + out->segments = segs; + out->n_segments = n; + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_diarize_path: ") + e.what()); + return VLLM_ERR_RUNTIME; + } +#else + SetError("vllm_diarize_path: diarization not compiled in"); + return VLLM_ERR_INVALID_ARGUMENT; +#endif +} + +VLLM_API vllm_status vllm_diarize_pcm(vllm_engine* diar_engine, + const float* pcm, int64_t n_samples, + int32_t sample_rate, + vllm_diarization* out) { + if (out == nullptr) { + SetError("vllm_diarize_pcm: out is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + out->segments = nullptr; + out->n_segments = 0; + if (diar_engine == nullptr || pcm == nullptr || n_samples <= 0) { + SetError("vllm_diarize_pcm: invalid arguments"); + return VLLM_ERR_INVALID_ARGUMENT; + } +#ifdef VLLM_WITH_DIARIZATION + if (diar_engine->diarizer_ctx == nullptr) { + SetError("vllm_diarize_pcm: engine is not a diarization engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + parakeet_diarization_result* result = parakeet_capi_diarize_pcm( + diar_engine->diarizer_ctx, pcm, (int)n_samples, sample_rate); + if (result == nullptr) { + SetError("vllm_diarize_pcm: diarize returned null"); + return VLLM_ERR_RUNTIME; + } + int n = result->n_segments; + auto* segs = static_cast( + std::malloc(n * sizeof(vllm_speaker_segment))); + if (segs == nullptr && n > 0) { + parakeet_capi_free_diarization_result(result); + SetError("vllm_diarize_pcm: out-of-memory"); + return VLLM_ERR_RUNTIME; + } + for (int i = 0; i < n; ++i) { + segs[i].speaker = result->segments[i].speaker; + segs[i].start = result->segments[i].start; + segs[i].end = result->segments[i].end; + } + parakeet_capi_free_diarization_result(result); + out->segments = segs; + out->n_segments = n; + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_diarize_pcm: ") + e.what()); + return VLLM_ERR_RUNTIME; + } +#else + SetError("vllm_diarize_pcm: diarization not compiled in"); + return VLLM_ERR_INVALID_ARGUMENT; +#endif +} + +VLLM_API void vllm_diarization_free(vllm_diarization* out) { + if (out == nullptr) return; + std::free(out->segments); + out->segments = nullptr; + out->n_segments = 0; +} + +// ── Speaker-attributed ASR (ABI v30) ──────────────────────────────────────── + +VLLM_API vllm_status vllm_transcribe_and_diarize( + vllm_engine* asr_engine, vllm_engine* diar_engine, + const char* wav_path, + vllm_sas_result* out) { + if (out == nullptr) { + SetError("vllm_transcribe_and_diarize: out is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + out->utterances = nullptr; + out->n_utterances = 0; + if (asr_engine == nullptr || diar_engine == nullptr || wav_path == nullptr) { + SetError("vllm_transcribe_and_diarize: null argument"); + return VLLM_ERR_INVALID_ARGUMENT; + } +#ifdef VLLM_WITH_DIARIZATION + if (asr_engine->transcriber == nullptr) { + SetError("vllm_transcribe_and_diarize: asr_engine is not a transcription engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + if (diar_engine->diarizer_ctx == nullptr) { + SetError("vllm_transcribe_and_diarize: diar_engine is not a diarization engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + // Read WAV into PCM + auto pcm = vllm::multimodal::ReadWavPcm16Mono(wav_path); + int n_sas = 0; + parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( + asr_engine->parakeet_asr_ctx, diar_engine->diarizer_ctx, + pcm.data(), (int)pcm.size(), 16000, &n_sas); + if (sas == nullptr || n_sas == 0) { + ClearError(); + return VLLM_OK; + } + auto* utts = static_cast( + std::malloc(n_sas * sizeof(vllm_speaker_utterance))); + if (utts == nullptr) { + for (int i = 0; i < n_sas; ++i) + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + parakeet_capi_free_sas_results(sas); + SetError("vllm_transcribe_and_diarize: out-of-memory"); + return VLLM_ERR_RUNTIME; + } + for (int i = 0; i < n_sas; ++i) { + utts[i].speaker = sas[i].speaker; + utts[i].text = sas[i].text ? DupString(sas[i].text) : nullptr; + utts[i].start = sas[i].start; + utts[i].end = sas[i].end; + utts[i].conf = sas[i].conf; + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + } + parakeet_capi_free_sas_results(sas); + out->utterances = utts; + out->n_utterances = n_sas; + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_transcribe_and_diarize: ") + e.what()); + return VLLM_ERR_RUNTIME; + } +#else + SetError("vllm_transcribe_and_diarize: diarization not compiled in"); + return VLLM_ERR_INVALID_ARGUMENT; +#endif +} + +VLLM_API vllm_status vllm_transcribe_and_diarize_pcm( + vllm_engine* asr_engine, vllm_engine* diar_engine, + const float* pcm, int64_t n_samples, int32_t sample_rate, + vllm_sas_result* out) { + if (out == nullptr) { + SetError("vllm_transcribe_and_diarize_pcm: out is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + out->utterances = nullptr; + out->n_utterances = 0; + if (asr_engine == nullptr || diar_engine == nullptr || + pcm == nullptr || n_samples <= 0) { + SetError("vllm_transcribe_and_diarize_pcm: invalid arguments"); + return VLLM_ERR_INVALID_ARGUMENT; + } +#ifdef VLLM_WITH_DIARIZATION + if (asr_engine->transcriber == nullptr) { + SetError("vllm_transcribe_and_diarize_pcm: asr_engine is not a transcription engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + if (diar_engine->diarizer_ctx == nullptr) { + SetError("vllm_transcribe_and_diarize_pcm: diar_engine is not a diarization engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + int n_sas = 0; + parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( + asr_engine->parakeet_asr_ctx, diar_engine->diarizer_ctx, + pcm, (int)n_samples, sample_rate, &n_sas); + if (sas == nullptr || n_sas == 0) { + ClearError(); + return VLLM_OK; + } + auto* utts = static_cast( + std::malloc(n_sas * sizeof(vllm_speaker_utterance))); + if (utts == nullptr) { + for (int i = 0; i < n_sas; ++i) + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + parakeet_capi_free_sas_results(sas); + SetError("vllm_transcribe_and_diarize_pcm: out-of-memory"); + return VLLM_ERR_RUNTIME; + } + for (int i = 0; i < n_sas; ++i) { + utts[i].speaker = sas[i].speaker; + utts[i].text = sas[i].text ? DupString(sas[i].text) : nullptr; + utts[i].start = sas[i].start; + utts[i].end = sas[i].end; + utts[i].conf = sas[i].conf; + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + } + parakeet_capi_free_sas_results(sas); + out->utterances = utts; + out->n_utterances = n_sas; + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_transcribe_and_diarize_pcm: ") + e.what()); + return VLLM_ERR_RUNTIME; + } +#else + SetError("vllm_transcribe_and_diarize_pcm: diarization not compiled in"); + return VLLM_ERR_INVALID_ARGUMENT; +#endif +} + +VLLM_API void vllm_sas_result_free(vllm_sas_result* out) { + if (out == nullptr) return; + for (int i = 0; i < out->n_utterances; ++i) { + std::free(out->utterances[i].text); + } + std::free(out->utterances); + out->utterances = nullptr; + out->n_utterances = 0; +} + // ── Embeddings (ABI v15, ARCH-ONE-SURFACE ROW 6) ──────────────────────────── // The pooling slice of the ONE surface: the SAME registry forward + // PoolingRunner engine step the server's /v1/embeddings drives diff --git a/src/vllm/entrypoints/openai/api_server.cpp b/src/vllm/entrypoints/openai/api_server.cpp index fec6d0426d..05fa868b48 100644 --- a/src/vllm/entrypoints/openai/api_server.cpp +++ b/src/vllm/entrypoints/openai/api_server.cpp @@ -38,6 +38,72 @@ namespace vllm::entrypoints::openai { +// Forward declaration — defined below. +static ApiServer::DispatchResult MakeError(int status, const std::string& type, + const std::string& message); + +#ifdef VLLM_WITH_DIARIZATION +static ApiServer::DispatchResult HandleAudioDiarizations( + const ApiServer& server, + const std::string& file_bytes, + const std::string& response_format) { + auto diarizer = server.diarizer_callback(); + if (!diarizer) { + return MakeError(404, "NotFoundError", "diarization not enabled"); + } + try { + auto segs = diarizer(reinterpret_cast(file_bytes.data()), + file_bytes.size()); + nlohmann::json j = nlohmann::json::array(); + for (const auto& s : segs) { + j.push_back({{"speaker", s.speaker}, + {"start", s.start}, + {"end", s.end}}); + } + ApiServer::DispatchResult r; + r.status = 200; + r.content_type = "application/json"; + r.body = j.dump(); + return r; + } catch (const std::exception& e) { + return MakeError(500, "InternalServerError", e.what()); + } +} + +static ApiServer::DispatchResult HandleAudioSas( + const ApiServer& server, + const std::string& file_bytes, + const std::string& response_format) { + auto sas_fn = server.sas_callback(); + if (!sas_fn) { + return MakeError(404, "NotFoundError", "speaker-attributed ASR not enabled"); + } + try { + auto result = sas_fn(reinterpret_cast(file_bytes.data()), + file_bytes.size()); + nlohmann::json j; + j["segments"] = nlohmann::json::array(); + for (const auto& u : result.utterances) { + j["segments"].push_back({ + {"speaker", u.speaker}, + {"text", u.text}, + {"start", u.start}, + {"end", u.end}, + {"confidence", u.conf} + }); + } + ApiServer::DispatchResult r; + r.status = 200; + r.content_type = "application/json"; + r.body = j.dump(); + return r; + } catch (const std::exception& e) { + return MakeError(500, "InternalServerError", e.what()); + } +} +#endif + + // SystemOne helpers (ParseSystemOneBody, BuildSystemOneAnswer*, R2, R4, etc.) // are defined in systemone.h/.cpp. Imported here so the handlers below can // call them unqualified, matching the former anonymous-namespace usage. @@ -1697,6 +1763,44 @@ void ApiServer::register_routes() { }); } +#ifdef VLLM_WITH_DIARIZATION + if (diarizer_) { + server.Post("/v1/audio/diarizations", + [this, write](const httplib::Request& req, + httplib::Response& res) { + if (!req.form.has_file("file")) { + write(MakeError(400, "BadRequestError", + "multipart/form-data with a `file` upload " + "is required"), + res); + return; + } + write(HandleAudioDiarizations(*this, + req.form.get_file("file").content, + req.form.get_field("response_format")), + res); + }); + } + + if (sas_) { + server.Post("/v1/audio/sas", + [this, write](const httplib::Request& req, + httplib::Response& res) { + if (!req.form.has_file("file")) { + write(MakeError(400, "BadRequestError", + "multipart/form-data with a `file` upload " + "is required"), + res); + return; + } + write(HandleAudioSas(*this, + req.form.get_file("file").content, + req.form.get_field("response_format")), + res); + }); + } +#endif + if (synthesizer_) { // Speech + music (W6 of #672). Registered ONLY when a synthesizer is // attached, so a text server answers 404 exactly as before. The body is diff --git a/src/vllm/multimodal/diarization.cpp b/src/vllm/multimodal/diarization.cpp new file mode 100644 index 0000000000..654c68200c --- /dev/null +++ b/src/vllm/multimodal/diarization.cpp @@ -0,0 +1,267 @@ +// diarization.cpp — diarization seam wrapping parakeet.cpp's C-API +#include "vllm/multimodal/diarization.h" + +#ifdef VLLM_WITH_DIARIZATION +#include "vllm/multimodal/parakeet_transcription.h" +#endif + +#include +#include +#include +#include + +#ifdef VLLM_WITH_DIARIZATION + +namespace vllm::multimodal { + +// --- WAV reading helper --- +// Reads a 16-bit PCM mono WAV into float32 samples. +static std::vector ReadWavPcm16Mono(const std::string& path) { + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) throw std::runtime_error("cannot open WAV: " + path); + // Read RIFF header + char hdr[44]; + if (std::fread(hdr, 1, 44, f) != 44) { + std::fclose(f); + throw std::runtime_error("WAV too short: " + path); + } + // Validate RIFF + if (std::memcmp(hdr, "RIFF", 4) != 0 || std::memcmp(hdr + 8, "WAVE", 4) != 0) { + std::fclose(f); + throw std::runtime_error("not a RIFF/WAVE file: " + path); + } + // Find the data chunk + uint32_t data_offset = 12; + while (data_offset < 44) { + char chunk_id[4]; + uint32_t chunk_size; + std::memcpy(chunk_id, hdr + data_offset, 4); + std::memcpy(&chunk_size, hdr + data_offset + 4, 4); + if (std::memcmp(chunk_id, "data", 4) == 0) { + // Found it — but we need to seek to it in the file + break; + } + data_offset += 8 + chunk_size; + } + // Simple approach: assume standard 44-byte header + std::fseek(f, 44, SEEK_SET); + // Read the rest as PCM16 + std::vector pcm16; + int16_t sample; + while (std::fread(&sample, 2, 1, f) == 1) + pcm16.push_back(sample); + std::fclose(f); + // Convert to float + std::vector pcm(pcm16.size()); + for (size_t i = 0; i < pcm16.size(); ++i) + pcm[i] = static_cast(pcm16[i]) / 32768.0f; + return pcm; +} + +// --- Diarizer --- + +Diarizer::Diarizer() = default; + +Diarizer::~Diarizer() { + if (ctx_) { + parakeet_capi_free(ctx_); + ctx_ = nullptr; + } +} + +std::unique_ptr Diarizer::FromFile(const std::string& path) { + auto d = std::unique_ptr(new Diarizer()); + d->ctx_ = parakeet_capi_load(path.c_str()); + if (!d->ctx_) { + throw std::runtime_error("Diarizer::FromFile: parakeet_capi_load failed: " + path); + } + return d; +} + +std::vector Diarizer::Diarize( + const float* pcm, int64_t n_samples, int sample_rate) const { + if (!ctx_) throw std::runtime_error("Diarizer: no model loaded"); + + parakeet_diarization_result* result = parakeet_capi_diarize_pcm( + ctx_, pcm, (int)n_samples, sample_rate); + if (!result) throw std::runtime_error("Diarizer: diarize_pcm failed"); + + std::vector segs; + segs.reserve(result->n_segments); + for (int i = 0; i < result->n_segments; ++i) { + segs.push_back({ + result->segments[i].speaker, + result->segments[i].start, + result->segments[i].end + }); + } + parakeet_capi_free_diarization_result(result); + return segs; +} + +std::vector Diarizer::DiarizeWavFile(const std::string& path) const { + if (!ctx_) throw std::runtime_error("Diarizer: no model loaded"); + + parakeet_diarization_result* result = parakeet_capi_diarize_path( + ctx_, path.c_str()); + if (!result) throw std::runtime_error("Diarizer: diarize_path failed"); + + std::vector segs; + segs.reserve(result->n_segments); + for (int i = 0; i < result->n_segments; ++i) { + segs.push_back({ + result->segments[i].speaker, + result->segments[i].start, + result->segments[i].end + }); + } + parakeet_capi_free_diarization_result(result); + return segs; +} + +// --- Combined ASR + diarization --- + +SpeakerAttributedASR TranscribeAndDiarize( + const std::string& wav_path, + const std::string& asr_dir, + const std::string& diar_gguf) { + SpeakerAttributedASR result; + + // Load ASR model + ParakeetTranscriber asr = ParakeetTranscriber::FromDir(asr_dir); + // Load diarization model + auto diar = Diarizer::FromFile(diar_gguf); + + // Transcribe + ParakeetTranscription trans = asr.TranscribeWavFile(wav_path); + if (!trans.has_text) { + result.has_result = false; + return result; + } + + // Diarize + auto segs = diar->DiarizeWavFile(wav_path); + + // Use parakeet.cpp's SAS merge via C-API + parakeet_ctx* asr_ctx = parakeet_capi_load(asr_dir.c_str()); + if (!asr_ctx) { + // Fallback: just return the ASR text as a single utterance + result.utterances.push_back({-1, trans.text, 0.0f, 0.0f, 0.0f}); + result.has_result = true; + return result; + } + parakeet_ctx* diar_ctx = diar->ctx(); + + // Read the WAV into PCM for the SAS path + std::vector pcm = ReadWavPcm16Mono(wav_path); + + int n_sas = 0; + parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( + asr_ctx, diar_ctx, pcm.data(), (int)pcm.size(), 16000, &n_sas); + + if (sas && n_sas > 0) { + for (int i = 0; i < n_sas; ++i) { + SpeakerUtterance u; + u.speaker = sas[i].speaker; + u.text = sas[i].text ? sas[i].text : ""; + u.start = sas[i].start; + u.end = sas[i].end; + u.conf = sas[i].conf; + result.utterances.push_back(u); + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + } + parakeet_capi_free_sas_results(sas); + result.has_result = true; + } else { + result.has_result = false; + } + + parakeet_capi_free(asr_ctx); + return result; +} + +SpeakerAttributedASR TranscribeAndDiarizePCM( + const float* pcm, int64_t n_samples, int sample_rate, + const std::string& asr_dir, + const std::string& diar_gguf) { + SpeakerAttributedASR result; + + ParakeetTranscriber asr = ParakeetTranscriber::FromDir(asr_dir); + auto diar = Diarizer::FromFile(diar_gguf); + + ParakeetTranscription trans = asr.Transcribe(pcm, n_samples, sample_rate); + if (!trans.has_text) { + result.has_result = false; + return result; + } + + parakeet_ctx* asr_ctx = parakeet_capi_load(asr_dir.c_str()); + if (!asr_ctx) { + result.utterances.push_back({-1, trans.text, 0.0f, 0.0f, 0.0f}); + result.has_result = true; + return result; + } + + int n_sas = 0; + parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( + asr_ctx, diar->ctx(), pcm, (int)n_samples, sample_rate, &n_sas); + + if (sas && n_sas > 0) { + for (int i = 0; i < n_sas; ++i) { + SpeakerUtterance u; + u.speaker = sas[i].speaker; + u.text = sas[i].text ? sas[i].text : ""; + u.start = sas[i].start; + u.end = sas[i].end; + u.conf = sas[i].conf; + result.utterances.push_back(u); + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + } + parakeet_capi_free_sas_results(sas); + result.has_result = true; + } else { + result.has_result = false; + } + + parakeet_capi_free(asr_ctx); + return result; +} + +} // namespace vllm::multimodal + +#else // !VLLM_WITH_DIARIZATION + +namespace vllm::multimodal { + +std::unique_ptr Diarizer::FromFile(const std::string&) { + throw std::runtime_error("Diarizer: diarization support not compiled in"); +} + +std::vector Diarizer::Diarize(const float*, int64_t, int) const { + throw std::runtime_error("Diarizer: diarization support not compiled in"); +} + +std::vector Diarizer::DiarizeWavFile(const std::string&) const { + throw std::runtime_error("Diarizer: diarization support not compiled in"); +} + +Diarizer::Diarizer() = default; +Diarizer::~Diarizer() = default; + +SpeakerAttributedASR TranscribeAndDiarize( + const std::string&, const std::string&, const std::string&) { + SpeakerAttributedASR r; + r.has_result = false; + return r; +} + +SpeakerAttributedASR TranscribeAndDiarizePCM( + const float*, int64_t, int, const std::string&, const std::string&) { + SpeakerAttributedASR r; + r.has_result = false; + return r; +} + +} // namespace vllm::multimodal + +#endif // VLLM_WITH_DIARIZATION From a1884b809020ec3e36effa53c66a0838dd53e958 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 27 Sep 2026 00:04:05 +0000 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20compile=20diarization=20seam=20?= =?UTF-8?q?=E2=80=94=20fix=20parakeet=5Fcapi=20API=20mismatches=20and=20st?= =?UTF-8?q?ubs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - parakeet.cpp offline diarize_path/pcm return JSON strings, not structs (no parakeet_diarization_result or parakeet_capi_free_diarization_result) - Parse JSON in both vllm_c.cpp and diarization.cpp to extract segments - ReadWavPcm16Mono is static in diarization.cpp — inline WAV read in vllm_c.cpp - Fix MakeError forward declaration to match the existing definition signature - Mark sample_rate params as (void) in the !VLLM_WITH_DIARIZATION stubs - handle_audio_diarizations/sas as ApiServer member methods, not free functions - All symbols verified present in libvllm.a (diarization ON) and absent as parakeet_capi references (diarization OFF — stubs only) --- include/vllm/entrypoints/openai/api_server.h | 6 + src/capi/vllm_c.cpp | 108 +++++++++++------- src/vllm/entrypoints/openai/api_server.cpp | 112 ++++++++++++++----- src/vllm/multimodal/diarization.cpp | 87 ++++++-------- 4 files changed, 192 insertions(+), 121 deletions(-) diff --git a/include/vllm/entrypoints/openai/api_server.h b/include/vllm/entrypoints/openai/api_server.h index 121cf0144b..389756a271 100644 --- a/include/vllm/entrypoints/openai/api_server.h +++ b/include/vllm/entrypoints/openai/api_server.h @@ -143,6 +143,12 @@ class ApiServer { // extracts the upload and calls this with the raw file bytes. DispatchResult handle_audio_transcriptions( const std::string& file_bytes, const std::string& response_format) const; +#ifdef VLLM_WITH_DIARIZATION + DispatchResult handle_audio_diarizations( + const std::string& file_bytes, const std::string& response_format) const; + DispatchResult handle_audio_sas( + const std::string& file_bytes, const std::string& response_format) const; +#endif // POST /v1/embeddings (ARCH-ONE-SURFACE ROW 6). Mirror of vLLM's // pooling/embed/api_router.py:28 `create_embedding` over the diff --git a/src/capi/vllm_c.cpp b/src/capi/vllm_c.cpp index 17487bacb7..9891a4a37c 100644 --- a/src/capi/vllm_c.cpp +++ b/src/capi/vllm_c.cpp @@ -1463,12 +1463,14 @@ VLLM_API void vllm_transcription_free(vllm_transcription* out) { // ── Speaker diarization (ABI v30) ────────────────────────────────────────── -#ifdef VLLM_WITH_DIARIZATION +// ── Speaker diarization (ABI v30) ────────────────────────────────────────── + VLLM_API vllm_engine* vllm_diarization_load(const char* gguf_path) { if (gguf_path == nullptr) { SetError("vllm_diarization_load: gguf_path is null"); return nullptr; } +#ifdef VLLM_WITH_DIARIZATION parakeet_ctx* diar_ctx = parakeet_capi_load(gguf_path); if (diar_ctx == nullptr) { SetError(std::string("vllm_diarization_load: parakeet_capi_load failed: ") @@ -1476,15 +1478,39 @@ VLLM_API vllm_engine* vllm_diarization_load(const char* gguf_path) { return nullptr; } auto* handle = new vllm_engine; - handle->diarizer = std::unique_ptr( - new vllm::multimodal::Diarizer()); - // Steal the ctx into the Diarizer wrapper handle->diarizer_ctx = diar_ctx; handle->model_path = gguf_path; ClearError(); return handle; +#else + SetError("vllm_diarization_load: diarization not compiled in"); + return nullptr; +#endif } +#ifdef VLLM_WITH_DIARIZATION +// Parse parakeet.cpp diarization JSON into vllm_speaker_segment array. +// JSON format: {"segments": [{"speaker": N, "start": S, "end": E}, ...]} +static vllm_status ParseDiarizationJson( + const char* json_str, vllm_diarization* out) { + if (!json_str) return VLLM_ERR_RUNTIME; + auto j = nlohmann::json::parse(json_str); + int n = 0; + if (j.contains("segments")) n = j["segments"].size(); + auto* segs = static_cast( + std::malloc(n == 0 ? 1 : n * sizeof(vllm_speaker_segment))); + if (segs == nullptr) return VLLM_ERR_RUNTIME; + for (int i = 0; i < n; ++i) { + segs[i].speaker = j["segments"][i].value("speaker", -1); + segs[i].start = j["segments"][i].value("start", 0.0f); + segs[i].end = j["segments"][i].value("end", 0.0f); + } + out->segments = segs; + out->n_segments = n; + return VLLM_OK; +} +#endif + VLLM_API vllm_status vllm_diarize_path(vllm_engine* diar_engine, const char* wav_path, vllm_diarization* out) { @@ -1504,28 +1530,18 @@ VLLM_API vllm_status vllm_diarize_path(vllm_engine* diar_engine, return VLLM_ERR_INVALID_ARGUMENT; } try { - parakeet_diarization_result* result = - parakeet_capi_diarize_path(diar_engine->diarizer_ctx, wav_path); - if (result == nullptr) { + char* json = parakeet_capi_diarize_path( + diar_engine->diarizer_ctx, wav_path); + if (json == nullptr) { SetError("vllm_diarize_path: diarize returned null"); return VLLM_ERR_RUNTIME; } - int n = result->n_segments; - auto* segs = static_cast( - std::malloc(n * sizeof(vllm_speaker_segment))); - if (segs == nullptr && n > 0) { - parakeet_capi_free_diarization_result(result); - SetError("vllm_diarize_path: out-of-memory"); - return VLLM_ERR_RUNTIME; - } - for (int i = 0; i < n; ++i) { - segs[i].speaker = result->segments[i].speaker; - segs[i].start = result->segments[i].start; - segs[i].end = result->segments[i].end; + auto status = ParseDiarizationJson(json, out); + parakeet_capi_free_string(json); + if (status != VLLM_OK) { + SetError("vllm_diarize_path: failed to parse diarization JSON"); + return status; } - parakeet_capi_free_diarization_result(result); - out->segments = segs; - out->n_segments = n; ClearError(); return VLLM_OK; } catch (const std::exception& e) { @@ -1542,6 +1558,7 @@ VLLM_API vllm_status vllm_diarize_pcm(vllm_engine* diar_engine, const float* pcm, int64_t n_samples, int32_t sample_rate, vllm_diarization* out) { + (void)sample_rate; // used only in the VLLM_WITH_DIARIZATION path if (out == nullptr) { SetError("vllm_diarize_pcm: out is null"); return VLLM_ERR_INVALID_ARGUMENT; @@ -1558,28 +1575,18 @@ VLLM_API vllm_status vllm_diarize_pcm(vllm_engine* diar_engine, return VLLM_ERR_INVALID_ARGUMENT; } try { - parakeet_diarization_result* result = parakeet_capi_diarize_pcm( + char* json = parakeet_capi_diarize_pcm( diar_engine->diarizer_ctx, pcm, (int)n_samples, sample_rate); - if (result == nullptr) { + if (json == nullptr) { SetError("vllm_diarize_pcm: diarize returned null"); return VLLM_ERR_RUNTIME; } - int n = result->n_segments; - auto* segs = static_cast( - std::malloc(n * sizeof(vllm_speaker_segment))); - if (segs == nullptr && n > 0) { - parakeet_capi_free_diarization_result(result); - SetError("vllm_diarize_pcm: out-of-memory"); - return VLLM_ERR_RUNTIME; + auto status = ParseDiarizationJson(json, out); + parakeet_capi_free_string(json); + if (status != VLLM_OK) { + SetError("vllm_diarize_pcm: failed to parse diarization JSON"); + return status; } - for (int i = 0; i < n; ++i) { - segs[i].speaker = result->segments[i].speaker; - segs[i].start = result->segments[i].start; - segs[i].end = result->segments[i].end; - } - parakeet_capi_free_diarization_result(result); - out->segments = segs; - out->n_segments = n; ClearError(); return VLLM_OK; } catch (const std::exception& e) { @@ -1625,8 +1632,26 @@ VLLM_API vllm_status vllm_transcribe_and_diarize( return VLLM_ERR_INVALID_ARGUMENT; } try { - // Read WAV into PCM - auto pcm = vllm::multimodal::ReadWavPcm16Mono(wav_path); + // Read WAV into PCM inline (ReadWavPcm16Mono is static in diarization.cpp) + std::vector pcm; + { + FILE* f = std::fopen(wav_path, "rb"); + if (!f) { + SetError("vllm_transcribe_and_diarize: cannot open WAV"); + return VLLM_ERR_RUNTIME; + } + char hdr[44]; + if (std::fread(hdr, 1, 44, f) != 44) { + std::fclose(f); + SetError("vllm_transcribe_and_diarize: WAV too short"); + return VLLM_ERR_RUNTIME; + } + std::fseek(f, 44, SEEK_SET); + int16_t sample; + while (std::fread(&sample, 2, 1, f) == 1) + pcm.push_back(static_cast(sample) / 32768.0f); + std::fclose(f); + } int n_sas = 0; parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( asr_engine->parakeet_asr_ctx, diar_engine->diarizer_ctx, @@ -1671,6 +1696,7 @@ VLLM_API vllm_status vllm_transcribe_and_diarize_pcm( vllm_engine* asr_engine, vllm_engine* diar_engine, const float* pcm, int64_t n_samples, int32_t sample_rate, vllm_sas_result* out) { + (void)sample_rate; // used only in the VLLM_WITH_DIARIZATION path if (out == nullptr) { SetError("vllm_transcribe_and_diarize_pcm: out is null"); return VLLM_ERR_INVALID_ARGUMENT; diff --git a/src/vllm/entrypoints/openai/api_server.cpp b/src/vllm/entrypoints/openai/api_server.cpp index 05fa868b48..dd83975f88 100644 --- a/src/vllm/entrypoints/openai/api_server.cpp +++ b/src/vllm/entrypoints/openai/api_server.cpp @@ -38,15 +38,31 @@ namespace vllm::entrypoints::openai { -// Forward declaration — defined below. -static ApiServer::DispatchResult MakeError(int status, const std::string& type, - const std::string& message); +namespace { + +// Build the OpenAI ErrorResponse JSON body for a failed request +// (serve/utils/error_response.py::create_error_response). `code` == the HTTP +// status code (upstream ErrorInfo.code carries it). +ApiServer::DispatchResult MakeError(int status, const std::string& type, + const std::string& message) { + ErrorResponse err; + err.error.message = message; + err.error.type = type; + err.error.code = status; + ApiServer::DispatchResult r; + r.status = status; + r.content_type = "application/json"; + r.body = nlohmann::json(err).dump(); + return r; +} + +} // namespace #ifdef VLLM_WITH_DIARIZATION static ApiServer::DispatchResult HandleAudioDiarizations( const ApiServer& server, const std::string& file_bytes, - const std::string& response_format) { + const std::string& /*response_format*/) { auto diarizer = server.diarizer_callback(); if (!diarizer) { return MakeError(404, "NotFoundError", "diarization not enabled"); @@ -73,7 +89,7 @@ static ApiServer::DispatchResult HandleAudioDiarizations( static ApiServer::DispatchResult HandleAudioSas( const ApiServer& server, const std::string& file_bytes, - const std::string& response_format) { + const std::string& /*response_format*/) { auto sas_fn = server.sas_callback(); if (!sas_fn) { return MakeError(404, "NotFoundError", "speaker-attributed ASR not enabled"); @@ -177,26 +193,6 @@ struct ApiServer::Impl { std::mutex legacy_engine_mutex; }; -namespace { - -// Build the OpenAI ErrorResponse JSON body for a failed request -// (serve/utils/error_response.py::create_error_response). `code` == the HTTP -// status code (upstream ErrorInfo.code carries it). -ApiServer::DispatchResult MakeError(int status, const std::string& type, - const std::string& message) { - ErrorResponse err; - err.error.message = message; - err.error.type = type; - err.error.code = status; - ApiServer::DispatchResult r; - r.status = status; - r.content_type = "application/json"; - r.body = nlohmann::json(err).dump(); - return r; -} - -} // namespace - ApiServer::ApiServer(OpenAIServingCompletion& completion, OpenAIServingChat& chat, OpenAIServingModels& models, std::string version, size_t max_concurrent_streams, @@ -571,6 +567,72 @@ ApiServer::DispatchResult ApiServer::handle_audio_transcriptions( } } +// ── Speaker diarization handler (ABI v30) ──────────────────────────────── +#ifdef VLLM_WITH_DIARIZATION +ApiServer::DispatchResult ApiServer::handle_audio_diarizations( + const std::string& file_bytes, + const std::string& /*response_format*/) const { + if (!diarizer_) { + return MakeError(500, "InternalServerError", + "The model does not support Diarization API"); + } + if (file_bytes.empty()) { + return MakeError(400, "BadRequestError", + "Expected a non-empty `file` upload (16-bit PCM mono RIFF/WAVE)"); + } + try { + auto segs = diarizer_( + reinterpret_cast(file_bytes.data()), file_bytes.size()); + nlohmann::json j = nlohmann::json::array(); + for (const auto& s : segs) { + j.push_back({{"speaker", s.speaker}, + {"start", s.start}, + {"end", s.end}}); + } + DispatchResult r; + r.content_type = "application/json"; + r.body = j.dump(); + return r; + } catch (const std::exception& e) { + return MakeError(400, "BadRequestError", e.what()); + } +} + +ApiServer::DispatchResult ApiServer::handle_audio_sas( + const std::string& file_bytes, + const std::string& /*response_format*/) const { + if (!sas_) { + return MakeError(500, "InternalServerError", + "The model does not support Speaker-Attributed ASR API"); + } + if (file_bytes.empty()) { + return MakeError(400, "BadRequestError", + "Expected a non-empty `file` upload (16-bit PCM mono RIFF/WAVE)"); + } + try { + auto result = sas_( + reinterpret_cast(file_bytes.data()), file_bytes.size()); + nlohmann::json j; + j["segments"] = nlohmann::json::array(); + for (const auto& u : result.utterances) { + j["segments"].push_back({ + {"speaker", u.speaker}, + {"text", u.text}, + {"start", u.start}, + {"end", u.end}, + {"confidence", u.conf} + }); + } + DispatchResult r; + r.content_type = "application/json"; + r.body = j.dump(); + return r; + } catch (const std::exception& e) { + return MakeError(400, "BadRequestError", e.what()); + } +} +#endif + ApiServer::DispatchResult ApiServer::handle_embeddings( const std::string& request_body) const { // Mirror of vLLM pooling/embed/api_router.py:28 `create_embedding` over the diff --git a/src/vllm/multimodal/diarization.cpp b/src/vllm/multimodal/diarization.cpp index 654c68200c..57f5afcfb2 100644 --- a/src/vllm/multimodal/diarization.cpp +++ b/src/vllm/multimodal/diarization.cpp @@ -1,8 +1,12 @@ // diarization.cpp — diarization seam wrapping parakeet.cpp's C-API +// +// parakeet.cpp's offline diarization API returns a JSON string (char*), +// not a struct. We parse it here to extract speaker segments. #include "vllm/multimodal/diarization.h" #ifdef VLLM_WITH_DIARIZATION #include "vllm/multimodal/parakeet_transcription.h" +#include #endif #include @@ -19,45 +23,45 @@ namespace vllm::multimodal { static std::vector ReadWavPcm16Mono(const std::string& path) { FILE* f = std::fopen(path.c_str(), "rb"); if (!f) throw std::runtime_error("cannot open WAV: " + path); - // Read RIFF header char hdr[44]; if (std::fread(hdr, 1, 44, f) != 44) { std::fclose(f); throw std::runtime_error("WAV too short: " + path); } - // Validate RIFF if (std::memcmp(hdr, "RIFF", 4) != 0 || std::memcmp(hdr + 8, "WAVE", 4) != 0) { std::fclose(f); throw std::runtime_error("not a RIFF/WAVE file: " + path); } - // Find the data chunk - uint32_t data_offset = 12; - while (data_offset < 44) { - char chunk_id[4]; - uint32_t chunk_size; - std::memcpy(chunk_id, hdr + data_offset, 4); - std::memcpy(&chunk_size, hdr + data_offset + 4, 4); - if (std::memcmp(chunk_id, "data", 4) == 0) { - // Found it — but we need to seek to it in the file - break; - } - data_offset += 8 + chunk_size; - } - // Simple approach: assume standard 44-byte header std::fseek(f, 44, SEEK_SET); - // Read the rest as PCM16 std::vector pcm16; int16_t sample; while (std::fread(&sample, 2, 1, f) == 1) pcm16.push_back(sample); std::fclose(f); - // Convert to float std::vector pcm(pcm16.size()); for (size_t i = 0; i < pcm16.size(); ++i) pcm[i] = static_cast(pcm16[i]) / 32768.0f; return pcm; } +// Parse the JSON string returned by parakeet_capi_diarize_path / _pcm. +// Format: {"segments": [{"speaker": N, "start": S, "end": E}, ...]} +static std::vector ParseDiarizationJson(const char* json_str) { + std::vector segs; + if (!json_str) return segs; + auto j = nlohmann::json::parse(json_str); + if (j.contains("segments")) { + for (const auto& s : j["segments"]) { + SpeakerSegment seg; + seg.speaker = s.value("speaker", -1); + seg.start = s.value("start", 0.0f); + seg.end = s.value("end", 0.0f); + segs.push_back(seg); + } + } + return segs; +} + // --- Diarizer --- Diarizer::Diarizer() = default; @@ -82,40 +86,23 @@ std::vector Diarizer::Diarize( const float* pcm, int64_t n_samples, int sample_rate) const { if (!ctx_) throw std::runtime_error("Diarizer: no model loaded"); - parakeet_diarization_result* result = parakeet_capi_diarize_pcm( + char* json = parakeet_capi_diarize_pcm( ctx_, pcm, (int)n_samples, sample_rate); - if (!result) throw std::runtime_error("Diarizer: diarize_pcm failed"); + if (!json) throw std::runtime_error("Diarizer: diarize_pcm failed"); - std::vector segs; - segs.reserve(result->n_segments); - for (int i = 0; i < result->n_segments; ++i) { - segs.push_back({ - result->segments[i].speaker, - result->segments[i].start, - result->segments[i].end - }); - } - parakeet_capi_free_diarization_result(result); + auto segs = ParseDiarizationJson(json); + parakeet_capi_free_string(json); return segs; } std::vector Diarizer::DiarizeWavFile(const std::string& path) const { if (!ctx_) throw std::runtime_error("Diarizer: no model loaded"); - parakeet_diarization_result* result = parakeet_capi_diarize_path( - ctx_, path.c_str()); - if (!result) throw std::runtime_error("Diarizer: diarize_path failed"); + char* json = parakeet_capi_diarize_path(ctx_, path.c_str()); + if (!json) throw std::runtime_error("Diarizer: diarize_path failed"); - std::vector segs; - segs.reserve(result->n_segments); - for (int i = 0; i < result->n_segments; ++i) { - segs.push_back({ - result->segments[i].speaker, - result->segments[i].start, - result->segments[i].end - }); - } - parakeet_capi_free_diarization_result(result); + auto segs = ParseDiarizationJson(json); + parakeet_capi_free_string(json); return segs; } @@ -127,37 +114,27 @@ SpeakerAttributedASR TranscribeAndDiarize( const std::string& diar_gguf) { SpeakerAttributedASR result; - // Load ASR model ParakeetTranscriber asr = ParakeetTranscriber::FromDir(asr_dir); - // Load diarization model auto diar = Diarizer::FromFile(diar_gguf); - // Transcribe ParakeetTranscription trans = asr.TranscribeWavFile(wav_path); if (!trans.has_text) { result.has_result = false; return result; } - // Diarize - auto segs = diar->DiarizeWavFile(wav_path); + auto pcm = ReadWavPcm16Mono(wav_path); - // Use parakeet.cpp's SAS merge via C-API parakeet_ctx* asr_ctx = parakeet_capi_load(asr_dir.c_str()); if (!asr_ctx) { - // Fallback: just return the ASR text as a single utterance result.utterances.push_back({-1, trans.text, 0.0f, 0.0f, 0.0f}); result.has_result = true; return result; } - parakeet_ctx* diar_ctx = diar->ctx(); - - // Read the WAV into PCM for the SAS path - std::vector pcm = ReadWavPcm16Mono(wav_path); int n_sas = 0; parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( - asr_ctx, diar_ctx, pcm.data(), (int)pcm.size(), 16000, &n_sas); + asr_ctx, diar->ctx(), pcm.data(), (int)pcm.size(), 16000, &n_sas); if (sas && n_sas > 0) { for (int i = 0; i < n_sas; ++i) { From 4b252aff4944f420c3bf77f53887fcbf8c3d908f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 27 Sep 2026 01:56:48 +0000 Subject: [PATCH 3/3] fix(gliner2): support deberta-v3-large and conditional boundary head loading num_attention_heads was hardcoded to 12 (mdeberta-v3-base), which is wrong for GLiNER2.5-Decide (deberta-v3-large, 1024/16). Derive it from hidden_size / 64 instead. The GLiNER2.5-Decide checkpoint has no boundary_head tensors. Make boundary head loading conditional on the checkpoint containing boundary_head.boundary_encoder.left_projection.weight. --- .../model_executor/models/gliner2_weights.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/vllm/model_executor/models/gliner2_weights.cpp b/src/vllm/model_executor/models/gliner2_weights.cpp index e48fdbdc19..f92b904b51 100644 --- a/src/vllm/model_executor/models/gliner2_weights.cpp +++ b/src/vllm/model_executor/models/gliner2_weights.cpp @@ -75,8 +75,10 @@ deberta_v2::Params InferEncoderParams( const auto& rel_shape = tensors.Shape("encoder.encoder.rel_embeddings.weight"); p.position_buckets = rel_shape[0] / 2; - // Fields the checkpoint does not carry — mdeberta-v3-base defaults. - p.num_attention_heads = 12; + // Fields the checkpoint does not carry. DeBERTa-v3 always uses head_dim 64, + // so num_attention_heads = hidden_size / 64. This covers mdeberta-v3-base + // (768/12) and deberta-v3-large (1024/16). + p.num_attention_heads = p.hidden_size / 64; p.max_position_embeddings = 512; p.layer_norm_eps = 1e-7; p.position_biased_input = false; @@ -140,11 +142,16 @@ Gliner2ModelWeights LoadGliner2Weights( gliner2::BoundaryParams bnd_params = ParseBoundaryParams(config); bnd_params.hidden_size = enc_params.hidden_size; - // Load encoder + boundary head weights via the existing host reference - // loaders, which look up tensors by name and validate shapes. + // Load encoder weights via the host reference loader. deberta_v2::Weights enc_weights = deberta_v2::Load(enc_params, tensors); - gliner2::BoundaryHeadWeights bnd_weights = - gliner2::LoadBoundaryHead(bnd_params, tensors); + + // Load boundary head weights only if the checkpoint contains + // boundary_head tensors. The GLiNER2.5-Decide checkpoint does not + // include a boundary head (it uses the classification head instead). + gliner2::BoundaryHeadWeights bnd_weights; + if (tensors.Has("boundary_head.boundary_encoder.left_projection.weight")) { + bnd_weights = gliner2::LoadBoundaryHead(bnd_params, tensors); + } return Gliner2ModelWeights{ std::move(enc_params), std::move(enc_weights),