diff --git a/CMakeLists.txt b/CMakeLists.txt index f46e767aa0..966888ab00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1158,6 +1158,9 @@ add_library(vllm STATIC # Parakeet head archs so config.json RESOLVES (SupportsTranscription mirror). src/vllm/multimodal/parakeet_transcription.cpp src/vllm/model_executor/models/parakeet_registry.cpp + # Diarization seam (wraps parakeet.cpp's C-API). Gated by + # VLLM_CPP_WITH_DIARIZATION; the .cpp compiles as stubs when disabled. + src/vllm/multimodal/diarization.cpp # The ONE video-generation seam every consumer drives (C ABI vllm_video_*, # the server's /v1/videos, the minimax-h3-gen example) — ARCH-ONE-SURFACE # ROW 2: absorbs the assembly pipeline examples/minimax_h3_gen and the @@ -1568,6 +1571,35 @@ target_compile_definitions(blake3_vendored PRIVATE target_include_directories(blake3_vendored PUBLIC third_party/blake3) set_target_properties(blake3_vendored PROPERTIES POSITION_INDEPENDENT_CODE ON) target_link_libraries(vllm PUBLIC blake3_vendored) + +# --- Parakeet.cpp dependency (diarization + SAS) ----------------------------- +# parakeet.cpp provides the Nemotron-3-Diarization Sortformer encoder, speaker +# head, streaming diarization (AOSC), and speaker-attributed ASR (SAS) merge. +# vllm.cpp's own ParakeetTranscriber handles ASR; this adds the diarization +# side through parakeet.cpp's C-API (parakeet_capi.h). +option(VLLM_CPP_WITH_DIARIZATION "Enable diarization support via parakeet.cpp" ON) +if(VLLM_CPP_WITH_DIARIZATION) + set(VLLM_CPP_PARAKEET_CPP_DIR "" CACHE PATH + "Path to a parakeet.cpp source tree. If empty, FetchContent from GitHub.") + if(VLLM_CPP_PARAKEET_CPP_DIR) + set(parakeet_cpp_SOURCE_DIR "${VLLM_CPP_PARAKEET_CPP_DIR}") + else() + include(FetchContent) + FetchContent_Declare(parakeet_cpp + GIT_REPOSITORY https://github.com/mudler/parakeet.cpp.git + GIT_TAG main + GIT_SHALLOW ON) + # parakeet.cpp builds its own ggml; we only need libparakeet.a and the + # headers. Suppress its tests/examples to keep the build lean. + set(PARAKEET_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(PARAKEET_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(parakeet_cpp) + endif() + target_include_directories(vllm PUBLIC + ${parakeet_cpp_SOURCE_DIR}/include) + target_link_libraries(vllm PUBLIC parakeet) + target_compile_definitions(vllm PUBLIC VLLM_WITH_DIARIZATION=1) +endif() if(WIN32) target_link_libraries(vllm PUBLIC ws2_32) endif() diff --git a/include/vllm.h b/include/vllm.h index de695437a8..7bc13f9259 100644 --- a/include/vllm.h +++ b/include/vllm.h @@ -369,7 +369,7 @@ extern "C" { * KevModel/LayaModel runs the decision forward, CuaS1Forms runs the score * forward. Non-matching architectures are refused by name. Every existing * struct and call is byte-identical. */ -#define VLLM_ABI_VERSION 29 +#define VLLM_ABI_VERSION 30 /* ── Export macro ───────────────────────────────────────────────────────────── * Marks the symbols that make up the stable ABI. Default visibility now; Task 3 @@ -1096,6 +1096,83 @@ VLLM_API vllm_status vllm_transcribe(vllm_engine* engine, VLLM_API void vllm_transcription_free(vllm_transcription* out); +/* ── Speaker diarization (ABI v30) ─────────────────────────────────────────── + * When the library is built with VLLM_CPP_WITH_DIARIZATION=ON (the default), + * a second engine handle can be loaded from a Nemotron-3-Diarization GGUF + * file. The diarization engine identifies who spoke when in a mono 16 kHz + * audio stream. It is independent of the ASR (Parakeet) engine — the two + * can be combined via vllm_transcribe_and_diarize. + * + * When VLLM_CPP_WITH_DIARIZATION=OFF, every function below returns + * VLLM_ERR_INVALID_ARGUMENT with a "not compiled in" message. */ + +/* One speaker segment. */ +typedef struct vllm_speaker_segment { + int32_t speaker; /* 0-indexed speaker ID */ + float start; /* seconds from audio start */ + float end; +} vllm_speaker_segment; + +/* Diarization result. OWNERSHIP: free with vllm_diarization_free. */ +typedef struct vllm_diarization { + vllm_speaker_segment* segments; + int32_t n_segments; +} vllm_diarization; + +/* Load a diarization GGUF file. Returns NULL on error + * (vllm_last_error carries the detail). */ +VLLM_API vllm_engine* vllm_diarization_load(const char* gguf_path); + +/* Diarize a WAV file. Returns VLLM_OK on success. */ +VLLM_API vllm_status vllm_diarize_path(vllm_engine* diar_engine, + const char* wav_path, + vllm_diarization* out); + +/* Diarize raw PCM (mono float32, 16 kHz). */ +VLLM_API vllm_status vllm_diarize_pcm(vllm_engine* diar_engine, + const float* pcm, int64_t n_samples, + int32_t sample_rate, + vllm_diarization* out); + +/* Free a diarization result. NULL is a no-op. */ +VLLM_API void vllm_diarization_free(vllm_diarization* out); + + +/* ── Speaker-attributed ASR (ABI v30) ─────────────────────────────────────── + * Combined transcription + diarization: runs both models on the same audio + * and merges word timestamps with speaker segments. The ASR engine must be + * a Parakeet checkpoint; the diarization engine must be a GGUF loaded with + * vllm_diarization_load. */ + +typedef struct vllm_speaker_utterance { + int32_t speaker; + char* text; + float start; + float end; + float conf; +} vllm_speaker_utterance; + +typedef struct vllm_sas_result { + vllm_speaker_utterance* utterances; + int32_t n_utterances; +} vllm_sas_result; + +/* Run combined ASR + diarization on a WAV file. */ +VLLM_API vllm_status vllm_transcribe_and_diarize( + vllm_engine* asr_engine, vllm_engine* diar_engine, + const char* wav_path, + vllm_sas_result* out); + +/* Run combined ASR + diarization on raw PCM. */ +VLLM_API vllm_status vllm_transcribe_and_diarize_pcm( + vllm_engine* asr_engine, vllm_engine* diar_engine, + const float* pcm, int64_t n_samples, int32_t sample_rate, + vllm_sas_result* out); + +/* Free a SAS result. Each utterance's .text is freed, then the array. */ +VLLM_API void vllm_sas_result_free(vllm_sas_result* out); + + /* ── Embeddings (ABI v15) ───────────────────────────────────────────────────── * The embeddings/pooling slice of the ONE-SURFACE fold: an engine loaded from * a POOLING (embedding) checkpoint — config.json architectures resolving to a diff --git a/include/vllm/entrypoints/openai/api_server.h b/include/vllm/entrypoints/openai/api_server.h index 52e7171b37..389756a271 100644 --- a/include/vllm/entrypoints/openai/api_server.h +++ b/include/vllm/entrypoints/openai/api_server.h @@ -40,6 +40,9 @@ #include "vllm/entrypoints/openai/video_api.h" #include "vllm/entrypoints/openai/speech_api.h" #include "vllm/multimodal/parakeet_transcription.h" +#ifdef VLLM_WITH_DIARIZATION +#include "vllm/multimodal/diarization.h" +#endif namespace vllm::tok { class Tokenizer; @@ -140,6 +143,12 @@ class ApiServer { // extracts the upload and calls this with the raw file bytes. DispatchResult handle_audio_transcriptions( const std::string& file_bytes, const std::string& response_format) const; +#ifdef VLLM_WITH_DIARIZATION + DispatchResult handle_audio_diarizations( + const std::string& file_bytes, const std::string& response_format) const; + DispatchResult handle_audio_sas( + const std::string& file_bytes, const std::string& response_format) const; +#endif // POST /v1/embeddings (ARCH-ONE-SURFACE ROW 6). Mirror of vLLM's // pooling/embed/api_router.py:28 `create_embedding` over the @@ -267,6 +276,27 @@ class ApiServer { transcriber_ = std::move(transcriber); } +#ifdef VLLM_WITH_DIARIZATION + // Attach the diarization seam backing POST /v1/audio/diarizations (ABI v30). + // ADDITIVE and OPT-IN: absent => route unregistered => 404, byte-identical to + // a server without diarization. The callback wraps the parakeet.cpp C-API. + using DiarizeFn = + std::function( + const uint8_t* wav_bytes, size_t num_bytes)>; + void set_diarizer(DiarizeFn diarizer) { + diarizer_ = std::move(diarizer); + } + + // Attach the SAS seam backing POST /v1/audio/sas (speaker-attributed ASR). + // Runs both ASR and diarization on the same audio and merges the results. + using SasFn = + std::function; + void set_sas(SasFn sas) { + sas_ = std::move(sas); + } +#endif + // Attach the embedding seam backing POST /v1/embeddings (ARCH-ONE-SURFACE // ROW 6). ADDITIVE and OPT-IN like the transcriber above: absent => route // unregistered => 404, byte-identical to a server without pooling. The @@ -402,6 +432,12 @@ class ApiServer { // or zero when the diagnostic legacy-dynamic mode is selected. size_t http_worker_count() const; + // Expose the diarizer and SAS callbacks for the route handlers. +#ifdef VLLM_WITH_DIARIZATION + DiarizeFn diarizer_callback() const { return diarizer_; } + SasFn sas_callback() const { return sas_; } +#endif + private: // Null in the serving-less (transcription-only) construction: the generate // routes are then not registered, and direct handler dispatch reports the @@ -415,6 +451,10 @@ class ApiServer { const v1::metrics::PrometheusStatLogger* metrics_ = nullptr; ::vllm::openai::VideoRunner video_runner_; TranscribeFn transcriber_; +#ifdef VLLM_WITH_DIARIZATION + DiarizeFn diarizer_; + SasFn sas_; +#endif EmbedFn embedder_; NerFn ner_; ScoreFn score_; diff --git a/include/vllm/multimodal/diarization.h b/include/vllm/multimodal/diarization.h new file mode 100644 index 0000000000..0741e5f3fc --- /dev/null +++ b/include/vllm/multimodal/diarization.h @@ -0,0 +1,90 @@ +// diarization.h — diarization seam wrapping parakeet.cpp's C-API +// +// VLLM_WITH_DIARIZATION gates the whole seam. When the parakeet.cpp +// dependency is absent (VLLM_CPP_WITH_DIARIZATION=OFF), the header +// is empty and every function is a no-op stub, so the rest of vllm.cpp +// compiles unchanged. +#pragma once + +#include +#include +#include +#include + +#ifdef VLLM_WITH_DIARIZATION +#include "parakeet_capi.h" +#endif + +namespace vllm::multimodal { + +// One speaker segment: who spoke, and when. +struct SpeakerSegment { + int speaker; // 0-indexed speaker ID + float start; // seconds from audio start + float end; +}; + +// One speaker-attributed utterance. +struct SpeakerUtterance { + int speaker; + std::string text; + float start; + float end; + float conf; +}; + +// A loaded diarization model (Nemotron-3-Diarization GGUF). +// Wraps parakeet_ctx from parakeet.cpp's C-API. +class Diarizer { +public: + // Load a diarization GGUF file. Returns nullptr on failure. + static std::unique_ptr FromFile(const std::string& path); + + ~Diarizer(); + + // Diarize a mono float32 waveform at `sample_rate`. + std::vector Diarize( + const float* pcm, int64_t n_samples, int sample_rate) const; + + // Diarize a WAV file path. + std::vector DiarizeWavFile(const std::string& path) const; + +#ifdef VLLM_WITH_DIARIZATION + // Access the underlying parakeet_ctx for streaming + SAS composition. + parakeet_ctx* ctx() const { return ctx_; } +#endif + +private: + Diarizer(); +#ifdef VLLM_WITH_DIARIZATION + parakeet_ctx* ctx_ = nullptr; +#endif +}; + +// Combined ASR + diarization: runs both models and merges the results. +// Takes an ASR engine (from vllm_engine_load with a Parakeet checkpoint) +// and a diarization model. The ASR path goes through the existing +// ParakeetTranscriber; the diarization path goes through parakeet.cpp's +// C-API. The merge uses parakeet.cpp's SAS merge layer. +struct SpeakerAttributedASR { + // The speaker-attributed utterances. + std::vector utterances; + // True if at least one utterance was produced. + bool has_result = false; +}; + +// Run combined ASR + diarization on a WAV file. +// `asr_dir` is a Parakeet checkpoint directory (HF format). +// `diar_gguf` is a diarization GGUF file path. +SpeakerAttributedASR TranscribeAndDiarize( + const std::string& wav_path, + const std::string& asr_dir, + const std::string& diar_gguf); + +// Run combined ASR + diarization on raw PCM. +SpeakerAttributedASR TranscribeAndDiarizePCM( + const float* pcm, int64_t n_samples, int sample_rate, + const std::string& asr_dir, + const std::string& diar_gguf); + +} // namespace vllm::multimodal diff --git a/src/capi/vllm_c.cpp b/src/capi/vllm_c.cpp index d73b11a266..9891a4a37c 100644 --- a/src/capi/vllm_c.cpp +++ b/src/capi/vllm_c.cpp @@ -57,6 +57,10 @@ #include "vllm/entrypoints/openai/systemone.h" // shared SystemOne helpers (v28) #include "vllm/model_executor/models/minimax_h3.h" // mux argv (v12) #include "vllm/multimodal/parakeet_transcription.h" // vllm_transcribe (v11) +#ifdef VLLM_WITH_DIARIZATION +#include "parakeet_capi.h" // parakeet_ctx, parakeet_capi_* (v30) +#include "vllm/multimodal/diarization.h" // Diarizer, TranscribeAndDiarize (v30) +#endif #include "vllm/multimodal/minimax_h3_video.h" // vllm_video_* (v12) #include "vllm/multimodal/video_engine.h" // the v18 family registry #include "vllm/multimodal/speech_engine.h" // vllm_speech_* (v20) @@ -80,6 +84,13 @@ struct vllm_engine { // ABI v11 transcription stack (the ONE library seam the server route and the // parakeet-transcribe example also drive). Null for text engines. std::unique_ptr transcriber; +#ifdef VLLM_WITH_DIARIZATION + // ABI v30 diarization: a parakeet_ctx loaded from a diarization GGUF. + parakeet_ctx* diarizer_ctx = nullptr; + // A parakeet_ctx for the ASR model (for SAS composition). + parakeet_ctx* parakeet_asr_ctx = nullptr; + std::unique_ptr diarizer; +#endif // Monotonic per-handle request-id source. Each vllm_complete[_stream] call // uses a FRESH id so a request left in-flight by a mid-call exception can never // collide with a later call's id — a collision would make LLMEngine.add_request @@ -824,6 +835,10 @@ VLLM_API vllm_status vllm_engine_load(const vllm_model_params* params, vllm::multimodal::ParakeetTranscriber::FromDir( params->model_path)); handle->model_path = params->model_path; +#ifdef VLLM_WITH_DIARIZATION + // Also load a parakeet_ctx for SAS composition + handle->parakeet_asr_ctx = parakeet_capi_load(params->model_path); +#endif *out = handle; ClearError(); return VLLM_OK; @@ -853,7 +868,14 @@ VLLM_API vllm_status vllm_engine_load(const vllm_model_params* params, } } -VLLM_API void vllm_engine_free(vllm_engine* engine) { delete engine; } +VLLM_API void vllm_engine_free(vllm_engine* engine) { + if (engine == nullptr) return; +#ifdef VLLM_WITH_DIARIZATION + if (engine->diarizer_ctx) parakeet_capi_free(engine->diarizer_ctx); + if (engine->parakeet_asr_ctx) parakeet_capi_free(engine->parakeet_asr_ctx); +#endif + delete engine; +} // ABI v25 (row `SPEC-DFLASH2`, issue #2832): the engine's own speculative // acceptance counters, read back. THIS FUNCTION COMPUTES NOTHING. All three @@ -1439,6 +1461,313 @@ VLLM_API void vllm_transcription_free(vllm_transcription* out) { out->has_text = 0; } +// ── Speaker diarization (ABI v30) ────────────────────────────────────────── + +// ── Speaker diarization (ABI v30) ────────────────────────────────────────── + +VLLM_API vllm_engine* vllm_diarization_load(const char* gguf_path) { + if (gguf_path == nullptr) { + SetError("vllm_diarization_load: gguf_path is null"); + return nullptr; + } +#ifdef VLLM_WITH_DIARIZATION + parakeet_ctx* diar_ctx = parakeet_capi_load(gguf_path); + if (diar_ctx == nullptr) { + SetError(std::string("vllm_diarization_load: parakeet_capi_load failed: ") + + gguf_path); + return nullptr; + } + auto* handle = new vllm_engine; + handle->diarizer_ctx = diar_ctx; + handle->model_path = gguf_path; + ClearError(); + return handle; +#else + SetError("vllm_diarization_load: diarization not compiled in"); + return nullptr; +#endif +} + +#ifdef VLLM_WITH_DIARIZATION +// Parse parakeet.cpp diarization JSON into vllm_speaker_segment array. +// JSON format: {"segments": [{"speaker": N, "start": S, "end": E}, ...]} +static vllm_status ParseDiarizationJson( + const char* json_str, vllm_diarization* out) { + if (!json_str) return VLLM_ERR_RUNTIME; + auto j = nlohmann::json::parse(json_str); + int n = 0; + if (j.contains("segments")) n = j["segments"].size(); + auto* segs = static_cast( + std::malloc(n == 0 ? 1 : n * sizeof(vllm_speaker_segment))); + if (segs == nullptr) return VLLM_ERR_RUNTIME; + for (int i = 0; i < n; ++i) { + segs[i].speaker = j["segments"][i].value("speaker", -1); + segs[i].start = j["segments"][i].value("start", 0.0f); + segs[i].end = j["segments"][i].value("end", 0.0f); + } + out->segments = segs; + out->n_segments = n; + return VLLM_OK; +} +#endif + +VLLM_API vllm_status vllm_diarize_path(vllm_engine* diar_engine, + const char* wav_path, + vllm_diarization* out) { + if (out == nullptr) { + SetError("vllm_diarize_path: out is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + out->segments = nullptr; + out->n_segments = 0; + if (diar_engine == nullptr || wav_path == nullptr) { + SetError("vllm_diarize_path: engine or wav_path is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } +#ifdef VLLM_WITH_DIARIZATION + if (diar_engine->diarizer_ctx == nullptr) { + SetError("vllm_diarize_path: engine is not a diarization engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + char* json = parakeet_capi_diarize_path( + diar_engine->diarizer_ctx, wav_path); + if (json == nullptr) { + SetError("vllm_diarize_path: diarize returned null"); + return VLLM_ERR_RUNTIME; + } + auto status = ParseDiarizationJson(json, out); + parakeet_capi_free_string(json); + if (status != VLLM_OK) { + SetError("vllm_diarize_path: failed to parse diarization JSON"); + return status; + } + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_diarize_path: ") + e.what()); + return VLLM_ERR_RUNTIME; + } +#else + SetError("vllm_diarize_path: diarization not compiled in"); + return VLLM_ERR_INVALID_ARGUMENT; +#endif +} + +VLLM_API vllm_status vllm_diarize_pcm(vllm_engine* diar_engine, + const float* pcm, int64_t n_samples, + int32_t sample_rate, + vllm_diarization* out) { + (void)sample_rate; // used only in the VLLM_WITH_DIARIZATION path + if (out == nullptr) { + SetError("vllm_diarize_pcm: out is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + out->segments = nullptr; + out->n_segments = 0; + if (diar_engine == nullptr || pcm == nullptr || n_samples <= 0) { + SetError("vllm_diarize_pcm: invalid arguments"); + return VLLM_ERR_INVALID_ARGUMENT; + } +#ifdef VLLM_WITH_DIARIZATION + if (diar_engine->diarizer_ctx == nullptr) { + SetError("vllm_diarize_pcm: engine is not a diarization engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + char* json = parakeet_capi_diarize_pcm( + diar_engine->diarizer_ctx, pcm, (int)n_samples, sample_rate); + if (json == nullptr) { + SetError("vllm_diarize_pcm: diarize returned null"); + return VLLM_ERR_RUNTIME; + } + auto status = ParseDiarizationJson(json, out); + parakeet_capi_free_string(json); + if (status != VLLM_OK) { + SetError("vllm_diarize_pcm: failed to parse diarization JSON"); + return status; + } + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_diarize_pcm: ") + e.what()); + return VLLM_ERR_RUNTIME; + } +#else + SetError("vllm_diarize_pcm: diarization not compiled in"); + return VLLM_ERR_INVALID_ARGUMENT; +#endif +} + +VLLM_API void vllm_diarization_free(vllm_diarization* out) { + if (out == nullptr) return; + std::free(out->segments); + out->segments = nullptr; + out->n_segments = 0; +} + +// ── Speaker-attributed ASR (ABI v30) ──────────────────────────────────────── + +VLLM_API vllm_status vllm_transcribe_and_diarize( + vllm_engine* asr_engine, vllm_engine* diar_engine, + const char* wav_path, + vllm_sas_result* out) { + if (out == nullptr) { + SetError("vllm_transcribe_and_diarize: out is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + out->utterances = nullptr; + out->n_utterances = 0; + if (asr_engine == nullptr || diar_engine == nullptr || wav_path == nullptr) { + SetError("vllm_transcribe_and_diarize: null argument"); + return VLLM_ERR_INVALID_ARGUMENT; + } +#ifdef VLLM_WITH_DIARIZATION + if (asr_engine->transcriber == nullptr) { + SetError("vllm_transcribe_and_diarize: asr_engine is not a transcription engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + if (diar_engine->diarizer_ctx == nullptr) { + SetError("vllm_transcribe_and_diarize: diar_engine is not a diarization engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + // Read WAV into PCM inline (ReadWavPcm16Mono is static in diarization.cpp) + std::vector pcm; + { + FILE* f = std::fopen(wav_path, "rb"); + if (!f) { + SetError("vllm_transcribe_and_diarize: cannot open WAV"); + return VLLM_ERR_RUNTIME; + } + char hdr[44]; + if (std::fread(hdr, 1, 44, f) != 44) { + std::fclose(f); + SetError("vllm_transcribe_and_diarize: WAV too short"); + return VLLM_ERR_RUNTIME; + } + std::fseek(f, 44, SEEK_SET); + int16_t sample; + while (std::fread(&sample, 2, 1, f) == 1) + pcm.push_back(static_cast(sample) / 32768.0f); + std::fclose(f); + } + int n_sas = 0; + parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( + asr_engine->parakeet_asr_ctx, diar_engine->diarizer_ctx, + pcm.data(), (int)pcm.size(), 16000, &n_sas); + if (sas == nullptr || n_sas == 0) { + ClearError(); + return VLLM_OK; + } + auto* utts = static_cast( + std::malloc(n_sas * sizeof(vllm_speaker_utterance))); + if (utts == nullptr) { + for (int i = 0; i < n_sas; ++i) + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + parakeet_capi_free_sas_results(sas); + SetError("vllm_transcribe_and_diarize: out-of-memory"); + return VLLM_ERR_RUNTIME; + } + for (int i = 0; i < n_sas; ++i) { + utts[i].speaker = sas[i].speaker; + utts[i].text = sas[i].text ? DupString(sas[i].text) : nullptr; + utts[i].start = sas[i].start; + utts[i].end = sas[i].end; + utts[i].conf = sas[i].conf; + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + } + parakeet_capi_free_sas_results(sas); + out->utterances = utts; + out->n_utterances = n_sas; + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_transcribe_and_diarize: ") + e.what()); + return VLLM_ERR_RUNTIME; + } +#else + SetError("vllm_transcribe_and_diarize: diarization not compiled in"); + return VLLM_ERR_INVALID_ARGUMENT; +#endif +} + +VLLM_API vllm_status vllm_transcribe_and_diarize_pcm( + vllm_engine* asr_engine, vllm_engine* diar_engine, + const float* pcm, int64_t n_samples, int32_t sample_rate, + vllm_sas_result* out) { + (void)sample_rate; // used only in the VLLM_WITH_DIARIZATION path + if (out == nullptr) { + SetError("vllm_transcribe_and_diarize_pcm: out is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + out->utterances = nullptr; + out->n_utterances = 0; + if (asr_engine == nullptr || diar_engine == nullptr || + pcm == nullptr || n_samples <= 0) { + SetError("vllm_transcribe_and_diarize_pcm: invalid arguments"); + return VLLM_ERR_INVALID_ARGUMENT; + } +#ifdef VLLM_WITH_DIARIZATION + if (asr_engine->transcriber == nullptr) { + SetError("vllm_transcribe_and_diarize_pcm: asr_engine is not a transcription engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + if (diar_engine->diarizer_ctx == nullptr) { + SetError("vllm_transcribe_and_diarize_pcm: diar_engine is not a diarization engine"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + int n_sas = 0; + parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( + asr_engine->parakeet_asr_ctx, diar_engine->diarizer_ctx, + pcm, (int)n_samples, sample_rate, &n_sas); + if (sas == nullptr || n_sas == 0) { + ClearError(); + return VLLM_OK; + } + auto* utts = static_cast( + std::malloc(n_sas * sizeof(vllm_speaker_utterance))); + if (utts == nullptr) { + for (int i = 0; i < n_sas; ++i) + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + parakeet_capi_free_sas_results(sas); + SetError("vllm_transcribe_and_diarize_pcm: out-of-memory"); + return VLLM_ERR_RUNTIME; + } + for (int i = 0; i < n_sas; ++i) { + utts[i].speaker = sas[i].speaker; + utts[i].text = sas[i].text ? DupString(sas[i].text) : nullptr; + utts[i].start = sas[i].start; + utts[i].end = sas[i].end; + utts[i].conf = sas[i].conf; + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + } + parakeet_capi_free_sas_results(sas); + out->utterances = utts; + out->n_utterances = n_sas; + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_transcribe_and_diarize_pcm: ") + e.what()); + return VLLM_ERR_RUNTIME; + } +#else + SetError("vllm_transcribe_and_diarize_pcm: diarization not compiled in"); + return VLLM_ERR_INVALID_ARGUMENT; +#endif +} + +VLLM_API void vllm_sas_result_free(vllm_sas_result* out) { + if (out == nullptr) return; + for (int i = 0; i < out->n_utterances; ++i) { + std::free(out->utterances[i].text); + } + std::free(out->utterances); + out->utterances = nullptr; + out->n_utterances = 0; +} + // ── Embeddings (ABI v15, ARCH-ONE-SURFACE ROW 6) ──────────────────────────── // The pooling slice of the ONE surface: the SAME registry forward + // PoolingRunner engine step the server's /v1/embeddings drives diff --git a/src/vllm/entrypoints/openai/api_server.cpp b/src/vllm/entrypoints/openai/api_server.cpp index fec6d0426d..dd83975f88 100644 --- a/src/vllm/entrypoints/openai/api_server.cpp +++ b/src/vllm/entrypoints/openai/api_server.cpp @@ -38,6 +38,88 @@ namespace vllm::entrypoints::openai { +namespace { + +// Build the OpenAI ErrorResponse JSON body for a failed request +// (serve/utils/error_response.py::create_error_response). `code` == the HTTP +// status code (upstream ErrorInfo.code carries it). +ApiServer::DispatchResult MakeError(int status, const std::string& type, + const std::string& message) { + ErrorResponse err; + err.error.message = message; + err.error.type = type; + err.error.code = status; + ApiServer::DispatchResult r; + r.status = status; + r.content_type = "application/json"; + r.body = nlohmann::json(err).dump(); + return r; +} + +} // namespace + +#ifdef VLLM_WITH_DIARIZATION +static ApiServer::DispatchResult HandleAudioDiarizations( + const ApiServer& server, + const std::string& file_bytes, + const std::string& /*response_format*/) { + auto diarizer = server.diarizer_callback(); + if (!diarizer) { + return MakeError(404, "NotFoundError", "diarization not enabled"); + } + try { + auto segs = diarizer(reinterpret_cast(file_bytes.data()), + file_bytes.size()); + nlohmann::json j = nlohmann::json::array(); + for (const auto& s : segs) { + j.push_back({{"speaker", s.speaker}, + {"start", s.start}, + {"end", s.end}}); + } + ApiServer::DispatchResult r; + r.status = 200; + r.content_type = "application/json"; + r.body = j.dump(); + return r; + } catch (const std::exception& e) { + return MakeError(500, "InternalServerError", e.what()); + } +} + +static ApiServer::DispatchResult HandleAudioSas( + const ApiServer& server, + const std::string& file_bytes, + const std::string& /*response_format*/) { + auto sas_fn = server.sas_callback(); + if (!sas_fn) { + return MakeError(404, "NotFoundError", "speaker-attributed ASR not enabled"); + } + try { + auto result = sas_fn(reinterpret_cast(file_bytes.data()), + file_bytes.size()); + nlohmann::json j; + j["segments"] = nlohmann::json::array(); + for (const auto& u : result.utterances) { + j["segments"].push_back({ + {"speaker", u.speaker}, + {"text", u.text}, + {"start", u.start}, + {"end", u.end}, + {"confidence", u.conf} + }); + } + ApiServer::DispatchResult r; + r.status = 200; + r.content_type = "application/json"; + r.body = j.dump(); + return r; + } catch (const std::exception& e) { + return MakeError(500, "InternalServerError", e.what()); + } +} +#endif + + // SystemOne helpers (ParseSystemOneBody, BuildSystemOneAnswer*, R2, R4, etc.) // are defined in systemone.h/.cpp. Imported here so the handlers below can // call them unqualified, matching the former anonymous-namespace usage. @@ -111,26 +193,6 @@ struct ApiServer::Impl { std::mutex legacy_engine_mutex; }; -namespace { - -// Build the OpenAI ErrorResponse JSON body for a failed request -// (serve/utils/error_response.py::create_error_response). `code` == the HTTP -// status code (upstream ErrorInfo.code carries it). -ApiServer::DispatchResult MakeError(int status, const std::string& type, - const std::string& message) { - ErrorResponse err; - err.error.message = message; - err.error.type = type; - err.error.code = status; - ApiServer::DispatchResult r; - r.status = status; - r.content_type = "application/json"; - r.body = nlohmann::json(err).dump(); - return r; -} - -} // namespace - ApiServer::ApiServer(OpenAIServingCompletion& completion, OpenAIServingChat& chat, OpenAIServingModels& models, std::string version, size_t max_concurrent_streams, @@ -505,6 +567,72 @@ ApiServer::DispatchResult ApiServer::handle_audio_transcriptions( } } +// ── Speaker diarization handler (ABI v30) ──────────────────────────────── +#ifdef VLLM_WITH_DIARIZATION +ApiServer::DispatchResult ApiServer::handle_audio_diarizations( + const std::string& file_bytes, + const std::string& /*response_format*/) const { + if (!diarizer_) { + return MakeError(500, "InternalServerError", + "The model does not support Diarization API"); + } + if (file_bytes.empty()) { + return MakeError(400, "BadRequestError", + "Expected a non-empty `file` upload (16-bit PCM mono RIFF/WAVE)"); + } + try { + auto segs = diarizer_( + reinterpret_cast(file_bytes.data()), file_bytes.size()); + nlohmann::json j = nlohmann::json::array(); + for (const auto& s : segs) { + j.push_back({{"speaker", s.speaker}, + {"start", s.start}, + {"end", s.end}}); + } + DispatchResult r; + r.content_type = "application/json"; + r.body = j.dump(); + return r; + } catch (const std::exception& e) { + return MakeError(400, "BadRequestError", e.what()); + } +} + +ApiServer::DispatchResult ApiServer::handle_audio_sas( + const std::string& file_bytes, + const std::string& /*response_format*/) const { + if (!sas_) { + return MakeError(500, "InternalServerError", + "The model does not support Speaker-Attributed ASR API"); + } + if (file_bytes.empty()) { + return MakeError(400, "BadRequestError", + "Expected a non-empty `file` upload (16-bit PCM mono RIFF/WAVE)"); + } + try { + auto result = sas_( + reinterpret_cast(file_bytes.data()), file_bytes.size()); + nlohmann::json j; + j["segments"] = nlohmann::json::array(); + for (const auto& u : result.utterances) { + j["segments"].push_back({ + {"speaker", u.speaker}, + {"text", u.text}, + {"start", u.start}, + {"end", u.end}, + {"confidence", u.conf} + }); + } + DispatchResult r; + r.content_type = "application/json"; + r.body = j.dump(); + return r; + } catch (const std::exception& e) { + return MakeError(400, "BadRequestError", e.what()); + } +} +#endif + ApiServer::DispatchResult ApiServer::handle_embeddings( const std::string& request_body) const { // Mirror of vLLM pooling/embed/api_router.py:28 `create_embedding` over the @@ -1697,6 +1825,44 @@ void ApiServer::register_routes() { }); } +#ifdef VLLM_WITH_DIARIZATION + if (diarizer_) { + server.Post("/v1/audio/diarizations", + [this, write](const httplib::Request& req, + httplib::Response& res) { + if (!req.form.has_file("file")) { + write(MakeError(400, "BadRequestError", + "multipart/form-data with a `file` upload " + "is required"), + res); + return; + } + write(HandleAudioDiarizations(*this, + req.form.get_file("file").content, + req.form.get_field("response_format")), + res); + }); + } + + if (sas_) { + server.Post("/v1/audio/sas", + [this, write](const httplib::Request& req, + httplib::Response& res) { + if (!req.form.has_file("file")) { + write(MakeError(400, "BadRequestError", + "multipart/form-data with a `file` upload " + "is required"), + res); + return; + } + write(HandleAudioSas(*this, + req.form.get_file("file").content, + req.form.get_field("response_format")), + res); + }); + } +#endif + if (synthesizer_) { // Speech + music (W6 of #672). Registered ONLY when a synthesizer is // attached, so a text server answers 404 exactly as before. The body is diff --git a/src/vllm/model_executor/models/gliner2_weights.cpp b/src/vllm/model_executor/models/gliner2_weights.cpp index e48fdbdc19..f92b904b51 100644 --- a/src/vllm/model_executor/models/gliner2_weights.cpp +++ b/src/vllm/model_executor/models/gliner2_weights.cpp @@ -75,8 +75,10 @@ deberta_v2::Params InferEncoderParams( const auto& rel_shape = tensors.Shape("encoder.encoder.rel_embeddings.weight"); p.position_buckets = rel_shape[0] / 2; - // Fields the checkpoint does not carry — mdeberta-v3-base defaults. - p.num_attention_heads = 12; + // Fields the checkpoint does not carry. DeBERTa-v3 always uses head_dim 64, + // so num_attention_heads = hidden_size / 64. This covers mdeberta-v3-base + // (768/12) and deberta-v3-large (1024/16). + p.num_attention_heads = p.hidden_size / 64; p.max_position_embeddings = 512; p.layer_norm_eps = 1e-7; p.position_biased_input = false; @@ -140,11 +142,16 @@ Gliner2ModelWeights LoadGliner2Weights( gliner2::BoundaryParams bnd_params = ParseBoundaryParams(config); bnd_params.hidden_size = enc_params.hidden_size; - // Load encoder + boundary head weights via the existing host reference - // loaders, which look up tensors by name and validate shapes. + // Load encoder weights via the host reference loader. deberta_v2::Weights enc_weights = deberta_v2::Load(enc_params, tensors); - gliner2::BoundaryHeadWeights bnd_weights = - gliner2::LoadBoundaryHead(bnd_params, tensors); + + // Load boundary head weights only if the checkpoint contains + // boundary_head tensors. The GLiNER2.5-Decide checkpoint does not + // include a boundary head (it uses the classification head instead). + gliner2::BoundaryHeadWeights bnd_weights; + if (tensors.Has("boundary_head.boundary_encoder.left_projection.weight")) { + bnd_weights = gliner2::LoadBoundaryHead(bnd_params, tensors); + } return Gliner2ModelWeights{ std::move(enc_params), std::move(enc_weights), diff --git a/src/vllm/multimodal/diarization.cpp b/src/vllm/multimodal/diarization.cpp new file mode 100644 index 0000000000..57f5afcfb2 --- /dev/null +++ b/src/vllm/multimodal/diarization.cpp @@ -0,0 +1,244 @@ +// diarization.cpp — diarization seam wrapping parakeet.cpp's C-API +// +// parakeet.cpp's offline diarization API returns a JSON string (char*), +// not a struct. We parse it here to extract speaker segments. +#include "vllm/multimodal/diarization.h" + +#ifdef VLLM_WITH_DIARIZATION +#include "vllm/multimodal/parakeet_transcription.h" +#include +#endif + +#include +#include +#include +#include + +#ifdef VLLM_WITH_DIARIZATION + +namespace vllm::multimodal { + +// --- WAV reading helper --- +// Reads a 16-bit PCM mono WAV into float32 samples. +static std::vector ReadWavPcm16Mono(const std::string& path) { + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) throw std::runtime_error("cannot open WAV: " + path); + char hdr[44]; + if (std::fread(hdr, 1, 44, f) != 44) { + std::fclose(f); + throw std::runtime_error("WAV too short: " + path); + } + if (std::memcmp(hdr, "RIFF", 4) != 0 || std::memcmp(hdr + 8, "WAVE", 4) != 0) { + std::fclose(f); + throw std::runtime_error("not a RIFF/WAVE file: " + path); + } + std::fseek(f, 44, SEEK_SET); + std::vector pcm16; + int16_t sample; + while (std::fread(&sample, 2, 1, f) == 1) + pcm16.push_back(sample); + std::fclose(f); + std::vector pcm(pcm16.size()); + for (size_t i = 0; i < pcm16.size(); ++i) + pcm[i] = static_cast(pcm16[i]) / 32768.0f; + return pcm; +} + +// Parse the JSON string returned by parakeet_capi_diarize_path / _pcm. +// Format: {"segments": [{"speaker": N, "start": S, "end": E}, ...]} +static std::vector ParseDiarizationJson(const char* json_str) { + std::vector segs; + if (!json_str) return segs; + auto j = nlohmann::json::parse(json_str); + if (j.contains("segments")) { + for (const auto& s : j["segments"]) { + SpeakerSegment seg; + seg.speaker = s.value("speaker", -1); + seg.start = s.value("start", 0.0f); + seg.end = s.value("end", 0.0f); + segs.push_back(seg); + } + } + return segs; +} + +// --- Diarizer --- + +Diarizer::Diarizer() = default; + +Diarizer::~Diarizer() { + if (ctx_) { + parakeet_capi_free(ctx_); + ctx_ = nullptr; + } +} + +std::unique_ptr Diarizer::FromFile(const std::string& path) { + auto d = std::unique_ptr(new Diarizer()); + d->ctx_ = parakeet_capi_load(path.c_str()); + if (!d->ctx_) { + throw std::runtime_error("Diarizer::FromFile: parakeet_capi_load failed: " + path); + } + return d; +} + +std::vector Diarizer::Diarize( + const float* pcm, int64_t n_samples, int sample_rate) const { + if (!ctx_) throw std::runtime_error("Diarizer: no model loaded"); + + char* json = parakeet_capi_diarize_pcm( + ctx_, pcm, (int)n_samples, sample_rate); + if (!json) throw std::runtime_error("Diarizer: diarize_pcm failed"); + + auto segs = ParseDiarizationJson(json); + parakeet_capi_free_string(json); + return segs; +} + +std::vector Diarizer::DiarizeWavFile(const std::string& path) const { + if (!ctx_) throw std::runtime_error("Diarizer: no model loaded"); + + char* json = parakeet_capi_diarize_path(ctx_, path.c_str()); + if (!json) throw std::runtime_error("Diarizer: diarize_path failed"); + + auto segs = ParseDiarizationJson(json); + parakeet_capi_free_string(json); + return segs; +} + +// --- Combined ASR + diarization --- + +SpeakerAttributedASR TranscribeAndDiarize( + const std::string& wav_path, + const std::string& asr_dir, + const std::string& diar_gguf) { + SpeakerAttributedASR result; + + ParakeetTranscriber asr = ParakeetTranscriber::FromDir(asr_dir); + auto diar = Diarizer::FromFile(diar_gguf); + + ParakeetTranscription trans = asr.TranscribeWavFile(wav_path); + if (!trans.has_text) { + result.has_result = false; + return result; + } + + auto pcm = ReadWavPcm16Mono(wav_path); + + parakeet_ctx* asr_ctx = parakeet_capi_load(asr_dir.c_str()); + if (!asr_ctx) { + result.utterances.push_back({-1, trans.text, 0.0f, 0.0f, 0.0f}); + result.has_result = true; + return result; + } + + int n_sas = 0; + parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( + asr_ctx, diar->ctx(), pcm.data(), (int)pcm.size(), 16000, &n_sas); + + if (sas && n_sas > 0) { + for (int i = 0; i < n_sas; ++i) { + SpeakerUtterance u; + u.speaker = sas[i].speaker; + u.text = sas[i].text ? sas[i].text : ""; + u.start = sas[i].start; + u.end = sas[i].end; + u.conf = sas[i].conf; + result.utterances.push_back(u); + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + } + parakeet_capi_free_sas_results(sas); + result.has_result = true; + } else { + result.has_result = false; + } + + parakeet_capi_free(asr_ctx); + return result; +} + +SpeakerAttributedASR TranscribeAndDiarizePCM( + const float* pcm, int64_t n_samples, int sample_rate, + const std::string& asr_dir, + const std::string& diar_gguf) { + SpeakerAttributedASR result; + + ParakeetTranscriber asr = ParakeetTranscriber::FromDir(asr_dir); + auto diar = Diarizer::FromFile(diar_gguf); + + ParakeetTranscription trans = asr.Transcribe(pcm, n_samples, sample_rate); + if (!trans.has_text) { + result.has_result = false; + return result; + } + + parakeet_ctx* asr_ctx = parakeet_capi_load(asr_dir.c_str()); + if (!asr_ctx) { + result.utterances.push_back({-1, trans.text, 0.0f, 0.0f, 0.0f}); + result.has_result = true; + return result; + } + + int n_sas = 0; + parakeet_sas_result* sas = parakeet_capi_transcribe_and_diarize( + asr_ctx, diar->ctx(), pcm, (int)n_samples, sample_rate, &n_sas); + + if (sas && n_sas > 0) { + for (int i = 0; i < n_sas; ++i) { + SpeakerUtterance u; + u.speaker = sas[i].speaker; + u.text = sas[i].text ? sas[i].text : ""; + u.start = sas[i].start; + u.end = sas[i].end; + u.conf = sas[i].conf; + result.utterances.push_back(u); + if (sas[i].text) parakeet_capi_free_string(sas[i].text); + } + parakeet_capi_free_sas_results(sas); + result.has_result = true; + } else { + result.has_result = false; + } + + parakeet_capi_free(asr_ctx); + return result; +} + +} // namespace vllm::multimodal + +#else // !VLLM_WITH_DIARIZATION + +namespace vllm::multimodal { + +std::unique_ptr Diarizer::FromFile(const std::string&) { + throw std::runtime_error("Diarizer: diarization support not compiled in"); +} + +std::vector Diarizer::Diarize(const float*, int64_t, int) const { + throw std::runtime_error("Diarizer: diarization support not compiled in"); +} + +std::vector Diarizer::DiarizeWavFile(const std::string&) const { + throw std::runtime_error("Diarizer: diarization support not compiled in"); +} + +Diarizer::Diarizer() = default; +Diarizer::~Diarizer() = default; + +SpeakerAttributedASR TranscribeAndDiarize( + const std::string&, const std::string&, const std::string&) { + SpeakerAttributedASR r; + r.has_result = false; + return r; +} + +SpeakerAttributedASR TranscribeAndDiarizePCM( + const float*, int64_t, int, const std::string&, const std::string&) { + SpeakerAttributedASR r; + r.has_result = false; + return r; +} + +} // namespace vllm::multimodal + +#endif // VLLM_WITH_DIARIZATION