From f0e22dd387f23fde671a09787521d77a30fc4cb8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 15:42:08 +0000 Subject: [PATCH 01/30] feat(parakeet-cpp): load diarization and CED models and companions Repin PARAKEET_VERSION to parakeet.cpp PR #75's head, which adds parakeet_capi_model_kind (ABI v8). Bind the new diarization, sound event and combined scene stream C symbols through the same purego.Dlsym probe pattern already used for the batched JSON entry point, so the backend still loads against an older libparakeet.so. Load now classifies the loaded GGUF by role (ASR, diarization or sound) via parakeet_capi_model_kind and can load up to two companion models from Options[] (asr_model:, diarization_model:, sound_model:, paths resolved against opts.ModelPath), verifying each companion's kind and freeing every context opened so far on any failure. Free releases the primary and every companion. AudioTranscription now names the loaded role when it is not ASR instead of a generic model not loaded error. The dynamic batcher starts only when an ASR context ends up loaded, primary or companion. This is groundwork only: the Diarize and SoundDetection RPCs and the live scene stream that actually use these new roles land in later commits. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/Makefile | 5 +- backend/go/parakeet-cpp/goparakeetcpp.go | 106 ++++++-- backend/go/parakeet-cpp/main.go | 28 +++ backend/go/parakeet-cpp/roles.go | 201 +++++++++++++++ backend/go/parakeet-cpp/roles_test.go | 304 +++++++++++++++++++++++ 5 files changed, 625 insertions(+), 19 deletions(-) create mode 100644 backend/go/parakeet-cpp/roles.go create mode 100644 backend/go/parakeet-cpp/roles_test.go diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile index e288f6fcc648..4c393f386b35 100644 --- a/backend/go/parakeet-cpp/Makefile +++ b/backend/go/parakeet-cpp/Makefile @@ -15,7 +15,10 @@ # That's what the L0 smoke test uses. The default target below does the # proper clone-at-pin + cmake build so CI doesn't need a side-checkout. -PARAKEET_VERSION?=2bf88954dc628b32835734e2e9159550a75a1dc6 +# Tracks mudler/parakeet.cpp PR #75 (diarization/CED model roles + sound and +# scene streaming, ABI v8) until it merges to master; repin to the merge +# commit once it lands. +PARAKEET_VERSION?=c018537fbe0bdd3ccb3ecb387a2119bbf901e90d PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp GOCMD?=go diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index 23e5c548ce0c..9b699433c770 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -74,8 +74,55 @@ var ( // libparakeet.so; nil falls back to the text-only CppStreamFeed/Finalize path. CppStreamFeedJSON func(s uintptr, pcm []float32, nSamples int32) uintptr CppStreamFinalizeJSON func(s uintptr) uintptr + + // CppModelKind reports which kind of model a loaded context holds + // (parakeet_capi_model_kind, ABI v8): see the modelKind* constants in + // roles.go. nil on an older libparakeet.so; Load then treats the primary + // as ASR (pre-v8 behavior) and rejects companion model options. + CppModelKind func(ctx uintptr) int32 + + // Speaker diarization (ABI v7). CppDiarizePCM runs offline diarization + // over in-memory mono float PCM; CppTranscribeAndDiarizeJSON pairs it with + // an ASR context for speaker-attributed text. Both return a malloc'd char* + // JSON document (uintptr, freed via CppFreeString). + CppDiarizePCM func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr + CppTranscribeAndDiarizeJSON func(asr, diar uintptr, samples *float32, n int32, sampleRate int32) uintptr + + // Sound-event detection (CED) and the combined scene stream (ABI v8). + // CppNumClasses/CppSoundOptsDefault/CppSoundStreamBegin.../ + // CppSceneOptsDefault/CppSceneStreamBegin... are only registered when + // CppModelKind is present (see main.go); nil otherwise. + CppNumClasses func(ctx uintptr) int32 + CppSoundOptsDefault func(o *cSoundOpts) + CppSoundStreamBegin func(tagger uintptr, o *cSoundOpts) uintptr + CppSoundStreamFeed func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 + CppSoundStreamDrainScoresJSON func(s uintptr) uintptr + CppFreeSoundSegments func(segs uintptr) + CppSoundStreamFree func(s uintptr) + CppSceneOptsDefault func(o *cSceneOpts) + CppSceneStreamBegin func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr + CppSceneStreamFeedJSON func(s uintptr, pcm *float32, n int32, isLast int32) uintptr + CppSceneStreamLastError func(s uintptr) string + CppSceneStreamFree func(s uintptr) ) +// cSoundOpts and cSceneOpts mirror parakeet_sound_opts / parakeet_scene_opts +// in parakeet_capi.h field-for-field (int -> int32, float -> float32); the +// C side sizes/versions them via the leading `size` field, set by the +// matching *_opts_default call. +type cSoundOpts struct { + Size int32 + WindowSec, HopSec, OnThreshold, OffThreshold, MinDurationSec float32 + TopK int32 +} + +type cSceneOpts struct { + Size int32 + DiarLatency int32 + Sound cSoundOpts + Flags int32 +} + // streamChunkSamples is how much 16 kHz mono PCM we hand to stream_feed per // call (1 s). The session buffers internally and decodes once a full // cache-aware encoder chunk is available, so this only bounds how often we @@ -140,10 +187,23 @@ type transcriptToken struct { // touch it concurrently. type ParakeetCpp struct { base.Base - ctxPtr uintptr - engineMu sync.Mutex // sole guard of the one C engine (dispatcher + streaming) - bat *batcher - batStop chan struct{} + ctxPtr uintptr // ASR context: the primary when it is an ASR model, or the asr_model companion + // diarCtx / tagCtx are the diarization and sound (CED) model contexts: + // the primary when it is that kind, or the diarization_model/sound_model + // companion. See roles.go. + diarCtx uintptr + tagCtx uintptr + // diarLatency is the PARAKEET_DIAR_LATENCY_* mode for diarization + // streaming (diarization_latency: option, default "low"). Unused until + // the diarization/scene streaming paths land. + diarLatency int32 + // companions holds every context this backend loaded itself beyond the + // primary (asr_model:/diarization_model:/sound_model: options), so Free + // can release them after the primary. + companions []uintptr + engineMu sync.Mutex // sole guard of the one C engine (dispatcher + streaming) + bat *batcher + batStop chan struct{} // segmentGapFrames is NeMo's segment_gap_threshold in ENCODER FRAMES (model // YAML option, default 0=off). When >0 it adds NeMo's silence-gap split on // top of the punctuation split; converted to seconds via the JSON frame_sec. @@ -151,21 +211,17 @@ type ParakeetCpp struct { } // Load is the LocalAI gRPC entry point for LoadModel: it calls -// parakeet_capi_load with the GGUF path and stashes the resulting -// opaque context pointer for AudioTranscription. +// parakeet_capi_load with the GGUF path, classifies it and any companion +// models named in Options[] by role (see roles.go), and starts the dynamic +// batcher when an ASR context (primary or companion) ends up loaded. func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error { if opts.ModelFile == "" { return errors.New("parakeet-cpp: ModelFile is required") } - ctx := CppLoad(opts.ModelFile) - if ctx == 0 { - // No ctx to ask for last_error (the C-API's last-error buffer - // lives on the ctx that was never returned). Surface the path - // so the operator at least knows which load failed. - return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for %q", opts.ModelFile) + if err := p.loadRoles(opts); err != nil { + return err } - p.ctxPtr = ctx // Dynamic batching knobs (model YAML options:, key:value form). Batching is // OFF by default (batch_max_size:1): each request runs on its own. On GPU, @@ -182,6 +238,12 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error { // default matches NeMo's default (punctuation-only segments); when set it // additionally splits segments on inter-word silence (see transcriptResultFromDoc). p.segmentGapFrames = optInt(opts, "segment_gap_threshold", 0) + + // The batcher only ever drives the ASR context; a diarization/sound + // primary with no asr_model companion has no ctxPtr and needs none. + if p.ctxPtr == 0 { + return nil + } if CppTranscribePcmBatchJSON != nil { p.batStop = make(chan struct{}) p.bat = newBatcher(maxSize, time.Duration(maxWaitMs)*time.Millisecond, p.runBatch) @@ -293,6 +355,9 @@ func (p *ParakeetCpp) runBatch(reqs []*batchRequest) { // (L2). func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.TranscriptRequest) (pb.TranscriptResult, error) { if p.ctxPtr == 0 { + if err := p.notASRError(); err != nil { + return pb.TranscriptResult{}, err + } return pb.TranscriptResult{}, grpcerrors.ModelNotLoaded("parakeet-cpp") } if opts.Dst == "" { @@ -817,8 +882,10 @@ func decodeWavMono16k(path string) ([]float32, float32, error) { return data, duration, nil } -// Free releases the underlying parakeet_ctx. Called by LocalAI when the -// model is unloaded. +// Free releases every parakeet_ctx this backend holds (the primary and any +// asr_model:/diarization_model:/sound_model: companions loaded in Load) and +// is idempotent: fields are zeroed as they are freed, so a second call frees +// nothing. Called by LocalAI when the model is unloaded. func (p *ParakeetCpp) Free() error { // Stop the dispatcher before releasing the engine so no in-flight runBatch // can touch a freed ctx (close leak / use-after-free on reload). @@ -830,10 +897,13 @@ func (p *ParakeetCpp) Free() error { // re-checks ctxPtr under the lock) can never feed into a freed ctx. p.engineMu.Lock() defer p.engineMu.Unlock() - if p.ctxPtr != 0 { - CppFree(p.ctxPtr) - p.ctxPtr = 0 + for _, ctxField := range [...]*uintptr{&p.ctxPtr, &p.diarCtx, &p.tagCtx} { + if *ctxField != 0 { + CppFree(*ctxField) + *ctxField = 0 + } } + p.companions = nil return nil } diff --git a/backend/go/parakeet-cpp/main.go b/backend/go/parakeet-cpp/main.go index 9c6466b131e5..865d5b6f1947 100644 --- a/backend/go/parakeet-cpp/main.go +++ b/backend/go/parakeet-cpp/main.go @@ -90,6 +90,34 @@ func main() { purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json") } + // Model roles + diarization/sound (ABI v7-v8): parakeet_capi_model_kind is + // what lets Load tell an ASR/diarization/sound context apart, so it gates + // every other new symbol below (an older libparakeet.so gets none of + // them, and companion model options are rejected in roles.go). Diarization + // itself (diarize_pcm, transcribe_and_diarize_json) predates model_kind + // (ABI v7), so it is probed on its own. + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_pcm"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppDiarizePCM, lib, "parakeet_capi_diarize_pcm") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_transcribe_and_diarize_json"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppTranscribeAndDiarizeJSON, lib, "parakeet_capi_transcribe_and_diarize_json") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_model_kind"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppModelKind, lib, "parakeet_capi_model_kind") + purego.RegisterLibFunc(&CppNumClasses, lib, "parakeet_capi_num_classes") + purego.RegisterLibFunc(&CppSoundOptsDefault, lib, "parakeet_capi_sound_opts_default") + purego.RegisterLibFunc(&CppSoundStreamBegin, lib, "parakeet_capi_sound_stream_begin") + purego.RegisterLibFunc(&CppSoundStreamFeed, lib, "parakeet_capi_sound_stream_feed") + purego.RegisterLibFunc(&CppSoundStreamDrainScoresJSON, lib, "parakeet_capi_sound_stream_drain_scores_json") + purego.RegisterLibFunc(&CppFreeSoundSegments, lib, "parakeet_capi_free_sound_segments") + purego.RegisterLibFunc(&CppSoundStreamFree, lib, "parakeet_capi_sound_stream_free") + purego.RegisterLibFunc(&CppSceneOptsDefault, lib, "parakeet_capi_scene_opts_default") + purego.RegisterLibFunc(&CppSceneStreamBegin, lib, "parakeet_capi_scene_stream_begin") + purego.RegisterLibFunc(&CppSceneStreamFeedJSON, lib, "parakeet_capi_scene_stream_feed_json") + purego.RegisterLibFunc(&CppSceneStreamLastError, lib, "parakeet_capi_scene_stream_last_error") + purego.RegisterLibFunc(&CppSceneStreamFree, lib, "parakeet_capi_scene_stream_free") + } + fmt.Fprintf(os.Stderr, "[parakeet-cpp] ABI=%d\n", CppAbiVersion()) flag.Parse() diff --git a/backend/go/parakeet-cpp/roles.go b/backend/go/parakeet-cpp/roles.go new file mode 100644 index 000000000000..b68b41be1489 --- /dev/null +++ b/backend/go/parakeet-cpp/roles.go @@ -0,0 +1,201 @@ +package main + +import ( + "errors" + "fmt" + "path/filepath" + "strings" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" +) + +// Model kinds returned by parakeet_capi_model_kind (ABI v8; mirrors the +// PARAKEET_MODEL_KIND_* defines in parakeet_capi.h). +const ( + modelKindNone = 0 + modelKindASR = 1 + modelKindDiarization = 2 + modelKindSound = 3 +) + +// Diarization streaming latency modes (mirrors PARAKEET_DIAR_LATENCY_* in +// parakeet_capi.h). diarLatencyLow is the spec's default when +// diarization_latency: is unset. +const ( + diarLatencyModel int32 = 0 + diarLatencyLow int32 = 1 + diarLatencyVeryLow int32 = 2 + diarLatencyUltraLow int32 = 3 +) + +// modelKindName renders a model kind for error messages. +func modelKindName(kind int32) string { + switch kind { + case modelKindASR: + return "ASR" + case modelKindDiarization: + return "diarization" + case modelKindSound: + return "sound" + default: + return "unknown" + } +} + +// optString reads a string model option (key:value form) from ModelOptions, +// returning "" when the key is absent. Same strings.Cut parsing as optInt. +func optString(opts *pb.ModelOptions, key string) string { + for _, o := range opts.GetOptions() { + k, v, ok := strings.Cut(o, ":") + if ok && strings.TrimSpace(k) == key { + return strings.TrimSpace(v) + } + } + return "" +} + +// resolveModelPath resolves a companion model option's path against +// modelPath (opts.ModelPath, the LocalAI models root): an absolute p, or an +// empty modelPath, passes through unchanged; anything else is joined onto +// modelPath. Mirrors vibevoice-cpp's resolvePath for tokenizer=/voice=/etc. +func resolveModelPath(modelPath, p string) string { + if p == "" || filepath.IsAbs(p) || modelPath == "" { + return p + } + return filepath.Join(modelPath, p) +} + +// parseDiarLatency maps the diarization_latency option value to a +// PARAKEET_DIAR_LATENCY_* mode. "" defaults to "low" (the spec's default); +// any other unrecognized value is a Load error. +func parseDiarLatency(s string) (int32, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "": + return diarLatencyLow, nil + case "model": + return diarLatencyModel, nil + case "low": + return diarLatencyLow, nil + case "very_low": + return diarLatencyVeryLow, nil + case "ultra_low": + return diarLatencyUltraLow, nil + default: + return 0, fmt.Errorf("parakeet-cpp: unknown diarization_latency %q (want model|low|very_low|ultra_low)", s) + } +} + +// companionSpec is one asr_model:/diarization_model:/sound_model: option: its +// name (for error messages and path resolution), the raw option value, the +// model kind the loaded companion must report, and the ParakeetCpp field it +// is assigned to on success. +type companionSpec struct { + optName string + value string + wantKind int32 + assign func(*ParakeetCpp, uintptr) +} + +// loadRoles loads opts.ModelFile as the primary parakeet_ctx, classifies it +// with parakeet_capi_model_kind (ABI v8) into ctxPtr/diarCtx/tagCtx, and +// loads any companion models named in Options[] (asr_model:, +// diarization_model:, sound_model:; paths resolved against opts.ModelPath). +// It also parses diarization_latency: into p.diarLatency. +// +// Against an older libparakeet.so (CppModelKind == nil) the primary is +// treated as ASR — the pre-v8 behavior — and companion model options are +// rejected outright, since there is no way to verify what they loaded. +// +// On any failure every context this call opened (primary and any companions +// loaded before the failure) is freed before the error is returned. +func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { + diarModelOpt := optString(opts, "diarization_model") + asrModelOpt := optString(opts, "asr_model") + soundModelOpt := optString(opts, "sound_model") + hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" + + if hasCompanionOpts && CppModelKind == nil { + return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model options need " + + "parakeet_capi_model_kind (ABI v8) to verify what they load; the loaded libparakeet.so " + + "is too old to report companion model roles") + } + + latency, err := parseDiarLatency(optString(opts, "diarization_latency")) + if err != nil { + return err + } + + primary := CppLoad(opts.ModelFile) + if primary == 0 { + // No ctx to ask for last_error (the C-API's last-error buffer lives on + // the ctx that was never returned). Surface the path so the operator + // at least knows which load failed. + return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for %q", opts.ModelFile) + } + loaded := []uintptr{primary} + freeLoaded := func() { + for _, c := range loaded { + CppFree(c) + } + } + + primaryKind := int32(modelKindASR) // old-library default: today's behavior + if CppModelKind != nil { + primaryKind = CppModelKind(primary) + } + switch primaryKind { + case modelKindDiarization: + p.diarCtx = primary + case modelKindSound: + p.tagCtx = primary + default: + p.ctxPtr = primary + } + + specs := []companionSpec{ + {"diarization_model", diarModelOpt, modelKindDiarization, func(pp *ParakeetCpp, c uintptr) { pp.diarCtx = c }}, + {"asr_model", asrModelOpt, modelKindASR, func(pp *ParakeetCpp, c uintptr) { pp.ctxPtr = c }}, + {"sound_model", soundModelOpt, modelKindSound, func(pp *ParakeetCpp, c uintptr) { pp.tagCtx = c }}, + } + for _, spec := range specs { + if spec.value == "" { + continue + } + resolved := resolveModelPath(opts.ModelPath, spec.value) + cctx := CppLoad(resolved) + if cctx == 0 { + freeLoaded() + return fmt.Errorf("parakeet-cpp: failed to load %s %q", spec.optName, resolved) + } + loaded = append(loaded, cctx) + if gotKind := CppModelKind(cctx); gotKind != spec.wantKind { + freeLoaded() + return fmt.Errorf("parakeet-cpp: %s %q is a %s model, expected a %s model", + spec.optName, resolved, modelKindName(gotKind), modelKindName(spec.wantKind)) + } + spec.assign(p, cctx) + p.companions = append(p.companions, cctx) + } + + p.diarLatency = latency + return nil +} + +// notASRError reports why AudioTranscription (and the streaming/live RPCs) +// cannot run when p.ctxPtr == 0: a loaded diarization or sound primary with +// no asr_model companion, named explicitly so the caller knows to use the +// right RPC instead of a generic "model not loaded". Returns nil when +// neither role is loaded (genuinely no model), leaving the caller to report +// the ordinary ModelNotLoaded error. +func (p *ParakeetCpp) notASRError() error { + switch { + case p.diarCtx != 0: + return errors.New("parakeet-cpp: loaded model is a diarization model, not ASR " + + "(use Diarize, or load with an asr_model: companion)") + case p.tagCtx != 0: + return errors.New("parakeet-cpp: loaded model is a sound model, not ASR " + + "(use SoundDetection)") + default: + return nil + } +} diff --git a/backend/go/parakeet-cpp/roles_test.go b/backend/go/parakeet-cpp/roles_test.go new file mode 100644 index 000000000000..36e29f2833af --- /dev/null +++ b/backend/go/parakeet-cpp/roles_test.go @@ -0,0 +1,304 @@ +package main + +import ( + "context" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The role-loading specs drive Load/Free entirely against stubbed +// CppLoad/CppModelKind/CppFree (the same seam live_test.go and +// batcher_test.go use), so they run without libparakeet.so. + +// fakeLib is a tiny in-memory stand-in for libparakeet.so: paths registered +// via withModel resolve to a fresh ctx handle of the given model kind on +// CppLoad, any other path fails the load, CppModelKind reads the kind back +// by ctx, and CppFree records every ctx it was asked to free, in order. +type fakeLib struct { + kinds map[string]int32 + ctxKind map[uintptr]int32 + next uintptr + loadedPaths []string + freed []uintptr +} + +func newFakeLib() *fakeLib { + return &fakeLib{kinds: map[string]int32{}, ctxKind: map[uintptr]int32{}, next: 1} +} + +func (f *fakeLib) withModel(path string, kind int32) *fakeLib { + f.kinds[path] = kind + return f +} + +// install swaps CppLoad/CppModelKind/CppFree for fakes backed by f and +// returns a restore func for AfterEach (mirrors live_test.go's liveStubs). +func (f *fakeLib) install() (restore func()) { + savedLoad, savedKind, savedFree := CppLoad, CppModelKind, CppFree + CppLoad = func(path string) uintptr { + f.loadedPaths = append(f.loadedPaths, path) + kind, ok := f.kinds[path] + if !ok { + return 0 + } + ctx := f.next + f.next++ + f.ctxKind[ctx] = kind + return ctx + } + CppModelKind = func(ctx uintptr) int32 { return f.ctxKind[ctx] } + CppFree = func(ctx uintptr) { f.freed = append(f.freed, ctx) } + return func() { + CppLoad, CppModelKind, CppFree = savedLoad, savedKind, savedFree + } +} + +var _ = Describe("model roles (stubbed C API)", func() { + var restore func() + + AfterEach(func() { + if restore != nil { + restore() + restore = nil + } + }) + + It("loads an ASR primary with no options", func() { + f := newFakeLib().withModel("asr.gguf", modelKindASR) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "asr.gguf"})).To(Succeed()) + + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + }) + + It("loads a diarization primary and rejects AudioTranscription without any further C call", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "diar.gguf"})).To(Succeed()) + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.ctxPtr).To(BeZero()) + + // CppTranscribePathJSON/CppTranscribePcmBatchJSON are left nil (the + // zero value): if AudioTranscription tried to call either, this would + // panic instead of returning cleanly, so a clean typed error here also + // proves no C call was made. + _, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: "x.wav"}) + Expect(err).To(MatchError(ContainSubstring("diarization model"))) + }) + + It("loads a sound primary", func() { + f := newFakeLib().withModel("sound.gguf", modelKindSound) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "sound.gguf"})).To(Succeed()) + + Expect(p.tagCtx).ToNot(BeZero()) + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + }) + + It("loads an ASR primary plus diarization and sound companions, and Free releases all three", func() { + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("/models/x.gguf", modelKindDiarization). + withModel("/abs/y.gguf", modelKindSound) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + ModelPath: "/models", + Options: []string{"diarization_model:x.gguf", "sound_model:/abs/y.gguf"}, + }) + Expect(err).ToNot(HaveOccurred()) + + Expect(f.loadedPaths).To(Equal([]string{"asr.gguf", "/models/x.gguf", "/abs/y.gguf"}), + "diarization_model resolves against ModelPath, sound_model's absolute path passes through") + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.tagCtx).ToNot(BeZero()) + Expect(p.companions).To(HaveLen(2)) + + asrCtx, diarCtx, tagCtx := p.ctxPtr, p.diarCtx, p.tagCtx + Expect(p.Free()).To(Succeed()) + Expect(f.freed).To(ConsistOf(asrCtx, diarCtx, tagCtx)) + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + }) + + It("loads a diarization primary plus an asr_model companion into ctxPtr", func() { + f := newFakeLib(). + withModel("diar.gguf", modelKindDiarization). + withModel("/models/z.gguf", modelKindASR) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "diar.gguf", + ModelPath: "/models", + Options: []string{"asr_model:z.gguf"}, + }) + Expect(err).ToNot(HaveOccurred()) + + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.ctxPtr).ToNot(Equal(p.diarCtx)) + }) + + It("fails a companion of the wrong kind and frees every ctx it opened", func() { + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("/models/wrong.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + ModelPath: "/models", + Options: []string{"sound_model:wrong.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sound_model")) + Expect(err.Error()).To(ContainSubstring("diarization")) + Expect(f.freed).To(HaveLen(2), "the primary and the wrong-kind companion must both be freed") + }) + + It("parses diarization_latency:very_low", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "diar.gguf", + Options: []string{"diarization_latency:very_low"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(p.diarLatency).To(Equal(int32(2))) + }) + + It("defaults diarization_latency to low (1) when unset", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "diar.gguf"})).To(Succeed()) + Expect(p.diarLatency).To(Equal(int32(1))) + }) + + It("rejects an invalid diarization_latency before any C call", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "diar.gguf", + Options: []string{"diarization_latency:bogus"}, + }) + Expect(err).To(HaveOccurred()) + Expect(f.loadedPaths).To(BeEmpty()) + }) + + Context("old library (no parakeet_capi_model_kind)", func() { + It("treats the primary as ASR, matching pre-v8 behavior", func() { + f := newFakeLib().withModel("model.gguf", modelKindDiarization) // kind is never consulted + restore = f.install() + CppModelKind = nil + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "model.gguf"})).To(Succeed()) + + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + }) + + It("rejects companion model options with an error naming the library as too old", func() { + f := newFakeLib().withModel("asr.gguf", modelKindASR) + restore = f.install() + CppModelKind = nil + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + Options: []string{"diarization_model:diar.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("too old")) + Expect(f.loadedPaths).To(BeEmpty(), "rejected before any C call") + }) + }) + + Context("batcher gating", func() { + It("does not start the batcher for a non-ASR primary with no asr_model companion", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + savedBatch := CppTranscribePcmBatchJSON + defer func() { CppTranscribePcmBatchJSON = savedBatch }() + CppTranscribePcmBatchJSON = func(uintptr, []float32, []int32, int32, int32, int32) uintptr { return 0 } + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "diar.gguf"})).To(Succeed()) + Expect(p.bat).To(BeNil()) + }) + + It("starts the batcher for an ASR ctxPtr", func() { + f := newFakeLib().withModel("asr.gguf", modelKindASR) + restore = f.install() + savedBatch := CppTranscribePcmBatchJSON + defer func() { CppTranscribePcmBatchJSON = savedBatch }() + CppTranscribePcmBatchJSON = func(uintptr, []float32, []int32, int32, int32, int32) uintptr { return 0 } + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "asr.gguf"})).To(Succeed()) + Expect(p.bat).ToNot(BeNil()) + Expect(p.Free()).To(Succeed()) // stop the dispatcher goroutine + }) + }) +}) + +var _ = Describe("resolveModelPath", func() { + It("keeps an absolute path unchanged", func() { + Expect(resolveModelPath("/models", "/abs/y.gguf")).To(Equal("/abs/y.gguf")) + }) + + It("joins a relative path onto modelPath", func() { + Expect(resolveModelPath("/models", "x.gguf")).To(Equal("/models/x.gguf")) + }) + + It("passes a relative path through when modelPath is empty", func() { + Expect(resolveModelPath("", "x.gguf")).To(Equal("x.gguf")) + }) +}) + +var _ = Describe("parseDiarLatency", func() { + It("defaults to low (1) for an empty value", func() { + v, err := parseDiarLatency("") + Expect(err).ToNot(HaveOccurred()) + Expect(v).To(Equal(int32(1))) + }) + + It("maps every named mode", func() { + for s, want := range map[string]int32{ + "model": 0, "low": 1, "very_low": 2, "ultra_low": 3, + } { + v, err := parseDiarLatency(s) + Expect(err).ToNot(HaveOccurred()) + Expect(v).To(Equal(want), "mode %q", s) + } + }) + + It("rejects an unknown value", func() { + _, err := parseDiarLatency("bogus") + Expect(err).To(HaveOccurred()) + }) +}) From 699c17b7dbadfa266db66c9ffb05ce09e1d459b8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 15:47:20 +0000 Subject: [PATCH 02/30] fix(parakeet-cpp): reset role fields on a failed companion load loadRoles' freeLoaded only released the C contexts it had opened; it left ctxPtr/diarCtx/tagCtx and companions pointing at those now-freed contexts, so a later Free() on the same instance would double-free. Zero all four alongside the CppFree calls. Also route AudioTranscriptionStream and AudioTranscriptionLive through notASRError when ctxPtr is unset but a diarization or sound model is loaded, matching AudioTranscription: both used to return the generic model-not-loaded error instead of naming the loaded role. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/goparakeetcpp.go | 3 +++ backend/go/parakeet-cpp/goparakeetcpp_test.go | 18 ++++++++++++++++++ backend/go/parakeet-cpp/live.go | 3 +++ backend/go/parakeet-cpp/live_test.go | 13 +++++++++++++ backend/go/parakeet-cpp/roles.go | 8 ++++++++ backend/go/parakeet-cpp/roles_test.go | 12 ++++++++++++ 6 files changed, 57 insertions(+) diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index 9b699433c770..3c6cb3ab2057 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -751,6 +751,9 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra defer close(results) if p.ctxPtr == 0 { + if err := p.notASRError(); err != nil { + return err + } return grpcerrors.ModelNotLoaded("parakeet-cpp") } if opts.Dst == "" { diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go index a6f6af1f04a0..02b24ece3e29 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp_test.go +++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go @@ -203,6 +203,24 @@ var _ = Describe("ParakeetCpp", func() { }) Context("AudioTranscriptionStream", func() { + It("names the loaded role instead of a generic model-not-loaded error for a diarization primary", func() { + // CppStreamBegin/CppStreamBeginLang are left nil (zero value): if + // AudioTranscriptionStream tried to call either, this would panic + // instead of returning cleanly, so a clean typed error here also + // proves no C call was made. + p := &ParakeetCpp{diarCtx: 1} + results := make(chan *pb.TranscriptStreamResponse, 8) + err := p.AudioTranscriptionStream(context.Background(), + &pb.TranscriptRequest{Dst: "ignored.wav"}, results) + Expect(err).To(MatchError(ContainSubstring("diarization model"))) + + var emitted []*pb.TranscriptStreamResponse + for r := range results { + emitted = append(emitted, r) + } + Expect(emitted).To(BeEmpty()) + }) + It("returns the typed Unimplemented signal for non-streaming models (no offline fallback)", func() { // stream_begin == 0 means the loaded model is not a cache-aware // streaming model. The backend must surface that, not silently diff --git a/backend/go/parakeet-cpp/live.go b/backend/go/parakeet-cpp/live.go index 3d68a2914be1..627492f1619d 100644 --- a/backend/go/parakeet-cpp/live.go +++ b/backend/go/parakeet-cpp/live.go @@ -41,6 +41,9 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest defer close(out) if p.ctxPtr == 0 { + if err := p.notASRError(); err != nil { + return err + } return grpcerrors.ModelNotLoaded("parakeet-cpp") } diff --git a/backend/go/parakeet-cpp/live_test.go b/backend/go/parakeet-cpp/live_test.go index 0462ee521b3a..ca900010f236 100644 --- a/backend/go/parakeet-cpp/live_test.go +++ b/backend/go/parakeet-cpp/live_test.go @@ -106,6 +106,19 @@ var _ = Describe("AudioTranscriptionLive (stubbed C API)", func() { AfterEach(func() { restore() }) + It("names the loaded role instead of a generic model-not-loaded error for a sound primary", func() { + // The ctxPtr==0 check returns before AudioTranscriptionLive ever reads + // from `in`, so nothing may be sent on it (unbuffered: a send would + // block forever waiting for a read that never happens). + p2 := &ParakeetCpp{tagCtx: 1} + in, out, errCh := runLive(p2) + close(in) + + err := <-errCh + Expect(err).To(MatchError(ContainSubstring("sound model"))) + Expect(collectLive(out)).To(BeEmpty()) + }) + It("rejects a stream whose first message is not a config", func() { in, out, errCh := runLive(p) in <- liveAudio([]float32{0.1}) diff --git a/backend/go/parakeet-cpp/roles.go b/backend/go/parakeet-cpp/roles.go index b68b41be1489..c05b0988989a 100644 --- a/backend/go/parakeet-cpp/roles.go +++ b/backend/go/parakeet-cpp/roles.go @@ -133,10 +133,18 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for %q", opts.ModelFile) } loaded := []uintptr{primary} + // freeLoaded undoes everything loadRoles opened this call: every context + // it freed AND every ParakeetCpp field it may have assigned (the primary + // lands in one of ctxPtr/diarCtx/tagCtx before the companion loop runs, + // and an earlier companion's spec.assign runs before a later one fails). + // Leaving a role field pointing at a freed ctx would double-free it on a + // later Free() call. freeLoaded := func() { for _, c := range loaded { CppFree(c) } + p.ctxPtr, p.diarCtx, p.tagCtx = 0, 0, 0 + p.companions = nil } primaryKind := int32(modelKindASR) // old-library default: today's behavior diff --git a/backend/go/parakeet-cpp/roles_test.go b/backend/go/parakeet-cpp/roles_test.go index 36e29f2833af..86eded3f133e 100644 --- a/backend/go/parakeet-cpp/roles_test.go +++ b/backend/go/parakeet-cpp/roles_test.go @@ -171,6 +171,18 @@ var _ = Describe("model roles (stubbed C API)", func() { Expect(err.Error()).To(ContainSubstring("sound_model")) Expect(err.Error()).To(ContainSubstring("diarization")) Expect(f.freed).To(HaveLen(2), "the primary and the wrong-kind companion must both be freed") + + // Every role field the failed load may have assigned (the primary + // lands in ctxPtr before the companion loop runs) must be reset, or a + // later Free() would double-free an already-freed context. + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + Expect(p.companions).To(BeEmpty()) + + freedBeforeFree := len(f.freed) + Expect(p.Free()).To(Succeed()) + Expect(f.freed).To(HaveLen(freedBeforeFree), "Free after a failed Load must not free anything again") }) It("parses diarization_latency:very_low", func() { From 30b4394db27bb49343bba30ebf16bc50a660a00a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 15:54:13 +0000 Subject: [PATCH 03/30] feat(parakeet-cpp): add speaker diarization Implement the Diarize RPC for the parakeet-cpp Go backend, wired to Nemotron-3-Diarization through libparakeet.so's diarization C-API. Plain diarization uses parakeet_capi_diarize_pcm; when include_text is set and an ASR companion is loaded, parakeet_capi_transcribe_and_ diarize_json fills each segment's text instead. Speaker labels are the decimal index, or "unknown" for -1 (no diarized speaker overlaps). min_duration_off merges same-speaker segments across a short gap before min_duration_on drops the segments still too short, then ids are renumbered. num_speakers/min_speakers/max_speakers/clustering_ threshold have no Sortformer equivalent and are logged at debug instead of rejected. Verified against the real Nemotron-3-Diarization + parakeet-tdt_ctc- 110m checkpoints on the two_speakers.wav fixture: correct A-B-A-B speaker segmentation and matching speaker-attributed transcripts. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/diarize.go | 275 ++++++++++++++++++++++++ backend/go/parakeet-cpp/diarize_test.go | 198 +++++++++++++++++ 2 files changed, 473 insertions(+) create mode 100644 backend/go/parakeet-cpp/diarize.go create mode 100644 backend/go/parakeet-cpp/diarize_test.go diff --git a/backend/go/parakeet-cpp/diarize.go b/backend/go/parakeet-cpp/diarize.go new file mode 100644 index 000000000000..d77308abda8f --- /dev/null +++ b/backend/go/parakeet-cpp/diarize.go @@ -0,0 +1,275 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// diarizeSegmentJSON mirrors one element of parakeet_capi_diarize_pcm's +// "segments" array: {"speaker":0,"start":0.50,"end":5.52}. +type diarizeSegmentJSON struct { + Speaker int `json:"speaker"` + Start float64 `json:"start"` + End float64 `json:"end"` +} + +// diarizePCMDoc mirrors the document parakeet_capi_diarize_pcm returns. +// "speakers" is the model's CAPACITY (e.g. 8 for Nemotron-3-Diarization), +// not the count of speakers actually present, so it is not read here; the +// response's num_speakers is computed from distinct segment labels instead. +type diarizePCMDoc struct { + Segments []diarizeSegmentJSON `json:"segments"` +} + +// diarizeUtteranceJSON mirrors one element of +// parakeet_capi_transcribe_and_diarize_json's "utterances" array. Speaker is +// -1 when no diarized speaker overlaps the utterance. +type diarizeUtteranceJSON struct { + Speaker int `json:"speaker"` + Text string `json:"text"` + Start float64 `json:"start"` + End float64 `json:"end"` +} + +// transcribeAndDiarizeDoc mirrors the document +// parakeet_capi_transcribe_and_diarize_json returns. Only "utterances" is +// consumed here; the per-word "words" detail belongs to a speaker-attributed +// transcript RPC, not Diarize. +type transcribeAndDiarizeDoc struct { + Utterances []diarizeUtteranceJSON `json:"utterances"` +} + +// speakerLabel renders a 0-based speaker index as the decimal string +// DiarizeSegment.speaker documents, or "unknown" for -1 (no diarized speaker +// overlaps this utterance; only transcribe_and_diarize_json can report this). +func speakerLabel(speaker int) string { + if speaker < 0 { + return "unknown" + } + return strconv.Itoa(speaker) +} + +// unsupportedDiarizeFields names the DiarizeRequest fields Sortformer has no +// equivalent for: it is an end-to-end model with a fixed speaker capacity and +// no clustering stage, so there is no config knob to target a speaker count +// or a clustering distance. Logged rather than rejected, so a request naming +// one of these still gets the diarization it can have. +func unsupportedDiarizeFields(req *pb.DiarizeRequest) []string { + var out []string + if req.GetNumSpeakers() != 0 { + out = append(out, "num_speakers") + } + if req.GetMinSpeakers() != 0 { + out = append(out, "min_speakers") + } + if req.GetMaxSpeakers() != 0 { + out = append(out, "max_speakers") + } + if req.GetClusteringThreshold() != 0 { + out = append(out, "clustering_threshold") + } + return out +} + +// Diarize labels who spoke when in the audio at req.Dst, using the loaded +// diarization model (p.diarCtx). When req.IncludeText is set and an ASR +// companion (p.ctxPtr) is loaded, each segment also carries its transcript +// (parakeet_capi_transcribe_and_diarize_json, one utterance per speaker +// turn); otherwise, or when no ASR companion is loaded, segments carry no +// text (parakeet_capi_diarize_pcm) and no error is raised. +func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error) { + if p.diarCtx == 0 { + return pb.DiarizeResponse{}, status.Error(codes.FailedPrecondition, + "parakeet-cpp: model is not a diarization model") + } + if CppDiarizePCM == nil { + return pb.DiarizeResponse{}, status.Error(codes.Unimplemented, + "parakeet-cpp: loaded libparakeet.so has no diarization support (parakeet_capi_diarize_pcm missing)") + } + if req.GetDst() == "" { + return pb.DiarizeResponse{}, status.Error(codes.InvalidArgument, + "parakeet-cpp: DiarizeRequest.dst (audio path) is required") + } + + if dropped := unsupportedDiarizeFields(req); len(dropped) > 0 { + xlog.Debug("parakeet-cpp: ignoring diarization request fields Sortformer has no equivalent for", + "fields", dropped) + } + + pcm, duration, err := decodeWavMono16k(req.GetDst()) + if err != nil { + return pb.DiarizeResponse{}, err + } + if len(pcm) == 0 { + return pb.DiarizeResponse{}, status.Error(codes.InvalidArgument, "parakeet-cpp: empty audio") + } + + wantText := req.GetIncludeText() && p.ctxPtr != 0 && CppTranscribeAndDiarizeJSON != nil + + raw, err := p.diarizeCall(pcm, wantText) + if err != nil { + return pb.DiarizeResponse{}, err + } + segments, err := parseDiarizeDoc(raw, wantText) + if err != nil { + return pb.DiarizeResponse{}, err + } + + segments = applyDurationFilters(segments, req.GetMinDurationOn(), req.GetMinDurationOff()) + renumberDiarizeSegments(segments) + + return pb.DiarizeResponse{ + Segments: segments, + NumSpeakers: distinctDiarizeSpeakers(segments), + Duration: duration, + }, nil +} + +// diarizeCall runs the single C call Diarize needs (transcribe_and_diarize_json +// when wantText, else diarize_pcm) under engineMu, and returns the raw JSON +// document. last_error is ctx-shared, so it is read under the same lock as the +// failing call, off p.diarCtx (the context both entry points share as their +// diarization side). +func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool) (string, error) { + p.engineMu.Lock() + defer p.engineMu.Unlock() + + var cstr uintptr + if wantText { + cstr = CppTranscribeAndDiarizeJSON(p.ctxPtr, p.diarCtx, &pcm[0], int32(len(pcm)), 16000) + } else { + cstr = CppDiarizePCM(p.diarCtx, &pcm[0], int32(len(pcm)), 16000) + } + if cstr == 0 { + msg := CppLastError(p.diarCtx) + if msg == "" { + msg = "unknown error" + } + return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", msg) + } + raw := goStringFromCPtr(cstr) + CppFreeString(cstr) + return raw, nil +} + +// parseDiarizeDoc decodes the raw JSON diarizeCall returned into +// DiarizeSegments (without ids: renumberDiarizeSegments assigns those after +// filtering). +func parseDiarizeDoc(raw string, wantText bool) ([]*pb.DiarizeSegment, error) { + if wantText { + var doc transcribeAndDiarizeDoc + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return nil, fmt.Errorf("parakeet-cpp: decode diarize json: %w", err) + } + segs := make([]*pb.DiarizeSegment, 0, len(doc.Utterances)) + for _, u := range doc.Utterances { + segs = append(segs, &pb.DiarizeSegment{ + Start: float32(u.Start), + End: float32(u.End), + Speaker: speakerLabel(u.Speaker), + Text: u.Text, + }) + } + return segs, nil + } + + var doc diarizePCMDoc + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return nil, fmt.Errorf("parakeet-cpp: decode diarize json: %w", err) + } + segs := make([]*pb.DiarizeSegment, 0, len(doc.Segments)) + for _, s := range doc.Segments { + segs = append(segs, &pb.DiarizeSegment{ + Start: float32(s.Start), + End: float32(s.End), + Speaker: speakerLabel(s.Speaker), + }) + } + return segs, nil +} + +// applyDurationFilters applies the request's postprocessing knobs, in the +// order NeMo's diarization postprocessing does: merge first +// (min_duration_off), then drop short segments (min_duration_on) — dropping +// first would leave short gaps unmerged that the drop step just created. +// Segments are assumed sorted by start time, as parakeet_capi_diarize_pcm and +// parakeet_capi_transcribe_and_diarize_json document. A non-positive value +// disables that filter (the proto's "0 = backend default" reads here as "no +// filtering"). +func applyDurationFilters(segs []*pb.DiarizeSegment, minOn, minOff float32) []*pb.DiarizeSegment { + segs = mergeCloseSegments(segs, minOff) + segs = dropShortSegments(segs, minOn) + return segs +} + +// mergeCloseSegments merges consecutive SAME-SPEAKER segments separated by a +// gap shorter than minOff into one segment spanning both (and concatenating +// any text). Segments from different speakers are never merged, regardless of +// gap: the gap only ever means "the same speaker paused", never "two speakers +// are actually one". +func mergeCloseSegments(segs []*pb.DiarizeSegment, minOff float32) []*pb.DiarizeSegment { + if minOff <= 0 || len(segs) < 2 { + return segs + } + out := make([]*pb.DiarizeSegment, 0, len(segs)) + out = append(out, segs[0]) + for _, s := range segs[1:] { + prev := out[len(out)-1] + if s.GetSpeaker() == prev.GetSpeaker() && s.GetStart()-prev.GetEnd() < minOff { + if s.GetEnd() > prev.GetEnd() { + prev.End = s.End + } + if s.GetText() != "" { + if prev.GetText() != "" { + prev.Text = prev.GetText() + " " + s.GetText() + } else { + prev.Text = s.GetText() + } + } + continue + } + out = append(out, s) + } + return out +} + +// dropShortSegments discards segments shorter than minOn. +func dropShortSegments(segs []*pb.DiarizeSegment, minOn float32) []*pb.DiarizeSegment { + if minOn <= 0 { + return segs + } + out := make([]*pb.DiarizeSegment, 0, len(segs)) + for _, s := range segs { + if s.GetEnd()-s.GetStart() < minOn { + continue + } + out = append(out, s) + } + return out +} + +// renumberDiarizeSegments assigns sequential ids (0..) to the final segment +// list, after filtering may have dropped or merged entries. +func renumberDiarizeSegments(segs []*pb.DiarizeSegment) { + for i, s := range segs { + s.Id = int32(i) + } +} + +// distinctDiarizeSpeakers counts the distinct speaker labels present in segs. +// This is what DiarizeResponse.num_speakers documents — the count of speakers +// actually present in the result — and is NOT the diarize_pcm JSON's +// top-level "speakers" field, which reports the model's fixed capacity. +func distinctDiarizeSpeakers(segs []*pb.DiarizeSegment) int32 { + seen := make(map[string]struct{}, len(segs)) + for _, s := range segs { + seen[s.GetSpeaker()] = struct{}{} + } + return int32(len(seen)) +} diff --git a/backend/go/parakeet-cpp/diarize_test.go b/backend/go/parakeet-cpp/diarize_test.go new file mode 100644 index 000000000000..735e8c9cf2b0 --- /dev/null +++ b/backend/go/parakeet-cpp/diarize_test.go @@ -0,0 +1,198 @@ +package main + +import ( + "path/filepath" + "sync" + "unsafe" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// The Diarize specs drive it entirely against stubbed CppDiarizePCM / +// CppTranscribeAndDiarizeJSON / CppFreeString / CppLastError (the same seam +// live_test.go and roles_test.go use), so they run without libparakeet.so. + +// diarizeCstrPool hands out NUL-terminated C-style strings backed by Go +// memory and keeps them alive for the duration of a spec (goStringFromCPtr +// reads through the raw pointer; mirrors live_test.go's liveCstrPool). +type diarizeCstrPool struct { + mu sync.Mutex + bufs [][]byte +} + +func (p *diarizeCstrPool) cstr(s string) uintptr { + p.mu.Lock() + defer p.mu.Unlock() + b := append([]byte(s), 0) + p.bufs = append(p.bufs, b) + return uintptr(unsafe.Pointer(&b[0])) +} + +// diarizeStubs swaps every C entry point Diarize touches and returns a +// restore func for AfterEach (mirrors live_test.go's liveStubs). +func diarizeStubs() (restore func()) { + savedDiarize := CppDiarizePCM + savedTranscribeAndDiarize := CppTranscribeAndDiarizeJSON + savedFreeString := CppFreeString + savedLastError := CppLastError + return func() { + CppDiarizePCM = savedDiarize + CppTranscribeAndDiarizeJSON = savedTranscribeAndDiarize + CppFreeString = savedFreeString + CppLastError = savedLastError + } +} + +// diarizeWav writes a silent 16 kHz mono WAV of the given duration (seconds) +// to a fresh temp file and returns its path. decodeWavMono16k reads real +// audio bytes off disk, so Diarize needs a file on disk even though the +// stubbed C calls never look at its samples. +func diarizeWav(seconds float64) string { + GinkgoHelper() + path := filepath.Join(GinkgoT().TempDir(), "diarize.wav") + writeMono16kWav(path, int(seconds*16000)) + return path +} + +var _ = Describe("ParakeetCpp.Diarize", func() { + var restore func() + var pool *diarizeCstrPool + + BeforeEach(func() { + restore = diarizeStubs() + pool = &diarizeCstrPool{} + }) + AfterEach(func() { restore() }) + + It("fails with FailedPrecondition when no diarization model is loaded", func() { + p := &ParakeetCpp{} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.FailedPrecondition)) + Expect(err.Error()).To(ContainSubstring("model is not a diarization model")) + }) + + It("fails with Unimplemented when the loaded libparakeet.so has no diarize_pcm symbol", func() { + CppDiarizePCM = nil + p := &ParakeetCpp{diarCtx: 42} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + }) + + It("maps plain segments with sequential ids, decimal speaker labels and distinct speaker count", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"segments":[` + + `{"speaker":0,"start":0.00,"end":3.00},` + + `{"speaker":1,"start":3.00,"end":6.00}]}`) + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(6)}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Segments).To(HaveLen(2)) + Expect(resp.Segments[0].Id).To(Equal(int32(0))) + Expect(resp.Segments[0].Speaker).To(Equal("0")) + Expect(resp.Segments[1].Id).To(Equal(int32(1))) + Expect(resp.Segments[1].Speaker).To(Equal("1")) + Expect(resp.Segments[0].Text).To(BeEmpty()) + Expect(resp.NumSpeakers).To(Equal(int32(2))) + Expect(resp.Duration).To(BeNumerically("~", 6.0, 0.01)) + }) + + It("fills text from utterances when include_text is set with an ASR companion, and maps speaker -1 to unknown", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + Fail("diarize_pcm must not be called when include_text has an ASR companion to pair with") + return 0 + } + CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"utterances":[` + + `{"speaker":0,"text":"hello there","start":0.00,"end":1.00,"conf":0.9},` + + `{"speaker":-1,"text":"mumble","start":1.00,"end":1.50,"conf":0.4}],"words":[]}`) + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42, ctxPtr: 7} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(2), IncludeText: true}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Segments).To(HaveLen(2)) + Expect(resp.Segments[0].Speaker).To(Equal("0")) + Expect(resp.Segments[0].Text).To(Equal("hello there")) + Expect(resp.Segments[1].Speaker).To(Equal("unknown")) + Expect(resp.Segments[1].Text).To(Equal("mumble")) + }) + + It("falls back to plain segments without error when include_text is set but no ASR companion is loaded", func() { + diarizeCalled := false + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + diarizeCalled = true + return pool.cstr(`{"speakers":8,"segments":[{"speaker":0,"start":0.00,"end":1.00}]}`) + } + CppFreeString = func(uintptr) {} + CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, samples *float32, n int32, sampleRate int32) uintptr { + Fail("transcribe_and_diarize_json must not be called without an ASR companion") + return 0 + } + + p := &ParakeetCpp{diarCtx: 42} // no ctxPtr companion + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1), IncludeText: true}) + Expect(err).ToNot(HaveOccurred()) + Expect(diarizeCalled).To(BeTrue()) + Expect(resp.Segments).To(HaveLen(1)) + Expect(resp.Segments[0].Text).To(BeEmpty()) + }) + + It("drops a segment shorter than min_duration_on", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"segments":[` + + `{"speaker":0,"start":0.30,"end":0.50},` + // 0.2s, at 0.3 + `{"speaker":0,"start":1.00,"end":2.00}]}`) // 1.0s, kept + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(3), MinDurationOn: 0.3}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Segments).To(HaveLen(1)) + Expect(resp.Segments[0].Id).To(Equal(int32(0))) + Expect(resp.Segments[0].Start).To(BeNumerically("~", 1.0, 0.001)) + }) + + It("merges same-speaker segments across a short gap but not across a speaker change", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"segments":[` + + `{"speaker":0,"start":0.00,"end":1.00},` + + `{"speaker":0,"start":1.30,"end":2.00},` + // 0.3s gap, same speaker: merges + `{"speaker":1,"start":2.10,"end":3.00}]}`) // 0.1s gap, different speaker: stays separate + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(3), MinDurationOff: 0.5}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Segments).To(HaveLen(2)) + Expect(resp.Segments[0].Id).To(Equal(int32(0))) + Expect(resp.Segments[0].Speaker).To(Equal("0")) + Expect(resp.Segments[0].Start).To(BeNumerically("~", 0.0, 0.001)) + Expect(resp.Segments[0].End).To(BeNumerically("~", 2.0, 0.001)) + Expect(resp.Segments[1].Id).To(Equal(int32(1))) + Expect(resp.Segments[1].Speaker).To(Equal("1")) + }) + + It("surfaces last_error when the C call returns NULL", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return 0 + } + CppLastError = func(ctx uintptr) string { return "boom" } + + p := &ParakeetCpp{diarCtx: 42} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("boom")) + }) +}) From 4eb8ed024cb066128d0dbcd5749870247706ba2d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:03:06 +0000 Subject: [PATCH 04/30] feat(parakeet-cpp): add sound event detection Wire the SoundDetection RPC to the CED tagger context (p.tagCtx) loaded by Task 1's role classification. It runs the whole clip through a one-shot parakeet_capi_sound_stream_* session (window 10s, hop 10s, top_k set to the tagger's class count so every drained window carries a full score list), averages each class's score across the drained windows, sorts descending, then applies the request's threshold and top_k (0 keeps every class). No tagCtx returns FailedPrecondition; a libparakeet.so missing the sound_stream symbols returns Unimplemented. Every C call runs under engineMu, and the stream is always freed, even when a feed or drain call fails partway through. Verified against a real ced-tiny-q8_0.gguf on the rooster.wav demo clip: "Chicken, rooster" tops the list at score 0.91. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/sound.go | 230 ++++++++++++++++++++ backend/go/parakeet-cpp/sound_test.go | 293 ++++++++++++++++++++++++++ 2 files changed, 523 insertions(+) create mode 100644 backend/go/parakeet-cpp/sound.go create mode 100644 backend/go/parakeet-cpp/sound_test.go diff --git a/backend/go/parakeet-cpp/sound.go b/backend/go/parakeet-cpp/sound.go new file mode 100644 index 000000000000..e85d1d58db1f --- /dev/null +++ b/backend/go/parakeet-cpp/sound.go @@ -0,0 +1,230 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// soundFeedChunkSamples is how much 16 kHz mono PCM soundStreamScores hands +// to sound_stream_feed per call (10 s), matching the window/hop it asks for +// below. The clip is fed in these pieces with is_last set on the final one, +// mirroring the streaming ASR path's chunked feed. +const soundFeedChunkSamples = 10 * 16000 + +// soundTagJSON mirrors one element of a soundWindowJSON's "tags" array. +type soundTagJSON struct { + Index int `json:"index"` + Label string `json:"label"` + Score float32 `json:"score"` +} + +// soundWindowJSON mirrors one element of the array +// parakeet_capi_sound_stream_drain_scores_json returns: +// +// [{"start":0.0,"end":10.0,"tags":[{"index":0,"label":"Speech","score":0.93}, ...]}] +type soundWindowJSON struct { + Start float64 `json:"start"` + End float64 `json:"end"` + Tags []soundTagJSON `json:"tags"` +} + +// classAvg is one class's score averaged across the drained windows, plus +// the label the tagger reported for it. +type classAvg struct { + Index int + Label string + Score float32 +} + +// SoundDetection runs the loaded CED model (p.tagCtx) over the clip at +// req.Src through a one-shot sound stream (window 10 s, hop 10 s, top_k set +// to the tagger's full class count so every window's drain carries a score +// for every class), averages each class's score across the drained windows, +// sorts descending, applies req.Threshold, then req.TopK (0 = all classes). +func (p *ParakeetCpp) SoundDetection(ctx context.Context, req *pb.SoundDetectionRequest) (*pb.SoundDetectionResponse, error) { + if p.tagCtx == 0 { + return nil, status.Error(codes.FailedPrecondition, + "parakeet-cpp: model is not a sound (CED) model") + } + if CppSoundStreamBegin == nil || CppSoundStreamFeed == nil || CppSoundStreamDrainScoresJSON == nil || + CppSoundStreamFree == nil || CppSoundOptsDefault == nil || CppNumClasses == nil { + return nil, status.Error(codes.Unimplemented, + "parakeet-cpp: loaded libparakeet.so has no sound-event detection support "+ + "(parakeet_capi_sound_stream_* missing)") + } + if req.GetSrc() == "" { + return nil, status.Error(codes.InvalidArgument, + "parakeet-cpp: SoundDetectionRequest.src (audio path) is required") + } + + pcm, _, err := decodeWavMono16k(req.GetSrc()) + if err != nil { + return nil, err + } + + windows, nClasses, err := p.soundStreamScores(pcm) + if err != nil { + return nil, err + } + + avgs := averageWindowScores(windows, nClasses) + sortSoundDetectionsDesc(avgs) + avgs = filterSoundDetections(avgs, req.GetThreshold(), req.GetTopK()) + + resp := &pb.SoundDetectionResponse{Detections: make([]*pb.SoundClass, 0, len(avgs))} + for _, a := range avgs { + resp.Detections = append(resp.Detections, &pb.SoundClass{ + Label: a.Label, + Score: a.Score, + Index: int32(a.Index), + }) + } + return resp, nil +} + +// soundStreamScores runs pcm through a fresh sound stream and returns the +// drained per-window scores plus the tagger's class count. Every C call runs +// under engineMu; the stream is freed (deferred right after a successful +// begin) even when a later feed or drain call fails. Each feed's returned +// segments array is freed with parakeet_capi_free_sound_segments even though +// SoundDetection has no use for the segments themselves (it only reads the +// drained window scores). +func (p *ParakeetCpp) soundStreamScores(pcm []float32) ([]soundWindowJSON, int, error) { + p.engineMu.Lock() + defer p.engineMu.Unlock() + + nClasses := int(CppNumClasses(p.tagCtx)) + + var opts cSoundOpts + CppSoundOptsDefault(&opts) + opts.WindowSec = 10 + opts.HopSec = 10 + opts.TopK = int32(nClasses) + + stream := CppSoundStreamBegin(p.tagCtx, &opts) + if stream == 0 { + return nil, nClasses, fmt.Errorf("parakeet-cpp: sound_stream_begin failed: %s", soundLastError(p.tagCtx)) + } + defer CppSoundStreamFree(stream) + + offset := 0 + for { + end := offset + soundFeedChunkSamples + isLast := int32(0) + if end >= len(pcm) { + end = len(pcm) + isLast = 1 + } + var samplePtr *float32 + if end > offset { + samplePtr = &pcm[offset] + } + + var segsOut uintptr + var nOut int32 + rc := CppSoundStreamFeed(stream, samplePtr, int32(end-offset), isLast, &segsOut, &nOut) + if segsOut != 0 && CppFreeSoundSegments != nil { + CppFreeSoundSegments(segsOut) + } + if rc != 0 { + return nil, nClasses, fmt.Errorf("parakeet-cpp: sound_stream_feed failed: %s", soundLastError(p.tagCtx)) + } + + offset = end + if isLast == 1 { + break + } + } + + raw := CppSoundStreamDrainScoresJSON(stream) + if raw == 0 { + return nil, nClasses, fmt.Errorf("parakeet-cpp: sound_stream_drain_scores_json failed: %s", soundLastError(p.tagCtx)) + } + doc := goStringFromCPtr(raw) + CppFreeString(raw) + + var windows []soundWindowJSON + if err := json.Unmarshal([]byte(doc), &windows); err != nil { + return nil, nClasses, fmt.Errorf("parakeet-cpp: decode sound scores json: %w", err) + } + return windows, nClasses, nil +} + +// soundLastError reads ctx's last_error, substituting a fallback message +// when the C side left it empty. +func soundLastError(ctx uintptr) string { + msg := CppLastError(ctx) + if msg == "" { + msg = "unknown error" + } + return msg +} + +// averageWindowScores averages each class's score across the drained +// per-window scores: CED's own long-clip method, summing a class's score +// over every window and dividing by the window count (a class absent from a +// window's tags counts as 0 in that window). Only classes that appeared in +// at least one window are returned, in no particular order; callers sort and +// filter afterward. A pure function so it is easy to unit test in isolation +// from the C stream. +func averageWindowScores(windows []soundWindowJSON, nClasses int) []classAvg { + if len(windows) == 0 { + return nil + } + + sums := make(map[int]float32) + labels := make(map[int]string) + for _, w := range windows { + for _, t := range w.Tags { + if t.Index < 0 || (nClasses > 0 && t.Index >= nClasses) { + continue + } + sums[t.Index] += t.Score + if _, ok := labels[t.Index]; !ok { + labels[t.Index] = t.Label + } + } + } + + n := float32(len(windows)) + out := make([]classAvg, 0, len(sums)) + for idx, sum := range sums { + out = append(out, classAvg{Index: idx, Label: labels[idx], Score: sum / n}) + } + return out +} + +// sortSoundDetectionsDesc sorts avgs by score descending, breaking ties by +// class index for a deterministic order (map iteration in +// averageWindowScores is otherwise unordered). +func sortSoundDetectionsDesc(avgs []classAvg) { + sort.Slice(avgs, func(i, j int) bool { + if avgs[i].Score != avgs[j].Score { + return avgs[i].Score > avgs[j].Score + } + return avgs[i].Index < avgs[j].Index + }) +} + +// filterSoundDetections drops entries scoring below threshold, then keeps +// only the first topK entries (0 = keep all). avgs is assumed already sorted +// descending by score. +func filterSoundDetections(avgs []classAvg, threshold float32, topK int32) []classAvg { + out := avgs[:0:0] + for _, a := range avgs { + if a.Score < threshold { + continue + } + out = append(out, a) + } + if topK > 0 && int32(len(out)) > topK { + out = out[:topK] + } + return out +} diff --git a/backend/go/parakeet-cpp/sound_test.go b/backend/go/parakeet-cpp/sound_test.go new file mode 100644 index 000000000000..8ecacd722cd7 --- /dev/null +++ b/backend/go/parakeet-cpp/sound_test.go @@ -0,0 +1,293 @@ +package main + +import ( + "context" + "path/filepath" + "sync" + "unsafe" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// The SoundDetection specs drive it entirely against stubbed +// CppSoundStreamBegin / CppSoundStreamFeed / CppSoundStreamDrainScoresJSON / +// CppSoundStreamFree / CppFreeSoundSegments / CppSoundOptsDefault / +// CppNumClasses / CppFreeString / CppLastError (the same seam diarize_test.go +// and live_test.go use), so they run without libparakeet.so. + +// soundCstrPool hands out NUL-terminated C-style strings backed by Go memory +// and keeps them alive for the duration of a spec (goStringFromCPtr reads +// through the raw pointer; mirrors diarize_test.go's diarizeCstrPool). +type soundCstrPool struct { + mu sync.Mutex + bufs [][]byte +} + +func (p *soundCstrPool) cstr(s string) uintptr { + p.mu.Lock() + defer p.mu.Unlock() + b := append([]byte(s), 0) + p.bufs = append(p.bufs, b) + return uintptr(unsafe.Pointer(&b[0])) +} + +// soundStubs swaps every C entry point SoundDetection touches and returns a +// restore func for AfterEach (mirrors diarize_test.go's diarizeStubs). +func soundStubs() (restore func()) { + savedBegin := CppSoundStreamBegin + savedFeed := CppSoundStreamFeed + savedDrain := CppSoundStreamDrainScoresJSON + savedFree := CppSoundStreamFree + savedFreeSegs := CppFreeSoundSegments + savedOptsDefault := CppSoundOptsDefault + savedNumClasses := CppNumClasses + savedFreeString := CppFreeString + savedLastError := CppLastError + return func() { + CppSoundStreamBegin = savedBegin + CppSoundStreamFeed = savedFeed + CppSoundStreamDrainScoresJSON = savedDrain + CppSoundStreamFree = savedFree + CppFreeSoundSegments = savedFreeSegs + CppSoundOptsDefault = savedOptsDefault + CppNumClasses = savedNumClasses + CppFreeString = savedFreeString + CppLastError = savedLastError + } +} + +// soundWav writes a silent 16 kHz mono WAV of the given duration (seconds) to +// a fresh temp file and returns its path. decodeWavMono16k reads real audio +// bytes off disk, so SoundDetection needs a file on disk even though the +// stubbed C calls never look at its samples. +func soundWav(seconds float64) string { + GinkgoHelper() + path := filepath.Join(GinkgoT().TempDir(), "sound.wav") + writeMono16kWav(path, int(seconds*16000)) + return path +} + +// noopFeed is a CppSoundStreamFeed stub that always succeeds and returns no +// segments, for specs that only care about the drained window scores. +func noopFeed(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + *out = 0 + *nOut = 0 + return 0 +} + +var _ = Describe("ParakeetCpp.SoundDetection", func() { + var restore func() + var pool *soundCstrPool + + BeforeEach(func() { + restore = soundStubs() + pool = &soundCstrPool{} + CppFreeString = func(uintptr) {} + CppFreeSoundSegments = func(uintptr) {} + CppSoundOptsDefault = func(o *cSoundOpts) { *o = cSoundOpts{} } + }) + AfterEach(func() { restore() }) + + It("fails with FailedPrecondition when no sound model is loaded", func() { + p := &ParakeetCpp{} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.FailedPrecondition)) + Expect(err.Error()).To(ContainSubstring("model is not a sound")) + }) + + It("fails with Unimplemented when the loaded libparakeet.so has no sound_stream_begin symbol", func() { + CppSoundStreamBegin = nil + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + }) + + It("averages two windows per class and sorts descending", func() { + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + return pool.cstr(`[` + + `{"start":0,"end":10,"tags":[{"index":0,"label":"Speech","score":0.8},{"index":1,"label":"Music","score":0.2}]},` + + `{"start":10,"end":20,"tags":[{"index":0,"label":"Speech","score":0.4},{"index":1,"label":"Music","score":0.6}]}` + + `]`) + } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(HaveLen(2)) + Expect(resp.Detections[0].Label).To(Equal("Speech")) + Expect(resp.Detections[0].Score).To(BeNumerically("~", 0.6, 1e-6)) + Expect(resp.Detections[1].Label).To(Equal("Music")) + Expect(resp.Detections[1].Score).To(BeNumerically("~", 0.4, 1e-6)) + }) + + It("drops classes scoring below threshold", func() { + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + return pool.cstr(`[{"start":0,"end":10,"tags":[` + + `{"index":0,"label":"Speech","score":0.8},` + + `{"index":1,"label":"Music","score":0.2}]}]`) + } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{ + Src: soundWav(1), Threshold: 0.5, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(HaveLen(1)) + Expect(resp.Detections[0].Label).To(Equal("Speech")) + }) + + It("keeps only the top_k entries", func() { + CppNumClasses = func(uintptr) int32 { return 4 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + return pool.cstr(`[{"start":0,"end":10,"tags":[` + + `{"index":0,"label":"A","score":0.9},` + + `{"index":1,"label":"B","score":0.7},` + + `{"index":2,"label":"C","score":0.5},` + + `{"index":3,"label":"D","score":0.3}]}]`) + } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{ + Src: soundWav(1), TopK: 3, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(HaveLen(3)) + Expect(resp.Detections[0].Label).To(Equal("A")) + Expect(resp.Detections[1].Label).To(Equal("B")) + Expect(resp.Detections[2].Label).To(Equal("C")) + }) + + It("keeps all classes when top_k is 0", func() { + CppNumClasses = func(uintptr) int32 { return 4 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + return pool.cstr(`[{"start":0,"end":10,"tags":[` + + `{"index":0,"label":"A","score":0.9},` + + `{"index":1,"label":"B","score":0.7},` + + `{"index":2,"label":"C","score":0.5},` + + `{"index":3,"label":"D","score":0.3}]}]`) + } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{ + Src: soundWav(1), TopK: 0, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(HaveLen(4)) + }) + + It("passes window 10s, hop 10s and top_k = the tagger's class count to sound_stream_begin", func() { + CppNumClasses = func(uintptr) int32 { return 527 } + var gotOpts cSoundOpts + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { + gotOpts = *o + return 1 + } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { return pool.cstr(`[]`) } + + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).ToNot(HaveOccurred()) + Expect(gotOpts.WindowSec).To(BeNumerically("==", 10)) + Expect(gotOpts.HopSec).To(BeNumerically("==", 10)) + Expect(gotOpts.TopK).To(Equal(int32(527))) + }) + + It("returns no detections without error for a short clip whose drain is empty", func() { + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { return pool.cstr(`[]`) } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(0.1)}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(BeEmpty()) + }) + + It("surfaces last_error and still frees the stream when feed fails", func() { + freed := false + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + *out = 0 + *nOut = 0 + return 1 + } + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + Fail("drain_scores_json must not be called when feed failed") + return 0 + } + CppSoundStreamFree = func(uintptr) { freed = true } + CppLastError = func(uintptr) string { return "boom" } + + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("boom")) + Expect(freed).To(BeTrue()) + }) +}) + +var _ = Describe("averageWindowScores", func() { + It("returns nil for no windows", func() { + Expect(averageWindowScores(nil, 2)).To(BeNil()) + }) + + It("treats a class absent from a window as 0 in that window's contribution", func() { + windows := []soundWindowJSON{ + {Tags: []soundTagJSON{{Index: 0, Label: "Speech", Score: 1.0}}}, + {Tags: []soundTagJSON{}}, // Speech absent this window + } + out := averageWindowScores(windows, 2) + Expect(out).To(HaveLen(1)) + Expect(out[0].Index).To(Equal(0)) + Expect(out[0].Score).To(BeNumerically("~", 0.5, 1e-6)) + }) + + It("ignores an out-of-range class index", func() { + windows := []soundWindowJSON{ + {Tags: []soundTagJSON{{Index: 5, Label: "Bogus", Score: 1.0}}}, + } + Expect(averageWindowScores(windows, 2)).To(BeEmpty()) + }) +}) + +var _ = Describe("filterSoundDetections", func() { + It("keeps everything when top_k is 0 and threshold is 0", func() { + in := []classAvg{{Index: 0, Score: 0.1}, {Index: 1, Score: 0.9}} + Expect(filterSoundDetections(in, 0, 0)).To(HaveLen(2)) + }) + + It("drops entries below threshold before applying top_k", func() { + in := []classAvg{ + {Index: 0, Score: 0.9}, + {Index: 1, Score: 0.4}, + {Index: 2, Score: 0.1}, + } + out := filterSoundDetections(in, 0.3, 5) + Expect(out).To(HaveLen(2)) + }) +}) From 4a219977530e72d4816de982e75db78b0ee87329 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:09:01 +0000 Subject: [PATCH 05/30] fix(parakeet-cpp): cancel sound detection mid-feed, shrink the lock SoundDetection now checks ctx before each 10 s feed slice (mirroring driver.go's feedSlices) and returns Canceled if the caller gave up, so a long clip can be interrupted instead of feeding to completion regardless. The stream is still freed on every path, cancellation included. Also narrow engineMu to the C calls: the drained JSON document is now decoded after the lock is released, splitting soundStreamScores into a locked soundStreamDrain (opts, begin, feed, drain, free) and an unlocked json.Unmarshal. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/sound.go | 56 +++++++++++++++++++-------- backend/go/parakeet-cpp/sound_test.go | 53 +++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 17 deletions(-) diff --git a/backend/go/parakeet-cpp/sound.go b/backend/go/parakeet-cpp/sound.go index e85d1d58db1f..e7dc36d6fe58 100644 --- a/backend/go/parakeet-cpp/sound.go +++ b/backend/go/parakeet-cpp/sound.go @@ -68,7 +68,7 @@ func (p *ParakeetCpp) SoundDetection(ctx context.Context, req *pb.SoundDetection return nil, err } - windows, nClasses, err := p.soundStreamScores(pcm) + windows, nClasses, err := p.soundStreamScores(ctx, pcm) if err != nil { return nil, err } @@ -89,13 +89,34 @@ func (p *ParakeetCpp) SoundDetection(ctx context.Context, req *pb.SoundDetection } // soundStreamScores runs pcm through a fresh sound stream and returns the -// drained per-window scores plus the tagger's class count. Every C call runs -// under engineMu; the stream is freed (deferred right after a successful -// begin) even when a later feed or drain call fails. Each feed's returned -// segments array is freed with parakeet_capi_free_sound_segments even though -// SoundDetection has no use for the segments themselves (it only reads the -// drained window scores). -func (p *ParakeetCpp) soundStreamScores(pcm []float32) ([]soundWindowJSON, int, error) { +// drained per-window scores plus the tagger's class count. The C calls run +// under engineMu (see soundStreamDrain); JSON decoding happens after the +// lock is released. +func (p *ParakeetCpp) soundStreamScores(ctx context.Context, pcm []float32) ([]soundWindowJSON, int, error) { + doc, nClasses, err := p.soundStreamDrain(ctx, pcm) + if err != nil { + return nil, nClasses, err + } + + var windows []soundWindowJSON + if err := json.Unmarshal([]byte(doc), &windows); err != nil { + return nil, nClasses, fmt.Errorf("parakeet-cpp: decode sound scores json: %w", err) + } + return windows, nClasses, nil +} + +// soundStreamDrain runs pcm through a fresh sound stream and returns the +// raw JSON document parakeet_capi_sound_stream_drain_scores_json drained, +// plus the tagger's class count. Every C call (opts default, begin, feed, +// free, drain) runs under engineMu; the stream is freed (deferred right +// after a successful begin) even when a later feed or drain call fails, or +// ctx is cancelled mid-feed. Each feed's returned segments array is freed +// with parakeet_capi_free_sound_segments even though SoundDetection has no +// use for the segments themselves (it only reads the drained window +// scores). ctx.Err() is checked before each feed slice, mirroring +// driver.go's feedSlices, so a long clip can be cancelled mid-feed; the +// caller decodes the returned JSON outside the lock. +func (p *ParakeetCpp) soundStreamDrain(ctx context.Context, pcm []float32) (string, int, error) { p.engineMu.Lock() defer p.engineMu.Unlock() @@ -109,12 +130,18 @@ func (p *ParakeetCpp) soundStreamScores(pcm []float32) ([]soundWindowJSON, int, stream := CppSoundStreamBegin(p.tagCtx, &opts) if stream == 0 { - return nil, nClasses, fmt.Errorf("parakeet-cpp: sound_stream_begin failed: %s", soundLastError(p.tagCtx)) + return "", nClasses, fmt.Errorf("parakeet-cpp: sound_stream_begin failed: %s", soundLastError(p.tagCtx)) } defer CppSoundStreamFree(stream) offset := 0 for { + if ctx != nil { + if err := ctx.Err(); err != nil { + return "", nClasses, status.Error(codes.Canceled, "parakeet-cpp: sound detection cancelled") + } + } + end := offset + soundFeedChunkSamples isLast := int32(0) if end >= len(pcm) { @@ -133,7 +160,7 @@ func (p *ParakeetCpp) soundStreamScores(pcm []float32) ([]soundWindowJSON, int, CppFreeSoundSegments(segsOut) } if rc != 0 { - return nil, nClasses, fmt.Errorf("parakeet-cpp: sound_stream_feed failed: %s", soundLastError(p.tagCtx)) + return "", nClasses, fmt.Errorf("parakeet-cpp: sound_stream_feed failed: %s", soundLastError(p.tagCtx)) } offset = end @@ -144,16 +171,11 @@ func (p *ParakeetCpp) soundStreamScores(pcm []float32) ([]soundWindowJSON, int, raw := CppSoundStreamDrainScoresJSON(stream) if raw == 0 { - return nil, nClasses, fmt.Errorf("parakeet-cpp: sound_stream_drain_scores_json failed: %s", soundLastError(p.tagCtx)) + return "", nClasses, fmt.Errorf("parakeet-cpp: sound_stream_drain_scores_json failed: %s", soundLastError(p.tagCtx)) } doc := goStringFromCPtr(raw) CppFreeString(raw) - - var windows []soundWindowJSON - if err := json.Unmarshal([]byte(doc), &windows); err != nil { - return nil, nClasses, fmt.Errorf("parakeet-cpp: decode sound scores json: %w", err) - } - return windows, nClasses, nil + return doc, nClasses, nil } // soundLastError reads ctx's last_error, substituting a fallback message diff --git a/backend/go/parakeet-cpp/sound_test.go b/backend/go/parakeet-cpp/sound_test.go index 8ecacd722cd7..0124118a43ad 100644 --- a/backend/go/parakeet-cpp/sound_test.go +++ b/backend/go/parakeet-cpp/sound_test.go @@ -249,6 +249,59 @@ var _ = Describe("ParakeetCpp.SoundDetection", func() { Expect(err.Error()).To(ContainSubstring("boom")) Expect(freed).To(BeTrue()) }) + + It("returns Canceled without feeding when ctx is already cancelled, and frees the stream", func() { + freed := false + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + Fail("sound_stream_feed must not be called when ctx is already cancelled") + return 0 + } + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + Fail("drain_scores_json must not be called when ctx is already cancelled") + return 0 + } + CppSoundStreamFree = func(uintptr) { freed = true } + + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(ctx, &pb.SoundDetectionRequest{Src: soundWav(15)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Canceled)) + Expect(freed).To(BeTrue()) + }) + + It("stops feeding and frees the stream when ctx is cancelled mid-feed", func() { + freed := false + feedCount := 0 + ctx, cancel := context.WithCancel(context.Background()) + + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + feedCount++ + cancel() // cancel after the first feed so a second chunk would exist if not stopped + *out = 0 + *nOut = 0 + return 0 + } + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + Fail("drain_scores_json must not be called when the feed loop was cancelled") + return 0 + } + CppSoundStreamFree = func(uintptr) { freed = true } + + // Two 10 s chunks, so a second feed call would happen without the cancel. + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(ctx, &pb.SoundDetectionRequest{Src: soundWav(15)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Canceled)) + Expect(feedCount).To(Equal(1)) + Expect(freed).To(BeTrue()) + }) }) var _ = Describe("averageWindowScores", func() { From dc53ef69dd31c57a0b88b654099e2ada424c8637 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:16:27 +0000 Subject: [PATCH 06/30] feat(parakeet-cpp): stream speaker and sound events during live transcription Add two additive proto fields, LiveSpeakerSegment and LiveSoundEvent, repeated on TranscriptLiveResponse. When a diarization or sound companion model is loaded, AudioTranscriptionLive now runs a no-ASR scene stream (parakeet_capi_scene_stream_begin) beside the ASR streaming session, feeding it the same PCM slices and forwarding any closed speaker or sound events alongside the matching ASR delta, or on their own when a slice has no ASR output. The scene stream is freed and reopened on a mid-stream Config reset, flushed with is_last before the closing FinalResult, and degrades gracefully (a warning, not an error) when begin or a later feed call fails, so live transcription keeps working ASR-only. Existing live behavior is unchanged when no companion is configured, and no scene C call is made in that case. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/backend.proto | 16 ++ backend/go/parakeet-cpp/live.go | 69 +++++++-- backend/go/parakeet-cpp/live_test.go | 214 +++++++++++++++++++++++++++ backend/go/parakeet-cpp/scene.go | 199 +++++++++++++++++++++++++ 4 files changed, 488 insertions(+), 10 deletions(-) create mode 100644 backend/go/parakeet-cpp/scene.go diff --git a/backend/backend.proto b/backend/backend.proto index 54255528e5ad..0aa0d253811c 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -641,6 +641,22 @@ message TranscriptLiveResponse { repeated TranscriptWord words = 4; // words finalized by this feed (stream-relative ns) TranscriptResult final_result = 5; // terminal message only, after the send side closes bool eob = 6; // fired: a backchannel ("uh-huh") ended — NOT a turn boundary + repeated LiveSpeakerSegment speakers = 7; // closed speaker segments from a companion diarization/scene stream + repeated LiveSoundEvent sounds = 8; // closed sound events from a companion sound/scene stream +} + +message LiveSpeakerSegment { + string speaker = 1; // decimal speaker index + int64 start = 2; // stream-relative nanoseconds + int64 end = 3; +} + +message LiveSoundEvent { + string label = 1; + int32 index = 2; + float peak = 3; + int64 start = 4; // stream-relative nanoseconds + int64 end = 5; } message TranscriptWord { diff --git a/backend/go/parakeet-cpp/live.go b/backend/go/parakeet-cpp/live.go index 627492f1619d..1fb48ffa6418 100644 --- a/backend/go/parakeet-cpp/live.go +++ b/backend/go/parakeet-cpp/live.go @@ -71,6 +71,21 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest // current when the RPC unwinds. defer func() { p.streamFree(stream) }() + // sceneStream runs a no-ASR scene stream (diarization/sound only) beside + // the ASR session when a diarization_model:/sound_model: companion is + // loaded (see scene.go). 0 means scene events are disabled: no + // companions, or the begin/a later feed call failed (logged below / + // in feedSlicesScene), in which case live transcription continues + // ASR-only. Reassigned on a mid-stream Config reset alongside stream. + var sceneStream uintptr + if p.sceneWanted() { + sceneStream = p.sceneBegin() + if sceneStream == 0 { + xlog.Warn("parakeet-cpp: scene stream begin failed; live continues without speaker/sound events") + } + } + defer func() { p.sceneFree(sceneStream) }() + out <- &pb.TranscriptLiveResponse{Ready: true} var ( @@ -87,21 +102,30 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest ) // emit forwards one decode increment: it streams the per-feed tokens the - // realtime turn detector consumes (delta/eou/eob/words) and accumulates the - // running transcript for the closing FinalResult. No segmentation or + // realtime turn detector consumes (delta/eou/eob/words), any closed + // speaker/sound events from the companion scene stream, and accumulates + // the running transcript for the closing FinalResult. No segmentation or // boundary latch here — the live consumer reads only the streamed tokens // and the final Text; per-utterance segments and the terminal flag // are an offline-path concern (see AudioTranscriptionStream / boundary.go). - emit := func(r streamFeedResult) error { + // + // A slice with no ASR output (delta/eou/eob/words) but a closed scene + // event still sends its own response, and a slice with both sends one + // response carrying everything. + emit := func(r streamFeedResult, scene sceneFeedJSON) error { if r.Delta != "" { full.WriteString(r.Delta) } - if r.Delta != "" || r.Eou || r.Eob || len(r.Words) > 0 { + speakers := liveSpeakersToProto(scene.Speakers) + sounds := liveSoundsToProto(scene.Sounds) + if r.Delta != "" || r.Eou || r.Eob || len(r.Words) > 0 || len(speakers) > 0 || len(sounds) > 0 { out <- &pb.TranscriptLiveResponse{ - Delta: r.Delta, - Eou: r.Eou, - Eob: r.Eob, - Words: liveWordsToProto(r.Words), + Delta: r.Delta, + Eou: r.Eou, + Eob: r.Eob, + Words: liveWordsToProto(r.Words), + Speakers: speakers, + Sounds: sounds, } } return nil @@ -123,6 +147,16 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest return grpcerrors.LiveTranscriptionUnsupported("parakeet-cpp", "loaded model is not a cache-aware streaming model") } + // The scene stream is freed and begun again alongside the ASR + // session, mirroring the reset above. + p.sceneFree(sceneStream) + sceneStream = 0 + if p.sceneWanted() { + sceneStream = p.sceneBegin() + if sceneStream == 0 { + xlog.Warn("parakeet-cpp: scene stream begin failed; live continues without speaker/sound events") + } + } full.Reset() fedSecs = 0 case *pb.TranscriptLiveRequest_Audio: @@ -132,7 +166,8 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest start := time.Now() // nil ctx: a live session is bounded by this request channel, not a // context — cancellation is the caller closing the stream. - if err := p.feedSlices(nil, stream, pcm, emit); err != nil { + sceneStream, err = p.feedSlicesScene(nil, stream, sceneStream, pcm, emit) + if err != nil { return err } wallSec := time.Since(start).Seconds() @@ -156,9 +191,23 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest // The live FinalResult carries only Text — the authoritative full-turn // transcript the realtime core commits. Per-utterance segments, duration, // and the terminal flag are not produced on the live path. - if err := p.flushTail(stream, emit); err != nil { + if err := p.flushTail(stream, func(r streamFeedResult) error { + return emit(r, sceneFeedJSON{}) + }); err != nil { return err } + // The scene stream gets its own is_last flush (it consumes no new audio + // here, so it is not part of flushTail above); its remaining events go + // out before the terminal FinalResult, then the stream is released by the + // deferred sceneFree above. + if sceneStream != 0 { + doc, err := p.sceneFeed(sceneStream, nil, true) + if err != nil { + xlog.Warn("parakeet-cpp: live scene finalize failed", "err", err) + } else if err := emit(streamFeedResult{}, doc); err != nil { + return err + } + } out <- &pb.TranscriptLiveResponse{ FinalResult: &pb.TranscriptResult{Text: strings.TrimSpace(full.String())}, } diff --git a/backend/go/parakeet-cpp/live_test.go b/backend/go/parakeet-cpp/live_test.go index ca900010f236..e807a521a1c8 100644 --- a/backend/go/parakeet-cpp/live_test.go +++ b/backend/go/parakeet-cpp/live_test.go @@ -42,15 +42,47 @@ func liveStubs() (restore func()) { savedFinalize, savedFinalizeJSON := CppStreamFinalize, CppStreamFinalizeJSON savedFree, savedLastError := CppStreamFree, CppLastError savedFreeString := CppFreeString + savedSceneOptsDefault := CppSceneOptsDefault + savedSceneBegin := CppSceneStreamBegin + savedSceneFeedJSON := CppSceneStreamFeedJSON + savedSceneLastError := CppSceneStreamLastError + savedSceneFree := CppSceneStreamFree return func() { CppStreamBegin, CppStreamBeginLang = savedBegin, savedBeginLang CppStreamFeed, CppStreamFeedJSON = savedFeed, savedFeedJSON CppStreamFinalize, CppStreamFinalizeJSON = savedFinalize, savedFinalizeJSON CppStreamFree, CppLastError = savedFree, savedLastError CppFreeString = savedFreeString + CppSceneOptsDefault = savedSceneOptsDefault + CppSceneStreamBegin = savedSceneBegin + CppSceneStreamFeedJSON = savedSceneFeedJSON + CppSceneStreamLastError = savedSceneLastError + CppSceneStreamFree = savedSceneFree } } +// liveSceneStubs wires a minimal scene stream stub set onto p (a diarization +// and/or sound companion context so sceneWanted() is true) and returns the +// call-count trackers the specs assert on. feedJSON is called once per scene +// feed (including the is_last flush) and must return the canned document for +// that call. +func liveSceneStubs(feedJSON func(calls int, isLast int32) uintptr) (begun, freed *int) { + begun, freed = new(int), new(int) + CppSceneOptsDefault = func(o *cSceneOpts) { *o = cSceneOpts{} } + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { + *begun++ + return uintptr(100 + *begun) + } + calls := 0 + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + calls++ + return feedJSON(calls, isLast) + } + CppSceneStreamLastError = func(s uintptr) string { return "scene stub error" } + CppSceneStreamFree = func(s uintptr) { *freed++ } + return begun, freed +} + // runLive starts the RPC on its own goroutine and returns the request // channel plus a collector for everything the backend emitted. func runLive(p *ParakeetCpp) (chan *pb.TranscriptLiveRequest, chan *pb.TranscriptLiveResponse, chan error) { @@ -381,6 +413,188 @@ var _ = Describe("AudioTranscriptionLive (stubbed C API)", func() { Expect(got).To(HaveLen(1)) // just the ready ack close(in) }) + + It("makes no scene C call and behaves unchanged when no companion is loaded", func() { + // p has ctxPtr only (no diarCtx/tagCtx): sceneWanted() must be false, + // and none of the scene entry points may be touched. + CppSceneOptsDefault = func(o *cSceneOpts) { Fail("scene_opts_default called with no companions loaded") } + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { + Fail("scene_stream_begin called with no companions loaded") + return 0 + } + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + Fail("scene_stream_feed_json called with no companions loaded") + return 0 + } + CppSceneStreamFree = func(s uintptr) { Fail("scene_stream_free called with no companions loaded") } + + CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { + return pool.cstr(`{"text":"hi","eou":0,"frame_sec":0.08,"words":[]}`) + } + CppStreamFinalizeJSON = func(s uintptr) uintptr { + return pool.cstr(`{"text":"","eou":0,"frame_sec":0.08,"words":[]}`) + } + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(3)) // ready, delta, final + Expect(got[1].Speakers).To(BeEmpty()) + Expect(got[1].Sounds).To(BeEmpty()) + }) +}) + +var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { + var ( + pool *liveCstrPool + restore func() + p *ParakeetCpp + ) + + BeforeEach(func() { + pool = &liveCstrPool{} + restore = liveStubs() + p = &ParakeetCpp{ctxPtr: 1, diarCtx: 2} + + CppStreamBeginLang = nil + CppStreamBegin = func(ctx uintptr) uintptr { return 7 } + CppStreamFree = func(s uintptr) {} + CppFreeString = func(s uintptr) {} + CppLastError = func(ctx uintptr) string { return "stub error" } + CppStreamFeed = nil + CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { + return pool.cstr(`{"text":"","eou":0,"frame_sec":0.08,"words":[]}`) + } + CppStreamFinalize = nil + CppStreamFinalizeJSON = func(s uintptr) uintptr { + return pool.cstr(`{"text":"","eou":0,"frame_sec":0.08,"words":[]}`) + } + }) + + AfterEach(func() { restore() }) + + It("emits a closed speaker segment as its own response", func() { + liveSceneStubs(func(calls int, isLast int32) uintptr { + if calls == 1 { + return pool.cstr(`{"speakers":[{"speaker":0,"start":0.1,"end":0.6}],"sounds":[]}`) + } + return pool.cstr(`{"speakers":[],"sounds":[]}`) + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(3)) // ready, speaker-only response, final + Expect(got[1].Delta).To(BeEmpty()) + Expect(got[1].Speakers).To(HaveLen(1)) + Expect(got[1].Speakers[0].Speaker).To(Equal("0")) + Expect(got[1].Speakers[0].Start).To(Equal(int64(0.1 * 1e9))) + Expect(got[1].Speakers[0].End).To(Equal(int64(0.6 * 1e9))) + }) + + It("combines an ASR delta and a scene sound event in one response", func() { + CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { + return pool.cstr(`{"text":"hello ","eou":0,"frame_sec":0.08,` + + `"words":[{"w":"hello","start":0.1,"end":0.4,"conf":0.9}]}`) + } + liveSceneStubs(func(calls int, isLast int32) uintptr { + if calls == 1 { + return pool.cstr(`{"speakers":[],"sounds":[{"index":365,` + + `"label":"Chicken, rooster","start":24.0,"end":30.0,"peak":0.86}]}`) + } + return pool.cstr(`{"speakers":[],"sounds":[]}`) + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(3)) // ready, combined delta+sound, final + Expect(got[1].Delta).To(Equal("hello ")) + Expect(got[1].Sounds).To(HaveLen(1)) + Expect(got[1].Sounds[0].Label).To(Equal("Chicken, rooster")) + Expect(got[1].Sounds[0].Index).To(Equal(int32(365))) + Expect(got[1].Sounds[0].Peak).To(BeNumerically("~", 0.86, 1e-6)) + Expect(got[1].Sounds[0].Start).To(Equal(int64(24.0 * 1e9))) + Expect(got[1].Sounds[0].End).To(Equal(int64(30.0 * 1e9))) + }) + + It("flushes the scene stream is_last before the final result, then frees it", func() { + begun, freed := liveSceneStubs(func(calls int, isLast int32) uintptr { + if calls == 2 { + Expect(isLast).To(Equal(int32(1))) + return pool.cstr(`{"speakers":[{"speaker":1,"start":1.0,"end":2.0}],"sounds":[]}`) + } + Expect(isLast).To(Equal(int32(0))) + return pool.cstr(`{"speakers":[],"sounds":[]}`) + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(3)) // ready, speaker from the is_last flush, final + Expect(got[1].Speakers).To(HaveLen(1)) + Expect(got[1].Speakers[0].Speaker).To(Equal("1")) + Expect(got[2].FinalResult).NotTo(BeNil()) + Expect(*begun).To(Equal(1)) + Expect(*freed).To(Equal(1)) + }) + + It("frees and begins the scene stream again on a mid-stream config reset", func() { + streamBegun := 0 + CppStreamBegin = func(ctx uintptr) uintptr { streamBegun++; return uintptr(10 + streamBegun) } + begun, freed := liveSceneStubs(func(calls int, isLast int32) uintptr { + return pool.cstr(`{"speakers":[],"sounds":[]}`) + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + in <- liveConfig("") // reset + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + collectLive(out) + + Expect(*begun).To(Equal(2), "scene stream begun again on reset") + Expect(*freed).To(Equal(2), "old scene stream freed on reset, new one on unwind") + }) + + It("continues without scene events when scene begin fails", func() { + CppSceneOptsDefault = func(o *cSceneOpts) { *o = cSceneOpts{} } + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { return 0 } + sceneFeedCalled := false + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + sceneFeedCalled = true + return 0 + } + CppSceneStreamFree = func(s uintptr) {} + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(2)) // ready, final only: no scene events, no ASR delta this stub sends + Expect(sceneFeedCalled).To(BeFalse(), "no feed call once begin failed") + }) }) var _ = Describe("stripEouMarker", func() { diff --git a/backend/go/parakeet-cpp/scene.go b/backend/go/parakeet-cpp/scene.go new file mode 100644 index 000000000000..cd9289b47a20 --- /dev/null +++ b/backend/go/parakeet-cpp/scene.go @@ -0,0 +1,199 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// sceneSpeakerJSON mirrors one element of a scene feed document's "speakers" +// array: {"speaker":0,"start":0.0,"end":0.6}. +type sceneSpeakerJSON struct { + Speaker int `json:"speaker"` + Start float64 `json:"start"` + End float64 `json:"end"` +} + +// sceneSoundJSON mirrors one element of a scene feed document's "sounds" +// array: {"index":365,"label":"Chicken, rooster","start":24.0,"end":30.0,"peak":0.86}. +type sceneSoundJSON struct { + Index int `json:"index"` + Label string `json:"label"` + Start float64 `json:"start"` + End float64 `json:"end"` + Peak float32 `json:"peak"` +} + +// sceneFeedJSON mirrors the subset of the document +// parakeet_capi_scene_stream_feed_json returns (docs/sound.md) that the live +// path consumes: the closed "speakers" and "sounds" arrays. "t", +// "utterances", "words" and "active" belong to an offline scene/SAS +// consumer, not the live path, and are not decoded here. +type sceneFeedJSON struct { + Speakers []sceneSpeakerJSON `json:"speakers"` + Sounds []sceneSoundJSON `json:"sounds"` +} + +// sceneWanted reports whether AudioTranscriptionLive should run a companion +// scene stream beside the ASR streaming session: at least one of the +// diarization/sound companions must be loaded, and the scene C-API symbols +// must be present. In practice the nil checks are defensive rather than +// live: loadRoles only ever sets diarCtx/tagCtx when parakeet_capi_model_kind +// (ABI v8) is present, and main.go registers every scene symbol in the same +// Dlsym-gated block as model_kind, so a companion being loaded already +// guarantees the scene symbols exist. +func (p *ParakeetCpp) sceneWanted() bool { + return (p.diarCtx != 0 || p.tagCtx != 0) && + CppSceneOptsDefault != nil && CppSceneStreamBegin != nil && + CppSceneStreamFeedJSON != nil && CppSceneStreamFree != nil +} + +// sceneBegin opens a no-ASR scene stream (diarization and/or sound events +// only; the live path's own ASR session already covers transcription) under +// engineMu. Call only when sceneWanted() is true. A 0 return means the C +// call itself failed; the caller logs a warning and continues the live +// session without speaker/sound events. +func (p *ParakeetCpp) sceneBegin() uintptr { + p.engineMu.Lock() + defer p.engineMu.Unlock() + var opts cSceneOpts + CppSceneOptsDefault(&opts) + opts.DiarLatency = p.diarLatency + return CppSceneStreamBegin(0, p.diarCtx, p.tagCtx, &opts) +} + +// sceneFree releases a scene stream opened by sceneBegin. A 0 stream (scene +// events disabled or never began) is a no-op. +func (p *ParakeetCpp) sceneFree(s uintptr) { + if s == 0 { + return + } + p.engineMu.Lock() + defer p.engineMu.Unlock() + CppSceneStreamFree(s) +} + +// sceneFeed runs one scene-stream feed (or the is_last flush) under +// engineMu and returns the parsed document. last_error is stream-scoped +// (parakeet_capi_scene_stream_last_error), so it is read under the same +// lock as the failing call. +func (p *ParakeetCpp) sceneFeed(s uintptr, pcm []float32, isLast bool) (sceneFeedJSON, error) { + p.engineMu.Lock() + defer p.engineMu.Unlock() + + var last int32 + if isLast { + last = 1 + } + var ptr *float32 + if len(pcm) > 0 { + ptr = &pcm[0] + } + ret := CppSceneStreamFeedJSON(s, ptr, int32(len(pcm)), last) + if ret == 0 { + msg := "" + if CppSceneStreamLastError != nil { + msg = CppSceneStreamLastError(s) + } + if msg == "" { + msg = "unknown error" + } + return sceneFeedJSON{}, fmt.Errorf("parakeet-cpp: scene stream feed failed: %s", msg) + } + raw := goStringFromCPtr(ret) + CppFreeString(ret) + var doc sceneFeedJSON + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return sceneFeedJSON{}, fmt.Errorf("parakeet-cpp: decode scene json: %w", err) + } + return doc, nil +} + +// feedSlicesScene mirrors driver.go's feedSlices but also feeds the same pcm +// slice to an optional companion scene stream right after each ASR slice, so +// the live path's speaker/sound events stay time-aligned with the ASR +// decode increments. sceneStream == 0 disables scene feeding for this call +// (no companions, or a previous scene feed already disabled it this +// session). +// +// A scene feed failure degrades gracefully rather than aborting live +// transcription over a secondary feature: it frees the broken stream, warns +// once, and returns 0 so the caller carries the ASR-only session forward. +// It returns the (possibly now-zeroed) scene stream for the caller to keep +// across the next call. +func (p *ParakeetCpp) feedSlicesScene(ctx context.Context, stream, sceneStream uintptr, pcm []float32, onFeed func(streamFeedResult, sceneFeedJSON) error) (uintptr, error) { + for off := 0; off < len(pcm); off += streamChunkSamples { + if ctx != nil { + if err := ctx.Err(); err != nil { + return sceneStream, status.Error(codes.Canceled, "transcription cancelled") + } + } + end := min(off+streamChunkSamples, len(pcm)) + chunk := pcm[off:end] + + res, err := p.feedChunk(stream, chunk, false) + if err != nil { + return sceneStream, err + } + + var sceneDoc sceneFeedJSON + if sceneStream != 0 { + sceneDoc, err = p.sceneFeed(sceneStream, chunk, false) + if err != nil { + xlog.Warn("parakeet-cpp: live scene feed failed; disabling speaker/sound events for this session", + "err", err) + p.sceneFree(sceneStream) + sceneStream = 0 + sceneDoc = sceneFeedJSON{} + } + } + + if err := onFeed(res, sceneDoc); err != nil { + return sceneStream, err + } + } + return sceneStream, nil +} + +// liveSpeakersToProto maps a scene feed document's closed "speakers" into +// TranscriptLiveResponse.speakers (stream-relative nanoseconds). Reuses +// diarize.go's speakerLabel so the live path renders speaker indices the +// same way the offline Diarize RPC does. +func liveSpeakersToProto(speakers []sceneSpeakerJSON) []*pb.LiveSpeakerSegment { + if len(speakers) == 0 { + return nil + } + out := make([]*pb.LiveSpeakerSegment, len(speakers)) + for i, s := range speakers { + out[i] = &pb.LiveSpeakerSegment{ + Speaker: speakerLabel(s.Speaker), + Start: secondsToNanos(s.Start), + End: secondsToNanos(s.End), + } + } + return out +} + +// liveSoundsToProto maps a scene feed document's closed "sounds" into +// TranscriptLiveResponse.sounds (stream-relative nanoseconds). +func liveSoundsToProto(sounds []sceneSoundJSON) []*pb.LiveSoundEvent { + if len(sounds) == 0 { + return nil + } + out := make([]*pb.LiveSoundEvent, len(sounds)) + for i, s := range sounds { + out[i] = &pb.LiveSoundEvent{ + Label: s.Label, + Index: int32(s.Index), + Peak: s.Peak, + Start: secondsToNanos(s.Start), + End: secondsToNanos(s.End), + } + } + return out +} From f156c5e5dc36524f6468e20a0ef2c87d2d695067 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:24:00 +0000 Subject: [PATCH 07/30] fix(parakeet-cpp): keep scene events off the ASR critical path in live Emit each slice's ASR result right after the ASR feed, before the scene feed for that slice runs, so a companion diarization/sound model never adds scene compute latency in front of the delta or that drives realtime turn detection. Closed speakers/sounds go out afterward as their own response, so a slice with both now produces two responses, ASR first. The live feed log line now reports ASR and scene wall time separately. Re-check the diarization/sound contexts a scene stream was begun with against the live contexts before every feed, under the same lock: Free() can race between an ASR feed and the matching scene feed and free the model the stream borrows. A mismatch now returns without touching the C side. Freeing the stream itself stays unconditional; the scene stream's destructor only releases its own buffers and never touches the borrowed contexts. Also recover a panicking stub inside the live test goroutine instead of crashing the test binary, and reset the live decode-lag tracker on a mid-stream config reset, matching what its own comment already promised. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/live.go | 68 +++++++------- backend/go/parakeet-cpp/live_test.go | 74 +++++++++++---- backend/go/parakeet-cpp/scene.go | 133 +++++++++++++++++++-------- 3 files changed, 186 insertions(+), 89 deletions(-) diff --git a/backend/go/parakeet-cpp/live.go b/backend/go/parakeet-cpp/live.go index 1fb48ffa6418..6497779d54ee 100644 --- a/backend/go/parakeet-cpp/live.go +++ b/backend/go/parakeet-cpp/live.go @@ -71,20 +71,22 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest // current when the RPC unwinds. defer func() { p.streamFree(stream) }() - // sceneStream runs a no-ASR scene stream (diarization/sound only) beside - // the ASR session when a diarization_model:/sound_model: companion is - // loaded (see scene.go). 0 means scene events are disabled: no - // companions, or the begin/a later feed call failed (logged below / - // in feedSlicesScene), in which case live transcription continues - // ASR-only. Reassigned on a mid-stream Config reset alongside stream. - var sceneStream uintptr + // scene runs a no-ASR scene stream (diarization/sound only) beside the + // ASR session when a diarization_model:/sound_model: companion is loaded + // (see scene.go). A zero handle means scene events are disabled: no + // companions, or the begin/a later feed call failed (logged below / in + // feedSlicesScene), in which case live transcription continues ASR-only. + // Reassigned on a mid-stream Config reset alongside stream, which also + // brings back a scene stream the session had disabled after an earlier + // scene error. + var scene sceneStreamHandle if p.sceneWanted() { - sceneStream = p.sceneBegin() - if sceneStream == 0 { + scene = p.sceneBegin() + if scene.s == 0 { xlog.Warn("parakeet-cpp: scene stream begin failed; live continues without speaker/sound events") } } - defer func() { p.sceneFree(sceneStream) }() + defer func() { p.sceneFree(scene) }() out <- &pb.TranscriptLiveResponse{Ready: true} @@ -101,23 +103,23 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest behindWarned bool ) - // emit forwards one decode increment: it streams the per-feed tokens the - // realtime turn detector consumes (delta/eou/eob/words), any closed - // speaker/sound events from the companion scene stream, and accumulates - // the running transcript for the closing FinalResult. No segmentation or - // boundary latch here — the live consumer reads only the streamed tokens - // and the final Text; per-utterance segments and the terminal flag - // are an offline-path concern (see AudioTranscriptionStream / boundary.go). - // - // A slice with no ASR output (delta/eou/eob/words) but a closed scene - // event still sends its own response, and a slice with both sends one - // response carrying everything. - emit := func(r streamFeedResult, scene sceneFeedJSON) error { + // emit sends one decode increment as its own response when it carries + // anything: either the ASR side (delta/eou/eob/words, accumulated into + // the running transcript for the closing FinalResult) or the scene + // side (closed speaker/sound events), never both at once — the live + // audio loop below calls it once for the ASR result right after the ASR + // feed and, separately, once more for the scene document after the + // scene feed (see feedSlicesScene), so a slice with both produces two + // responses, ASR first. No segmentation or boundary latch here — the + // live consumer reads only the streamed tokens and the final Text; + // per-utterance segments and the terminal flag are an + // offline-path concern (see AudioTranscriptionStream / boundary.go). + emit := func(r streamFeedResult, sceneDoc sceneFeedJSON) error { if r.Delta != "" { full.WriteString(r.Delta) } - speakers := liveSpeakersToProto(scene.Speakers) - sounds := liveSoundsToProto(scene.Sounds) + speakers := liveSpeakersToProto(sceneDoc.Speakers) + sounds := liveSoundsToProto(sceneDoc.Sounds) if r.Delta != "" || r.Eou || r.Eob || len(r.Words) > 0 || len(speakers) > 0 || len(sounds) > 0 { out <- &pb.TranscriptLiveResponse{ Delta: r.Delta, @@ -149,16 +151,18 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest } // The scene stream is freed and begun again alongside the ASR // session, mirroring the reset above. - p.sceneFree(sceneStream) - sceneStream = 0 + p.sceneFree(scene) + scene = sceneStreamHandle{} if p.sceneWanted() { - sceneStream = p.sceneBegin() - if sceneStream == 0 { + scene = p.sceneBegin() + if scene.s == 0 { xlog.Warn("parakeet-cpp: scene stream begin failed; live continues without speaker/sound events") } } full.Reset() fedSecs = 0 + behindSec = 0 + behindWarned = false case *pb.TranscriptLiveRequest_Audio: pcm := payload.Audio.GetPcm() audioSec := float64(len(pcm)) / liveSampleRate @@ -166,7 +170,8 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest start := time.Now() // nil ctx: a live session is bounded by this request channel, not a // context — cancellation is the caller closing the stream. - sceneStream, err = p.feedSlicesScene(nil, stream, sceneStream, pcm, emit) + var asrWall, sceneWall time.Duration + scene, asrWall, sceneWall, err = p.feedSlicesScene(nil, stream, scene, pcm, emit) if err != nil { return err } @@ -177,6 +182,7 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest } xlog.Debug("parakeet-cpp: live feed", "audio_ms", int(audioSec*1000), "wall_ms", int(wallSec*1000), + "asr_wall_ms", int(asrWall.Seconds()*1000), "scene_wall_ms", int(sceneWall.Seconds()*1000), "behind_ms", int(behindSec*1000), "fed_s", fedSecs) if behindSec > 1 && !behindWarned { behindWarned = true @@ -200,8 +206,8 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest // here, so it is not part of flushTail above); its remaining events go // out before the terminal FinalResult, then the stream is released by the // deferred sceneFree above. - if sceneStream != 0 { - doc, err := p.sceneFeed(sceneStream, nil, true) + if scene.s != 0 { + doc, err := p.sceneFeed(scene, nil, true) if err != nil { xlog.Warn("parakeet-cpp: live scene finalize failed", "err", err) } else if err := emit(streamFeedResult{}, doc); err != nil { diff --git a/backend/go/parakeet-cpp/live_test.go b/backend/go/parakeet-cpp/live_test.go index e807a521a1c8..5b70725bab5e 100644 --- a/backend/go/parakeet-cpp/live_test.go +++ b/backend/go/parakeet-cpp/live_test.go @@ -64,9 +64,10 @@ func liveStubs() (restore func()) { // liveSceneStubs wires a minimal scene stream stub set onto p (a diarization // and/or sound companion context so sceneWanted() is true) and returns the // call-count trackers the specs assert on. feedJSON is called once per scene -// feed (including the is_last flush) and must return the canned document for -// that call. -func liveSceneStubs(feedJSON func(calls int, isLast int32) uintptr) (begun, freed *int) { +// feed (including the is_last flush) with the stream handle the C side would +// have received (so a reset spec can tell a pre-reset feed from a post-reset +// one) and must return the canned document for that call. +func liveSceneStubs(feedJSON func(calls int, s uintptr, isLast int32) uintptr) (begun, freed *int) { begun, freed = new(int), new(int) CppSceneOptsDefault = func(o *cSceneOpts) { *o = cSceneOpts{} } CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { @@ -76,7 +77,7 @@ func liveSceneStubs(feedJSON func(calls int, isLast int32) uintptr) (begun, free calls := 0 CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { calls++ - return feedJSON(calls, isLast) + return feedJSON(calls, s, isLast) } CppSceneStreamLastError = func(s uintptr) string { return "scene stub error" } CppSceneStreamFree = func(s uintptr) { *freed++ } @@ -84,12 +85,18 @@ func liveSceneStubs(feedJSON func(calls int, isLast int32) uintptr) (begun, free } // runLive starts the RPC on its own goroutine and returns the request -// channel plus a collector for everything the backend emitted. +// channel plus a collector for everything the backend emitted. GinkgoRecover +// turns an Expect/Fail failure inside a stub called from this goroutine into +// a normal spec failure instead of a panic that would crash the whole test +// binary (Ginkgo's failure handling is goroutine-local). func runLive(p *ParakeetCpp) (chan *pb.TranscriptLiveRequest, chan *pb.TranscriptLiveResponse, chan error) { in := make(chan *pb.TranscriptLiveRequest) out := make(chan *pb.TranscriptLiveResponse, 32) errCh := make(chan error, 1) - go func() { errCh <- p.AudioTranscriptionLive(in, out) }() + go func() { + defer GinkgoRecover() + errCh <- p.AudioTranscriptionLive(in, out) + }() return in, out, errCh } @@ -478,7 +485,7 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { AfterEach(func() { restore() }) It("emits a closed speaker segment as its own response", func() { - liveSceneStubs(func(calls int, isLast int32) uintptr { + liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { if calls == 1 { return pool.cstr(`{"speakers":[{"speaker":0,"start":0.1,"end":0.6}],"sounds":[]}`) } @@ -500,12 +507,12 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { Expect(got[1].Speakers[0].End).To(Equal(int64(0.6 * 1e9))) }) - It("combines an ASR delta and a scene sound event in one response", func() { + It("sends the ASR delta and a scene sound event as two responses, ASR first", func() { CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { return pool.cstr(`{"text":"hello ","eou":0,"frame_sec":0.08,` + `"words":[{"w":"hello","start":0.1,"end":0.4,"conf":0.9}]}`) } - liveSceneStubs(func(calls int, isLast int32) uintptr { + liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { if calls == 1 { return pool.cstr(`{"speakers":[],"sounds":[{"index":365,` + `"label":"Chicken, rooster","start":24.0,"end":30.0,"peak":0.86}]}`) @@ -520,18 +527,20 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { Expect(<-errCh).NotTo(HaveOccurred()) got := collectLive(out) - Expect(got).To(HaveLen(3)) // ready, combined delta+sound, final + Expect(got).To(HaveLen(4)) // ready, ASR delta, sound-only, final Expect(got[1].Delta).To(Equal("hello ")) - Expect(got[1].Sounds).To(HaveLen(1)) - Expect(got[1].Sounds[0].Label).To(Equal("Chicken, rooster")) - Expect(got[1].Sounds[0].Index).To(Equal(int32(365))) - Expect(got[1].Sounds[0].Peak).To(BeNumerically("~", 0.86, 1e-6)) - Expect(got[1].Sounds[0].Start).To(Equal(int64(24.0 * 1e9))) - Expect(got[1].Sounds[0].End).To(Equal(int64(30.0 * 1e9))) + Expect(got[1].Sounds).To(BeEmpty(), "the ASR response must not wait on the scene feed") + Expect(got[2].Delta).To(BeEmpty()) + Expect(got[2].Sounds).To(HaveLen(1)) + Expect(got[2].Sounds[0].Label).To(Equal("Chicken, rooster")) + Expect(got[2].Sounds[0].Index).To(Equal(int32(365))) + Expect(got[2].Sounds[0].Peak).To(BeNumerically("~", 0.86, 1e-6)) + Expect(got[2].Sounds[0].Start).To(Equal(int64(24.0 * 1e9))) + Expect(got[2].Sounds[0].End).To(Equal(int64(30.0 * 1e9))) }) It("flushes the scene stream is_last before the final result, then frees it", func() { - begun, freed := liveSceneStubs(func(calls int, isLast int32) uintptr { + begun, freed := liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { if calls == 2 { Expect(isLast).To(Equal(int32(1))) return pool.cstr(`{"speakers":[{"speaker":1,"start":1.0,"end":2.0}],"sounds":[]}`) @@ -558,7 +567,9 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { It("frees and begins the scene stream again on a mid-stream config reset", func() { streamBegun := 0 CppStreamBegin = func(ctx uintptr) uintptr { streamBegun++; return uintptr(10 + streamBegun) } - begun, freed := liveSceneStubs(func(calls int, isLast int32) uintptr { + var seenStreams []uintptr + begun, freed := liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { + seenStreams = append(seenStreams, s) return pool.cstr(`{"speakers":[],"sounds":[]}`) }) @@ -573,6 +584,33 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { Expect(*begun).To(Equal(2), "scene stream begun again on reset") Expect(*freed).To(Equal(2), "old scene stream freed on reset, new one on unwind") + // One scene feed per audio message (pre-reset, post-reset) plus the + // close is_last flush, which runs on the post-reset stream. + Expect(seenStreams).To(HaveLen(3)) + Expect(seenStreams[0]).NotTo(Equal(seenStreams[1]), "post-reset audio must go to the new scene stream handle") + Expect(seenStreams[2]).To(Equal(seenStreams[1]), "the close flush uses the post-reset stream too") + }) + + It("returns without a C call when Free() ran between begin and a scene feed", func() { + feedCalls := 0 + CppSceneOptsDefault = func(o *cSceneOpts) { *o = cSceneOpts{} } + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { return 999 } + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + feedCalls++ + return pool.cstr(`{"speakers":[],"sounds":[]}`) + } + + h := p.sceneBegin() + Expect(h.s).NotTo(BeZero()) + + // Simulate a Free() racing in between the begin and the next feed: it + // zeroes the companion context under engineMu, exactly as the real + // Free() does. + p.diarCtx = 0 + + _, err := p.sceneFeed(h, make([]float32, 10), false) + Expect(grpcerrors.IsModelNotLoaded(err)).To(BeTrue()) + Expect(feedCalls).To(Equal(0), "no C call once the scene stream's contexts were freed") }) It("continues without scene events when scene begin fails", func() { diff --git a/backend/go/parakeet-cpp/scene.go b/backend/go/parakeet-cpp/scene.go index cd9289b47a20..97dbdb6d903b 100644 --- a/backend/go/parakeet-cpp/scene.go +++ b/backend/go/parakeet-cpp/scene.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "fmt" + "time" + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/xlog" "google.golang.org/grpc/codes" @@ -53,39 +55,75 @@ func (p *ParakeetCpp) sceneWanted() bool { CppSceneStreamFeedJSON != nil && CppSceneStreamFree != nil } +// sceneStreamHandle bundles the C scene_stream pointer with the diar/tag +// contexts it was begun with. sceneFeed re-checks those against p.diarCtx/ +// p.tagCtx under engineMu before every call, so a Free() racing between the +// begin and a later feed (freeing the very contexts the stream borrows) is +// caught instead of handed to the C side — mirroring streamFeedDoc's re-check +// of p.ctxPtr (see the "Per-C-call engine serialization" comment in +// goparakeetcpp.go). The zero value (s == 0) means "no scene stream". +type sceneStreamHandle struct { + s uintptr + diar uintptr + tag uintptr +} + // sceneBegin opens a no-ASR scene stream (diarization and/or sound events // only; the live path's own ASR session already covers transcription) under -// engineMu. Call only when sceneWanted() is true. A 0 return means the C -// call itself failed; the caller logs a warning and continues the live -// session without speaker/sound events. -func (p *ParakeetCpp) sceneBegin() uintptr { +// engineMu. Call only when sceneWanted() is true. Refuses to begin with both +// contexts 0 (defensive: sceneWanted() already guards this). A zero handle +// means the C call itself failed; the caller logs a warning and continues +// the live session without speaker/sound events. +func (p *ParakeetCpp) sceneBegin() sceneStreamHandle { p.engineMu.Lock() defer p.engineMu.Unlock() + diar, tag := p.diarCtx, p.tagCtx + if diar == 0 && tag == 0 { + return sceneStreamHandle{} + } var opts cSceneOpts CppSceneOptsDefault(&opts) opts.DiarLatency = p.diarLatency - return CppSceneStreamBegin(0, p.diarCtx, p.tagCtx, &opts) + s := CppSceneStreamBegin(0, diar, tag, &opts) + if s == 0 { + return sceneStreamHandle{} + } + return sceneStreamHandle{s: s, diar: diar, tag: tag} } -// sceneFree releases a scene stream opened by sceneBegin. A 0 stream (scene -// events disabled or never began) is a no-op. -func (p *ParakeetCpp) sceneFree(s uintptr) { - if s == 0 { +// sceneFree releases a scene stream opened by sceneBegin. A zero handle +// (scene events disabled or never began) is a no-op. Safe to call even after +// the contexts the stream borrowed have been freed: parakeet_scene_stream's +// destructor only releases its own buffers and never dereferences the +// borrowed asr/diar/tagger pointers (verified against +// parakeet.cpp's parakeet_capi_scene_stream_free / SceneStream::~SceneStream +// / DiarPcmStream::~DiarPcmStream, all `= default`), unlike a feed call. +func (p *ParakeetCpp) sceneFree(h sceneStreamHandle) { + if h.s == 0 { return } p.engineMu.Lock() defer p.engineMu.Unlock() - CppSceneStreamFree(s) + CppSceneStreamFree(h.s) } // sceneFeed runs one scene-stream feed (or the is_last flush) under -// engineMu and returns the parsed document. last_error is stream-scoped -// (parakeet_capi_scene_stream_last_error), so it is read under the same -// lock as the failing call. -func (p *ParakeetCpp) sceneFeed(s uintptr, pcm []float32, isLast bool) (sceneFeedJSON, error) { +// engineMu and returns the parsed document. Before touching the C side it +// re-checks that p.diarCtx/p.tagCtx still match what the stream was begun +// with: Free() can run between the caller's ASR feed and this call (both +// take engineMu individually, never for a session's lifetime, so nothing +// blocks a concurrent Free()) and free the very model the stream borrows. +// A mismatch returns ModelNotLoaded without making the C call; last_error is +// otherwise stream-scoped (parakeet_capi_scene_stream_last_error), read +// under the same lock as the failing call. +func (p *ParakeetCpp) sceneFeed(h sceneStreamHandle, pcm []float32, isLast bool) (sceneFeedJSON, error) { p.engineMu.Lock() defer p.engineMu.Unlock() + if p.diarCtx != h.diar || p.tagCtx != h.tag { + return sceneFeedJSON{}, grpcerrors.ModelNotLoaded("parakeet-cpp") + } + var last int32 if isLast { last = 1 @@ -94,11 +132,11 @@ func (p *ParakeetCpp) sceneFeed(s uintptr, pcm []float32, isLast bool) (sceneFee if len(pcm) > 0 { ptr = &pcm[0] } - ret := CppSceneStreamFeedJSON(s, ptr, int32(len(pcm)), last) + ret := CppSceneStreamFeedJSON(h.s, ptr, int32(len(pcm)), last) if ret == 0 { msg := "" if CppSceneStreamLastError != nil { - msg = CppSceneStreamLastError(s) + msg = CppSceneStreamLastError(h.s) } if msg == "" { msg = "unknown error" @@ -116,48 +154,63 @@ func (p *ParakeetCpp) sceneFeed(s uintptr, pcm []float32, isLast bool) (sceneFee // feedSlicesScene mirrors driver.go's feedSlices but also feeds the same pcm // slice to an optional companion scene stream right after each ASR slice, so -// the live path's speaker/sound events stay time-aligned with the ASR -// decode increments. sceneStream == 0 disables scene feeding for this call -// (no companions, or a previous scene feed already disabled it this -// session). +// the live path's speaker/sound events stay time-aligned with the ASR decode +// increments. scene.s == 0 disables scene feeding for this call (no +// companions, or a previous scene feed already disabled it this session). +// +// The ASR result is emitted immediately after the ASR feed — the same +// response contents/timing a no-companion session would produce — before the +// scene feed for that slice runs, so a companion model never adds scene +// compute latency in front of the ASR delta/ that drives realtime turn +// detection. Any closed speakers/sounds from the scene feed are emitted +// afterward as their own response, so a slice with both produces two +// responses, ASR first. // // A scene feed failure degrades gracefully rather than aborting live // transcription over a secondary feature: it frees the broken stream, warns -// once, and returns 0 so the caller carries the ASR-only session forward. -// It returns the (possibly now-zeroed) scene stream for the caller to keep -// across the next call. -func (p *ParakeetCpp) feedSlicesScene(ctx context.Context, stream, sceneStream uintptr, pcm []float32, onFeed func(streamFeedResult, sceneFeedJSON) error) (uintptr, error) { +// once, and zeroes the handle so the caller carries the ASR-only session +// forward. Returns the (possibly now-zeroed) scene handle plus the +// cumulative ASR and scene wall time this call spent in feedChunk/sceneFeed, +// for the caller's lag log line. +func (p *ParakeetCpp) feedSlicesScene(ctx context.Context, stream uintptr, scene sceneStreamHandle, pcm []float32, onFeed func(streamFeedResult, sceneFeedJSON) error) (sceneStreamHandle, time.Duration, time.Duration, error) { + var asrWall, sceneWall time.Duration for off := 0; off < len(pcm); off += streamChunkSamples { if ctx != nil { if err := ctx.Err(); err != nil { - return sceneStream, status.Error(codes.Canceled, "transcription cancelled") + return scene, asrWall, sceneWall, status.Error(codes.Canceled, "transcription cancelled") } } end := min(off+streamChunkSamples, len(pcm)) chunk := pcm[off:end] + asrStart := time.Now() res, err := p.feedChunk(stream, chunk, false) + asrWall += time.Since(asrStart) if err != nil { - return sceneStream, err + return scene, asrWall, sceneWall, err } - - var sceneDoc sceneFeedJSON - if sceneStream != 0 { - sceneDoc, err = p.sceneFeed(sceneStream, chunk, false) - if err != nil { - xlog.Warn("parakeet-cpp: live scene feed failed; disabling speaker/sound events for this session", - "err", err) - p.sceneFree(sceneStream) - sceneStream = 0 - sceneDoc = sceneFeedJSON{} - } + if err := onFeed(res, sceneFeedJSON{}); err != nil { + return scene, asrWall, sceneWall, err } - if err := onFeed(res, sceneDoc); err != nil { - return sceneStream, err + if scene.s == 0 { + continue + } + sceneStart := time.Now() + sceneDoc, serr := p.sceneFeed(scene, chunk, false) + sceneWall += time.Since(sceneStart) + if serr != nil { + xlog.Warn("parakeet-cpp: live scene feed failed; disabling speaker/sound events for this session", + "err", serr) + p.sceneFree(scene) + scene = sceneStreamHandle{} + continue + } + if err := onFeed(streamFeedResult{}, sceneDoc); err != nil { + return scene, asrWall, sceneWall, err } } - return sceneStream, nil + return scene, asrWall, sceneWall, nil } // liveSpeakersToProto maps a scene feed document's closed "speakers" into From bacaf14aa20e5d1e9f306be4954acb8e496f3043 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:30:50 +0000 Subject: [PATCH 08/30] feat(realtime): surface live speaker and sound events Carry the backend's closed speaker segments and sound events (TranscriptLiveResponse fields 7/8) through LiveTranscriptionEvent as LiveSpeakerSegment/LiveSoundEvent (nanoseconds mapped to seconds), and forward them from the semantic_vad live path. Each speaker segment emits conversation.item.input_audio_transcription.segment with speaker, start, end and empty text under the turn's item id. Each sound event emits conversation.item.sound_detection with one tag (label, score = peak, index) and the event's new optional start/end seconds fields, omitted when unset so the existing unary/windowed sound-detection path is unaffected. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- core/backend/transcript_live.go | 48 +++++++++++++-- core/backend/transcript_live_internal_test.go | 28 +++++++++ .../endpoints/openai/realtime_semantic_vad.go | 29 +++++++++ .../openai/realtime_semantic_vad_test.go | 61 +++++++++++++++++++ .../openai/realtime_sound_detection_test.go | 38 ++++++++++++ .../endpoints/openai/types/server_events.go | 9 +++ 6 files changed, 208 insertions(+), 5 deletions(-) diff --git a/core/backend/transcript_live.go b/core/backend/transcript_live.go index 0ad6d72e1456..310723df909b 100644 --- a/core/backend/transcript_live.go +++ b/core/backend/transcript_live.go @@ -26,11 +26,33 @@ import ( // backchannel ("uh-huh") ended — callers must NOT treat Eob as a turn // boundary. type LiveTranscriptionEvent struct { - Delta string - Eou bool - Eob bool - Words []schema.TranscriptionWord - Final *schema.TranscriptionResult + Delta string + Eou bool + Eob bool + Words []schema.TranscriptionWord + Speakers []LiveSpeakerSegment + Sounds []LiveSoundEvent + Final *schema.TranscriptionResult +} + +// LiveSpeakerSegment is one closed speaker segment from a companion +// diarization/scene stream running alongside live transcription. Start/End +// are stream-relative seconds (mapped from the backend's nanoseconds). +type LiveSpeakerSegment struct { + Speaker string + Start float64 + End float64 +} + +// LiveSoundEvent is one closed sound event from a companion sound/scene +// stream running alongside live transcription. Start/End are stream-relative +// seconds (mapped from the backend's nanoseconds). +type LiveSoundEvent struct { + Label string + Index int + Peak float32 + Start float64 + End float64 } // LiveTranscriptionSession is a handle on an open live transcription stream. @@ -303,6 +325,22 @@ func liveEventFromProto(r *proto.TranscriptLiveResponse) LiveTranscriptionEvent Text: w.Text, }) } + for _, s := range r.GetSpeakers() { + ev.Speakers = append(ev.Speakers, LiveSpeakerSegment{ + Speaker: s.GetSpeaker(), + Start: time.Duration(s.GetStart()).Seconds(), + End: time.Duration(s.GetEnd()).Seconds(), + }) + } + for _, s := range r.GetSounds() { + ev.Sounds = append(ev.Sounds, LiveSoundEvent{ + Label: s.GetLabel(), + Index: int(s.GetIndex()), + Peak: s.GetPeak(), + Start: time.Duration(s.GetStart()).Seconds(), + End: time.Duration(s.GetEnd()).Seconds(), + }) + } if r.GetFinalResult() != nil { ev.Final = transcriptResultFromProto(r.GetFinalResult()) } diff --git a/core/backend/transcript_live_internal_test.go b/core/backend/transcript_live_internal_test.go index 6f6bed6a4b5c..c0e1ca0ceca6 100644 --- a/core/backend/transcript_live_internal_test.go +++ b/core/backend/transcript_live_internal_test.go @@ -59,6 +59,34 @@ var _ = Describe("liveEventFromProto", func() { Expect(ev.Eob).To(BeTrue()) Expect(ev.Eou).To(BeFalse()) }) + + It("maps speaker segments and sound events (ns -> seconds)", func() { + ev := liveEventFromProto(&proto.TranscriptLiveResponse{ + Speakers: []*proto.LiveSpeakerSegment{ + {Speaker: "1", Start: int64(1500 * time.Millisecond), End: int64(3200 * time.Millisecond)}, + }, + Sounds: []*proto.LiveSoundEvent{ + {Label: "Dog bark", Index: 5, Peak: 0.8, Start: int64(500 * time.Millisecond), End: int64(900 * time.Millisecond)}, + }, + }) + Expect(ev.Speakers).To(HaveLen(1)) + Expect(ev.Speakers[0].Speaker).To(Equal("1")) + Expect(ev.Speakers[0].Start).To(BeNumerically("~", 1.5, 1e-9)) + Expect(ev.Speakers[0].End).To(BeNumerically("~", 3.2, 1e-9)) + + Expect(ev.Sounds).To(HaveLen(1)) + Expect(ev.Sounds[0].Label).To(Equal("Dog bark")) + Expect(ev.Sounds[0].Index).To(Equal(5)) + Expect(ev.Sounds[0].Peak).To(BeNumerically("~", 0.8, 1e-6)) + Expect(ev.Sounds[0].Start).To(BeNumerically("~", 0.5, 1e-9)) + Expect(ev.Sounds[0].End).To(BeNumerically("~", 0.9, 1e-9)) + }) + + It("leaves speakers and sounds nil when the proto carries none", func() { + ev := liveEventFromProto(&proto.TranscriptLiveResponse{Delta: "hi"}) + Expect(ev.Speakers).To(BeNil()) + Expect(ev.Sounds).To(BeNil()) + }) }) // liveTraceState is what makes streaming-only pipelines visible on the diff --git a/core/http/endpoints/openai/realtime_semantic_vad.go b/core/http/endpoints/openai/realtime_semantic_vad.go index 75a71ba25a85..df56abf19783 100644 --- a/core/http/endpoints/openai/realtime_semantic_vad.go +++ b/core/http/endpoints/openai/realtime_semantic_vad.go @@ -209,6 +209,35 @@ func (l *liveTurnState) drainEvents(audioSec float64) { if ev.Final != nil && strings.TrimSpace(ev.Final.Text) != "" { l.finalText = ev.Final.Text } + // Speaker and sound events from a companion diarization/scene + // stream: forward each as its own event under the turn's item + // id, same as caption deltas. Text is empty — the event exists + // to carry the speaker/segment boundary, not transcript text. + if l.transport != nil && l.itemID != "" { + for _, seg := range ev.Speakers { + sendEvent(l.transport, types.ConversationItemInputAudioTranscriptionSegmentEvent{ + ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, + ItemID: l.itemID, + ContentIndex: 0, + Speaker: seg.Speaker, + Start: seg.Start, + End: seg.End, + }) + } + for _, sound := range ev.Sounds { + start, end := sound.Start, sound.End + sendEvent(l.transport, types.ConversationItemSoundDetectionEvent{ + ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, + ItemID: l.itemID, + ContentIndex: 0, + Detections: []types.SoundDetectionTag{ + {Label: sound.Label, Score: sound.Peak, Index: sound.Index}, + }, + Start: &start, + End: &end, + }) + } + } default: return } diff --git a/core/http/endpoints/openai/realtime_semantic_vad_test.go b/core/http/endpoints/openai/realtime_semantic_vad_test.go index b1107c1f6442..aa6c49b12933 100644 --- a/core/http/endpoints/openai/realtime_semantic_vad_test.go +++ b/core/http/endpoints/openai/realtime_semantic_vad_test.go @@ -291,6 +291,67 @@ var _ = Describe("liveTurnState", func() { Expect(ftr.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionFailed)).To(Equal(0)) }) }) + + Describe("scene events (speakers and sounds)", func() { + It("emits a segment event per speaker with empty text under the turn's item id", func() { + Expect(lts.openTurn(context.Background(), "item1")).To(BeTrue()) + turnID := lts.itemID + + m.liveSession.onEvent(backend.LiveTranscriptionEvent{ + Speakers: []backend.LiveSpeakerSegment{{Speaker: "1", Start: 1.2, End: 3.4}}, + }) + lts.drainEvents(3.4) + + var got []types.ConversationItemInputAudioTranscriptionSegmentEvent + for _, e := range ftr.events { + if seg, ok := e.(types.ConversationItemInputAudioTranscriptionSegmentEvent); ok { + got = append(got, seg) + } + } + Expect(got).To(HaveLen(1)) + Expect(got[0].ItemID).To(Equal(turnID)) + Expect(got[0].Speaker).To(Equal("1")) + Expect(got[0].Start).To(BeNumerically("~", 1.2, 1e-9)) + Expect(got[0].End).To(BeNumerically("~", 3.4, 1e-9)) + Expect(got[0].Text).To(BeEmpty()) + }) + + It("emits a sound_detection event per sound with one tag and start/end", func() { + Expect(lts.openTurn(context.Background(), "item1")).To(BeTrue()) + turnID := lts.itemID + + m.liveSession.onEvent(backend.LiveTranscriptionEvent{ + Sounds: []backend.LiveSoundEvent{{Label: "Dog bark", Index: 5, Peak: 0.8, Start: 0.5, End: 0.9}}, + }) + lts.drainEvents(1.0) + + var got []types.ConversationItemSoundDetectionEvent + for _, e := range ftr.events { + if sd, ok := e.(types.ConversationItemSoundDetectionEvent); ok { + got = append(got, sd) + } + } + Expect(got).To(HaveLen(1)) + Expect(got[0].ItemID).To(Equal(turnID)) + Expect(got[0].Detections).To(HaveLen(1)) + Expect(got[0].Detections[0].Label).To(Equal("Dog bark")) + Expect(got[0].Detections[0].Score).To(BeNumerically("~", 0.8, 1e-6)) + Expect(got[0].Detections[0].Index).To(Equal(5)) + Expect(got[0].Start).NotTo(BeNil()) + Expect(*got[0].Start).To(BeNumerically("~", 0.5, 1e-9)) + Expect(got[0].End).NotTo(BeNil()) + Expect(*got[0].End).To(BeNumerically("~", 0.9, 1e-9)) + }) + + It("sends neither event when a live event carries no speakers or sounds", func() { + Expect(lts.openTurn(context.Background(), "item1")).To(BeTrue()) + m.liveSession.onEvent(backend.LiveTranscriptionEvent{Delta: "hi"}) + lts.drainEvents(1.0) + + Expect(ftr.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionSegment)).To(Equal(0)) + Expect(ftr.countEvents(types.ServerEventTypeConversationItemSoundDetection)).To(Equal(0)) + }) + }) }) // commitUtteranceWithTranscript routes the three transcript sources: the diff --git a/core/http/endpoints/openai/realtime_sound_detection_test.go b/core/http/endpoints/openai/realtime_sound_detection_test.go index 058c74076ec7..d36d91f7948f 100644 --- a/core/http/endpoints/openai/realtime_sound_detection_test.go +++ b/core/http/endpoints/openai/realtime_sound_detection_test.go @@ -3,6 +3,7 @@ package openai import ( "context" "encoding/binary" + "encoding/json" "errors" "os" @@ -14,6 +15,43 @@ import ( "github.com/mudler/LocalAI/core/schema" ) +// ConversationItemSoundDetectionEvent gained optional Start/End (seconds) +// for the live scene-event path; the unary/windowed paths never set them, +// so existing consumers must see no start/end keys at all. +var _ = Describe("ConversationItemSoundDetectionEvent JSON", func() { + It("omits start and end when nil", func() { + ev := types.ConversationItemSoundDetectionEvent{ + ItemID: "item1", + Detections: []types.SoundDetectionTag{{Label: "Speech", Score: 0.5, Index: 7}}, + } + b, err := json.Marshal(ev) + Expect(err).ToNot(HaveOccurred()) + + var got map[string]any + Expect(json.Unmarshal(b, &got)).To(Succeed()) + _, hasStart := got["start"] + _, hasEnd := got["end"] + Expect(hasStart).To(BeFalse()) + Expect(hasEnd).To(BeFalse()) + }) + + It("includes start and end when set", func() { + start, end := 0.5, 0.9 + ev := types.ConversationItemSoundDetectionEvent{ + ItemID: "item1", + Start: &start, + End: &end, + } + b, err := json.Marshal(ev) + Expect(err).ToNot(HaveOccurred()) + + var got map[string]any + Expect(json.Unmarshal(b, &got)).To(Succeed()) + Expect(got["start"]).To(BeNumerically("~", 0.5, 1e-9)) + Expect(got["end"]).To(BeNumerically("~", 0.9, 1e-9)) + }) +}) + // emitSoundDetection classifies a committed utterance and emits a single // conversation.item.sound_detection event carrying the scored AudioSet tags. var _ = Describe("emitSoundDetection", func() { diff --git a/core/http/endpoints/openai/types/server_events.go b/core/http/endpoints/openai/types/server_events.go index 114a7065a055..afddff254f58 100644 --- a/core/http/endpoints/openai/types/server_events.go +++ b/core/http/endpoints/openai/types/server_events.go @@ -512,6 +512,15 @@ type ConversationItemSoundDetectionEvent struct { // The scored sound-event tags, in score-descending order. Detections []SoundDetectionTag `json:"detections"` + + // The start time of the detection window in seconds, when known. Set by + // the live scene-event path (a companion sound stream alongside live + // transcription); omitted by the unary/windowed sound-detection paths, + // which have no per-event timing. + Start *float64 `json:"start,omitempty"` + + // The end time of the detection window in seconds, when known. + End *float64 `json:"end,omitempty"` } func (m ConversationItemSoundDetectionEvent) ServerEventType() ServerEventType { From 694b410d36f3fe66d7b38fbb03228b7e3421bfae Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:34:40 +0000 Subject: [PATCH 09/30] fix(realtime): keep start/end on a zero-second transcription segment ConversationItemInputAudioTranscriptionSegmentEvent.Start/End used omitempty, so a speaker segment starting at 0.0s dropped its "start" key. Nothing emitted this event before the live scene-event path, so drop omitempty: the segment always carries real times. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- .../openai/realtime_sound_detection_test.go | 26 +++++++++++++++++++ .../endpoints/openai/types/server_events.go | 10 ++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/core/http/endpoints/openai/realtime_sound_detection_test.go b/core/http/endpoints/openai/realtime_sound_detection_test.go index d36d91f7948f..94406e7cc6aa 100644 --- a/core/http/endpoints/openai/realtime_sound_detection_test.go +++ b/core/http/endpoints/openai/realtime_sound_detection_test.go @@ -52,6 +52,32 @@ var _ = Describe("ConversationItemSoundDetectionEvent JSON", func() { }) }) +// ConversationItemInputAudioTranscriptionSegmentEvent.Start/End are plain +// float64 (no omitempty): a speaker segment starting at 0.0s must still +// carry "start" in the JSON, unlike the sound-detection event's optional +// pointer fields above. +var _ = Describe("ConversationItemInputAudioTranscriptionSegmentEvent JSON", func() { + It("marshals start:0 and end:1.5 even when start is the zero value", func() { + ev := types.ConversationItemInputAudioTranscriptionSegmentEvent{ + ItemID: "item1", + Speaker: "1", + Start: 0, + End: 1.5, + } + b, err := json.Marshal(ev) + Expect(err).ToNot(HaveOccurred()) + + var got map[string]any + Expect(json.Unmarshal(b, &got)).To(Succeed()) + _, hasStart := got["start"] + _, hasEnd := got["end"] + Expect(hasStart).To(BeTrue()) + Expect(hasEnd).To(BeTrue()) + Expect(got["start"]).To(BeNumerically("~", 0.0, 1e-9)) + Expect(got["end"]).To(BeNumerically("~", 1.5, 1e-9)) + }) +}) + // emitSoundDetection classifies a committed utterance and emits a single // conversation.item.sound_detection event carrying the scored AudioSet tags. var _ = Describe("emitSoundDetection", func() { diff --git a/core/http/endpoints/openai/types/server_events.go b/core/http/endpoints/openai/types/server_events.go index afddff254f58..4cab30a61d80 100644 --- a/core/http/endpoints/openai/types/server_events.go +++ b/core/http/endpoints/openai/types/server_events.go @@ -595,11 +595,13 @@ type ConversationItemInputAudioTranscriptionSegmentEvent struct { // The speaker label for the segment, if available. Speaker string `json:"speaker,omitempty"` - // The start time of the segment in seconds. - Start float64 `json:"start,omitempty"` + // The start time of the segment in seconds. Always present (not + // omitempty: a segment starting at 0.0s must still carry "start"). + Start float64 `json:"start"` - // The end time of the segment in seconds. - End float64 `json:"end,omitempty"` + // The end time of the segment in seconds. Always present (not + // omitempty: see Start). + End float64 `json:"end"` // The text content of the segment. Text string `json:"text,omitempty"` From cfef2c887a93b57d01aeec848c8902f6624fbb2f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:37:50 +0000 Subject: [PATCH 10/30] chore(gallery): add parakeet-cpp diarization, CED and realtime scene models Add gallery entries for the new parakeet-cpp capabilities: standalone Nemotron-3-Diarization, the same paired with the Parakeet TDT+CTC 110M ASR model for speaker-attributed text, CED-Tiny and CED-Base sound classifiers, and a realtime scene bundle combining the streaming EOU ASR model with diarization and sound companions. SHA256 taken from the Hub API; licenses from each model card (openmdw-1.1 for Nemotron-3-Diarization, apache-2.0 for CED, cc-by-4.0 for the Parakeet ASR models). Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 201 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 4e37b0e78aa6..096b0b126bef 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -53670,6 +53670,207 @@ - filename: parakeet-cpp/nemotron-3.5-asr-streaming-0.6b-q8_0.gguf uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3.5-asr-streaming-0.6b-q8_0.gguf sha256: ba2f13eccd4a5245be728f77e6149bd6a4fdcdd133ff2e08ac6005bcef7a99f1 +- name: parakeet-cpp-nemotron-3-diarization + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://github.com/mudler/parakeet.cpp + description: | + Nemotron-3-Diarization (Sortformer), Q8_0 GGUF for the parakeet-cpp backend + (C++/ggml port of NVIDIA NeMo). Speaker diarization only: served through + /v1/audio/diarization, returns per-segment start, end and speaker label + ("0", "1", ...). It does not transcribe; pair it with an ASR model and set + asr_model to get speaker-attributed text from the same call. num_speakers, + min_speakers, max_speakers and clustering_threshold are not supported by + Sortformer and are ignored. + license: openmdw-1.1 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - diarization + - speaker-diarization + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - diarization + name: parakeet-cpp-nemotron-3-diarization + parameters: + model: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + files: + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 +- name: parakeet-cpp-nemotron-3-diarization-asr + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/nvidia/parakeet-tdt_ctc-110m + - https://github.com/mudler/parakeet.cpp + description: | + Nemotron-3-Diarization (Sortformer) paired with the Parakeet TDT+CTC 110M + ASR model through the asr_model option, both Q8_0/F16 GGUF for the + parakeet-cpp backend (C++/ggml port of NVIDIA NeMo). Served through + /v1/audio/diarization with include_text: each speaker segment comes back + with its transcribed text in one call. Diarization model is + OpenMDW-1.1, ASR model is CC-BY-4.0. + license: openmdw-1.1 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - asr + - diarization + - speaker-diarization + - speech-recognition + - stt + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - diarization + name: parakeet-cpp-nemotron-3-diarization-asr + options: + - asr_model:parakeet-cpp/tdt_ctc-110m-f16.gguf + parameters: + model: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + files: + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/tdt_ctc-110m-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/tdt_ctc-110m-f16.gguf + sha256: 7f9a6376edde6a74592ace48b2ebdc27a1ac972d0be9dfcc29e668d99381faf1 +- name: parakeet-cpp-ced-tiny + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/mispeech/ced-tiny + - https://github.com/mudler/parakeet.cpp + description: | + CED-Tiny sound event tagger, Q8_0 GGUF for the parakeet-cpp backend + (C++/ggml, loaded through third_party/ced.cpp). Served through + /v1/audio/classification: 10 s windows are scored and averaged over the + clip, then sorted by score with threshold and top_k applied. Smallest and + fastest of the CED sizes; use ced-base for higher accuracy. + license: apache-2.0 + tags: + - parakeet-cpp + - ced + - sound-classification + - audio-tagging + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - sound_classification + name: parakeet-cpp-ced-tiny + parameters: + model: parakeet-cpp/ced-tiny-q8_0.gguf + files: + - filename: parakeet-cpp/ced-tiny-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-tiny-q8_0.gguf + sha256: 48bee4e2fc3cc85d7806e03471db24e77fda6c2a2e81ffe9ef67caebaf2bd674 +- name: parakeet-cpp-ced-base + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/mispeech/ced-base + - https://github.com/mudler/parakeet.cpp + description: | + CED-Base sound event tagger, Q8_0 GGUF for the parakeet-cpp backend + (C++/ggml, loaded through third_party/ced.cpp). Served through + /v1/audio/classification: 10 s windows are scored and averaged over the + clip, then sorted by score with threshold and top_k applied. Larger and + more accurate than ced-tiny, still CPU-friendly. + license: apache-2.0 + tags: + - parakeet-cpp + - ced + - sound-classification + - audio-tagging + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - sound_classification + name: parakeet-cpp-ced-base + parameters: + model: parakeet-cpp/ced-base-q8_0.gguf + files: + - filename: parakeet-cpp/ced-base-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-base-q8_0.gguf + sha256: bd34a7710169f0047fea17267965d211f967828ab25ba6fb9d3768481393f6e2 +- name: parakeet-cpp-realtime-scene + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/nvidia/parakeet_realtime_eou_120m-v1 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-tiny + - https://github.com/mudler/parakeet.cpp + description: | + Cache-aware streaming RNNT FastConformer with end-of-utterance (EOU) + detection, 120M, paired with Nemotron-3-Diarization and CED-Tiny through + the diarization_model and sound_model options. F16/Q8_0 GGUF for the + parakeet-cpp backend (C++/ggml port of NVIDIA NeMo). Use with streaming + transcription: while a turn is live, closed speaker segments and sound + events are surfaced alongside the ASR text (realtime + conversation.item.input_audio_transcription.segment and + conversation.item.sound_detection events). Live speaker/sound events only + fire during speech turns under semantic_vad; sounds between turns are not + seen by this path. License per model: transcription model CC-BY-4.0, + diarization model OpenMDW-1.1, CED-Tiny Apache-2.0. + license: cc-by-4.0 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - asr + - speech-recognition + - diarization + - sound-classification + - streaming + - realtime + - stt + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + name: parakeet-cpp-realtime-scene + options: + - diarization_model:parakeet-cpp/nemotron-3-diarization-q8_0.gguf + - sound_model:parakeet-cpp/ced-tiny-q8_0.gguf + parameters: + model: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + files: + - filename: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/realtime_eou_120m-v1-f16.gguf + sha256: d1a2b12f12b8a096a57499c9111ed13b442a2b786e17a292c168be45088f0edc + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/ced-tiny-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-tiny-q8_0.gguf + sha256: 48bee4e2fc3cc85d7806e03471db24e77fda6c2a2e81ffe9ef67caebaf2bd674 - name: moss-transcribe-cpp-0.9b url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: From 57497dbed878c80b7692170fa939c7c5cf6a0b6a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:39:50 +0000 Subject: [PATCH 11/30] docs: document parakeet-cpp diarization, sound detection and live scene events Cover the new parakeet-cpp capabilities across the feature pages: Nemotron-3-Diarization as a diarization backend (with and without speaker text, the ignored speaker-count hints, the Sortformer voice-like-sound quirk), CED as a sound classification backend, the asr_model/diarization_model/sound_model/diarization_latency companion options, and the realtime live speaker/sound events (event shapes, the speech-turn-only limitation, and using this or pipeline.sound_detection but not both). Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- docs/content/features/audio-classification.md | 21 +++++++ docs/content/features/audio-diarization.md | 33 +++++++++- docs/content/features/audio-to-text.md | 15 ++++- docs/content/features/openai-realtime.md | 60 +++++++++++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) diff --git a/docs/content/features/audio-classification.md b/docs/content/features/audio-classification.md index 09759ab72da3..4f9c58fb9d61 100644 --- a/docs/content/features/audio-classification.md +++ b/docs/content/features/audio-classification.md @@ -9,6 +9,8 @@ Sound-event classification (audio tagging) answers the question **"what am I hea LocalAI exposes this through the `/v1/audio/classification` endpoint, modelled after `/v1/audio/transcriptions`. The reference backend is **[ced.cpp](https://github.com/localai-org/ced.cpp)** (CED, a 527-class AudioSet tagger), a small ViT over a log-mel spectrogram ported to ggml with full PyTorch parity. Apache-2.0 weights are redistributable as GGUF. +**[parakeet.cpp](https://github.com/mudler/parakeet.cpp)** can also load a CED model (through `third_party/ced.cpp`) and serve `/v1/audio/classification` from the same backend used for ASR and diarization. It scores the clip in 10 s windows and averages each class's score across the windows before sorting and applying `top_k`/`threshold` - CED's own method for clips longer than one window. Install `parakeet-cpp-ced-tiny` or `parakeet-cpp-ced-base` from the gallery, or point `parameters.model` at a CED GGUF under `backend: parakeet-cpp`. A parakeet-cpp ASR model can also point `sound_model` at a CED GGUF to add live sound events during realtime transcription - see [Realtime API]({{% relref "openai-realtime" %}}). + Because classification is exposed as a regular OpenAI-style endpoint, any HTTP client works - there is no Python dependency on the consumer side. In distributed mode, LocalAI stages uploaded audio and realtime sound-detection @@ -59,6 +61,25 @@ curl http://localhost:8080/v1/audio/classification \ -F top_k=10 ``` +The same request works unchanged against a parakeet-cpp CED model: + +```yaml +name: parakeet-ced-tiny +backend: parakeet-cpp +parameters: + model: ced-tiny-q8_0.gguf +known_usecases: + - sound_classification +``` + +```bash +curl http://localhost:8080/v1/audio/classification \ + -H "Content-Type: multipart/form-data" \ + -F file="@/path/to/clip.wav" \ + -F model="parakeet-ced-tiny" \ + -F top_k=10 +``` + ## See also - [Audio to Text]({{% relref "audio-to-text" %}}) - speech transcription diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md index 83a51343e561..df5f95a82371 100644 --- a/docs/content/features/audio-diarization.md +++ b/docs/content/features/audio-diarization.md @@ -9,12 +9,13 @@ url = "/features/audio-diarization/" Speaker diarization answers the question **"who spoke when?"** - given an audio clip with multiple speakers, it returns time-stamped segments labelled with a stable speaker ID (`SPEAKER_00`, `SPEAKER_01`, …). -LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Four backends are supported today: +LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Five backends are supported today: - **[sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx)** - pyannote-3.0 segmentation + a speaker-embedding extractor (3D-Speaker, NeMo, WeSpeaker) + fast clustering. Pure diarization - no transcription cost. Recommended when you only need speaker turns. - **[vibevoice.cpp](https://github.com/microsoft/VibeVoice)** - produces speaker-labelled segments as a by-product of its long-form ASR pass, so you can optionally get a transcript per segment for free. - **[NeMo-Speech.cpp](https://github.com/NVIDIA/NeMo-Speech.cpp)** - NVIDIA Sortformer, served standalone by the [NeMo-Speech.cpp backend]({{%relref "features/nemo-speech-cpp" %}}). It is end to end, so the speaker capacity is fixed by the checkpoint and the count hints are ignored. The same backend can instead put speaker tags on a transcript, by attaching a Sortformer model to an ASR one. - **[audio.cpp](https://github.com/0xShug0/audio.cpp)** - the `sortformer_diar` family, served by the multi-modality [audio.cpp backend]({{%relref "features/audio-cpp" %}}). +- **[parakeet.cpp](https://github.com/mudler/parakeet.cpp)** - NVIDIA Nemotron-3-Diarization (Sortformer), served standalone or paired with a Parakeet ASR model for per-segment text. See the [Audio to Text]({{% relref "audio-to-text" %}}) page for the parakeet-cpp option reference. Because diarization is exposed as a regular OpenAI-compatible endpoint, any HTTP client works. There is no Python dependency on pyannote or NeMo on the consumer side. @@ -157,10 +158,38 @@ curl http://localhost:8080/v1/audio/diarization \ -F response_format=verbose_json ``` +## Backend setup - parakeet-cpp (Nemotron-3-Diarization) + +Nemotron-3-Diarization is Sortformer, served standalone or paired with a Parakeet ASR model. Install `parakeet-cpp-nemotron-3-diarization` from the gallery for diarization only, or `parakeet-cpp-nemotron-3-diarization-asr` for the same model paired with `parakeet-cpp-tdt_ctc-110m` through the `asr_model` option: + +```yaml +name: parakeet-diarize +backend: parakeet-cpp +parameters: + model: nemotron-3-diarization-q8_0.gguf +options: + - asr_model:tdt_ctc-110m-f16.gguf +known_usecases: + - diarization +``` + +`asr_model` is what turns on `include_text`: without it, segments carry no text and no error is raised. Sortformer has a fixed speaker capacity and no clustering stage, so `num_speakers`, `min_speakers`, `max_speakers` and `clustering_threshold` are ignored (logged at debug); `min_duration_on` and `min_duration_off` are honored. Speaker labels are the decimal index the model assigned (`"0"`, `"1"`, …), or `"unknown"` when a segment has no diarized speaker. + +```bash +curl http://localhost:8080/v1/audio/diarization \ + -H "Content-Type: multipart/form-data" \ + -F file="@meeting.wav" \ + -F model="parakeet-diarize" \ + -F include_text=true \ + -F response_format=verbose_json +``` + +Sortformer clusters on voice-like characteristics, not on "is this a human". A loud non-speech sound with voice-like pitch and rhythm (a rooster crow, in one test clip) can come back as its own speaker segment alongside the real speakers. This is model behavior, not a bug in the LocalAI integration: treat an unexpected extra speaker as a hint the clip may contain a non-speech sound, and use [Sound Classification]({{% relref "audio-classification" %}}) to confirm what it is. + ## Notes - **Speaker identity across files**: speaker IDs (`SPEAKER_00`, `SPEAKER_01`, …) are local to each request. To track the same person across multiple recordings, combine `/v1/audio/diarization` with `/v1/voice/embed` (speaker embedding) and maintain your own embedding store. -- **Hints vs. forces**: `num_speakers` overrides clustering when set; `min_speakers` / `max_speakers` are advisory and only honored by backends that expose a range hint. vibevoice.cpp ignores them - its model picks the count itself. +- **Hints vs. forces**: `num_speakers` overrides clustering when set; `min_speakers` / `max_speakers` are advisory and only honored by backends that expose a range hint. vibevoice.cpp and parakeet-cpp (Sortformer) ignore them - the model picks the count itself. - **Sample rate**: input is automatically converted to 16 kHz mono via ffmpeg before the backend sees it; sherpa-onnx pyannote-3.0 requires 16 kHz. ## See also diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 5a5e833cf3a3..a26ce46e1c28 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -12,7 +12,7 @@ The transcription endpoint allows to convert audio files to text. The endpoint s - **moonshine**: Ultra-fast transcription engine optimized for low-end devices - **faster-whisper**: Fast Whisper implementation with CTranslate2 - **WhisperX**: Whisper transcription with word alignment and optional speaker diarization. Set `HF_TOKEN` and pass `diarize=true` to load WhisperX's gated pyannote diarization pipeline. -- **[parakeet-cpp](https://github.com/mudler/parakeet.cpp)**: A C++/ggml port of NVIDIA NeMo Parakeet (FastConformer TDT/CTC/RNNT/hybrid). Runs quantized GGUFs on CPU or GPU, emits word-level timestamps, and supports cache-aware streaming (the `realtime_eou` model surfaces end-of-utterance events). +- **[parakeet-cpp](https://github.com/mudler/parakeet.cpp)**: A C++/ggml port of NVIDIA NeMo Parakeet (FastConformer TDT/CTC/RNNT/hybrid). Runs quantized GGUFs on CPU or GPU, emits word-level timestamps, and supports cache-aware streaming (the `realtime_eou` model surfaces end-of-utterance events). The same backend also loads Nemotron-3-Diarization (`/v1/audio/diarization`) and CED sound models (`/v1/audio/classification`), and can attach either as a companion to a transcription model. - **llama-cpp**: Route transcription to any multimodal-audio GGUF model served by the `llama-cpp` backend (e.g. [Qwen3-ASR](https://huggingface.co/ggml-org/Qwen3-ASR-0.6B-GGUF), Voxtral, Qwen2-Audio). Under the hood the request is converted into a chat completion with the audio attached via the model's audio encoder - the same path the upstream llama.cpp server uses. Set `backend: llama-cpp` in the model YAML and point `mmproj` at the matching audio encoder. - **voxtral**: Voxtral-family models served by a dedicated backend - **[NeMo-Speech.cpp](https://github.com/NVIDIA/NeMo-Speech.cpp)**: NVIDIA's C++/ggml runtime for the Nemotron Speech models. Serves offline, streaming and live transcription, with VAD, punctuation, inverse text normalization and Sortformer speaker tags attached through model options, and covers diarization, speech synthesis and translation from the same backend. See the [NeMo-Speech.cpp backend]({{%relref "features/nemo-speech-cpp" %}}) page for the model options. @@ -190,6 +190,19 @@ curl http://localhost:8080/v1/audio/transcriptions \ For real-time use, load a cache-aware streaming model (e.g. `realtime_eou_120m-v1-*.gguf`) and pass `-F stream=true`. Deltas are emitted as the audio is decoded, with end-of-utterance events closing each segment. +### Diarization and sound classification + +The same backend also serves the `/v1/audio/diarization` and `/v1/audio/classification` endpoints, and can attach a diarization or sound model to a live transcription session. `options:` accepts paths relative to the models directory, or absolute: + +| Option | Allowed on | Used for | +|---|---|---| +| `asr_model:` | a diarization model | `include_text` on `/v1/audio/diarization` | +| `diarization_model:` | an ASR model | speaker segments during realtime live transcription | +| `sound_model:` | an ASR model | sound events during realtime live transcription | +| `diarization_latency:` | any model with a diarization companion | latency mode for the live speaker stream; default `low` | + +A companion model of the wrong kind (for example `sound_model:` pointing at an ASR GGUF) fails to load, naming the kind it expected. See [Speaker Diarization]({{% relref "audio-diarization" %}}) for the `Diarize` RPC and [Sound Classification]({{% relref "audio-classification" %}}) for `SoundDetection`, and [Realtime API]({{% relref "openai-realtime" %}}) for the live speaker/sound events emitted during a realtime session. + ### Segment timestamps Transcriptions are split into segments the same way NVIDIA NeMo does: a new segment starts after sentence-ending punctuation (`.`, `?`, `!`), and each segment carries `start`/`end` times. This is the default (NeMo's punctuation-only segmentation) and needs no configuration. While streaming, each end-of-utterance closes a segment, now with timestamps. diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index ac7afb484796..bf4234fab573 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -129,6 +129,66 @@ A client `session.update` still overrides `type` and `eagerness` per session. - `false` (default): the transcript accumulated from the live stream is used as-is - the model runs once per utterance and the LLM starts immediately at commit. - `true`: the committed audio is re-transcribed offline. If the batch decode also ends with the end-of-utterance token the turn proceeds (using the batch transcript); if it does **not**, the commit is cancelled and the session keeps listening - treating the streaming token as a false positive. Both transcripts are compared and logged, which makes this mode a useful diagnostic for how well the streaming and batch decodes align, at the cost of one extra decode per turn. +### Live speaker and sound events (parakeet-cpp) + +When the `semantic_vad` transcription model is a parakeet-cpp model loaded with a `diarization_model` and/or `sound_model` companion (see [Audio to Text]({{% relref "audio-to-text" %}})), the realtime session also streams speaker and sound events while a turn is live, alongside the transcript deltas. Nothing needs to change on the client: unrecognized event types are ignored by standard OpenAI Realtime clients. + +The transcription model, with its companions: + +```yaml +name: parakeet-realtime-scene +backend: parakeet-cpp +parameters: + model: realtime_eou_120m-v1-f16.gguf +options: + - diarization_model:nemotron-3-diarization-q8_0.gguf + - sound_model:ced-tiny-q8_0.gguf +``` + +The realtime pipeline that uses it: + +```yaml +name: gpt-realtime +pipeline: + vad: silero-vad-ggml + transcription: parakeet-realtime-scene + llm: qwen3-4b + tts: tts-1 + turn_detection: + type: semantic_vad +``` + +Each closed speaker segment emits a `conversation.item.input_audio_transcription.segment` event under the turn's item id, with an empty `text` (the event exists to carry the speaker boundary, not a transcript - the transcript still comes from the ordinary delta/completed events): + +```json +{ + "type": "conversation.item.input_audio_transcription.segment", + "item_id": "item_abc", + "content_index": 0, + "speaker": "0", + "start": 1.92, + "end": 4.10, + "text": "" +} +``` + +Each sound event emits a `conversation.item.sound_detection` event with one tag and the detection window's `start`/`end` (seconds, stream-relative - the same clock as the live words): + +```json +{ + "type": "conversation.item.sound_detection", + "item_id": "item_abc", + "content_index": 0, + "detections": [{"label": "Rooster", "score": 0.91, "index": 17}], + "start": 24.0, + "end": 30.0 +} +``` + +`score` is the peak score seen for that tag while the sound was live, not an average. On session close the companion stream is flushed and its remaining events are sent before the final result; a mid-stream `session.update` that changes the transcription model resets it along with the ASR session. + +**Limitation**: under `semantic_vad`, live transcription (and so this companion stream) only runs during speech turns - it does not see audio between turns. A sound that happens while nobody is speaking is not detected this way. If you need sound events independent of speech turns, use the pipeline's `sound_detection` model instead (see [Sound Classification]({{% relref "audio-classification" %}})), which classifies each VAD-committed utterance on its own. Use one or the other, not both, on the same session - they overlap in purpose and would emit sound detections twice. + ### Disabling thinking For reasoning models, you can force the pipeline LLM's thinking off without editing the LLM model config: From 5354cb4f7c5d8e686bd6f21cb0e2b81e91a4717f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:44:05 +0000 Subject: [PATCH 12/30] fix(gallery): correct the realtime-scene license and wording nits parakeet-cpp-realtime-scene mistakenly copied cc-by-4.0 from the existing realtime_eou_120m-v1 entry; the model card lists the NVIDIA open model license instead. Switch to the gallery's usual spelling for that license and keep the diarization/CED licenses called out in the description. Also: audio-diarization.md now says getting per-segment text needs both an asr_model companion and include_text=true on the request, and audio-to-text.md's option table reads "Use on" (a pairing the loader does not enforce) instead of "Allowed on". Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- docs/content/features/audio-diarization.md | 2 +- docs/content/features/audio-to-text.md | 6 +++--- gallery/index.yaml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md index df5f95a82371..e2f085de1bac 100644 --- a/docs/content/features/audio-diarization.md +++ b/docs/content/features/audio-diarization.md @@ -173,7 +173,7 @@ known_usecases: - diarization ``` -`asr_model` is what turns on `include_text`: without it, segments carry no text and no error is raised. Sortformer has a fixed speaker capacity and no clustering stage, so `num_speakers`, `min_speakers`, `max_speakers` and `clustering_threshold` are ignored (logged at debug); `min_duration_on` and `min_duration_off` are honored. Speaker labels are the decimal index the model assigned (`"0"`, `"1"`, …), or `"unknown"` when a segment has no diarized speaker. +Getting text on each segment needs both: an `asr_model` companion loaded on the model, and `include_text=true` on the request. With only one of the two, segments carry no text and no error is raised. Sortformer has a fixed speaker capacity and no clustering stage, so `num_speakers`, `min_speakers`, `max_speakers` and `clustering_threshold` are ignored (logged at debug); `min_duration_on` and `min_duration_off` are honored. Speaker labels are the decimal index the model assigned (`"0"`, `"1"`, …), or `"unknown"` when a segment has no diarized speaker. ```bash curl http://localhost:8080/v1/audio/diarization \ diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index a26ce46e1c28..6a4076c15aba 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -194,14 +194,14 @@ For real-time use, load a cache-aware streaming model (e.g. `realtime_eou_120m-v The same backend also serves the `/v1/audio/diarization` and `/v1/audio/classification` endpoints, and can attach a diarization or sound model to a live transcription session. `options:` accepts paths relative to the models directory, or absolute: -| Option | Allowed on | Used for | +| Option | Use on | Used for | |---|---|---| | `asr_model:` | a diarization model | `include_text` on `/v1/audio/diarization` | | `diarization_model:` | an ASR model | speaker segments during realtime live transcription | | `sound_model:` | an ASR model | sound events during realtime live transcription | -| `diarization_latency:` | any model with a diarization companion | latency mode for the live speaker stream; default `low` | +| `diarization_latency:` | a model with a diarization companion | latency mode for the live speaker stream; default `low` | -A companion model of the wrong kind (for example `sound_model:` pointing at an ASR GGUF) fails to load, naming the kind it expected. See [Speaker Diarization]({{% relref "audio-diarization" %}}) for the `Diarize` RPC and [Sound Classification]({{% relref "audio-classification" %}}) for `SoundDetection`, and [Realtime API]({{% relref "openai-realtime" %}}) for the live speaker/sound events emitted during a realtime session. +"Use on" is the intended pairing; the loader does not check the primary model's kind before accepting an option, it only checks that the companion GGUF it loads matches the expected kind (for example `sound_model:` pointing at an ASR GGUF fails to load, naming the kind it expected). See [Speaker Diarization]({{% relref "audio-diarization" %}}) for the `Diarize` RPC and [Sound Classification]({{% relref "audio-classification" %}}) for `SoundDetection`, and [Realtime API]({{% relref "openai-realtime" %}}) for the live speaker/sound events emitted during a realtime session. ### Segment timestamps diff --git a/gallery/index.yaml b/gallery/index.yaml index 096b0b126bef..731773646abb 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -53833,9 +53833,9 @@ conversation.item.input_audio_transcription.segment and conversation.item.sound_detection events). Live speaker/sound events only fire during speech turns under semantic_vad; sounds between turns are not - seen by this path. License per model: transcription model CC-BY-4.0, - diarization model OpenMDW-1.1, CED-Tiny Apache-2.0. - license: cc-by-4.0 + seen by this path. License per model: transcription model NVIDIA Open + Model License, diarization model OpenMDW-1.1, CED-Tiny Apache-2.0. + license: nvidia-open-model-license tags: - parakeet - parakeet-cpp From 9e66c71c1164eabcd887c66a29fad27552c91484 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:57:31 +0000 Subject: [PATCH 13/30] fix(parakeet-cpp): reject a companion role that duplicates the primary's loadRoles let a companion option (asr_model:/diarization_model:/ sound_model:) assign into a role field the primary already occupied, for example asr_model: on an already-ASR primary. The companion's context silently overwrote ctxPtr/diarCtx/tagCtx, and Free() only walks those three fields, so the original primary context was never freed again. Reject a companion whose role the primary already holds before its GGUF is even loaded, freeing everything loadRoles opened so far, the same way a wrong-kind companion is already rejected. Also warn, rather than silently fall through, when parakeet_capi_model_kind reports PARAKEET_MODEL_KIND_NONE for a successfully loaded primary; the primary is still treated as ASR, matching today's behavior. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/roles.go | 46 ++++++++++-- backend/go/parakeet-cpp/roles_test.go | 101 ++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 5 deletions(-) diff --git a/backend/go/parakeet-cpp/roles.go b/backend/go/parakeet-cpp/roles.go index c05b0988989a..c6a1e912e1c1 100644 --- a/backend/go/parakeet-cpp/roles.go +++ b/backend/go/parakeet-cpp/roles.go @@ -7,6 +7,7 @@ import ( "strings" pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" ) // Model kinds returned by parakeet_capi_model_kind (ABI v8; mirrors the @@ -87,13 +88,29 @@ func parseDiarLatency(s string) (int32, error) { // companionSpec is one asr_model:/diarization_model:/sound_model: option: its // name (for error messages and path resolution), the raw option value, the -// model kind the loaded companion must report, and the ParakeetCpp field it -// is assigned to on success. +// model kind the loaded companion must report, the ParakeetCpp field it is +// assigned to on success, and a getter for that same field's current value +// (used to reject a companion whose role the primary already occupies). type companionSpec struct { optName string value string wantKind int32 assign func(*ParakeetCpp, uintptr) + current func(*ParakeetCpp) uintptr +} + +// indefiniteArticle returns "an" for a word starting with a vowel sound and +// "a" otherwise, for grammatical error messages built from modelKindName. +func indefiniteArticle(word string) string { + if len(word) == 0 { + return "a" + } + switch word[0] { + case 'A', 'E', 'I', 'O', 'U', 'a', 'e', 'i', 'o', 'u': + return "an" + default: + return "a" + } } // loadRoles loads opts.ModelFile as the primary parakeet_ctx, classifies it @@ -150,6 +167,10 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { primaryKind := int32(modelKindASR) // old-library default: today's behavior if CppModelKind != nil { primaryKind = CppModelKind(primary) + if primaryKind == modelKindNone { + xlog.Warn("parakeet-cpp: parakeet_capi_model_kind reported PARAKEET_MODEL_KIND_NONE " + + "for a successfully loaded primary; treating it as an ASR model") + } } switch primaryKind { case modelKindDiarization: @@ -161,14 +182,29 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { } specs := []companionSpec{ - {"diarization_model", diarModelOpt, modelKindDiarization, func(pp *ParakeetCpp, c uintptr) { pp.diarCtx = c }}, - {"asr_model", asrModelOpt, modelKindASR, func(pp *ParakeetCpp, c uintptr) { pp.ctxPtr = c }}, - {"sound_model", soundModelOpt, modelKindSound, func(pp *ParakeetCpp, c uintptr) { pp.tagCtx = c }}, + {"diarization_model", diarModelOpt, modelKindDiarization, + func(pp *ParakeetCpp, c uintptr) { pp.diarCtx = c }, + func(pp *ParakeetCpp) uintptr { return pp.diarCtx }}, + {"asr_model", asrModelOpt, modelKindASR, + func(pp *ParakeetCpp, c uintptr) { pp.ctxPtr = c }, + func(pp *ParakeetCpp) uintptr { return pp.ctxPtr }}, + {"sound_model", soundModelOpt, modelKindSound, + func(pp *ParakeetCpp, c uintptr) { pp.tagCtx = c }, + func(pp *ParakeetCpp) uintptr { return pp.tagCtx }}, } for _, spec := range specs { if spec.value == "" { continue } + // A companion whose role the primary already occupies (e.g. asr_model: + // on an already-ASR primary) would overwrite that role field below, + // leaking the primary ctx: Free() only walks ctxPtr/diarCtx/tagCtx, so + // the overwritten pointer is never freed. Reject it before loading. + if spec.current(p) != 0 { + freeLoaded() + return fmt.Errorf("parakeet-cpp: %s is not allowed on %s %s model", + spec.optName, indefiniteArticle(modelKindName(spec.wantKind)), modelKindName(spec.wantKind)) + } resolved := resolveModelPath(opts.ModelPath, spec.value) cctx := CppLoad(resolved) if cctx == 0 { diff --git a/backend/go/parakeet-cpp/roles_test.go b/backend/go/parakeet-cpp/roles_test.go index 86eded3f133e..7b2a892b333d 100644 --- a/backend/go/parakeet-cpp/roles_test.go +++ b/backend/go/parakeet-cpp/roles_test.go @@ -185,6 +185,107 @@ var _ = Describe("model roles (stubbed C API)", func() { Expect(f.freed).To(HaveLen(freedBeforeFree), "Free after a failed Load must not free anything again") }) + It("rejects an asr_model companion on an already-ASR primary and frees everything it opened", func() { + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("/models/other.gguf", modelKindASR) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + ModelPath: "/models", + Options: []string{"asr_model:other.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(`parakeet-cpp: asr_model is not allowed on an ASR model`)) + Expect(f.loadedPaths).To(Equal([]string{"asr.gguf"})) + Expect(f.freed).To(HaveLen(1), "the primary must be freed too, or it leaks") + + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + Expect(p.companions).To(BeEmpty()) + }) + + It("rejects a diarization_model companion on an already-diarization primary", func() { + f := newFakeLib(). + withModel("diar.gguf", modelKindDiarization). + withModel("/models/other.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "diar.gguf", + ModelPath: "/models", + Options: []string{"diarization_model:other.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(`parakeet-cpp: diarization_model is not allowed on a diarization model`)) + Expect(f.loadedPaths).To(Equal([]string{"diar.gguf"})) + Expect(p.diarCtx).To(BeZero()) + }) + + It("rejects a sound_model companion on an already-sound (CED) primary", func() { + f := newFakeLib(). + withModel("sound.gguf", modelKindSound). + withModel("/models/other.gguf", modelKindSound) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "sound.gguf", + ModelPath: "/models", + Options: []string{"sound_model:other.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(`parakeet-cpp: sound_model is not allowed on a sound model`)) + Expect(f.loadedPaths).To(Equal([]string{"sound.gguf"})) + Expect(p.tagCtx).To(BeZero()) + }) + + It("does not overwrite ctxPtr (and so does not leak the primary) when an asr_model companion "+ + "duplicates an ASR primary loaded alongside other companions", func() { + // Regression for the leak this whole check exists to close: before + // the fix, spec.assign(p, cctx) overwrote p.ctxPtr with the + // companion's ctx, and Free() (which only walks + // ctxPtr/diarCtx/tagCtx) never saw the original primary again. + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("/models/diar.gguf", modelKindDiarization). + withModel("/models/dup.gguf", modelKindASR) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + ModelPath: "/models", + // diarization_model loads first (declared first in loadRoles' + // specs) and succeeds; asr_model then collides with the primary. + Options: []string{"diarization_model:diar.gguf", "asr_model:dup.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(f.freed).To(HaveLen(2), "the primary and the already-loaded diarization companion") + Expect(f.loadedPaths).To(Equal([]string{"asr.gguf", "/models/diar.gguf"}), + "dup.gguf must never be loaded: the role check runs before CppLoad") + + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.companions).To(BeEmpty()) + }) + + It("treats a primary reporting PARAKEET_MODEL_KIND_NONE as ASR", func() { + f := newFakeLib().withModel("model.gguf", modelKindNone) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "model.gguf"})).To(Succeed()) + + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + }) + It("parses diarization_latency:very_low", func() { f := newFakeLib().withModel("diar.gguf", modelKindDiarization) restore = f.install() From c524b7b88b6e8cf328368b139347de063f86401f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:57:35 +0000 Subject: [PATCH 14/30] fix(parakeet-cpp): cap live scene sound score retention sceneBegin started the live diarization/sound companion stream with the C API's default sound options, whose top_k keeps 5 scores per window forever until drained. The live scene path never drains sound scores (only the offline SoundDetection RPC does, with its own fresh stream), so this window queue on the C side grew for the whole session's lifetime. Set opts.Sound.TopK = 0 before starting the scene stream: this disables score retention while leaving sound event detection (onset/ offset), which the live path actually consumes, unaffected. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/scene.go | 6 ++++ backend/go/parakeet-cpp/scene_test.go | 42 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 backend/go/parakeet-cpp/scene_test.go diff --git a/backend/go/parakeet-cpp/scene.go b/backend/go/parakeet-cpp/scene.go index 97dbdb6d903b..cac6e7b31b77 100644 --- a/backend/go/parakeet-cpp/scene.go +++ b/backend/go/parakeet-cpp/scene.go @@ -84,6 +84,12 @@ func (p *ParakeetCpp) sceneBegin() sceneStreamHandle { var opts cSceneOpts CppSceneOptsDefault(&opts) opts.DiarLatency = p.diarLatency + // The live scene path never drains sound scores (unlike the offline + // SoundDetection RPC, see sound.go), so the default top_k of 5 would + // leave the C side's per-window score queue growing for the session's + // whole lifetime. 0 disables per-class score retention; sound EVENTS + // (onset/offset, what the live path actually consumes) are unaffected. + opts.Sound.TopK = 0 s := CppSceneStreamBegin(0, diar, tag, &opts) if s == 0 { return sceneStreamHandle{} diff --git a/backend/go/parakeet-cpp/scene_test.go b/backend/go/parakeet-cpp/scene_test.go new file mode 100644 index 000000000000..9dbfa22055a2 --- /dev/null +++ b/backend/go/parakeet-cpp/scene_test.go @@ -0,0 +1,42 @@ +package main + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The sceneBegin spec drives it entirely against stubbed +// CppSceneOptsDefault/CppSceneStreamBegin (the same seam live_test.go uses +// for the full live-session scene specs), so it runs without libparakeet.so. + +var _ = Describe("ParakeetCpp.sceneBegin", func() { + It("forces Sound.TopK to 0 and carries p.diarLatency into the begin opts", func() { + savedOptsDefault := CppSceneOptsDefault + savedBegin := CppSceneStreamBegin + defer func() { + CppSceneOptsDefault = savedOptsDefault + CppSceneStreamBegin = savedBegin + }() + + // parakeet_capi_scene_opts_default's real default is top_k = 5 (a + // sound-window score history); simulate that here so the test proves + // sceneBegin overrides it rather than merely never setting it. + CppSceneOptsDefault = func(o *cSceneOpts) { + *o = cSceneOpts{Sound: cSoundOpts{TopK: 5}} + } + var gotOpts cSceneOpts + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { + gotOpts = *o + return 1 + } + + p := &ParakeetCpp{diarCtx: 42, diarLatency: diarLatencyVeryLow} + h := p.sceneBegin() + Expect(h.s).ToNot(BeZero()) + Expect(gotOpts.Sound.TopK).To(Equal(int32(0)), + "the live scene path never drains sound scores (see sound.go's SoundDetection, "+ + "which does); a nonzero top_k leaves the C side's per-window score queue "+ + "growing for the session's lifetime") + Expect(gotOpts.DiarLatency).To(Equal(diarLatencyVeryLow)) + }) +}) From 6eec4f8020acdd2a6dbbd319277e6a0fbf0d9643 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:57:40 +0000 Subject: [PATCH 15/30] fix(parakeet-cpp): merge diarization segments per speaker, harden Diarize mergeCloseSegments only compared neighbors in the single start-sorted segment list, so two same-speaker segments never merged once another speaker's turn fell between them (A, B, A): the short B segment broke the adjacency the merge relied on. Group segments by speaker first, merge within each speaker's own start-ordered run, then re-sort the result by start so interleaved speakers come back out in timeline order. Also harden Diarize's entry points the same way streamFeedDoc/ sceneFeed already are: diarizeCall re-checks p.diarCtx (and, on the include_text path, p.ctxPtr) under engineMu right before the C call, so a Free() racing between Diarize's own checks and the lock can no longer reach the C side with a freed context. When the include_text call returns NULL, last_error is now read from both contexts and whichever came back non-empty is reported, since either side of the pairing can be the one that failed. A WAV decode failure is reported as InvalidArgument instead of an unwrapped/untyped error. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/diarize.go | 109 ++++++++++++++++++------ backend/go/parakeet-cpp/diarize_test.go | 77 +++++++++++++++++ 2 files changed, 158 insertions(+), 28 deletions(-) diff --git a/backend/go/parakeet-cpp/diarize.go b/backend/go/parakeet-cpp/diarize.go index d77308abda8f..1853aa666314 100644 --- a/backend/go/parakeet-cpp/diarize.go +++ b/backend/go/parakeet-cpp/diarize.go @@ -3,8 +3,11 @@ package main import ( "encoding/json" "fmt" + "sort" "strconv" + "strings" + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/xlog" "google.golang.org/grpc/codes" @@ -104,7 +107,7 @@ func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error pcm, duration, err := decodeWavMono16k(req.GetDst()) if err != nil { - return pb.DiarizeResponse{}, err + return pb.DiarizeResponse{}, status.Errorf(codes.InvalidArgument, "parakeet-cpp: decode audio: %s", err) } if len(pcm) == 0 { return pb.DiarizeResponse{}, status.Error(codes.InvalidArgument, "parakeet-cpp: empty audio") @@ -133,13 +136,20 @@ func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error // diarizeCall runs the single C call Diarize needs (transcribe_and_diarize_json // when wantText, else diarize_pcm) under engineMu, and returns the raw JSON -// document. last_error is ctx-shared, so it is read under the same lock as the -// failing call, off p.diarCtx (the context both entry points share as their -// diarization side). +// document. p.diarCtx (and, on the include_text path, p.ctxPtr) is re-checked +// under the lock before the C call: Diarize's own p.diarCtx==0/wantText checks +// run before this lock is taken, so a Free() racing in between (which zeroes +// those fields under the same engineMu) would otherwise reach the C side with +// a freed context. last_error is ctx-shared, so it is read under the same +// lock as the failing call. func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool) (string, error) { p.engineMu.Lock() defer p.engineMu.Unlock() + if p.diarCtx == 0 || (wantText && p.ctxPtr == 0) { + return "", grpcerrors.ModelNotLoaded("parakeet-cpp") + } + var cstr uintptr if wantText { cstr = CppTranscribeAndDiarizeJSON(p.ctxPtr, p.diarCtx, &pcm[0], int32(len(pcm)), 16000) @@ -147,17 +157,34 @@ func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool) (string, error) cstr = CppDiarizePCM(p.diarCtx, &pcm[0], int32(len(pcm)), 16000) } if cstr == 0 { - msg := CppLastError(p.diarCtx) - if msg == "" { - msg = "unknown error" - } - return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", msg) + return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", diarizeLastError(p, wantText)) } raw := goStringFromCPtr(cstr) CppFreeString(cstr) return raw, nil } +// diarizeLastError reads last_error off p.diarCtx and, on the include_text +// path, p.ctxPtr too — the failing call is CppTranscribeAndDiarizeJSON there, +// and either side of the pairing may be the one that set it — then joins +// whichever came back non-empty. Called under the same engineMu as the +// failing call (last_error is ctx-shared state). +func diarizeLastError(p *ParakeetCpp, wantText bool) string { + var msgs []string + if m := CppLastError(p.diarCtx); m != "" { + msgs = append(msgs, m) + } + if wantText { + if m := CppLastError(p.ctxPtr); m != "" { + msgs = append(msgs, m) + } + } + if len(msgs) == 0 { + return "unknown error" + } + return strings.Join(msgs, "; ") +} + // parseDiarizeDoc decodes the raw JSON diarizeCall returned into // DiarizeSegments (without ids: renumberDiarizeSegments assigns those after // filtering). @@ -208,34 +235,60 @@ func applyDurationFilters(segs []*pb.DiarizeSegment, minOn, minOff float32) []*p return segs } -// mergeCloseSegments merges consecutive SAME-SPEAKER segments separated by a -// gap shorter than minOff into one segment spanning both (and concatenating -// any text). Segments from different speakers are never merged, regardless of -// gap: the gap only ever means "the same speaker paused", never "two speakers -// are actually one". +// mergeCloseSegments merges SAME-SPEAKER segments separated by a gap shorter +// than minOff into one segment spanning both (and concatenating any text). +// Segments from different speakers are never merged, regardless of gap: the +// gap only ever means "the same speaker paused", never "two speakers are +// actually one". +// +// Merging runs per speaker rather than on the single start-sorted list: two +// segments of the same speaker are not necessarily adjacent in that list once +// another speaker's turn falls between them (A, B, A), and a start-sorted +// walk would then never compare the two A's at all. Grouping by speaker first +// keeps each group's own start order (segs is assumed start-sorted, as +// parakeet_capi_diarize_pcm and parakeet_capi_transcribe_and_diarize_json +// document), merges within the group, then the merged segments are re-sorted +// by start so interleaved speakers come back out in timeline order. func mergeCloseSegments(segs []*pb.DiarizeSegment, minOff float32) []*pb.DiarizeSegment { if minOff <= 0 || len(segs) < 2 { return segs } + + bySpeaker := make(map[string][]*pb.DiarizeSegment) + var order []string // first-seen speaker order, for a deterministic group walk + for _, s := range segs { + if _, ok := bySpeaker[s.GetSpeaker()]; !ok { + order = append(order, s.GetSpeaker()) + } + bySpeaker[s.GetSpeaker()] = append(bySpeaker[s.GetSpeaker()], s) + } + out := make([]*pb.DiarizeSegment, 0, len(segs)) - out = append(out, segs[0]) - for _, s := range segs[1:] { - prev := out[len(out)-1] - if s.GetSpeaker() == prev.GetSpeaker() && s.GetStart()-prev.GetEnd() < minOff { - if s.GetEnd() > prev.GetEnd() { - prev.End = s.End - } - if s.GetText() != "" { - if prev.GetText() != "" { - prev.Text = prev.GetText() + " " + s.GetText() - } else { - prev.Text = s.GetText() + for _, speaker := range order { + group := bySpeaker[speaker] + merged := make([]*pb.DiarizeSegment, 0, len(group)) + merged = append(merged, group[0]) + for _, s := range group[1:] { + prev := merged[len(merged)-1] + if s.GetStart()-prev.GetEnd() < minOff { + if s.GetEnd() > prev.GetEnd() { + prev.End = s.End + } + if s.GetText() != "" { + if prev.GetText() != "" { + prev.Text = prev.GetText() + " " + s.GetText() + } else { + prev.Text = s.GetText() + } } + continue } - continue + merged = append(merged, s) } - out = append(out, s) + out = append(out, merged...) } + + sort.Slice(out, func(i, j int) bool { return out[i].GetStart() < out[j].GetStart() }) return out } diff --git a/backend/go/parakeet-cpp/diarize_test.go b/backend/go/parakeet-cpp/diarize_test.go index 735e8c9cf2b0..1db076771d62 100644 --- a/backend/go/parakeet-cpp/diarize_test.go +++ b/backend/go/parakeet-cpp/diarize_test.go @@ -5,6 +5,7 @@ import ( "sync" "unsafe" + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" pb "github.com/mudler/LocalAI/pkg/grpc/proto" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -195,4 +196,80 @@ var _ = Describe("ParakeetCpp.Diarize", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("boom")) }) + + It("reports last_error from both contexts when the include_text C call returns NULL", func() { + // Diarize's Unimplemented gate checks CppDiarizePCM regardless of + // wantText, so it needs a non-nil (never called) stub here too. + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + Fail("diarize_pcm must not be called when include_text has an ASR companion to pair with") + return 0 + } + CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return 0 + } + CppLastError = func(ctx uintptr) string { + if ctx == 7 { + return "asr side broke" + } + return "diar side broke" + } + + p := &ParakeetCpp{diarCtx: 42, ctxPtr: 7} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1), IncludeText: true}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("asr side broke")) + Expect(err.Error()).To(ContainSubstring("diar side broke")) + }) + + It("wraps a decode failure as InvalidArgument", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + Fail("decode must fail before any C call is made") + return 0 + } + + p := &ParakeetCpp{diarCtx: 42} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: filepath.Join(GinkgoT().TempDir(), "missing.wav")}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + }) + + It("returns ModelNotLoaded without a C call when diarCtx is zeroed between the entry check and the call", func() { + called := false + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + called = true + return pool.cstr(`{"speakers":8,"segments":[]}`) + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + // Simulate a Free() racing between Diarize's own diarCtx==0 check and + // diarizeCall's lock, exactly as it zeroes diarCtx under engineMu. + p.diarCtx = 0 + _, err := p.diarizeCall(make([]float32, 10), false) + Expect(grpcerrors.IsModelNotLoaded(err)).To(BeTrue()) + Expect(called).To(BeFalse(), "no C call once diarCtx was cleared") + }) + + It("merges same-speaker segments across an intervening different speaker (A, B, A)", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"segments":[` + + `{"speaker":0,"start":0.00,"end":1.00},` + + `{"speaker":1,"start":1.05,"end":1.20},` + // short B segment sits between the two A's + `{"speaker":0,"start":1.30,"end":2.00}]}`) // 0.1s gap from the first A: same speaker, merges + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(3), MinDurationOff: 0.5}) + Expect(err).ToNot(HaveOccurred()) + // The two speaker-0 segments merge into one spanning 0.00-2.00, and + // the timeline re-sort puts speaker 1's untouched segment in between. + Expect(resp.Segments).To(HaveLen(2)) + Expect(resp.Segments[0].Speaker).To(Equal("0")) + Expect(resp.Segments[0].Start).To(BeNumerically("~", 0.0, 0.001)) + Expect(resp.Segments[0].End).To(BeNumerically("~", 2.0, 0.001)) + Expect(resp.Segments[1].Speaker).To(Equal("1")) + Expect(resp.Segments[1].Start).To(BeNumerically("~", 1.05, 0.001)) + Expect(resp.Segments[1].End).To(BeNumerically("~", 1.20, 0.001)) + }) }) From 7eac36f9775ec673c02827baaf4acd07c5f83cea Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:57:44 +0000 Subject: [PATCH 16/30] fix(parakeet-cpp): harden SoundDetection's engine checks soundStreamDrain ran every C call under engineMu but never re-checked p.tagCtx there, so a Free() racing between SoundDetection's own tagCtx==0 check and this lock could still reach the C side with a freed context. Re-check p.tagCtx under the lock and return ModelNotLoaded when it was cleared, mirroring diarizeCall's own re-check. A WAV decode failure is now reported as InvalidArgument instead of an unwrapped/untyped error. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/sound.go | 11 +++++++- backend/go/parakeet-cpp/sound_test.go | 37 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/backend/go/parakeet-cpp/sound.go b/backend/go/parakeet-cpp/sound.go index e7dc36d6fe58..1a3a358ad09d 100644 --- a/backend/go/parakeet-cpp/sound.go +++ b/backend/go/parakeet-cpp/sound.go @@ -6,6 +6,7 @@ import ( "fmt" "sort" + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -65,7 +66,7 @@ func (p *ParakeetCpp) SoundDetection(ctx context.Context, req *pb.SoundDetection pcm, _, err := decodeWavMono16k(req.GetSrc()) if err != nil { - return nil, err + return nil, status.Errorf(codes.InvalidArgument, "parakeet-cpp: decode audio: %s", err) } windows, nClasses, err := p.soundStreamScores(ctx, pcm) @@ -120,6 +121,14 @@ func (p *ParakeetCpp) soundStreamDrain(ctx context.Context, pcm []float32) (stri p.engineMu.Lock() defer p.engineMu.Unlock() + // SoundDetection's own p.tagCtx==0 check runs before this lock is taken; + // re-check here so a Free() racing in between (which zeroes p.tagCtx + // under this same engineMu) is caught instead of handed to the C side, + // mirroring streamFeedDoc's/sceneFeed's re-check. + if p.tagCtx == 0 { + return "", 0, grpcerrors.ModelNotLoaded("parakeet-cpp") + } + nClasses := int(CppNumClasses(p.tagCtx)) var opts cSoundOpts diff --git a/backend/go/parakeet-cpp/sound_test.go b/backend/go/parakeet-cpp/sound_test.go index 0124118a43ad..1162a84ac6e2 100644 --- a/backend/go/parakeet-cpp/sound_test.go +++ b/backend/go/parakeet-cpp/sound_test.go @@ -6,6 +6,7 @@ import ( "sync" "unsafe" + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" pb "github.com/mudler/LocalAI/pkg/grpc/proto" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -302,6 +303,42 @@ var _ = Describe("ParakeetCpp.SoundDetection", func() { Expect(feedCount).To(Equal(1)) Expect(freed).To(BeTrue()) }) + + It("wraps a decode failure as InvalidArgument", func() { + // Every required symbol must be non-nil to clear SoundDetection's own + // Unimplemented gate and reach the decode step this spec targets; + // none of them may actually be called. + fail := func(string) { Fail("no C call once the decode itself has failed") } + CppNumClasses = func(uintptr) int32 { fail("num_classes"); return 0 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { fail("begin"); return 0 } + CppSoundStreamFeed = func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + fail("feed") + return 0 + } + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { fail("drain"); return 0 } + CppSoundStreamFree = func(uintptr) { fail("free") } + + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{ + Src: filepath.Join(GinkgoT().TempDir(), "missing.wav"), + }) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + }) + + It("returns ModelNotLoaded without a C call when tagCtx is zeroed between the entry check and the call", func() { + called := false + CppNumClasses = func(uintptr) int32 { called = true; return 2 } + + p := &ParakeetCpp{tagCtx: 42} + // Simulate a Free() racing between SoundDetection's own tagCtx==0 + // check and soundStreamDrain's lock, exactly as it zeroes tagCtx + // under engineMu. + p.tagCtx = 0 + _, _, err := p.soundStreamDrain(context.Background(), make([]float32, 10)) + Expect(grpcerrors.IsModelNotLoaded(err)).To(BeTrue()) + Expect(called).To(BeFalse(), "no C call once tagCtx was cleared") + }) }) var _ = Describe("averageWindowScores", func() { From a63b37e8540fa339322d7b7108351b63d4758cf4 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:57:46 +0000 Subject: [PATCH 17/30] test(parakeet-cpp): cover a mid-session scene feed failure feedSlicesScene already degrades gracefully when a scene feed call fails mid-session: it frees the broken stream and carries the ASR-only session forward. Add a spec covering that path end to end: the scene stream is freed exactly once, later audio slices still produce ASR responses, and no speaker/sound events appear before or after the failure. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/live_test.go | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/backend/go/parakeet-cpp/live_test.go b/backend/go/parakeet-cpp/live_test.go index 5b70725bab5e..861e5a7779ca 100644 --- a/backend/go/parakeet-cpp/live_test.go +++ b/backend/go/parakeet-cpp/live_test.go @@ -613,6 +613,43 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { Expect(feedCalls).To(Equal(0), "no C call once the scene stream's contexts were freed") }) + It("degrades to ASR-only after a mid-session scene feed failure: freed once, no more scene events, ASR keeps working", func() { + CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { + return pool.cstr(`{"text":"hi ","eou":0,"frame_sec":0.08,` + + `"words":[{"w":"hi","start":0.1,"end":0.3,"conf":0.9}]}`) + } + sceneFeedCalls := 0 + begun, freed := liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { + sceneFeedCalls++ + return 0 // fails every call; only the first should ever be reached + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) // scene feed fails here: warn, free, zero the handle + in <- liveAudio(make([]float32, 10)) // ASR-only: no scene C call at all + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + // ready, ASR delta (msg 1), ASR delta (msg 2), final: no scene-only + // response ever appears, before or after the failure. + Expect(got).To(HaveLen(4)) + Expect(got[0].Ready).To(BeTrue()) + Expect(got[1].Delta).To(Equal("hi ")) + Expect(got[2].Delta).To(Equal("hi ")) + for _, r := range got { + Expect(r.Speakers).To(BeEmpty(), "no speaker events once the scene stream has failed") + Expect(r.Sounds).To(BeEmpty(), "no sound events once the scene stream has failed") + } + Expect(got[3].FinalResult).NotTo(BeNil()) + Expect(got[3].FinalResult.Text).To(Equal("hi hi")) + + Expect(*begun).To(Equal(1)) + Expect(sceneFeedCalls).To(Equal(1), "the second audio message must not retry the broken scene stream") + Expect(*freed).To(Equal(1), "the broken scene stream is freed exactly once, not again at RPC unwind") + }) + It("continues without scene events when scene begin fails", func() { CppSceneOptsDefault = func(o *cSceneOpts) { *o = cSceneOpts{} } CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { return 0 } From 39a993411680905d65102bd1423d8a7929ddca8b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 16:57:52 +0000 Subject: [PATCH 18/30] docs: fix the parakeet-cpp companion role table and realtime scene docs audio-to-text.md's companion option table read "Use on" with a note that the loader did not enforce the pairing; it now rejects a companion whose role duplicates the primary's, so restore the "Allowed on" wording and describe the real enforcement. openai-realtime.md's live speaker/sound section claimed a mid-stream session.update resets the companion stream and that it flushes on session close; neither happens, since the realtime core opens one live stream (and so one scene stream) per speech turn and closes it at that turn's commit, with no mid-stream Config in between. Document that lifecycle instead, state precisely that start/end are seconds from the start of the turn's own audio, and note that the diarization model starts a fresh session every turn, so a speaker index is only meaningful within one turn. The example sound tag ("Rooster", index 17) did not match any real CED label; index 17 in ced-tiny-q8_0.gguf is "Baby laughter". Replaced with "Chicken, rooster" at its real index, 99. Assisted-by: Claude:claude-sonnet-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- docs/content/features/audio-to-text.md | 4 ++-- docs/content/features/openai-realtime.md | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 6a4076c15aba..a73b5d009f9e 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -194,14 +194,14 @@ For real-time use, load a cache-aware streaming model (e.g. `realtime_eou_120m-v The same backend also serves the `/v1/audio/diarization` and `/v1/audio/classification` endpoints, and can attach a diarization or sound model to a live transcription session. `options:` accepts paths relative to the models directory, or absolute: -| Option | Use on | Used for | +| Option | Allowed on | Used for | |---|---|---| | `asr_model:` | a diarization model | `include_text` on `/v1/audio/diarization` | | `diarization_model:` | an ASR model | speaker segments during realtime live transcription | | `sound_model:` | an ASR model | sound events during realtime live transcription | | `diarization_latency:` | a model with a diarization companion | latency mode for the live speaker stream; default `low` | -"Use on" is the intended pairing; the loader does not check the primary model's kind before accepting an option, it only checks that the companion GGUF it loads matches the expected kind (for example `sound_model:` pointing at an ASR GGUF fails to load, naming the kind it expected). See [Speaker Diarization]({{% relref "audio-diarization" %}}) for the `Diarize` RPC and [Sound Classification]({{% relref "audio-classification" %}}) for `SoundDetection`, and [Realtime API]({{% relref "openai-realtime" %}}) for the live speaker/sound events emitted during a realtime session. +The loader rejects a companion whose role duplicates the primary's own (for example `asr_model:` on an already-ASR primary, or `sound_model:` on a CED primary), and rejects a companion GGUF that does not match the role its option names (for example `sound_model:` pointing at an ASR GGUF fails to load, naming the kind it expected). See [Speaker Diarization]({{% relref "audio-diarization" %}}) for the `Diarize` RPC and [Sound Classification]({{% relref "audio-classification" %}}) for `SoundDetection`, and [Realtime API]({{% relref "openai-realtime" %}}) for the live speaker/sound events emitted during a realtime session. ### Segment timestamps diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index bf4234fab573..535c47144389 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -172,20 +172,24 @@ Each closed speaker segment emits a `conversation.item.input_audio_transcription } ``` -Each sound event emits a `conversation.item.sound_detection` event with one tag and the detection window's `start`/`end` (seconds, stream-relative - the same clock as the live words): +Each sound event emits a `conversation.item.sound_detection` event with one tag and the detection window's `start`/`end`: ```json { "type": "conversation.item.sound_detection", "item_id": "item_abc", "content_index": 0, - "detections": [{"label": "Rooster", "score": 0.91, "index": 17}], + "detections": [{"label": "Chicken, rooster", "score": 0.91, "index": 99}], "start": 24.0, "end": 30.0 } ``` -`score` is the peak score seen for that tag while the sound was live, not an average. On session close the companion stream is flushed and its remaining events are sent before the final result; a mid-stream `session.update` that changes the transcription model resets it along with the ASR session. +The `start`/`end` on both event types are seconds measured from the start of the current turn's own audio, not the session or the WebSocket connection - the same base the streamed transcript words use. + +The companion stream is opened fresh for each speech turn, alongside that turn's ASR live session, and closed when the turn commits: whatever it had not yet emitted is drained and sent at that point. Because the diarization model runs a brand new session every turn, its speaker indices are scoped to the turn too - `"speaker": "0"` in one turn and `"speaker": "0"` in the next are not guaranteed to be the same person, even within the same conversation. + +`score` is the peak score seen for that tag while the sound was live, not an average. **Limitation**: under `semantic_vad`, live transcription (and so this companion stream) only runs during speech turns - it does not see audio between turns. A sound that happens while nobody is speaking is not detected this way. If you need sound events independent of speech turns, use the pipeline's `sound_detection` model instead (see [Sound Classification]({{% relref "audio-classification" %}})), which classifies each VAD-committed utterance on its own. Use one or the other, not both, on the same session - they overlap in purpose and would emit sound detections twice. From f1f33c7232aa7948ee5ddfd7e040e92518e9d173 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 19:39:55 +0000 Subject: [PATCH 19/30] fix(parakeet-cpp): use CED's real index for Chicken, rooster The scene feed comment and the live test's canned document gave "Chicken, rooster" index 365. In CED's AudioSet label list it is 99, which is also what the realtime docs show. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/live_test.go | 4 ++-- backend/go/parakeet-cpp/scene.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/go/parakeet-cpp/live_test.go b/backend/go/parakeet-cpp/live_test.go index 861e5a7779ca..11dc496b62d4 100644 --- a/backend/go/parakeet-cpp/live_test.go +++ b/backend/go/parakeet-cpp/live_test.go @@ -514,7 +514,7 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { } liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { if calls == 1 { - return pool.cstr(`{"speakers":[],"sounds":[{"index":365,` + + return pool.cstr(`{"speakers":[],"sounds":[{"index":99,` + `"label":"Chicken, rooster","start":24.0,"end":30.0,"peak":0.86}]}`) } return pool.cstr(`{"speakers":[],"sounds":[]}`) @@ -533,7 +533,7 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { Expect(got[2].Delta).To(BeEmpty()) Expect(got[2].Sounds).To(HaveLen(1)) Expect(got[2].Sounds[0].Label).To(Equal("Chicken, rooster")) - Expect(got[2].Sounds[0].Index).To(Equal(int32(365))) + Expect(got[2].Sounds[0].Index).To(Equal(int32(99))) Expect(got[2].Sounds[0].Peak).To(BeNumerically("~", 0.86, 1e-6)) Expect(got[2].Sounds[0].Start).To(Equal(int64(24.0 * 1e9))) Expect(got[2].Sounds[0].End).To(Equal(int64(30.0 * 1e9))) diff --git a/backend/go/parakeet-cpp/scene.go b/backend/go/parakeet-cpp/scene.go index cac6e7b31b77..590740f13a05 100644 --- a/backend/go/parakeet-cpp/scene.go +++ b/backend/go/parakeet-cpp/scene.go @@ -22,7 +22,7 @@ type sceneSpeakerJSON struct { } // sceneSoundJSON mirrors one element of a scene feed document's "sounds" -// array: {"index":365,"label":"Chicken, rooster","start":24.0,"end":30.0,"peak":0.86}. +// array: {"index":99,"label":"Chicken, rooster","start":24.0,"end":30.0,"peak":0.86}. type sceneSoundJSON struct { Index int `json:"index"` Label string `json:"label"` From f85fe9ac1997366741071e386d8ca74458b6818c Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Tue, 29 Sep 2026 03:04:26 +0000 Subject: [PATCH 20/30] fix(realtime): call the test event accessor The scene-event tests range over a method instead of its returned slice. Call the synchronized accessor so the OpenAI test package compiles. Assisted-by: Codex:gpt-6 Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/openai/realtime_semantic_vad_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/http/endpoints/openai/realtime_semantic_vad_test.go b/core/http/endpoints/openai/realtime_semantic_vad_test.go index aa6c49b12933..5a92e3b15000 100644 --- a/core/http/endpoints/openai/realtime_semantic_vad_test.go +++ b/core/http/endpoints/openai/realtime_semantic_vad_test.go @@ -303,7 +303,7 @@ var _ = Describe("liveTurnState", func() { lts.drainEvents(3.4) var got []types.ConversationItemInputAudioTranscriptionSegmentEvent - for _, e := range ftr.events { + for _, e := range ftr.events() { if seg, ok := e.(types.ConversationItemInputAudioTranscriptionSegmentEvent); ok { got = append(got, seg) } @@ -326,7 +326,7 @@ var _ = Describe("liveTurnState", func() { lts.drainEvents(1.0) var got []types.ConversationItemSoundDetectionEvent - for _, e := range ftr.events { + for _, e := range ftr.events() { if sd, ok := e.(types.ConversationItemSoundDetectionEvent); ok { got = append(got, sd) } From 54a869c8b72770e68c04ba05c7d95e39448fa109 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 29 Sep 2026 15:54:26 +0000 Subject: [PATCH 21/30] chore(parakeet-cpp): pin parakeet.cpp master with sound events mudler/parakeet.cpp#75 (sound events, scene stream, model kinds) and #74 (the missing include that broke the image builds) are on master now. Pin 6dea76a instead of the #75 PR head, and update the header comment the bump bot reads. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile index 4c393f386b35..4d462d11b46b 100644 --- a/backend/go/parakeet-cpp/Makefile +++ b/backend/go/parakeet-cpp/Makefile @@ -1,6 +1,6 @@ # parakeet-cpp backend Makefile. # -# Upstream pin lives below as PARAKEET_VERSION?=2bf88954dc628b32835734e2e9159550a75a1dc6 +# Upstream pin lives below as PARAKEET_VERSION?=6dea76a81ee0bba83c6dad320dccc2fbeef111e2 # (.github/bump_deps.sh) can find and update it - matches the # whisper.cpp / ds4 / vibevoice-cpp convention. # @@ -15,10 +15,7 @@ # That's what the L0 smoke test uses. The default target below does the # proper clone-at-pin + cmake build so CI doesn't need a side-checkout. -# Tracks mudler/parakeet.cpp PR #75 (diarization/CED model roles + sound and -# scene streaming, ABI v8) until it merges to master; repin to the merge -# commit once it lands. -PARAKEET_VERSION?=c018537fbe0bdd3ccb3ecb387a2119bbf901e90d +PARAKEET_VERSION?=6dea76a81ee0bba83c6dad320dccc2fbeef111e2 PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp GOCMD?=go From 1dd8b737110ed5743ad8c21260114c132afab5f2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 29 Sep 2026 15:56:20 +0000 Subject: [PATCH 22/30] chore(parakeet-cpp): pin parakeet.cpp with ced.cpp on main parakeet.cpp #76 moved its ced.cpp submodule from the head of localai-org/ced.cpp#3 (a branch-only commit) to ced.cpp main, where #3 landed with an identical tree. Pin 623a968 so the image builds no longer depend on that branch. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile index 4d462d11b46b..59fe567170ca 100644 --- a/backend/go/parakeet-cpp/Makefile +++ b/backend/go/parakeet-cpp/Makefile @@ -1,6 +1,6 @@ # parakeet-cpp backend Makefile. # -# Upstream pin lives below as PARAKEET_VERSION?=6dea76a81ee0bba83c6dad320dccc2fbeef111e2 +# Upstream pin lives below as PARAKEET_VERSION?=623a968bccbd2214588df398fcce687cd4218dea # (.github/bump_deps.sh) can find and update it - matches the # whisper.cpp / ds4 / vibevoice-cpp convention. # @@ -15,7 +15,7 @@ # That's what the L0 smoke test uses. The default target below does the # proper clone-at-pin + cmake build so CI doesn't need a side-checkout. -PARAKEET_VERSION?=6dea76a81ee0bba83c6dad320dccc2fbeef111e2 +PARAKEET_VERSION?=623a968bccbd2214588df398fcce687cd4218dea PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp GOCMD?=go From a22322a0f4d7953a0713b616130bc28eb1b4668e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 28 Sep 2026 09:40:09 +0000 Subject: [PATCH 23/30] feat(transcription): carry speaker labels on words and streamed segments A diarizing backend could label transcript segments, but two paths dropped the label: TranscriptWord had no speaker field, so live transcription words and word-level timestamps could not carry one, and the stream=true transcript.text.done event left the speaker out of its segments. TranscriptWord gains an optional speaker (proto field 4, additive). It flows through the live event and result mapping, the JSON word output of the endpoint and the CLI, and transcript.text.done now includes a segment's speaker when there is one. Empty labels are omitted, so responses without diarization are unchanged. Assisted-by: Claude:claude-opus-5-5 [Claude Code] (cherry picked from commit 2f0049f979496c3428c08c9b1d5bf4315403dccf) Signed-off-by: Ettore Di Giacinto --- backend/backend.proto | 1 + core/backend/transcript.go | 7 +++--- core/backend/transcript_live.go | 7 +++--- core/backend/transcript_live_internal_test.go | 14 ++++++++++++ core/cli/transcript.go | 14 +++++++----- core/http/endpoints/openai/transcription.go | 22 ++++++++++++------- core/schema/transcription.go | 14 +++++++----- 7 files changed, 53 insertions(+), 26 deletions(-) diff --git a/backend/backend.proto b/backend/backend.proto index 0aa0d253811c..6a09b98eb39b 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -663,6 +663,7 @@ message TranscriptWord { int64 start = 1; int64 end = 2; string text = 3; + string speaker = 4; // backend speaker label when diarizing; empty otherwise } message TranscriptSegment { diff --git a/core/backend/transcript.go b/core/backend/transcript.go index 0ddee3adddff..27b13bc55316 100644 --- a/core/backend/transcript.go +++ b/core/backend/transcript.go @@ -213,9 +213,10 @@ func transcriptResultFromProto(r *proto.TranscriptResult) *schema.TranscriptionR var words []schema.TranscriptionWord for _, w := range s.Words { var word = schema.TranscriptionWord{ - Start: time.Duration(w.Start), - End: time.Duration(w.End), - Text: w.Text, + Start: time.Duration(w.Start), + End: time.Duration(w.End), + Text: w.Text, + Speaker: w.Speaker, } words = append(words, word) tr.Words = append(tr.Words, word) diff --git a/core/backend/transcript_live.go b/core/backend/transcript_live.go index 310723df909b..e64e54a578e2 100644 --- a/core/backend/transcript_live.go +++ b/core/backend/transcript_live.go @@ -320,9 +320,10 @@ func liveEventFromProto(r *proto.TranscriptLiveResponse) LiveTranscriptionEvent } for _, w := range r.GetWords() { ev.Words = append(ev.Words, schema.TranscriptionWord{ - Start: time.Duration(w.Start), - End: time.Duration(w.End), - Text: w.Text, + Start: time.Duration(w.Start), + End: time.Duration(w.End), + Text: w.Text, + Speaker: w.Speaker, }) } for _, s := range r.GetSpeakers() { diff --git a/core/backend/transcript_live_internal_test.go b/core/backend/transcript_live_internal_test.go index c0e1ca0ceca6..8b2aee91e0c2 100644 --- a/core/backend/transcript_live_internal_test.go +++ b/core/backend/transcript_live_internal_test.go @@ -54,6 +54,20 @@ var _ = Describe("liveEventFromProto", func() { Expect(ev.Final).To(BeNil()) }) + It("carries word speakers and final segment speakers from a diarizing backend", func() { + ev := liveEventFromProto(&proto.TranscriptLiveResponse{ + Words: []*proto.TranscriptWord{{Text: "hi", Speaker: "1"}}, + }) + Expect(ev.Words[0].Speaker).To(Equal("1")) + ev = liveEventFromProto(&proto.TranscriptLiveResponse{ + FinalResult: &proto.TranscriptResult{ + Text: "hi there", + Segments: []*proto.TranscriptSegment{{Text: "hi", Speaker: "0"}, {Text: "there", Speaker: "1"}}, + }, + }) + Expect(ev.Final.Segments[1].Speaker).To(Equal("1")) + }) + It("maps the eob backchannel flag separately from eou", func() { ev := liveEventFromProto(&proto.TranscriptLiveResponse{Delta: "uh-huh", Eob: true}) Expect(ev.Eob).To(BeTrue()) diff --git a/core/cli/transcript.go b/core/cli/transcript.go index 06764f4dd277..e187453a6fa9 100644 --- a/core/cli/transcript.go +++ b/core/cli/transcript.go @@ -93,18 +93,20 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error { } for _, word := range(tr.Words) { trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } for _, seg := range(tr.Segments) { segWords := []schema.TranscriptionWordSeconds{} for _, word := range(seg.Words) { segWords = append(segWords, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{ diff --git a/core/http/endpoints/openai/transcription.go b/core/http/endpoints/openai/transcription.go index 6c99d7afcaaa..49825c3a13cd 100644 --- a/core/http/endpoints/openai/transcription.go +++ b/core/http/endpoints/openai/transcription.go @@ -210,18 +210,20 @@ func TranscriptEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app } for _, word := range tr.Words { trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } for _, seg := range tr.Segments { segWords := []schema.TranscriptionWordSeconds{} for _, word := range seg.Words { segWords = append(segWords, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{ @@ -338,12 +340,16 @@ func streamTranscription(c echo.Context, req backend.TranscriptionRequest, ml *m if len(finalResult.Segments) > 0 { segs := make([]map[string]any, 0, len(finalResult.Segments)) for _, seg := range finalResult.Segments { - segs = append(segs, map[string]any{ + entry := map[string]any{ "id": seg.Id, "start": seg.Start.Seconds(), "end": seg.End.Seconds(), "text": seg.Text, - }) + } + if seg.Speaker != "" { + entry["speaker"] = seg.Speaker + } + segs = append(segs, entry) } doneEvent["segments"] = segs } diff --git a/core/schema/transcription.go b/core/schema/transcription.go index 8414fd0ba805..aab914f704e9 100644 --- a/core/schema/transcription.go +++ b/core/schema/transcription.go @@ -13,9 +13,10 @@ type TranscriptionSegment struct { } type TranscriptionWord struct { - Start time.Duration `json:"start"` - End time.Duration `json:"end"` - Text string `json:"text"` + Start time.Duration `json:"start"` + End time.Duration `json:"end"` + Text string `json:"text"` + Speaker string `json:"speaker,omitempty"` } type TranscriptionResult struct { @@ -42,9 +43,10 @@ type TranscriptionSegmentSeconds struct { } type TranscriptionWordSeconds struct { - Start float64 `json:"start"` - End float64 `json:"end"` - Text string `json:"text"` + Start float64 `json:"start"` + End float64 `json:"end"` + Text string `json:"text"` + Speaker string `json:"speaker,omitempty"` } type TranscriptionResultSeconds struct { From 2cc809e502165601f87c2a12452e41ebe5410d20 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 29 Sep 2026 16:49:46 +0000 Subject: [PATCH 24/30] feat(importers): detect the parakeet.cpp diarization GGUF The Nemotron-3-Diarization GGUFs are published in mudler/parakeet-cpp-gguf as nemotron-3-diarization-.gguf. The parakeet-cpp importer did not recognise that name, so a direct `local-ai models import` of the file fell through to another importer. A direct URL to the file now imports with the diarization usecase. A repo import still picks ASR weights when the repo also ships the diarization model, and falls back to the diarization weights only when there are no others. Ported from #12323. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- core/gallery/importers/parakeet-cpp.go | 31 +++++++++++++-- core/gallery/importers/parakeet-cpp_test.go | 42 +++++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/core/gallery/importers/parakeet-cpp.go b/core/gallery/importers/parakeet-cpp.go index aa732aa3e7fc..5b758167b457 100644 --- a/core/gallery/importers/parakeet-cpp.go +++ b/core/gallery/importers/parakeet-cpp.go @@ -108,6 +108,11 @@ func (i *ParakeetCppImporter) Import(details Details) (gallery.ModelConfig, erro uri := downloader.URI(details.URI) directGGUF := isParakeetGGUF(filepath.Base(details.URI)) + // A speaker diarization GGUF is served by the same backend but answers + // /v1/audio/diarization, not transcription. + if directGGUF && isParakeetDiarGGUF(filepath.Base(details.URI)) { + modelConfig.KnownUsecaseStrings = []string{"diarization"} + } switch { case uri.LooksLikeURL() && directGGUF: // Direct file URL (e.g. .../resolve/main/tdt_ctc-110m-f16.gguf). The @@ -128,12 +133,23 @@ func (i *ParakeetCppImporter) Import(details Details) (gallery.ModelConfig, erro // HF repo: collect every parakeet GGUF, pick the preferred quant, and // nest under parakeet-cpp/models// so a multi-quant repo doesn't // collide on disk. - var ggufFiles []hfapi.ModelFile + // Prefer ASR weights: a repo that also ships the diarization model + // (mudler/parakeet-cpp-gguf) imports as a transcription model, and the + // diarization GGUF is imported by its direct URL. A repo with only + // diarization weights imports as a diarization model. + var ggufFiles, diarFiles []hfapi.ModelFile for _, f := range details.HuggingFace.Files { - if isParakeetGGUF(filepath.Base(f.Path)) { + switch base := filepath.Base(f.Path); { + case isParakeetDiarGGUF(base): + diarFiles = append(diarFiles, f) + case isParakeetGGUF(base): ggufFiles = append(ggufFiles, f) } } + if len(ggufFiles) == 0 && len(diarFiles) > 0 { + ggufFiles = diarFiles + modelConfig.KnownUsecaseStrings = []string{"diarization"} + } if chosen, ok := pickPreferredGGMLFile(ggufFiles, quants); ok { target := filepath.Join("parakeet-cpp", "models", name, filepath.Base(chosen.Path)) cfg.Files = append(cfg.Files, gallery.File{ @@ -176,5 +192,14 @@ func isParakeetGGUF(name string) bool { return true } } - return false + return isParakeetDiarGGUF(name) +} + +// isParakeetDiarGGUF reports whether name is the parakeet.cpp speaker +// diarization GGUF (nemotron-3-diarization-.gguf). Matched by its +// published name only, so diarization weights for other backends are not +// claimed. +func isParakeetDiarGGUF(name string) bool { + lower := strings.ToLower(name) + return strings.HasSuffix(lower, ".gguf") && strings.Contains(lower, "nemotron-3-diarization") } diff --git a/core/gallery/importers/parakeet-cpp_test.go b/core/gallery/importers/parakeet-cpp_test.go index 4aa87c4110e6..8636c929de6d 100644 --- a/core/gallery/importers/parakeet-cpp_test.go +++ b/core/gallery/importers/parakeet-cpp_test.go @@ -50,6 +50,11 @@ var _ = Describe("ParakeetCppImporter", func() { Expect(imp.Match(d)).To(BeTrue()) }) + It("matches a direct URL to the diarization GGUF", func() { + d := parakeetDetails("https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf", `{}`) + Expect(imp.Match(d)).To(BeTrue()) + }) + It("does NOT claim a generic llama-style GGUF", func() { d := parakeetDetails("huggingface://someorg/some-llm-gguf", `{}`, hfapi.ModelFile{Path: "llama-3-8b-instruct-q4_k_m.gguf"}, @@ -66,6 +71,43 @@ var _ = Describe("ParakeetCppImporter", func() { }) Context("import (Import)", func() { + It("imports the diarization GGUF as a diarization model", func() { + d := parakeetDetails("https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf", + `{"name":"nemotron-diarization"}`) + cfg, err := imp.Import(d) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.ConfigFile).To(ContainSubstring("backend: parakeet-cpp")) + Expect(cfg.ConfigFile).To(ContainSubstring("diarization")) + Expect(cfg.ConfigFile).ToNot(ContainSubstring("transcript")) + Expect(cfg.Files).To(HaveLen(1)) + Expect(cfg.Files[0].Filename).To(HaveSuffix("nemotron-3-diarization-q8_0.gguf")) + }) + + It("keeps picking ASR weights from a repo that also ships the diarization model", func() { + d := parakeetDetails("huggingface://mudler/parakeet-cpp-gguf", `{"name":"parakeet-110m"}`, + hfapi.ModelFile{Path: "nemotron-3-diarization-f16.gguf", URL: "https://hf/diar-f16", SHA256: "ddd"}, + hfapi.ModelFile{Path: "tdt_ctc-110m-f16.gguf", URL: "https://hf/f16", SHA256: "aaa"}, + hfapi.ModelFile{Path: "nemotron-3-diarization-q8_0.gguf", URL: "https://hf/diar-q8", SHA256: "eee"}, + ) + cfg, err := imp.Import(d) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.Files).To(HaveLen(1)) + Expect(cfg.Files[0].URI).To(Equal("https://hf/f16")) + Expect(cfg.ConfigFile).To(ContainSubstring("transcript")) + }) + + It("imports a diarization-only repo as a diarization model", func() { + d := parakeetDetails("huggingface://someone/diar-gguf", `{"name":"diar"}`, + hfapi.ModelFile{Path: "nemotron-3-diarization-f16.gguf", URL: "https://hf/diar-f16", SHA256: "ddd"}, + hfapi.ModelFile{Path: "nemotron-3-diarization-q8_0.gguf", URL: "https://hf/diar-q8", SHA256: "eee"}, + ) + cfg, err := imp.Import(d) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.Files).To(HaveLen(1)) + Expect(cfg.Files[0].URI).To(Equal("https://hf/diar-q8"), "default quant ladder picks q8_0 before f16") + Expect(cfg.ConfigFile).To(ContainSubstring("diarization")) + }) + It("picks the default quant (q4_k) from a multi-quant HF repo", func() { d := parakeetDetails("huggingface://mudler/parakeet-cpp-gguf", `{"name":"parakeet-110m"}`, hfapi.ModelFile{Path: "tdt_ctc-110m-f16.gguf", URL: "https://hf/f16", SHA256: "aaa"}, From 5c3ba9df90158642d3c20725dcc47e8b046c5b48 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 29 Sep 2026 16:50:15 +0000 Subject: [PATCH 25/30] fix(config): advertise diarization and sound detection for parakeet-cpp The capability table listed parakeet-cpp as transcription only, though the backend now answers Diarize (Nemotron-3-Diarization) and SoundDetection (CED) depending on the model kind it loads. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- core/config/backend_capabilities.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index 886ddd22a185..b48266ead4df 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -477,11 +477,15 @@ var BackendCapabilities = map[string]BackendCapability{ DefaultUsecases: []string{UsecaseTranscript}, Description: "NVIDIA NeMo speech recognition", }, + // parakeet-cpp loads three model kinds, picked from the GGUF: an ASR model + // transcribes (and labels speakers when a diarization_model companion is + // attached), a Nemotron-3-Diarization model answers Diarize, and a CED model + // answers SoundDetection. PossibleUsecases is their union. "parakeet-cpp": { - GRPCMethods: []GRPCMethod{MethodAudioTranscription}, - PossibleUsecases: []string{UsecaseTranscript}, + GRPCMethods: []GRPCMethod{MethodAudioTranscription, MethodDiarize, MethodSoundDetection}, + PossibleUsecases: []string{UsecaseTranscript, UsecaseDiarization, UsecaseSoundClassification}, DefaultUsecases: []string{UsecaseTranscript}, - Description: "NVIDIA NeMo Parakeet ASR (parakeet.cpp)", + Description: "NVIDIA NeMo Parakeet ASR, Nemotron-3-Diarization speaker diarization and CED sound-event detection (parakeet.cpp)", }, // nemo-speech-cpp is one gRPC server in front of four NeMo-Speech.cpp model // families, picked at load time from the GGUF general.architecture key, so From 5f956b36250c285ad273ac76d0515d9d590ea25a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 29 Sep 2026 16:55:17 +0000 Subject: [PATCH 26/30] feat(parakeet-cpp): label transcript segments with the diarization companion A diarization_model companion only fed live speaker events and Diarize; /v1/audio/transcriptions ignored it. With the companion attached and diarize=true (the OpenAI endpoint's default), unary transcription now labels each segment with its speaker and splits segments at speaker turns; with word timestamps each word carries its speaker. The stream=true final result labels each utterance with the speaker who said most of it. Both use the checkpoint's own diarization over the whole clip, as NeMo's diarize() does. Words take the speaker whose segments overlap them most, or the nearest segment within 0.5 s, the same rule as parakeet.cpp's speaker-attributed ASR. Docs: the diarization_model row and a paragraph on transcript speakers; Nemotron-3-Diarization handles up to 8 speakers. Ported from #12323. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/goparakeetcpp.go | 73 +++++++++-- backend/go/parakeet-cpp/goparakeetcpp_test.go | 10 ++ backend/go/parakeet-cpp/speakers.go | 121 +++++++++++++++++ backend/go/parakeet-cpp/speakers_test.go | 123 ++++++++++++++++++ docs/content/features/audio-diarization.md | 2 +- docs/content/features/audio-to-text.md | 4 +- 6 files changed, 321 insertions(+), 12 deletions(-) create mode 100644 backend/go/parakeet-cpp/speakers.go create mode 100644 backend/go/parakeet-cpp/speakers_test.go diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index 3c6cb3ab2057..e8ed1c5253f7 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -349,10 +349,11 @@ func (p *ParakeetCpp) runBatch(reqs []*batchRequest) { // OpenAI API, whose default is segment-level); token ids always populate // Segment.Tokens. // -// translate/diarize/prompt/temperature/threads are not applicable to parakeet -// and are ignored; language is honored on the batched + streaming paths (see -// opts.GetLanguage() below); streaming is handled by AudioTranscriptionStream -// (L2). +// With a diarization_model companion, diarize=true labels segments with their +// speaker (speakers.go). translate/prompt/temperature/threads are not +// applicable to parakeet and are ignored; language is honored on the batched + +// streaming paths (see opts.GetLanguage() below); streaming is handled by +// AudioTranscriptionStream (L2). func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.TranscriptRequest) (pb.TranscriptResult, error) { if p.ctxPtr == 0 { if err := p.notASRError(); err != nil { @@ -415,7 +416,17 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip if err := json.Unmarshal([]byte(res.json), &doc); err != nil { return pb.TranscriptResult{}, fmt.Errorf("parakeet-cpp: decode transcript json: %w", err) } - return transcriptResultFromDoc(doc, opts, p.segmentGapFrames), nil + + // With a diarization_model companion, label each segment with its speaker. + var speakers []int + if p.wantSpeakers(opts.GetDiarize()) && len(doc.Words) > 0 { + segs, err := p.diarizeSegmentsPCM(pcm) + if err != nil { + return pb.TranscriptResult{}, err + } + speakers = assignSpeakers(doc.Words, segs) + } + return transcriptResultWithSpeakers(doc, opts, p.segmentGapFrames, speakers), nil } // segmentSeparators is NeMo's default segment_seperators (sentence-ending @@ -430,6 +441,14 @@ var segmentSeparators = []rune{'.', '?', '!'} // the caller requested word granularity; token ids populate each segment's // Tokens by time-window membership. Shared by the batched and direct paths. func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gapFrames int) pb.TranscriptResult { + return transcriptResultWithSpeakers(doc, opts, gapFrames, nil) +} + +// transcriptResultWithSpeakers is transcriptResultFromDoc plus optional +// per-word speakers (indexed like doc.Words, -1 = none; see speakers.go): +// segments additionally split wherever the speaker changes, and segments and +// words carry the speaker's label. +func transcriptResultWithSpeakers(doc transcriptJSON, opts *pb.TranscriptRequest, gapFrames int, speakers []int) pb.TranscriptResult { text, eou := stripEouMarker(strings.TrimSpace(doc.Text)) // Frame-unit gap threshold -> seconds (NeMo segment_gap_threshold). 0 = off. @@ -453,6 +472,11 @@ func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gap } } + var groupSpeakers []int + if speakers != nil && len(speakers) == len(doc.Words) { + groups, groupSpeakers = splitAtSpeakerChanges(groups, speakers) + } + wantWords := wordsRequested(opts.TimestampGranularities) segments := make([]*pb.TranscriptSegment, 0, len(groups)) for id, group := range groups { @@ -467,10 +491,14 @@ func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gap Text: strings.TrimSpace(strings.Join(parts, " ")), Tokens: tokensInWindow(doc.Tokens, group[0].Start, group[len(group)-1].End), } + if groupSpeakers != nil { + seg.Speaker = transcriptSpeaker(groupSpeakers[id]) + } if wantWords { ws := make([]*pb.TranscriptWord, len(group)) for i, gw := range group { - ws[i] = &pb.TranscriptWord{Start: secondsToNanos(gw.Start), End: secondsToNanos(gw.End), Text: gw.W} + ws[i] = &pb.TranscriptWord{Start: secondsToNanos(gw.Start), End: secondsToNanos(gw.End), Text: gw.W, + Speaker: seg.Speaker} } seg.Words = ws } @@ -568,10 +596,11 @@ func tokensInWindow(tokens []transcriptToken, start, end float64) []int32 { // text-only library (no words) it falls back to segmenting the delta text, so // the same assembler serves both paths. type streamSegmenter struct { - segs []*pb.TranscriptSegment - cur []transcriptWord // words for the open segment (ABI v4 JSON path) - curText []string // delta text for the open segment (text-only path) - nextID int32 + segs []*pb.TranscriptSegment + segWords [][]transcriptWord // words of each segment (nil for text-only ones) + cur []transcriptWord // words for the open segment (ABI v4 JSON path) + curText []string // delta text for the open segment (text-only path) + nextID int32 } func (s *streamSegmenter) add(r streamFeedResult) { @@ -599,12 +628,14 @@ func (s *streamSegmenter) flush() { End: secondsToNanos(s.cur[len(s.cur)-1].End), Text: strings.TrimSpace(strings.Join(parts, " ")), }) + s.segWords = append(s.segWords, s.cur) s.nextID++ case len(s.curText) > 0: // No words this segment: emit a text-only segment (no timestamps), // skipping a purely-whitespace one as the legacy text path did. if t := strings.TrimSpace(strings.Join(s.curText, "")); t != "" { s.segs = append(s.segs, &pb.TranscriptSegment{Id: s.nextID, Text: t}) + s.segWords = append(s.segWords, nil) s.nextID++ } } @@ -821,6 +852,28 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra // The single-segment fallback stays trimmed. fullText := full.String() segments := seg.segments() + + // With a diarization_model companion, label each utterance with the + // speaker who said most of it. The whole file is available, so this runs + // the same diarization as the unary path. + if p.wantSpeakers(opts.GetDiarize()) && len(seg.segWords) == len(segments) { + var all []transcriptWord + for _, ws := range seg.segWords { + all = append(all, ws...) + } + if len(all) > 0 { + segs, err := p.diarizeSegmentsPCM(data) + if err != nil { + return err + } + speakers := assignSpeakers(all, segs) + k := 0 + for i, ws := range seg.segWords { + segments[i].Speaker = transcriptSpeaker(majoritySpeaker(ws, speakers[k:k+len(ws)])) + k += len(ws) + } + } + } if trimmed := strings.TrimSpace(fullText); len(segments) == 0 && trimmed != "" { segments = append(segments, &pb.TranscriptSegment{Id: 0, Text: trimmed}) } diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go index 02b24ece3e29..8dc437f3e937 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp_test.go +++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go @@ -59,6 +59,16 @@ func ensureLibLoaded() { purego.RegisterLibFunc(&CppStreamFeedJSON, lib, "parakeet_capi_stream_feed_json") purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json") } + // Diarization and model roles, probed like main.go (speakers_test.go). + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_pcm"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppDiarizePCM, lib, "parakeet_capi_diarize_pcm") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_transcribe_and_diarize_json"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppTranscribeAndDiarizeJSON, lib, "parakeet_capi_transcribe_and_diarize_json") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_model_kind"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppModelKind, lib, "parakeet_capi_model_kind") + } purego.RegisterLibFunc(&CppFreeString, lib, "parakeet_capi_free_string") purego.RegisterLibFunc(&CppLastError, lib, "parakeet_capi_last_error") }) diff --git a/backend/go/parakeet-cpp/speakers.go b/backend/go/parakeet-cpp/speakers.go new file mode 100644 index 000000000000..ddd2c4292b6e --- /dev/null +++ b/backend/go/parakeet-cpp/speakers.go @@ -0,0 +1,121 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" +) + +// Speaker labels on transcripts. With a diarization_model companion attached +// to an ASR model, unary transcription tags each segment (and, with word +// timestamps, each word) with its speaker, and the stream=true final result +// tags each utterance. Live transcription carries speakers through the scene +// stream instead (scene.go). + +// speakerSnapSeconds mirrors parakeet.cpp's merge_asr_diarization: a word that +// overlaps no speaker segment takes the nearest segment's speaker when that +// segment is this close. ASR word boundaries and diarization boundaries can +// disagree by a frame or two; a word farther than this has no speaker. +const speakerSnapSeconds = 0.5 + +// transcriptSpeaker renders a 0-based speaker for a transcript segment or +// word; -1 (no speaker) is left empty so the field is omitted. +func transcriptSpeaker(spk int) string { + if spk < 0 { + return "" + } + return strconv.Itoa(spk) +} + +// wantSpeakers reports whether a transcription should carry speaker labels: +// a diarization companion is attached, the library can diarize, and the +// request did not turn it off (the OpenAI endpoint sends diarize=true unless +// the client passes diarize=false). +func (p *ParakeetCpp) wantSpeakers(diarize bool) bool { + return diarize && p.ctxPtr != 0 && p.diarCtx != 0 && CppDiarizePCM != nil +} + +// diarizeSegmentsPCM runs the diarization companion over 16 kHz PCM +// (parakeet_capi_diarize_pcm, the checkpoint's own mode, as NeMo's +// diarize()) and returns its segments. +func (p *ParakeetCpp) diarizeSegmentsPCM(pcm []float32) ([]diarizeSegmentJSON, error) { + if len(pcm) == 0 { + return nil, nil + } + raw, err := p.diarizeCall(pcm, false) + if err != nil { + return nil, err + } + var doc diarizePCMDoc + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return nil, fmt.Errorf("parakeet-cpp: decode diarization json: %w", err) + } + return doc.Segments, nil +} + +// assignSpeakers gives each word the speaker whose segments overlap it most, +// falling back to the nearest segment within speakerSnapSeconds; -1 when none. +// Same rule as parakeet.cpp's merge_asr_diarization, so the labels match the +// library's own speaker-attributed ASR. +func assignSpeakers(words []transcriptWord, segs []diarizeSegmentJSON) []int { + out := make([]int, len(words)) + for i, w := range words { + best, bestOverlap := -1, 0.0 + for _, s := range segs { + if ov := min(w.End, s.End) - max(w.Start, s.Start); ov > bestOverlap { + best, bestOverlap = s.Speaker, ov + } + } + if best < 0 { + bestDist := speakerSnapSeconds + for _, s := range segs { + dist := s.Start - w.End + if s.End <= w.Start { + dist = w.Start - s.End + } + if dist >= 0 && dist <= bestDist { + best, bestDist = s.Speaker, dist + } + } + } + out[i] = best + } + return out +} + +// splitAtSpeakerChanges splits each word group wherever the speaker changes, +// so every segment has one speaker. speakers is indexed like the +// concatenation of groups. Returns the new groups and each group's speaker. +func splitAtSpeakerChanges(groups [][]transcriptWord, speakers []int) ([][]transcriptWord, []int) { + var outGroups [][]transcriptWord + var outSpk []int + k := 0 + for _, g := range groups { + start := 0 + for i := 1; i <= len(g); i++ { + if i == len(g) || speakers[k+i] != speakers[k+start] { + outGroups = append(outGroups, g[start:i]) + outSpk = append(outSpk, speakers[k+start]) + start = i + } + } + k += len(g) + } + return outGroups, outSpk +} + +// majoritySpeaker is the speaker covering most of the words' duration, or -1. +func majoritySpeaker(words []transcriptWord, speakers []int) int { + dur := map[int]float64{} + best, bestDur := -1, 0.0 + for i, w := range words { + if speakers[i] < 0 { + continue + } + dur[speakers[i]] += max(w.End-w.Start, 1e-3) + if dur[speakers[i]] > bestDur { + best, bestDur = speakers[i], dur[speakers[i]] + } + } + return best +} diff --git a/backend/go/parakeet-cpp/speakers_test.go b/backend/go/parakeet-cpp/speakers_test.go new file mode 100644 index 000000000000..041bda6103a0 --- /dev/null +++ b/backend/go/parakeet-cpp/speakers_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "os" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func sseg(spk int, start, end float64) diarizeSegmentJSON { + return diarizeSegmentJSON{Speaker: spk, Start: start, End: end} +} + +// speakerTurnsOf collapses consecutive repeats: [0 0 1 0] -> [0 1 0]. +func speakerTurnsOf(labels []string) []string { + var out []string + for _, l := range labels { + if len(out) == 0 || out[len(out)-1] != l { + out = append(out, l) + } + } + return out +} + +var _ = Describe("transcript speaker labels", func() { + Context("assignSpeakers", func() { + It("picks the speaker with the largest overlap", func() { + words := []transcriptWord{tw("a", 0.5, 0.8)} + Expect(assignSpeakers(words, []diarizeSegmentJSON{sseg(0, 0.0, 0.6), sseg(1, 0.55, 2.0)})). + To(Equal([]int{1})) + }) + + It("snaps a word just outside a segment to it, but not a far one", func() { + words := []transcriptWord{tw("well", 19.92, 20.00), tw("far", 25.0, 25.2)} + segs := []diarizeSegmentJSON{sseg(0, 14.78, 18.75), sseg(1, 20.10, 23.60)} + Expect(assignSpeakers(words, segs)).To(Equal([]int{1, -1})) + }) + }) + + It("splits segments at speaker turns and labels segments and words", func() { + doc := transcriptJSON{ + Text: "hi there. hello", + Words: []transcriptWord{tw("hi", 0.0, 0.3), tw("there.", 0.3, 0.6), tw("hello", 1.0, 1.4)}, + } + // Punctuation gives [hi there.] [hello]; the turn after "hi" splits the first. + opts := &pb.TranscriptRequest{TimestampGranularities: []string{"word"}} + res := transcriptResultWithSpeakers(doc, opts, 0, []int{0, 1, 1}) + Expect(res.Segments).To(HaveLen(3)) + Expect([]string{res.Segments[0].Speaker, res.Segments[1].Speaker, res.Segments[2].Speaker}). + To(Equal([]string{"0", "1", "1"})) + Expect(res.Segments[1].Text).To(Equal("there.")) + Expect(res.Segments[2].Words[0].Speaker).To(Equal("1")) + for i, seg := range res.Segments { + Expect(seg.Id).To(Equal(int32(i))) + } + }) + + It("leaves speakers empty without diarization and for unknown words", func() { + doc := transcriptJSON{Text: "hi.", Words: []transcriptWord{tw("hi.", 0, 0.3)}} + Expect(transcriptResultFromDoc(doc, &pb.TranscriptRequest{}, 0).Segments[0].Speaker).To(BeEmpty()) + Expect(transcriptResultWithSpeakers(doc, &pb.TranscriptRequest{}, 0, []int{-1}).Segments[0].Speaker). + To(BeEmpty()) + }) + + It("picks the speaker covering most of an utterance", func() { + words := []transcriptWord{tw("a", 0, 0.2), tw("b", 0.2, 1.5), tw("c", 1.5, 1.6)} + Expect(majoritySpeaker(words, []int{0, 1, 0})).To(Equal(1)) + Expect(majoritySpeaker(words, []int{-1, -1, -1})).To(Equal(-1)) + }) + + It("only diarizes with a companion, a capable library and diarize=true", func() { + saved := CppDiarizePCM + defer func() { CppDiarizePCM = saved }() + CppDiarizePCM = func(uintptr, *float32, int32, int32) uintptr { return 0 } + p := &ParakeetCpp{ctxPtr: 1, diarCtx: 2} + Expect(p.wantSpeakers(true)).To(BeTrue()) + Expect(p.wantSpeakers(false)).To(BeFalse()) + Expect((&ParakeetCpp{ctxPtr: 1}).wantSpeakers(true)).To(BeFalse()) + CppDiarizePCM = nil + Expect(p.wantSpeakers(true)).To(BeFalse()) + }) +}) + +var _ = Describe("ParakeetCpp transcript speakers (real models)", func() { + It("labels unary and stream=true transcripts with a diarization_model companion", func() { + diarModel := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_MODEL") + asrModel := os.Getenv("PARAKEET_BACKEND_TEST_MODEL") + wavPath := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_WAV") + if diarModel == "" || asrModel == "" || wavPath == "" { + Skip("set PARAKEET_BACKEND_TEST_DIAR_MODEL, PARAKEET_BACKEND_TEST_MODEL and " + + "PARAKEET_BACKEND_TEST_DIAR_WAV (a multi-speaker 16 kHz WAV)") + } + ensureLibLoaded() + if CppDiarizePCM == nil || CppModelKind == nil { + Skip("libparakeet.so has no diarization / model-kind C-API") + } + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ + ModelFile: asrModel, + Options: []string{"diarization_model:" + diarModel}, + })).To(Succeed()) + defer func() { _ = p.Free() }() + + res, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath, Diarize: true}) + Expect(err).ToNot(HaveOccurred()) + labels := make([]string, len(res.Segments)) + for i, s := range res.Segments { + Expect(s.Speaker).ToNot(BeEmpty(), "segment %d %q", i, s.Text) + labels[i] = s.Speaker + } + // parakeet.cpp's tests/fixtures/two_speakers.wav alternates A-B-A-B. + Expect(speakerTurnsOf(labels)).To(Equal([]string{"0", "1", "0", "1"})) + + plain, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath}) + Expect(err).ToNot(HaveOccurred()) + for _, s := range plain.Segments { + Expect(s.Speaker).To(BeEmpty()) + } + Expect(plain.Text).To(Equal(res.Text)) + }) +}) diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md index e2f085de1bac..37f8c81596b8 100644 --- a/docs/content/features/audio-diarization.md +++ b/docs/content/features/audio-diarization.md @@ -15,7 +15,7 @@ LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled afte - **[vibevoice.cpp](https://github.com/microsoft/VibeVoice)** - produces speaker-labelled segments as a by-product of its long-form ASR pass, so you can optionally get a transcript per segment for free. - **[NeMo-Speech.cpp](https://github.com/NVIDIA/NeMo-Speech.cpp)** - NVIDIA Sortformer, served standalone by the [NeMo-Speech.cpp backend]({{%relref "features/nemo-speech-cpp" %}}). It is end to end, so the speaker capacity is fixed by the checkpoint and the count hints are ignored. The same backend can instead put speaker tags on a transcript, by attaching a Sortformer model to an ASR one. - **[audio.cpp](https://github.com/0xShug0/audio.cpp)** - the `sortformer_diar` family, served by the multi-modality [audio.cpp backend]({{%relref "features/audio-cpp" %}}). -- **[parakeet.cpp](https://github.com/mudler/parakeet.cpp)** - NVIDIA Nemotron-3-Diarization (Sortformer), served standalone or paired with a Parakeet ASR model for per-segment text. See the [Audio to Text]({{% relref "audio-to-text" %}}) page for the parakeet-cpp option reference. +- **[parakeet.cpp](https://github.com/mudler/parakeet.cpp)** - NVIDIA Nemotron-3-Diarization (Sortformer, up to 8 speakers), served standalone or paired with a Parakeet ASR model for per-segment text. See the [Audio to Text]({{% relref "audio-to-text" %}}) page for the parakeet-cpp option reference. Because diarization is exposed as a regular OpenAI-compatible endpoint, any HTTP client works. There is no Python dependency on pyannote or NeMo on the consumer side. diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index a73b5d009f9e..d01edec4e391 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -197,10 +197,12 @@ The same backend also serves the `/v1/audio/diarization` and `/v1/audio/classifi | Option | Allowed on | Used for | |---|---|---| | `asr_model:` | a diarization model | `include_text` on `/v1/audio/diarization` | -| `diarization_model:` | an ASR model | speaker segments during realtime live transcription | +| `diarization_model:` | an ASR model | a `speaker` on transcript segments (and words), and speaker segments during realtime live transcription | | `sound_model:` | an ASR model | sound events during realtime live transcription | | `diarization_latency:` | a model with a diarization companion | latency mode for the live speaker stream; default `low` | +With a `diarization_model` companion, `/v1/audio/transcriptions` labels each segment with its `speaker` (`"0"`, `"1"`, ... in order of first appearance) and splits segments where the speaker changes; with `timestamp_granularities[]=word` each word carries its speaker too. With `stream=true` the closing `transcript.text.done` event lists the segments with their speakers. Pass `-F diarize=false` to skip diarization for one request. The diarization GGUF can also be imported directly: `local-ai models import https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-f16.gguf`. + The loader rejects a companion whose role duplicates the primary's own (for example `asr_model:` on an already-ASR primary, or `sound_model:` on a CED primary), and rejects a companion GGUF that does not match the role its option names (for example `sound_model:` pointing at an ASR GGUF fails to load, naming the kind it expected). See [Speaker Diarization]({{% relref "audio-diarization" %}}) for the `Diarize` RPC and [Sound Classification]({{% relref "audio-classification" %}}) for `SoundDetection`, and [Realtime API]({{% relref "openai-realtime" %}}) for the live speaker/sound events emitted during a realtime session. ### Segment timestamps From 1aab6ff091519f0e24bef54b9311c5bcdb6eea93 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 29 Sep 2026 17:11:35 +0000 Subject: [PATCH 27/30] feat(realtime): speaker segments from committed-turn transcription Speaker events reached a realtime session only from the live semantic_vad path, which needs a cache-aware streaming transcription model. Committed-turn transcription (server_vad, or any offline model) always asked the backend for diarize=false and dropped the segments' speakers. pipeline.diarization (off by default) asks the transcription model for speaker labels on each committed turn and emits every labelled segment as a conversation.item.input_audio_transcription.segment event, with its text, before the turn's completed event. It is opt-in because some backends fail a diarization request they cannot serve. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- core/config/model_config.go | 8 ++ .../endpoints/openai/realtime_doubles_test.go | 7 +- .../openai/realtime_transcription.go | 42 +++++++++- .../openai/realtime_transcription_test.go | 83 +++++++++++++++++++ 4 files changed, 136 insertions(+), 4 deletions(-) diff --git a/core/config/model_config.go b/core/config/model_config.go index 3c8987920149..c8502fae5865 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -833,6 +833,14 @@ type Pipeline struct { SoundDetectionWindowMs int `yaml:"sound_detection_window_ms,omitempty" json:"sound_detection_window_ms,omitempty"` SoundDetectionHopMs int `yaml:"sound_detection_hop_ms,omitempty" json:"sound_detection_hop_ms,omitempty"` + // Diarization asks the transcription model for speaker labels on each + // VAD-committed utterance and emits every labelled segment as a + // conversation.item.input_audio_transcription.segment event. It needs a + // transcription model that diarizes (e.g. parakeet-cpp with a + // diarization_model companion); off by default because some backends fail + // a diarize request they cannot serve. Speaker labels are per turn. + Diarization bool `yaml:"diarization,omitempty" json:"diarization,omitempty"` + // ReasoningEffort sets the reasoning effort (none|minimal|low|medium|high) for // the pipeline's LLM without editing the LLM model config. Overrides the LLM's // own reasoning_effort. Unset leaves the LLM model config in charge. diff --git a/core/http/endpoints/openai/realtime_doubles_test.go b/core/http/endpoints/openai/realtime_doubles_test.go index e82a4a13056f..9ddc1299fdfa 100644 --- a/core/http/endpoints/openai/realtime_doubles_test.go +++ b/core/http/endpoints/openai/realtime_doubles_test.go @@ -88,6 +88,7 @@ type fakeModel struct { transcribeDeltas []string transcribeFinal *schema.TranscriptionResult transcribeErr error + lastDiarize bool // diarize flag of the last Transcribe/TranscribeStream call // TranscribeLive scripting: liveErr makes the open fail (degrade path); // liveEvents are delivered to onEvent synchronously at open; @@ -189,7 +190,8 @@ func (m *fakeModel) VAD(_ context.Context, req *schema.VADRequest) (*schema.VADR return &schema.VADResponse{Segments: m.vadSegments}, nil } -func (m *fakeModel) Transcribe(context.Context, string, string, bool, bool, string) (*schema.TranscriptionResult, error) { +func (m *fakeModel) Transcribe(_ context.Context, _, _ string, _, diarize bool, _ string) (*schema.TranscriptionResult, error) { + m.lastDiarize = diarize return m.transcribeFinal, m.transcribeErr } @@ -236,7 +238,8 @@ func (m *fakeModel) TTSStream(_ context.Context, _, _, _ string, onAudio func(pc return nil } -func (m *fakeModel) TranscribeStream(_ context.Context, _, _ string, _, _ bool, _ string, onDelta func(text string)) (*schema.TranscriptionResult, error) { +func (m *fakeModel) TranscribeStream(_ context.Context, _, _ string, _, diarize bool, _ string, onDelta func(text string)) (*schema.TranscriptionResult, error) { + m.lastDiarize = diarize for _, d := range m.transcribeDeltas { onDelta(d) } diff --git a/core/http/endpoints/openai/realtime_transcription.go b/core/http/endpoints/openai/realtime_transcription.go index 28a5147c17e0..c10535f18809 100644 --- a/core/http/endpoints/openai/realtime_transcription.go +++ b/core/http/endpoints/openai/realtime_transcription.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/mudler/LocalAI/core/http/endpoints/openai/types" + "github.com/mudler/LocalAI/core/schema" ) // emitPrecomputedTranscription emits the transcription events for a turn @@ -42,9 +43,10 @@ func emitPrecomputedTranscription(t Transport, itemID string, deltas []string, t // a single completed event. delta and completed events share itemID. func emitTranscription(ctx context.Context, t Transport, session *Session, itemID, audioPath string) (string, error) { cfg := session.InputAudioTranscription + diarize := session.ModelConfig != nil && session.ModelConfig.Pipeline.Diarization if session.ModelConfig != nil && session.ModelConfig.Pipeline.StreamTranscription() { - final, err := session.ModelInterface.TranscribeStream(ctx, audioPath, cfg.Language, false, false, cfg.Prompt, func(delta string) { + final, err := session.ModelInterface.TranscribeStream(ctx, audioPath, cfg.Language, false, diarize, cfg.Prompt, func(delta string) { _ = t.SendEvent(types.ConversationItemInputAudioTranscriptionDeltaEvent{ ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, ItemID: itemID, @@ -58,6 +60,11 @@ func emitTranscription(ctx context.Context, t Transport, session *Session, itemI transcript := "" if final != nil { transcript = final.Text + if diarize { + if err := emitSpeakerSegments(t, itemID, final); err != nil { + return "", err + } + } } if err := t.SendEvent(types.ConversationItemInputAudioTranscriptionCompletedEvent{ ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, @@ -71,13 +78,18 @@ func emitTranscription(ctx context.Context, t Transport, session *Session, itemI } // Unary fallback: transcribe the whole utterance, emit one completed event. - tr, err := session.ModelInterface.Transcribe(ctx, audioPath, cfg.Language, false, false, cfg.Prompt) + tr, err := session.ModelInterface.Transcribe(ctx, audioPath, cfg.Language, false, diarize, cfg.Prompt) if err != nil { return "", err } if tr == nil { return "", fmt.Errorf("transcribe result is nil") } + if diarize { + if err := emitSpeakerSegments(t, itemID, tr); err != nil { + return "", err + } + } if err := t.SendEvent(types.ConversationItemInputAudioTranscriptionCompletedEvent{ ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, ItemID: itemID, @@ -88,3 +100,29 @@ func emitTranscription(ctx context.Context, t Transport, session *Session, itemI } return tr.Text, nil } + +// emitSpeakerSegments forwards each speaker-labelled segment of a committed +// turn's transcript as a conversation.item.input_audio_transcription.segment +// event (pipeline.diarization), before the turn's completed event. Times are +// relative to the turn's audio and speaker labels are only consistent within +// the turn, as on the live path. Segments without a speaker are skipped. +func emitSpeakerSegments(t Transport, itemID string, tr *schema.TranscriptionResult) error { + for _, seg := range tr.Segments { + if seg.Speaker == "" { + continue + } + if err := t.SendEvent(types.ConversationItemInputAudioTranscriptionSegmentEvent{ + ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, + ItemID: itemID, + ContentIndex: 0, + ID: fmt.Sprintf("seg_%d", seg.Id), + Speaker: seg.Speaker, + Start: seg.Start.Seconds(), + End: seg.End.Seconds(), + Text: seg.Text, + }); err != nil { + return err + } + } + return nil +} diff --git a/core/http/endpoints/openai/realtime_transcription_test.go b/core/http/endpoints/openai/realtime_transcription_test.go index f3f760fd8a5f..e8ab399fd87b 100644 --- a/core/http/endpoints/openai/realtime_transcription_test.go +++ b/core/http/endpoints/openai/realtime_transcription_test.go @@ -2,6 +2,7 @@ package openai import ( "context" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -51,4 +52,86 @@ var _ = Describe("emitTranscription", func() { Expect(t.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionDelta)).To(Equal(0)) Expect(t.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionCompleted)).To(Equal(1)) }) + + Context("pipeline.diarization", func() { + labelled := &schema.TranscriptionResult{ + Text: "hi there. hello", + Segments: []schema.TranscriptionSegment{ + {Id: 0, Text: "hi there.", Start: 0, End: 600 * time.Millisecond, Speaker: "0"}, + {Id: 1, Text: "hello", Start: time.Second, End: 1400 * time.Millisecond, Speaker: "1"}, + {Id: 2, Text: "unlabelled"}, + }, + } + + segmentEvents := func(t *fakeTransport) []types.ConversationItemInputAudioTranscriptionSegmentEvent { + var out []types.ConversationItemInputAudioTranscriptionSegmentEvent + for _, e := range t.sent { + if seg, ok := e.(types.ConversationItemInputAudioTranscriptionSegmentEvent); ok { + out = append(out, seg) + } + } + return out + } + + It("requests speakers and emits one segment event per labelled segment", func() { + m := &fakeModel{transcribeFinal: labelled} + session := &Session{ + InputAudioTranscription: &types.AudioTranscription{}, + ModelConfig: &config.ModelConfig{Pipeline: config.Pipeline{Diarization: true}}, + ModelInterface: m, + } + t := &fakeTransport{} + + transcript, err := emitTranscription(context.Background(), t, session, "item1", "/tmp/x.wav") + + Expect(err).ToNot(HaveOccurred()) + Expect(transcript).To(Equal("hi there. hello")) + Expect(m.lastDiarize).To(BeTrue()) + segs := segmentEvents(t) + Expect(segs).To(HaveLen(2)) + Expect(segs[0].ItemID).To(Equal("item1")) + Expect(segs[0].Speaker).To(Equal("0")) + Expect(segs[0].Text).To(Equal("hi there.")) + Expect(segs[1].Speaker).To(Equal("1")) + Expect(segs[1].Start).To(BeNumerically("~", 1.0, 1e-9)) + Expect(segs[1].End).To(BeNumerically("~", 1.4, 1e-9)) + Expect(t.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionCompleted)).To(Equal(1)) + }) + + It("also emits segment events on the streaming transcription path", func() { + on := true + m := &fakeModel{transcribeDeltas: []string{"hi"}, transcribeFinal: labelled} + session := &Session{ + InputAudioTranscription: &types.AudioTranscription{}, + ModelConfig: &config.ModelConfig{Pipeline: config.Pipeline{ + Diarization: true, + Streaming: config.PipelineStreaming{Transcription: &on}, + }}, + ModelInterface: m, + } + t := &fakeTransport{} + + _, err := emitTranscription(context.Background(), t, session, "item1", "/tmp/x.wav") + + Expect(err).ToNot(HaveOccurred()) + Expect(m.lastDiarize).To(BeTrue()) + Expect(segmentEvents(t)).To(HaveLen(2)) + }) + + It("neither asks for speakers nor emits segments when off", func() { + m := &fakeModel{transcribeFinal: labelled} + session := &Session{ + InputAudioTranscription: &types.AudioTranscription{}, + ModelConfig: &config.ModelConfig{}, + ModelInterface: m, + } + t := &fakeTransport{} + + _, err := emitTranscription(context.Background(), t, session, "item1", "/tmp/x.wav") + + Expect(err).ToNot(HaveOccurred()) + Expect(m.lastDiarize).To(BeFalse()) + Expect(segmentEvents(t)).To(BeEmpty()) + }) + }) }) From f82120bc3198e0f0039ccc91929d38daa8075621 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 29 Sep 2026 17:11:36 +0000 Subject: [PATCH 28/30] feat(gallery): add parakeet-cpp-realtime-scene-tdt parakeet-cpp-realtime-scene pairs the streaming EOU model with the diarization and CED companions; its speaker and sound events need a cache-aware streaming model. This entry does the same with Parakeet TDT 0.6B v3 (multilingual, offline) for realtime under server_vad: set it as both transcription and sound_detection and turn on pipeline.diarization, and each committed turn gets speaker segments and sound tags from one parakeet-cpp backend. Files and sha256 match the Hub and are shared with the existing TDT v3, diarization and CED-Tiny entries. A real-model spec checks the combination on a clip with two speakers and a rooster: A-B-A-B speaker turns, and "Chicken, rooster" among the sound tags. The test loader now binds the sound entry points like main.go. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/go/parakeet-cpp/goparakeetcpp_test.go | 7 +++ backend/go/parakeet-cpp/speakers_test.go | 43 +++++++++++++ docs/content/features/openai-realtime.md | 32 ++++++++++ gallery/index.yaml | 60 +++++++++++++++++++ 4 files changed, 142 insertions(+) diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go index 8dc437f3e937..eda5eaa36ce3 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp_test.go +++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go @@ -68,6 +68,13 @@ func ensureLibLoaded() { } if sym, err := purego.Dlsym(lib, "parakeet_capi_model_kind"); err == nil && sym != 0 { purego.RegisterLibFunc(&CppModelKind, lib, "parakeet_capi_model_kind") + purego.RegisterLibFunc(&CppNumClasses, lib, "parakeet_capi_num_classes") + purego.RegisterLibFunc(&CppSoundOptsDefault, lib, "parakeet_capi_sound_opts_default") + purego.RegisterLibFunc(&CppSoundStreamBegin, lib, "parakeet_capi_sound_stream_begin") + purego.RegisterLibFunc(&CppSoundStreamFeed, lib, "parakeet_capi_sound_stream_feed") + purego.RegisterLibFunc(&CppSoundStreamDrainScoresJSON, lib, "parakeet_capi_sound_stream_drain_scores_json") + purego.RegisterLibFunc(&CppFreeSoundSegments, lib, "parakeet_capi_free_sound_segments") + purego.RegisterLibFunc(&CppSoundStreamFree, lib, "parakeet_capi_sound_stream_free") } purego.RegisterLibFunc(&CppFreeString, lib, "parakeet_capi_free_string") purego.RegisterLibFunc(&CppLastError, lib, "parakeet_capi_last_error") diff --git a/backend/go/parakeet-cpp/speakers_test.go b/backend/go/parakeet-cpp/speakers_test.go index 041bda6103a0..b302bb6e9356 100644 --- a/backend/go/parakeet-cpp/speakers_test.go +++ b/backend/go/parakeet-cpp/speakers_test.go @@ -121,3 +121,46 @@ var _ = Describe("ParakeetCpp transcript speakers (real models)", func() { Expect(plain.Text).To(Equal(res.Text)) }) }) + +var _ = Describe("ParakeetCpp scene companions on an offline ASR model (real models)", func() { + It("labels speakers and detects sounds from one model with both companions", func() { + asrModel := os.Getenv("PARAKEET_BACKEND_TEST_SCENE_ASR_MODEL") // e.g. tdt-0.6b-v3 + diarModel := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_MODEL") + soundModel := os.Getenv("PARAKEET_BACKEND_TEST_SOUND_MODEL") + wavPath := os.Getenv("PARAKEET_BACKEND_TEST_SCENE_WAV") // speech + a non-speech sound + wantLabel := os.Getenv("PARAKEET_BACKEND_TEST_SCENE_LABEL") // e.g. "Chicken, rooster" + if asrModel == "" || diarModel == "" || soundModel == "" || wavPath == "" || wantLabel == "" { + Skip("set PARAKEET_BACKEND_TEST_SCENE_ASR_MODEL, _DIAR_MODEL, _SOUND_MODEL, " + + "PARAKEET_BACKEND_TEST_SCENE_WAV and PARAKEET_BACKEND_TEST_SCENE_LABEL") + } + ensureLibLoaded() + if CppDiarizePCM == nil || CppModelKind == nil { + Skip("libparakeet.so has no diarization / model-kind C-API") + } + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ + ModelFile: asrModel, + Options: []string{"diarization_model:" + diarModel, "sound_model:" + soundModel}, + })).To(Succeed()) + defer func() { _ = p.Free() }() + + res, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath, Diarize: true}) + Expect(err).ToNot(HaveOccurred()) + labels := make([]string, 0, len(res.Segments)) + for _, s := range res.Segments { + GinkgoWriter.Printf("[%5.1f-%5.1f] spk %q: %s\n", float64(s.Start)/1e9, float64(s.End)/1e9, s.Speaker, s.Text) + labels = append(labels, s.Speaker) + } + Expect(len(speakerTurnsOf(labels))).To(BeNumerically(">=", 3), "speaker turns: %v", labels) + Expect(labels).To(ContainElements("0", "1")) + + sd, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: wavPath, TopK: 5}) + Expect(err).ToNot(HaveOccurred()) + var got []string + for _, d := range sd.GetDetections() { + GinkgoWriter.Printf("sound %q %.2f\n", d.GetLabel(), d.GetScore()) + got = append(got, d.GetLabel()) + } + Expect(got).To(ContainElement(wantLabel)) + }) +}) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 535c47144389..ca8759e672bc 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -193,6 +193,38 @@ The companion stream is opened fresh for each speech turn, alongside that turn's **Limitation**: under `semantic_vad`, live transcription (and so this companion stream) only runs during speech turns - it does not see audio between turns. A sound that happens while nobody is speaking is not detected this way. If you need sound events independent of speech turns, use the pipeline's `sound_detection` model instead (see [Sound Classification]({{% relref "audio-classification" %}})), which classifies each VAD-committed utterance on its own. Use one or the other, not both, on the same session - they overlap in purpose and would emit sound detections twice. +### Speaker and sound events with an offline model (Parakeet TDT v3) + +The live events above need a cache-aware streaming transcription model. An offline model such as Parakeet TDT 0.6B v3 (25 languages) runs under `server_vad` instead: each VAD-committed turn is transcribed as a whole. The gallery model `parakeet-cpp-realtime-scene-tdt` bundles it with Nemotron-3-Diarization and CED-Tiny, so one parakeet-cpp backend handles transcription, speakers and sounds. Point both `transcription` and `sound_detection` at it and turn on `diarization`: + +```yaml +name: gpt-realtime-scene +pipeline: + vad: silero-vad-ggml + transcription: parakeet-cpp-realtime-scene-tdt + sound_detection: parakeet-cpp-realtime-scene-tdt + diarization: true + llm: qwen3-4b + tts: tts-1 +``` + +`pipeline.diarization` asks the transcription model for speaker labels on each committed turn and emits every labelled segment as a `conversation.item.input_audio_transcription.segment` event before the turn's `completed` event. Unlike the live path, these segments carry their `text`: + +```json +{ + "type": "conversation.item.input_audio_transcription.segment", + "item_id": "item_abc", + "content_index": 0, + "id": "seg_1", + "speaker": "1", + "start": 6.85, + "end": 10.82, + "text": "Well, I don't wish to see it any more, observed Phoebe, turning away her eyes." +} +``` + +`sound_detection` classifies the same committed audio and emits one `conversation.item.sound_detection` event per turn (see [Sound Classification]({{% relref "audio-classification" %}})). As on the live path, times are relative to the turn's audio and speaker labels are only consistent within a turn. `pipeline.diarization` is off by default: it needs a transcription model that diarizes (parakeet-cpp with a `diarization_model` companion), and some other backends fail a diarization request they cannot serve. + ### Disabling thinking For reasoning models, you can force the pipeline LLM's thinking off without editing the LLM model config: diff --git a/gallery/index.yaml b/gallery/index.yaml index 731773646abb..7298128d9055 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -53871,6 +53871,66 @@ - filename: parakeet-cpp/ced-tiny-q8_0.gguf uri: huggingface://mudler/ced-gguf/ced-tiny-q8_0.gguf sha256: 48bee4e2fc3cc85d7806e03471db24e77fda6c2a2e81ffe9ef67caebaf2bd674 +- name: parakeet-cpp-realtime-scene-tdt + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-tiny + - https://github.com/mudler/parakeet.cpp + description: | + Parakeet TDT 0.6B v3 (multilingual, 25 European languages) paired with + Nemotron-3-Diarization and CED-Tiny through the diarization_model and + sound_model options: one parakeet-cpp backend transcribes, labels speakers + and tags sound events. GGUF for the parakeet-cpp backend (C++/ggml port of + NVIDIA NeMo). TDT is not a streaming model, so in a realtime pipeline use + it with server_vad: set it as both transcription and sound_detection and + turn on pipeline.diarization, and each committed turn gets speaker segments + (conversation.item.input_audio_transcription.segment, with text) and + sound tags (conversation.item.sound_detection). Also labels speakers on + /v1/audio/transcriptions. Speaker labels are per turn. License per model: + transcription model CC-BY-4.0, diarization model OpenMDW-1.1, CED-Tiny + Apache-2.0. + license: cc-by-4.0 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - asr + - speech-recognition + - diarization + - sound-classification + - multilingual + - realtime + - stt + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + - diarization + - sound_classification + name: parakeet-cpp-realtime-scene-tdt + options: + - diarization_model:parakeet-cpp/nemotron-3-diarization-q8_0.gguf + - sound_model:parakeet-cpp/ced-tiny-q8_0.gguf + parameters: + model: parakeet-cpp/tdt-0.6b-v3-f16.gguf + files: + - filename: parakeet-cpp/tdt-0.6b-v3-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/tdt-0.6b-v3-f16.gguf + sha256: 8ba47343e1e919895aca90e099150a01ed203ee0942d8ed31e27295efc5abb22 + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/ced-tiny-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-tiny-q8_0.gguf + sha256: 48bee4e2fc3cc85d7806e03471db24e77fda6c2a2e81ffe9ef67caebaf2bd674 - name: moss-transcribe-cpp-0.9b url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: From febca11e4e7e07a00b44f4fe60bff5d1b674bf16 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 29 Sep 2026 19:39:16 +0000 Subject: [PATCH 29/30] feat(gallery): add CED-Base variants of the parakeet-cpp scene models parakeet-cpp-realtime-scene and parakeet-cpp-realtime-scene-tdt ship with CED-Tiny. The -base variants use CED-Base (86M), which tags sounds more confidently (on the rooster clip "Crowing" 0.65 against 0.49 for Tiny). Measured on CPU over a 37 s clip: the live diarization + sound stream runs at 0.125 of real time with CED-Base against 0.103 with CED-Tiny, because diarization dominates; sound detection per committed turn costs 0.031 against 0.005. The realtime docs list both and note that any CED size works as sound_model. Files and sha256 match the Hub and are shared with the existing parakeet-cpp-ced-base entry. The TDT variant passes the real-model scene spec with CED-Base (A-B-A-B speakers, "Chicken, rooster" found). Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- docs/content/features/openai-realtime.md | 11 ++ gallery/index.yaml | 122 +++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index ca8759e672bc..150966cc452a 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -225,6 +225,17 @@ pipeline: `sound_detection` classifies the same committed audio and emits one `conversation.item.sound_detection` event per turn (see [Sound Classification]({{% relref "audio-classification" %}})). As on the live path, times are relative to the turn's audio and speaker labels are only consistent within a turn. `pipeline.diarization` is off by default: it needs a transcription model that diarizes (parakeet-cpp with a `diarization_model` companion), and some other backends fail a diarization request they cannot serve. +#### Choosing the sound model + +Both scene models ship with CED-Tiny, the cheapest to run all the time. `parakeet-cpp-realtime-scene-base` and `parakeet-cpp-realtime-scene-tdt-base` are the same pipelines with CED-Base (86M, the largest CED), which tags sounds more confidently. Any CED GGUF from [`mudler/ced-gguf`](https://huggingface.co/mudler/ced-gguf) (tiny, mini, small, base) works as `sound_model`. Measured on CPU (Ryzen 9 9950X3D) over a 37 s clip with two speakers and a rooster, as a fraction of real time: + +| | CED-Tiny | CED-Base | +|---|---|---| +| Live scene stream (diarization `low` + sound), EOU path | 0.103 | 0.125 | +| Sound detection per committed turn, TDT path | 0.005 | 0.031 | + +The EOU model's own ASR stream adds 0.016. Diarization dominates the live cost, so CED-Base keeps the live path about 7x faster than real time. + ### Disabling thinking For reasoning models, you can force the pipeline LLM's thinking off without editing the LLM model config: diff --git a/gallery/index.yaml b/gallery/index.yaml index 7298128d9055..71c572e6bad2 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -53931,6 +53931,128 @@ - filename: parakeet-cpp/ced-tiny-q8_0.gguf uri: huggingface://mudler/ced-gguf/ced-tiny-q8_0.gguf sha256: 48bee4e2fc3cc85d7806e03471db24e77fda6c2a2e81ffe9ef67caebaf2bd674 +- name: parakeet-cpp-realtime-scene-base + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/nvidia/parakeet_realtime_eou_120m-v1 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-base + - https://github.com/mudler/parakeet.cpp + description: | + Cache-aware streaming RNNT FastConformer with end-of-utterance (EOU) + detection, 120M, paired with Nemotron-3-Diarization and CED-Base (86M, the largest CED; + more confident sound tags than CED-Tiny at a small extra cost: on CPU the + live diarization + sound stream runs at 0.125 of real time against 0.103 + with CED-Tiny) through + the diarization_model and sound_model options. F16/Q8_0 GGUF for the + parakeet-cpp backend (C++/ggml port of NVIDIA NeMo). Use with streaming + transcription: while a turn is live, closed speaker segments and sound + events are surfaced alongside the ASR text (realtime + conversation.item.input_audio_transcription.segment and + conversation.item.sound_detection events). Live speaker/sound events only + fire during speech turns under semantic_vad; sounds between turns are not + seen by this path. License per model: transcription model NVIDIA Open + Model License, diarization model OpenMDW-1.1, CED-Base Apache-2.0. + license: nvidia-open-model-license + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - asr + - speech-recognition + - diarization + - sound-classification + - streaming + - realtime + - stt + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + name: parakeet-cpp-realtime-scene-base + options: + - diarization_model:parakeet-cpp/nemotron-3-diarization-q8_0.gguf + - sound_model:parakeet-cpp/ced-base-q8_0.gguf + parameters: + model: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + files: + - filename: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/realtime_eou_120m-v1-f16.gguf + sha256: d1a2b12f12b8a096a57499c9111ed13b442a2b786e17a292c168be45088f0edc + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/ced-base-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-base-q8_0.gguf + sha256: bd34a7710169f0047fea17267965d211f967828ab25ba6fb9d3768481393f6e2 +- name: parakeet-cpp-realtime-scene-tdt-base + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-base + - https://github.com/mudler/parakeet.cpp + description: | + Parakeet TDT 0.6B v3 (multilingual, 25 European languages) paired with + Nemotron-3-Diarization and CED-Base (86M, the largest CED; about 0.03 s of + CPU per second of audio per committed turn, against 0.005 for CED-Tiny) + through the diarization_model and + sound_model options: one parakeet-cpp backend transcribes, labels speakers + and tags sound events. GGUF for the parakeet-cpp backend (C++/ggml port of + NVIDIA NeMo). TDT is not a streaming model, so in a realtime pipeline use + it with server_vad: set it as both transcription and sound_detection and + turn on pipeline.diarization, and each committed turn gets speaker segments + (conversation.item.input_audio_transcription.segment, with text) and + sound tags (conversation.item.sound_detection). Also labels speakers on + /v1/audio/transcriptions. Speaker labels are per turn. License per model: + transcription model CC-BY-4.0, diarization model OpenMDW-1.1, CED-Tiny + Apache-2.0. + license: cc-by-4.0 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - asr + - speech-recognition + - diarization + - sound-classification + - multilingual + - realtime + - stt + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + - diarization + - sound_classification + name: parakeet-cpp-realtime-scene-tdt-base + options: + - diarization_model:parakeet-cpp/nemotron-3-diarization-q8_0.gguf + - sound_model:parakeet-cpp/ced-base-q8_0.gguf + parameters: + model: parakeet-cpp/tdt-0.6b-v3-f16.gguf + files: + - filename: parakeet-cpp/tdt-0.6b-v3-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/tdt-0.6b-v3-f16.gguf + sha256: 8ba47343e1e919895aca90e099150a01ed203ee0942d8ed31e27295efc5abb22 + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/ced-base-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-base-q8_0.gguf + sha256: bd34a7710169f0047fea17267965d211f967828ab25ba6fb9d3768481393f6e2 - name: moss-transcribe-cpp-0.9b url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: From 7b91c7f264695ce0847a50dc525d3a465668f48d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 29 Sep 2026 20:13:02 +0000 Subject: [PATCH 30/30] fix(config): register pipeline.diarization in the config metadata TestAllFieldsHaveRegistryEntries fails on the branch because the new pipeline.diarization field has no registry entry. Add one so the model editor shows it as a toggle next to the sound detection options. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- core/config/meta/registry.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index c98514a785d5..dfc2dc61aca4 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -509,6 +509,13 @@ func DefaultRegistry() map[string]FieldMetaOverride { Min: f64(0), Order: 66, }, + "pipeline.diarization": { + Section: "pipeline", + Label: "Speaker Diarization", + Description: "Label speakers on each committed utterance and emit every labelled segment as a conversation.item.input_audio_transcription.segment event. Needs a transcription model that diarizes (e.g. parakeet-cpp with a diarization_model companion). Speaker labels are per turn.", + Component: "toggle", + Order: 67, + }, "pipeline.reasoning_effort": { Section: "pipeline", Label: "Reasoning Effort",