diff --git a/backend/backend.proto b/backend/backend.proto index 54255528e5ad..6a09b98eb39b 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -641,12 +641,29 @@ message TranscriptLiveResponse { repeated TranscriptWord words = 4; // words finalized by this feed (stream-relative ns) TranscriptResult final_result = 5; // terminal message only, after the send side closes bool eob = 6; // fired: a backchannel ("uh-huh") ended — NOT a turn boundary + repeated LiveSpeakerSegment speakers = 7; // closed speaker segments from a companion diarization/scene stream + repeated LiveSoundEvent sounds = 8; // closed sound events from a companion sound/scene stream +} + +message LiveSpeakerSegment { + string speaker = 1; // decimal speaker index + int64 start = 2; // stream-relative nanoseconds + int64 end = 3; +} + +message LiveSoundEvent { + string label = 1; + int32 index = 2; + float peak = 3; + int64 start = 4; // stream-relative nanoseconds + int64 end = 5; } message TranscriptWord { int64 start = 1; int64 end = 2; string text = 3; + string speaker = 4; // backend speaker label when diarizing; empty otherwise } message TranscriptSegment { diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile index e288f6fcc648..59fe567170ca 100644 --- a/backend/go/parakeet-cpp/Makefile +++ b/backend/go/parakeet-cpp/Makefile @@ -1,6 +1,6 @@ # parakeet-cpp backend Makefile. # -# Upstream pin lives below as PARAKEET_VERSION?=2bf88954dc628b32835734e2e9159550a75a1dc6 +# Upstream pin lives below as PARAKEET_VERSION?=623a968bccbd2214588df398fcce687cd4218dea # (.github/bump_deps.sh) can find and update it - matches the # whisper.cpp / ds4 / vibevoice-cpp convention. # @@ -15,7 +15,7 @@ # That's what the L0 smoke test uses. The default target below does the # proper clone-at-pin + cmake build so CI doesn't need a side-checkout. -PARAKEET_VERSION?=2bf88954dc628b32835734e2e9159550a75a1dc6 +PARAKEET_VERSION?=623a968bccbd2214588df398fcce687cd4218dea PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp GOCMD?=go diff --git a/backend/go/parakeet-cpp/diarize.go b/backend/go/parakeet-cpp/diarize.go new file mode 100644 index 000000000000..1853aa666314 --- /dev/null +++ b/backend/go/parakeet-cpp/diarize.go @@ -0,0 +1,328 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// diarizeSegmentJSON mirrors one element of parakeet_capi_diarize_pcm's +// "segments" array: {"speaker":0,"start":0.50,"end":5.52}. +type diarizeSegmentJSON struct { + Speaker int `json:"speaker"` + Start float64 `json:"start"` + End float64 `json:"end"` +} + +// diarizePCMDoc mirrors the document parakeet_capi_diarize_pcm returns. +// "speakers" is the model's CAPACITY (e.g. 8 for Nemotron-3-Diarization), +// not the count of speakers actually present, so it is not read here; the +// response's num_speakers is computed from distinct segment labels instead. +type diarizePCMDoc struct { + Segments []diarizeSegmentJSON `json:"segments"` +} + +// diarizeUtteranceJSON mirrors one element of +// parakeet_capi_transcribe_and_diarize_json's "utterances" array. Speaker is +// -1 when no diarized speaker overlaps the utterance. +type diarizeUtteranceJSON struct { + Speaker int `json:"speaker"` + Text string `json:"text"` + Start float64 `json:"start"` + End float64 `json:"end"` +} + +// transcribeAndDiarizeDoc mirrors the document +// parakeet_capi_transcribe_and_diarize_json returns. Only "utterances" is +// consumed here; the per-word "words" detail belongs to a speaker-attributed +// transcript RPC, not Diarize. +type transcribeAndDiarizeDoc struct { + Utterances []diarizeUtteranceJSON `json:"utterances"` +} + +// speakerLabel renders a 0-based speaker index as the decimal string +// DiarizeSegment.speaker documents, or "unknown" for -1 (no diarized speaker +// overlaps this utterance; only transcribe_and_diarize_json can report this). +func speakerLabel(speaker int) string { + if speaker < 0 { + return "unknown" + } + return strconv.Itoa(speaker) +} + +// unsupportedDiarizeFields names the DiarizeRequest fields Sortformer has no +// equivalent for: it is an end-to-end model with a fixed speaker capacity and +// no clustering stage, so there is no config knob to target a speaker count +// or a clustering distance. Logged rather than rejected, so a request naming +// one of these still gets the diarization it can have. +func unsupportedDiarizeFields(req *pb.DiarizeRequest) []string { + var out []string + if req.GetNumSpeakers() != 0 { + out = append(out, "num_speakers") + } + if req.GetMinSpeakers() != 0 { + out = append(out, "min_speakers") + } + if req.GetMaxSpeakers() != 0 { + out = append(out, "max_speakers") + } + if req.GetClusteringThreshold() != 0 { + out = append(out, "clustering_threshold") + } + return out +} + +// Diarize labels who spoke when in the audio at req.Dst, using the loaded +// diarization model (p.diarCtx). When req.IncludeText is set and an ASR +// companion (p.ctxPtr) is loaded, each segment also carries its transcript +// (parakeet_capi_transcribe_and_diarize_json, one utterance per speaker +// turn); otherwise, or when no ASR companion is loaded, segments carry no +// text (parakeet_capi_diarize_pcm) and no error is raised. +func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error) { + if p.diarCtx == 0 { + return pb.DiarizeResponse{}, status.Error(codes.FailedPrecondition, + "parakeet-cpp: model is not a diarization model") + } + if CppDiarizePCM == nil { + return pb.DiarizeResponse{}, status.Error(codes.Unimplemented, + "parakeet-cpp: loaded libparakeet.so has no diarization support (parakeet_capi_diarize_pcm missing)") + } + if req.GetDst() == "" { + return pb.DiarizeResponse{}, status.Error(codes.InvalidArgument, + "parakeet-cpp: DiarizeRequest.dst (audio path) is required") + } + + if dropped := unsupportedDiarizeFields(req); len(dropped) > 0 { + xlog.Debug("parakeet-cpp: ignoring diarization request fields Sortformer has no equivalent for", + "fields", dropped) + } + + pcm, duration, err := decodeWavMono16k(req.GetDst()) + if err != nil { + return pb.DiarizeResponse{}, status.Errorf(codes.InvalidArgument, "parakeet-cpp: decode audio: %s", err) + } + if len(pcm) == 0 { + return pb.DiarizeResponse{}, status.Error(codes.InvalidArgument, "parakeet-cpp: empty audio") + } + + wantText := req.GetIncludeText() && p.ctxPtr != 0 && CppTranscribeAndDiarizeJSON != nil + + raw, err := p.diarizeCall(pcm, wantText) + if err != nil { + return pb.DiarizeResponse{}, err + } + segments, err := parseDiarizeDoc(raw, wantText) + if err != nil { + return pb.DiarizeResponse{}, err + } + + segments = applyDurationFilters(segments, req.GetMinDurationOn(), req.GetMinDurationOff()) + renumberDiarizeSegments(segments) + + return pb.DiarizeResponse{ + Segments: segments, + NumSpeakers: distinctDiarizeSpeakers(segments), + Duration: duration, + }, nil +} + +// diarizeCall runs the single C call Diarize needs (transcribe_and_diarize_json +// when wantText, else diarize_pcm) under engineMu, and returns the raw JSON +// document. p.diarCtx (and, on the include_text path, p.ctxPtr) is re-checked +// under the lock before the C call: Diarize's own p.diarCtx==0/wantText checks +// run before this lock is taken, so a Free() racing in between (which zeroes +// those fields under the same engineMu) would otherwise reach the C side with +// a freed context. last_error is ctx-shared, so it is read under the same +// lock as the failing call. +func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool) (string, error) { + p.engineMu.Lock() + defer p.engineMu.Unlock() + + if p.diarCtx == 0 || (wantText && p.ctxPtr == 0) { + return "", grpcerrors.ModelNotLoaded("parakeet-cpp") + } + + var cstr uintptr + if wantText { + cstr = CppTranscribeAndDiarizeJSON(p.ctxPtr, p.diarCtx, &pcm[0], int32(len(pcm)), 16000) + } else { + cstr = CppDiarizePCM(p.diarCtx, &pcm[0], int32(len(pcm)), 16000) + } + if cstr == 0 { + return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", diarizeLastError(p, wantText)) + } + raw := goStringFromCPtr(cstr) + CppFreeString(cstr) + return raw, nil +} + +// diarizeLastError reads last_error off p.diarCtx and, on the include_text +// path, p.ctxPtr too — the failing call is CppTranscribeAndDiarizeJSON there, +// and either side of the pairing may be the one that set it — then joins +// whichever came back non-empty. Called under the same engineMu as the +// failing call (last_error is ctx-shared state). +func diarizeLastError(p *ParakeetCpp, wantText bool) string { + var msgs []string + if m := CppLastError(p.diarCtx); m != "" { + msgs = append(msgs, m) + } + if wantText { + if m := CppLastError(p.ctxPtr); m != "" { + msgs = append(msgs, m) + } + } + if len(msgs) == 0 { + return "unknown error" + } + return strings.Join(msgs, "; ") +} + +// parseDiarizeDoc decodes the raw JSON diarizeCall returned into +// DiarizeSegments (without ids: renumberDiarizeSegments assigns those after +// filtering). +func parseDiarizeDoc(raw string, wantText bool) ([]*pb.DiarizeSegment, error) { + if wantText { + var doc transcribeAndDiarizeDoc + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return nil, fmt.Errorf("parakeet-cpp: decode diarize json: %w", err) + } + segs := make([]*pb.DiarizeSegment, 0, len(doc.Utterances)) + for _, u := range doc.Utterances { + segs = append(segs, &pb.DiarizeSegment{ + Start: float32(u.Start), + End: float32(u.End), + Speaker: speakerLabel(u.Speaker), + Text: u.Text, + }) + } + return segs, nil + } + + var doc diarizePCMDoc + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return nil, fmt.Errorf("parakeet-cpp: decode diarize json: %w", err) + } + segs := make([]*pb.DiarizeSegment, 0, len(doc.Segments)) + for _, s := range doc.Segments { + segs = append(segs, &pb.DiarizeSegment{ + Start: float32(s.Start), + End: float32(s.End), + Speaker: speakerLabel(s.Speaker), + }) + } + return segs, nil +} + +// applyDurationFilters applies the request's postprocessing knobs, in the +// order NeMo's diarization postprocessing does: merge first +// (min_duration_off), then drop short segments (min_duration_on) — dropping +// first would leave short gaps unmerged that the drop step just created. +// Segments are assumed sorted by start time, as parakeet_capi_diarize_pcm and +// parakeet_capi_transcribe_and_diarize_json document. A non-positive value +// disables that filter (the proto's "0 = backend default" reads here as "no +// filtering"). +func applyDurationFilters(segs []*pb.DiarizeSegment, minOn, minOff float32) []*pb.DiarizeSegment { + segs = mergeCloseSegments(segs, minOff) + segs = dropShortSegments(segs, minOn) + return segs +} + +// mergeCloseSegments merges SAME-SPEAKER segments separated by a gap shorter +// than minOff into one segment spanning both (and concatenating any text). +// Segments from different speakers are never merged, regardless of gap: the +// gap only ever means "the same speaker paused", never "two speakers are +// actually one". +// +// Merging runs per speaker rather than on the single start-sorted list: two +// segments of the same speaker are not necessarily adjacent in that list once +// another speaker's turn falls between them (A, B, A), and a start-sorted +// walk would then never compare the two A's at all. Grouping by speaker first +// keeps each group's own start order (segs is assumed start-sorted, as +// parakeet_capi_diarize_pcm and parakeet_capi_transcribe_and_diarize_json +// document), merges within the group, then the merged segments are re-sorted +// by start so interleaved speakers come back out in timeline order. +func mergeCloseSegments(segs []*pb.DiarizeSegment, minOff float32) []*pb.DiarizeSegment { + if minOff <= 0 || len(segs) < 2 { + return segs + } + + bySpeaker := make(map[string][]*pb.DiarizeSegment) + var order []string // first-seen speaker order, for a deterministic group walk + for _, s := range segs { + if _, ok := bySpeaker[s.GetSpeaker()]; !ok { + order = append(order, s.GetSpeaker()) + } + bySpeaker[s.GetSpeaker()] = append(bySpeaker[s.GetSpeaker()], s) + } + + out := make([]*pb.DiarizeSegment, 0, len(segs)) + for _, speaker := range order { + group := bySpeaker[speaker] + merged := make([]*pb.DiarizeSegment, 0, len(group)) + merged = append(merged, group[0]) + for _, s := range group[1:] { + prev := merged[len(merged)-1] + if s.GetStart()-prev.GetEnd() < minOff { + if s.GetEnd() > prev.GetEnd() { + prev.End = s.End + } + if s.GetText() != "" { + if prev.GetText() != "" { + prev.Text = prev.GetText() + " " + s.GetText() + } else { + prev.Text = s.GetText() + } + } + continue + } + merged = append(merged, s) + } + out = append(out, merged...) + } + + sort.Slice(out, func(i, j int) bool { return out[i].GetStart() < out[j].GetStart() }) + return out +} + +// dropShortSegments discards segments shorter than minOn. +func dropShortSegments(segs []*pb.DiarizeSegment, minOn float32) []*pb.DiarizeSegment { + if minOn <= 0 { + return segs + } + out := make([]*pb.DiarizeSegment, 0, len(segs)) + for _, s := range segs { + if s.GetEnd()-s.GetStart() < minOn { + continue + } + out = append(out, s) + } + return out +} + +// renumberDiarizeSegments assigns sequential ids (0..) to the final segment +// list, after filtering may have dropped or merged entries. +func renumberDiarizeSegments(segs []*pb.DiarizeSegment) { + for i, s := range segs { + s.Id = int32(i) + } +} + +// distinctDiarizeSpeakers counts the distinct speaker labels present in segs. +// This is what DiarizeResponse.num_speakers documents — the count of speakers +// actually present in the result — and is NOT the diarize_pcm JSON's +// top-level "speakers" field, which reports the model's fixed capacity. +func distinctDiarizeSpeakers(segs []*pb.DiarizeSegment) int32 { + seen := make(map[string]struct{}, len(segs)) + for _, s := range segs { + seen[s.GetSpeaker()] = struct{}{} + } + return int32(len(seen)) +} diff --git a/backend/go/parakeet-cpp/diarize_test.go b/backend/go/parakeet-cpp/diarize_test.go new file mode 100644 index 000000000000..1db076771d62 --- /dev/null +++ b/backend/go/parakeet-cpp/diarize_test.go @@ -0,0 +1,275 @@ +package main + +import ( + "path/filepath" + "sync" + "unsafe" + + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// The Diarize specs drive it entirely against stubbed CppDiarizePCM / +// CppTranscribeAndDiarizeJSON / CppFreeString / CppLastError (the same seam +// live_test.go and roles_test.go use), so they run without libparakeet.so. + +// diarizeCstrPool hands out NUL-terminated C-style strings backed by Go +// memory and keeps them alive for the duration of a spec (goStringFromCPtr +// reads through the raw pointer; mirrors live_test.go's liveCstrPool). +type diarizeCstrPool struct { + mu sync.Mutex + bufs [][]byte +} + +func (p *diarizeCstrPool) cstr(s string) uintptr { + p.mu.Lock() + defer p.mu.Unlock() + b := append([]byte(s), 0) + p.bufs = append(p.bufs, b) + return uintptr(unsafe.Pointer(&b[0])) +} + +// diarizeStubs swaps every C entry point Diarize touches and returns a +// restore func for AfterEach (mirrors live_test.go's liveStubs). +func diarizeStubs() (restore func()) { + savedDiarize := CppDiarizePCM + savedTranscribeAndDiarize := CppTranscribeAndDiarizeJSON + savedFreeString := CppFreeString + savedLastError := CppLastError + return func() { + CppDiarizePCM = savedDiarize + CppTranscribeAndDiarizeJSON = savedTranscribeAndDiarize + CppFreeString = savedFreeString + CppLastError = savedLastError + } +} + +// diarizeWav writes a silent 16 kHz mono WAV of the given duration (seconds) +// to a fresh temp file and returns its path. decodeWavMono16k reads real +// audio bytes off disk, so Diarize needs a file on disk even though the +// stubbed C calls never look at its samples. +func diarizeWav(seconds float64) string { + GinkgoHelper() + path := filepath.Join(GinkgoT().TempDir(), "diarize.wav") + writeMono16kWav(path, int(seconds*16000)) + return path +} + +var _ = Describe("ParakeetCpp.Diarize", func() { + var restore func() + var pool *diarizeCstrPool + + BeforeEach(func() { + restore = diarizeStubs() + pool = &diarizeCstrPool{} + }) + AfterEach(func() { restore() }) + + It("fails with FailedPrecondition when no diarization model is loaded", func() { + p := &ParakeetCpp{} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.FailedPrecondition)) + Expect(err.Error()).To(ContainSubstring("model is not a diarization model")) + }) + + It("fails with Unimplemented when the loaded libparakeet.so has no diarize_pcm symbol", func() { + CppDiarizePCM = nil + p := &ParakeetCpp{diarCtx: 42} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + }) + + It("maps plain segments with sequential ids, decimal speaker labels and distinct speaker count", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"segments":[` + + `{"speaker":0,"start":0.00,"end":3.00},` + + `{"speaker":1,"start":3.00,"end":6.00}]}`) + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(6)}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Segments).To(HaveLen(2)) + Expect(resp.Segments[0].Id).To(Equal(int32(0))) + Expect(resp.Segments[0].Speaker).To(Equal("0")) + Expect(resp.Segments[1].Id).To(Equal(int32(1))) + Expect(resp.Segments[1].Speaker).To(Equal("1")) + Expect(resp.Segments[0].Text).To(BeEmpty()) + Expect(resp.NumSpeakers).To(Equal(int32(2))) + Expect(resp.Duration).To(BeNumerically("~", 6.0, 0.01)) + }) + + It("fills text from utterances when include_text is set with an ASR companion, and maps speaker -1 to unknown", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + Fail("diarize_pcm must not be called when include_text has an ASR companion to pair with") + return 0 + } + CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"utterances":[` + + `{"speaker":0,"text":"hello there","start":0.00,"end":1.00,"conf":0.9},` + + `{"speaker":-1,"text":"mumble","start":1.00,"end":1.50,"conf":0.4}],"words":[]}`) + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42, ctxPtr: 7} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(2), IncludeText: true}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Segments).To(HaveLen(2)) + Expect(resp.Segments[0].Speaker).To(Equal("0")) + Expect(resp.Segments[0].Text).To(Equal("hello there")) + Expect(resp.Segments[1].Speaker).To(Equal("unknown")) + Expect(resp.Segments[1].Text).To(Equal("mumble")) + }) + + It("falls back to plain segments without error when include_text is set but no ASR companion is loaded", func() { + diarizeCalled := false + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + diarizeCalled = true + return pool.cstr(`{"speakers":8,"segments":[{"speaker":0,"start":0.00,"end":1.00}]}`) + } + CppFreeString = func(uintptr) {} + CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, samples *float32, n int32, sampleRate int32) uintptr { + Fail("transcribe_and_diarize_json must not be called without an ASR companion") + return 0 + } + + p := &ParakeetCpp{diarCtx: 42} // no ctxPtr companion + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1), IncludeText: true}) + Expect(err).ToNot(HaveOccurred()) + Expect(diarizeCalled).To(BeTrue()) + Expect(resp.Segments).To(HaveLen(1)) + Expect(resp.Segments[0].Text).To(BeEmpty()) + }) + + It("drops a segment shorter than min_duration_on", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"segments":[` + + `{"speaker":0,"start":0.30,"end":0.50},` + // 0.2s, at 0.3 + `{"speaker":0,"start":1.00,"end":2.00}]}`) // 1.0s, kept + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(3), MinDurationOn: 0.3}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Segments).To(HaveLen(1)) + Expect(resp.Segments[0].Id).To(Equal(int32(0))) + Expect(resp.Segments[0].Start).To(BeNumerically("~", 1.0, 0.001)) + }) + + It("merges same-speaker segments across a short gap but not across a speaker change", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"segments":[` + + `{"speaker":0,"start":0.00,"end":1.00},` + + `{"speaker":0,"start":1.30,"end":2.00},` + // 0.3s gap, same speaker: merges + `{"speaker":1,"start":2.10,"end":3.00}]}`) // 0.1s gap, different speaker: stays separate + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(3), MinDurationOff: 0.5}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Segments).To(HaveLen(2)) + Expect(resp.Segments[0].Id).To(Equal(int32(0))) + Expect(resp.Segments[0].Speaker).To(Equal("0")) + Expect(resp.Segments[0].Start).To(BeNumerically("~", 0.0, 0.001)) + Expect(resp.Segments[0].End).To(BeNumerically("~", 2.0, 0.001)) + Expect(resp.Segments[1].Id).To(Equal(int32(1))) + Expect(resp.Segments[1].Speaker).To(Equal("1")) + }) + + It("surfaces last_error when the C call returns NULL", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return 0 + } + CppLastError = func(ctx uintptr) string { return "boom" } + + p := &ParakeetCpp{diarCtx: 42} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("boom")) + }) + + It("reports last_error from both contexts when the include_text C call returns NULL", func() { + // Diarize's Unimplemented gate checks CppDiarizePCM regardless of + // wantText, so it needs a non-nil (never called) stub here too. + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + Fail("diarize_pcm must not be called when include_text has an ASR companion to pair with") + return 0 + } + CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return 0 + } + CppLastError = func(ctx uintptr) string { + if ctx == 7 { + return "asr side broke" + } + return "diar side broke" + } + + p := &ParakeetCpp{diarCtx: 42, ctxPtr: 7} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(1), IncludeText: true}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("asr side broke")) + Expect(err.Error()).To(ContainSubstring("diar side broke")) + }) + + It("wraps a decode failure as InvalidArgument", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + Fail("decode must fail before any C call is made") + return 0 + } + + p := &ParakeetCpp{diarCtx: 42} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: filepath.Join(GinkgoT().TempDir(), "missing.wav")}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + }) + + It("returns ModelNotLoaded without a C call when diarCtx is zeroed between the entry check and the call", func() { + called := false + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + called = true + return pool.cstr(`{"speakers":8,"segments":[]}`) + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + // Simulate a Free() racing between Diarize's own diarCtx==0 check and + // diarizeCall's lock, exactly as it zeroes diarCtx under engineMu. + p.diarCtx = 0 + _, err := p.diarizeCall(make([]float32, 10), false) + Expect(grpcerrors.IsModelNotLoaded(err)).To(BeTrue()) + Expect(called).To(BeFalse(), "no C call once diarCtx was cleared") + }) + + It("merges same-speaker segments across an intervening different speaker (A, B, A)", func() { + CppDiarizePCM = func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr { + return pool.cstr(`{"speakers":8,"segments":[` + + `{"speaker":0,"start":0.00,"end":1.00},` + + `{"speaker":1,"start":1.05,"end":1.20},` + // short B segment sits between the two A's + `{"speaker":0,"start":1.30,"end":2.00}]}`) // 0.1s gap from the first A: same speaker, merges + } + CppFreeString = func(uintptr) {} + + p := &ParakeetCpp{diarCtx: 42} + resp, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(3), MinDurationOff: 0.5}) + Expect(err).ToNot(HaveOccurred()) + // The two speaker-0 segments merge into one spanning 0.00-2.00, and + // the timeline re-sort puts speaker 1's untouched segment in between. + Expect(resp.Segments).To(HaveLen(2)) + Expect(resp.Segments[0].Speaker).To(Equal("0")) + Expect(resp.Segments[0].Start).To(BeNumerically("~", 0.0, 0.001)) + Expect(resp.Segments[0].End).To(BeNumerically("~", 2.0, 0.001)) + Expect(resp.Segments[1].Speaker).To(Equal("1")) + Expect(resp.Segments[1].Start).To(BeNumerically("~", 1.05, 0.001)) + Expect(resp.Segments[1].End).To(BeNumerically("~", 1.20, 0.001)) + }) +}) diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index 23e5c548ce0c..e8ed1c5253f7 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -74,8 +74,55 @@ var ( // libparakeet.so; nil falls back to the text-only CppStreamFeed/Finalize path. CppStreamFeedJSON func(s uintptr, pcm []float32, nSamples int32) uintptr CppStreamFinalizeJSON func(s uintptr) uintptr + + // CppModelKind reports which kind of model a loaded context holds + // (parakeet_capi_model_kind, ABI v8): see the modelKind* constants in + // roles.go. nil on an older libparakeet.so; Load then treats the primary + // as ASR (pre-v8 behavior) and rejects companion model options. + CppModelKind func(ctx uintptr) int32 + + // Speaker diarization (ABI v7). CppDiarizePCM runs offline diarization + // over in-memory mono float PCM; CppTranscribeAndDiarizeJSON pairs it with + // an ASR context for speaker-attributed text. Both return a malloc'd char* + // JSON document (uintptr, freed via CppFreeString). + CppDiarizePCM func(ctx uintptr, samples *float32, n int32, sampleRate int32) uintptr + CppTranscribeAndDiarizeJSON func(asr, diar uintptr, samples *float32, n int32, sampleRate int32) uintptr + + // Sound-event detection (CED) and the combined scene stream (ABI v8). + // CppNumClasses/CppSoundOptsDefault/CppSoundStreamBegin.../ + // CppSceneOptsDefault/CppSceneStreamBegin... are only registered when + // CppModelKind is present (see main.go); nil otherwise. + CppNumClasses func(ctx uintptr) int32 + CppSoundOptsDefault func(o *cSoundOpts) + CppSoundStreamBegin func(tagger uintptr, o *cSoundOpts) uintptr + CppSoundStreamFeed func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 + CppSoundStreamDrainScoresJSON func(s uintptr) uintptr + CppFreeSoundSegments func(segs uintptr) + CppSoundStreamFree func(s uintptr) + CppSceneOptsDefault func(o *cSceneOpts) + CppSceneStreamBegin func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr + CppSceneStreamFeedJSON func(s uintptr, pcm *float32, n int32, isLast int32) uintptr + CppSceneStreamLastError func(s uintptr) string + CppSceneStreamFree func(s uintptr) ) +// cSoundOpts and cSceneOpts mirror parakeet_sound_opts / parakeet_scene_opts +// in parakeet_capi.h field-for-field (int -> int32, float -> float32); the +// C side sizes/versions them via the leading `size` field, set by the +// matching *_opts_default call. +type cSoundOpts struct { + Size int32 + WindowSec, HopSec, OnThreshold, OffThreshold, MinDurationSec float32 + TopK int32 +} + +type cSceneOpts struct { + Size int32 + DiarLatency int32 + Sound cSoundOpts + Flags int32 +} + // streamChunkSamples is how much 16 kHz mono PCM we hand to stream_feed per // call (1 s). The session buffers internally and decodes once a full // cache-aware encoder chunk is available, so this only bounds how often we @@ -140,10 +187,23 @@ type transcriptToken struct { // touch it concurrently. type ParakeetCpp struct { base.Base - ctxPtr uintptr - engineMu sync.Mutex // sole guard of the one C engine (dispatcher + streaming) - bat *batcher - batStop chan struct{} + ctxPtr uintptr // ASR context: the primary when it is an ASR model, or the asr_model companion + // diarCtx / tagCtx are the diarization and sound (CED) model contexts: + // the primary when it is that kind, or the diarization_model/sound_model + // companion. See roles.go. + diarCtx uintptr + tagCtx uintptr + // diarLatency is the PARAKEET_DIAR_LATENCY_* mode for diarization + // streaming (diarization_latency: option, default "low"). Unused until + // the diarization/scene streaming paths land. + diarLatency int32 + // companions holds every context this backend loaded itself beyond the + // primary (asr_model:/diarization_model:/sound_model: options), so Free + // can release them after the primary. + companions []uintptr + engineMu sync.Mutex // sole guard of the one C engine (dispatcher + streaming) + bat *batcher + batStop chan struct{} // segmentGapFrames is NeMo's segment_gap_threshold in ENCODER FRAMES (model // YAML option, default 0=off). When >0 it adds NeMo's silence-gap split on // top of the punctuation split; converted to seconds via the JSON frame_sec. @@ -151,21 +211,17 @@ type ParakeetCpp struct { } // Load is the LocalAI gRPC entry point for LoadModel: it calls -// parakeet_capi_load with the GGUF path and stashes the resulting -// opaque context pointer for AudioTranscription. +// parakeet_capi_load with the GGUF path, classifies it and any companion +// models named in Options[] by role (see roles.go), and starts the dynamic +// batcher when an ASR context (primary or companion) ends up loaded. func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error { if opts.ModelFile == "" { return errors.New("parakeet-cpp: ModelFile is required") } - ctx := CppLoad(opts.ModelFile) - if ctx == 0 { - // No ctx to ask for last_error (the C-API's last-error buffer - // lives on the ctx that was never returned). Surface the path - // so the operator at least knows which load failed. - return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for %q", opts.ModelFile) + if err := p.loadRoles(opts); err != nil { + return err } - p.ctxPtr = ctx // Dynamic batching knobs (model YAML options:, key:value form). Batching is // OFF by default (batch_max_size:1): each request runs on its own. On GPU, @@ -182,6 +238,12 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error { // default matches NeMo's default (punctuation-only segments); when set it // additionally splits segments on inter-word silence (see transcriptResultFromDoc). p.segmentGapFrames = optInt(opts, "segment_gap_threshold", 0) + + // The batcher only ever drives the ASR context; a diarization/sound + // primary with no asr_model companion has no ctxPtr and needs none. + if p.ctxPtr == 0 { + return nil + } if CppTranscribePcmBatchJSON != nil { p.batStop = make(chan struct{}) p.bat = newBatcher(maxSize, time.Duration(maxWaitMs)*time.Millisecond, p.runBatch) @@ -287,12 +349,16 @@ func (p *ParakeetCpp) runBatch(reqs []*batchRequest) { // OpenAI API, whose default is segment-level); token ids always populate // Segment.Tokens. // -// translate/diarize/prompt/temperature/threads are not applicable to parakeet -// and are ignored; language is honored on the batched + streaming paths (see -// opts.GetLanguage() below); streaming is handled by AudioTranscriptionStream -// (L2). +// With a diarization_model companion, diarize=true labels segments with their +// speaker (speakers.go). translate/prompt/temperature/threads are not +// applicable to parakeet and are ignored; language is honored on the batched + +// streaming paths (see opts.GetLanguage() below); streaming is handled by +// AudioTranscriptionStream (L2). func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.TranscriptRequest) (pb.TranscriptResult, error) { if p.ctxPtr == 0 { + if err := p.notASRError(); err != nil { + return pb.TranscriptResult{}, err + } return pb.TranscriptResult{}, grpcerrors.ModelNotLoaded("parakeet-cpp") } if opts.Dst == "" { @@ -350,7 +416,17 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip if err := json.Unmarshal([]byte(res.json), &doc); err != nil { return pb.TranscriptResult{}, fmt.Errorf("parakeet-cpp: decode transcript json: %w", err) } - return transcriptResultFromDoc(doc, opts, p.segmentGapFrames), nil + + // With a diarization_model companion, label each segment with its speaker. + var speakers []int + if p.wantSpeakers(opts.GetDiarize()) && len(doc.Words) > 0 { + segs, err := p.diarizeSegmentsPCM(pcm) + if err != nil { + return pb.TranscriptResult{}, err + } + speakers = assignSpeakers(doc.Words, segs) + } + return transcriptResultWithSpeakers(doc, opts, p.segmentGapFrames, speakers), nil } // segmentSeparators is NeMo's default segment_seperators (sentence-ending @@ -365,6 +441,14 @@ var segmentSeparators = []rune{'.', '?', '!'} // the caller requested word granularity; token ids populate each segment's // Tokens by time-window membership. Shared by the batched and direct paths. func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gapFrames int) pb.TranscriptResult { + return transcriptResultWithSpeakers(doc, opts, gapFrames, nil) +} + +// transcriptResultWithSpeakers is transcriptResultFromDoc plus optional +// per-word speakers (indexed like doc.Words, -1 = none; see speakers.go): +// segments additionally split wherever the speaker changes, and segments and +// words carry the speaker's label. +func transcriptResultWithSpeakers(doc transcriptJSON, opts *pb.TranscriptRequest, gapFrames int, speakers []int) pb.TranscriptResult { text, eou := stripEouMarker(strings.TrimSpace(doc.Text)) // Frame-unit gap threshold -> seconds (NeMo segment_gap_threshold). 0 = off. @@ -388,6 +472,11 @@ func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gap } } + var groupSpeakers []int + if speakers != nil && len(speakers) == len(doc.Words) { + groups, groupSpeakers = splitAtSpeakerChanges(groups, speakers) + } + wantWords := wordsRequested(opts.TimestampGranularities) segments := make([]*pb.TranscriptSegment, 0, len(groups)) for id, group := range groups { @@ -402,10 +491,14 @@ func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gap Text: strings.TrimSpace(strings.Join(parts, " ")), Tokens: tokensInWindow(doc.Tokens, group[0].Start, group[len(group)-1].End), } + if groupSpeakers != nil { + seg.Speaker = transcriptSpeaker(groupSpeakers[id]) + } if wantWords { ws := make([]*pb.TranscriptWord, len(group)) for i, gw := range group { - ws[i] = &pb.TranscriptWord{Start: secondsToNanos(gw.Start), End: secondsToNanos(gw.End), Text: gw.W} + ws[i] = &pb.TranscriptWord{Start: secondsToNanos(gw.Start), End: secondsToNanos(gw.End), Text: gw.W, + Speaker: seg.Speaker} } seg.Words = ws } @@ -503,10 +596,11 @@ func tokensInWindow(tokens []transcriptToken, start, end float64) []int32 { // text-only library (no words) it falls back to segmenting the delta text, so // the same assembler serves both paths. type streamSegmenter struct { - segs []*pb.TranscriptSegment - cur []transcriptWord // words for the open segment (ABI v4 JSON path) - curText []string // delta text for the open segment (text-only path) - nextID int32 + segs []*pb.TranscriptSegment + segWords [][]transcriptWord // words of each segment (nil for text-only ones) + cur []transcriptWord // words for the open segment (ABI v4 JSON path) + curText []string // delta text for the open segment (text-only path) + nextID int32 } func (s *streamSegmenter) add(r streamFeedResult) { @@ -534,12 +628,14 @@ func (s *streamSegmenter) flush() { End: secondsToNanos(s.cur[len(s.cur)-1].End), Text: strings.TrimSpace(strings.Join(parts, " ")), }) + s.segWords = append(s.segWords, s.cur) s.nextID++ case len(s.curText) > 0: // No words this segment: emit a text-only segment (no timestamps), // skipping a purely-whitespace one as the legacy text path did. if t := strings.TrimSpace(strings.Join(s.curText, "")); t != "" { s.segs = append(s.segs, &pb.TranscriptSegment{Id: s.nextID, Text: t}) + s.segWords = append(s.segWords, nil) s.nextID++ } } @@ -686,6 +782,9 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra defer close(results) if p.ctxPtr == 0 { + if err := p.notASRError(); err != nil { + return err + } return grpcerrors.ModelNotLoaded("parakeet-cpp") } if opts.Dst == "" { @@ -753,6 +852,28 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra // The single-segment fallback stays trimmed. fullText := full.String() segments := seg.segments() + + // With a diarization_model companion, label each utterance with the + // speaker who said most of it. The whole file is available, so this runs + // the same diarization as the unary path. + if p.wantSpeakers(opts.GetDiarize()) && len(seg.segWords) == len(segments) { + var all []transcriptWord + for _, ws := range seg.segWords { + all = append(all, ws...) + } + if len(all) > 0 { + segs, err := p.diarizeSegmentsPCM(data) + if err != nil { + return err + } + speakers := assignSpeakers(all, segs) + k := 0 + for i, ws := range seg.segWords { + segments[i].Speaker = transcriptSpeaker(majoritySpeaker(ws, speakers[k:k+len(ws)])) + k += len(ws) + } + } + } if trimmed := strings.TrimSpace(fullText); len(segments) == 0 && trimmed != "" { segments = append(segments, &pb.TranscriptSegment{Id: 0, Text: trimmed}) } @@ -817,8 +938,10 @@ func decodeWavMono16k(path string) ([]float32, float32, error) { return data, duration, nil } -// Free releases the underlying parakeet_ctx. Called by LocalAI when the -// model is unloaded. +// Free releases every parakeet_ctx this backend holds (the primary and any +// asr_model:/diarization_model:/sound_model: companions loaded in Load) and +// is idempotent: fields are zeroed as they are freed, so a second call frees +// nothing. Called by LocalAI when the model is unloaded. func (p *ParakeetCpp) Free() error { // Stop the dispatcher before releasing the engine so no in-flight runBatch // can touch a freed ctx (close leak / use-after-free on reload). @@ -830,10 +953,13 @@ func (p *ParakeetCpp) Free() error { // re-checks ctxPtr under the lock) can never feed into a freed ctx. p.engineMu.Lock() defer p.engineMu.Unlock() - if p.ctxPtr != 0 { - CppFree(p.ctxPtr) - p.ctxPtr = 0 + for _, ctxField := range [...]*uintptr{&p.ctxPtr, &p.diarCtx, &p.tagCtx} { + if *ctxField != 0 { + CppFree(*ctxField) + *ctxField = 0 + } } + p.companions = nil return nil } diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go index a6f6af1f04a0..eda5eaa36ce3 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp_test.go +++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go @@ -59,6 +59,23 @@ func ensureLibLoaded() { purego.RegisterLibFunc(&CppStreamFeedJSON, lib, "parakeet_capi_stream_feed_json") purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json") } + // Diarization and model roles, probed like main.go (speakers_test.go). + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_pcm"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppDiarizePCM, lib, "parakeet_capi_diarize_pcm") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_transcribe_and_diarize_json"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppTranscribeAndDiarizeJSON, lib, "parakeet_capi_transcribe_and_diarize_json") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_model_kind"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppModelKind, lib, "parakeet_capi_model_kind") + purego.RegisterLibFunc(&CppNumClasses, lib, "parakeet_capi_num_classes") + purego.RegisterLibFunc(&CppSoundOptsDefault, lib, "parakeet_capi_sound_opts_default") + purego.RegisterLibFunc(&CppSoundStreamBegin, lib, "parakeet_capi_sound_stream_begin") + purego.RegisterLibFunc(&CppSoundStreamFeed, lib, "parakeet_capi_sound_stream_feed") + purego.RegisterLibFunc(&CppSoundStreamDrainScoresJSON, lib, "parakeet_capi_sound_stream_drain_scores_json") + purego.RegisterLibFunc(&CppFreeSoundSegments, lib, "parakeet_capi_free_sound_segments") + purego.RegisterLibFunc(&CppSoundStreamFree, lib, "parakeet_capi_sound_stream_free") + } purego.RegisterLibFunc(&CppFreeString, lib, "parakeet_capi_free_string") purego.RegisterLibFunc(&CppLastError, lib, "parakeet_capi_last_error") }) @@ -203,6 +220,24 @@ var _ = Describe("ParakeetCpp", func() { }) Context("AudioTranscriptionStream", func() { + It("names the loaded role instead of a generic model-not-loaded error for a diarization primary", func() { + // CppStreamBegin/CppStreamBeginLang are left nil (zero value): if + // AudioTranscriptionStream tried to call either, this would panic + // instead of returning cleanly, so a clean typed error here also + // proves no C call was made. + p := &ParakeetCpp{diarCtx: 1} + results := make(chan *pb.TranscriptStreamResponse, 8) + err := p.AudioTranscriptionStream(context.Background(), + &pb.TranscriptRequest{Dst: "ignored.wav"}, results) + Expect(err).To(MatchError(ContainSubstring("diarization model"))) + + var emitted []*pb.TranscriptStreamResponse + for r := range results { + emitted = append(emitted, r) + } + Expect(emitted).To(BeEmpty()) + }) + It("returns the typed Unimplemented signal for non-streaming models (no offline fallback)", func() { // stream_begin == 0 means the loaded model is not a cache-aware // streaming model. The backend must surface that, not silently diff --git a/backend/go/parakeet-cpp/live.go b/backend/go/parakeet-cpp/live.go index 3d68a2914be1..6497779d54ee 100644 --- a/backend/go/parakeet-cpp/live.go +++ b/backend/go/parakeet-cpp/live.go @@ -41,6 +41,9 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest defer close(out) if p.ctxPtr == 0 { + if err := p.notASRError(); err != nil { + return err + } return grpcerrors.ModelNotLoaded("parakeet-cpp") } @@ -68,6 +71,23 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest // current when the RPC unwinds. defer func() { p.streamFree(stream) }() + // scene runs a no-ASR scene stream (diarization/sound only) beside the + // ASR session when a diarization_model:/sound_model: companion is loaded + // (see scene.go). A zero handle means scene events are disabled: no + // companions, or the begin/a later feed call failed (logged below / in + // feedSlicesScene), in which case live transcription continues ASR-only. + // Reassigned on a mid-stream Config reset alongside stream, which also + // brings back a scene stream the session had disabled after an earlier + // scene error. + var scene sceneStreamHandle + if p.sceneWanted() { + scene = p.sceneBegin() + if scene.s == 0 { + xlog.Warn("parakeet-cpp: scene stream begin failed; live continues without speaker/sound events") + } + } + defer func() { p.sceneFree(scene) }() + out <- &pb.TranscriptLiveResponse{Ready: true} var ( @@ -83,22 +103,31 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest behindWarned bool ) - // emit forwards one decode increment: it streams the per-feed tokens the - // realtime turn detector consumes (delta/eou/eob/words) and accumulates the - // running transcript for the closing FinalResult. No segmentation or - // boundary latch here — the live consumer reads only the streamed tokens - // and the final Text; per-utterance segments and the terminal flag - // are an offline-path concern (see AudioTranscriptionStream / boundary.go). - emit := func(r streamFeedResult) error { + // emit sends one decode increment as its own response when it carries + // anything: either the ASR side (delta/eou/eob/words, accumulated into + // the running transcript for the closing FinalResult) or the scene + // side (closed speaker/sound events), never both at once — the live + // audio loop below calls it once for the ASR result right after the ASR + // feed and, separately, once more for the scene document after the + // scene feed (see feedSlicesScene), so a slice with both produces two + // responses, ASR first. No segmentation or boundary latch here — the + // live consumer reads only the streamed tokens and the final Text; + // per-utterance segments and the terminal flag are an + // offline-path concern (see AudioTranscriptionStream / boundary.go). + emit := func(r streamFeedResult, sceneDoc sceneFeedJSON) error { if r.Delta != "" { full.WriteString(r.Delta) } - if r.Delta != "" || r.Eou || r.Eob || len(r.Words) > 0 { + speakers := liveSpeakersToProto(sceneDoc.Speakers) + sounds := liveSoundsToProto(sceneDoc.Sounds) + if r.Delta != "" || r.Eou || r.Eob || len(r.Words) > 0 || len(speakers) > 0 || len(sounds) > 0 { out <- &pb.TranscriptLiveResponse{ - Delta: r.Delta, - Eou: r.Eou, - Eob: r.Eob, - Words: liveWordsToProto(r.Words), + Delta: r.Delta, + Eou: r.Eou, + Eob: r.Eob, + Words: liveWordsToProto(r.Words), + Speakers: speakers, + Sounds: sounds, } } return nil @@ -120,8 +149,20 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest return grpcerrors.LiveTranscriptionUnsupported("parakeet-cpp", "loaded model is not a cache-aware streaming model") } + // The scene stream is freed and begun again alongside the ASR + // session, mirroring the reset above. + p.sceneFree(scene) + scene = sceneStreamHandle{} + if p.sceneWanted() { + scene = p.sceneBegin() + if scene.s == 0 { + xlog.Warn("parakeet-cpp: scene stream begin failed; live continues without speaker/sound events") + } + } full.Reset() fedSecs = 0 + behindSec = 0 + behindWarned = false case *pb.TranscriptLiveRequest_Audio: pcm := payload.Audio.GetPcm() audioSec := float64(len(pcm)) / liveSampleRate @@ -129,7 +170,9 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest start := time.Now() // nil ctx: a live session is bounded by this request channel, not a // context — cancellation is the caller closing the stream. - if err := p.feedSlices(nil, stream, pcm, emit); err != nil { + var asrWall, sceneWall time.Duration + scene, asrWall, sceneWall, err = p.feedSlicesScene(nil, stream, scene, pcm, emit) + if err != nil { return err } wallSec := time.Since(start).Seconds() @@ -139,6 +182,7 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest } xlog.Debug("parakeet-cpp: live feed", "audio_ms", int(audioSec*1000), "wall_ms", int(wallSec*1000), + "asr_wall_ms", int(asrWall.Seconds()*1000), "scene_wall_ms", int(sceneWall.Seconds()*1000), "behind_ms", int(behindSec*1000), "fed_s", fedSecs) if behindSec > 1 && !behindWarned { behindWarned = true @@ -153,9 +197,23 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest // The live FinalResult carries only Text — the authoritative full-turn // transcript the realtime core commits. Per-utterance segments, duration, // and the terminal flag are not produced on the live path. - if err := p.flushTail(stream, emit); err != nil { + if err := p.flushTail(stream, func(r streamFeedResult) error { + return emit(r, sceneFeedJSON{}) + }); err != nil { return err } + // The scene stream gets its own is_last flush (it consumes no new audio + // here, so it is not part of flushTail above); its remaining events go + // out before the terminal FinalResult, then the stream is released by the + // deferred sceneFree above. + if scene.s != 0 { + doc, err := p.sceneFeed(scene, nil, true) + if err != nil { + xlog.Warn("parakeet-cpp: live scene finalize failed", "err", err) + } else if err := emit(streamFeedResult{}, doc); err != nil { + return err + } + } out <- &pb.TranscriptLiveResponse{ FinalResult: &pb.TranscriptResult{Text: strings.TrimSpace(full.String())}, } diff --git a/backend/go/parakeet-cpp/live_test.go b/backend/go/parakeet-cpp/live_test.go index 0462ee521b3a..11dc496b62d4 100644 --- a/backend/go/parakeet-cpp/live_test.go +++ b/backend/go/parakeet-cpp/live_test.go @@ -42,22 +42,61 @@ func liveStubs() (restore func()) { savedFinalize, savedFinalizeJSON := CppStreamFinalize, CppStreamFinalizeJSON savedFree, savedLastError := CppStreamFree, CppLastError savedFreeString := CppFreeString + savedSceneOptsDefault := CppSceneOptsDefault + savedSceneBegin := CppSceneStreamBegin + savedSceneFeedJSON := CppSceneStreamFeedJSON + savedSceneLastError := CppSceneStreamLastError + savedSceneFree := CppSceneStreamFree return func() { CppStreamBegin, CppStreamBeginLang = savedBegin, savedBeginLang CppStreamFeed, CppStreamFeedJSON = savedFeed, savedFeedJSON CppStreamFinalize, CppStreamFinalizeJSON = savedFinalize, savedFinalizeJSON CppStreamFree, CppLastError = savedFree, savedLastError CppFreeString = savedFreeString + CppSceneOptsDefault = savedSceneOptsDefault + CppSceneStreamBegin = savedSceneBegin + CppSceneStreamFeedJSON = savedSceneFeedJSON + CppSceneStreamLastError = savedSceneLastError + CppSceneStreamFree = savedSceneFree } } +// liveSceneStubs wires a minimal scene stream stub set onto p (a diarization +// and/or sound companion context so sceneWanted() is true) and returns the +// call-count trackers the specs assert on. feedJSON is called once per scene +// feed (including the is_last flush) with the stream handle the C side would +// have received (so a reset spec can tell a pre-reset feed from a post-reset +// one) and must return the canned document for that call. +func liveSceneStubs(feedJSON func(calls int, s uintptr, isLast int32) uintptr) (begun, freed *int) { + begun, freed = new(int), new(int) + CppSceneOptsDefault = func(o *cSceneOpts) { *o = cSceneOpts{} } + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { + *begun++ + return uintptr(100 + *begun) + } + calls := 0 + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + calls++ + return feedJSON(calls, s, isLast) + } + CppSceneStreamLastError = func(s uintptr) string { return "scene stub error" } + CppSceneStreamFree = func(s uintptr) { *freed++ } + return begun, freed +} + // runLive starts the RPC on its own goroutine and returns the request -// channel plus a collector for everything the backend emitted. +// channel plus a collector for everything the backend emitted. GinkgoRecover +// turns an Expect/Fail failure inside a stub called from this goroutine into +// a normal spec failure instead of a panic that would crash the whole test +// binary (Ginkgo's failure handling is goroutine-local). func runLive(p *ParakeetCpp) (chan *pb.TranscriptLiveRequest, chan *pb.TranscriptLiveResponse, chan error) { in := make(chan *pb.TranscriptLiveRequest) out := make(chan *pb.TranscriptLiveResponse, 32) errCh := make(chan error, 1) - go func() { errCh <- p.AudioTranscriptionLive(in, out) }() + go func() { + defer GinkgoRecover() + errCh <- p.AudioTranscriptionLive(in, out) + }() return in, out, errCh } @@ -106,6 +145,19 @@ var _ = Describe("AudioTranscriptionLive (stubbed C API)", func() { AfterEach(func() { restore() }) + It("names the loaded role instead of a generic model-not-loaded error for a sound primary", func() { + // The ctxPtr==0 check returns before AudioTranscriptionLive ever reads + // from `in`, so nothing may be sent on it (unbuffered: a send would + // block forever waiting for a read that never happens). + p2 := &ParakeetCpp{tagCtx: 1} + in, out, errCh := runLive(p2) + close(in) + + err := <-errCh + Expect(err).To(MatchError(ContainSubstring("sound model"))) + Expect(collectLive(out)).To(BeEmpty()) + }) + It("rejects a stream whose first message is not a config", func() { in, out, errCh := runLive(p) in <- liveAudio([]float32{0.1}) @@ -368,6 +420,256 @@ var _ = Describe("AudioTranscriptionLive (stubbed C API)", func() { Expect(got).To(HaveLen(1)) // just the ready ack close(in) }) + + It("makes no scene C call and behaves unchanged when no companion is loaded", func() { + // p has ctxPtr only (no diarCtx/tagCtx): sceneWanted() must be false, + // and none of the scene entry points may be touched. + CppSceneOptsDefault = func(o *cSceneOpts) { Fail("scene_opts_default called with no companions loaded") } + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { + Fail("scene_stream_begin called with no companions loaded") + return 0 + } + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + Fail("scene_stream_feed_json called with no companions loaded") + return 0 + } + CppSceneStreamFree = func(s uintptr) { Fail("scene_stream_free called with no companions loaded") } + + CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { + return pool.cstr(`{"text":"hi","eou":0,"frame_sec":0.08,"words":[]}`) + } + CppStreamFinalizeJSON = func(s uintptr) uintptr { + return pool.cstr(`{"text":"","eou":0,"frame_sec":0.08,"words":[]}`) + } + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(3)) // ready, delta, final + Expect(got[1].Speakers).To(BeEmpty()) + Expect(got[1].Sounds).To(BeEmpty()) + }) +}) + +var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { + var ( + pool *liveCstrPool + restore func() + p *ParakeetCpp + ) + + BeforeEach(func() { + pool = &liveCstrPool{} + restore = liveStubs() + p = &ParakeetCpp{ctxPtr: 1, diarCtx: 2} + + CppStreamBeginLang = nil + CppStreamBegin = func(ctx uintptr) uintptr { return 7 } + CppStreamFree = func(s uintptr) {} + CppFreeString = func(s uintptr) {} + CppLastError = func(ctx uintptr) string { return "stub error" } + CppStreamFeed = nil + CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { + return pool.cstr(`{"text":"","eou":0,"frame_sec":0.08,"words":[]}`) + } + CppStreamFinalize = nil + CppStreamFinalizeJSON = func(s uintptr) uintptr { + return pool.cstr(`{"text":"","eou":0,"frame_sec":0.08,"words":[]}`) + } + }) + + AfterEach(func() { restore() }) + + It("emits a closed speaker segment as its own response", func() { + liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { + if calls == 1 { + return pool.cstr(`{"speakers":[{"speaker":0,"start":0.1,"end":0.6}],"sounds":[]}`) + } + return pool.cstr(`{"speakers":[],"sounds":[]}`) + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(3)) // ready, speaker-only response, final + Expect(got[1].Delta).To(BeEmpty()) + Expect(got[1].Speakers).To(HaveLen(1)) + Expect(got[1].Speakers[0].Speaker).To(Equal("0")) + Expect(got[1].Speakers[0].Start).To(Equal(int64(0.1 * 1e9))) + Expect(got[1].Speakers[0].End).To(Equal(int64(0.6 * 1e9))) + }) + + It("sends the ASR delta and a scene sound event as two responses, ASR first", func() { + CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { + return pool.cstr(`{"text":"hello ","eou":0,"frame_sec":0.08,` + + `"words":[{"w":"hello","start":0.1,"end":0.4,"conf":0.9}]}`) + } + liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { + if calls == 1 { + return pool.cstr(`{"speakers":[],"sounds":[{"index":99,` + + `"label":"Chicken, rooster","start":24.0,"end":30.0,"peak":0.86}]}`) + } + return pool.cstr(`{"speakers":[],"sounds":[]}`) + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(4)) // ready, ASR delta, sound-only, final + Expect(got[1].Delta).To(Equal("hello ")) + Expect(got[1].Sounds).To(BeEmpty(), "the ASR response must not wait on the scene feed") + Expect(got[2].Delta).To(BeEmpty()) + Expect(got[2].Sounds).To(HaveLen(1)) + Expect(got[2].Sounds[0].Label).To(Equal("Chicken, rooster")) + Expect(got[2].Sounds[0].Index).To(Equal(int32(99))) + Expect(got[2].Sounds[0].Peak).To(BeNumerically("~", 0.86, 1e-6)) + Expect(got[2].Sounds[0].Start).To(Equal(int64(24.0 * 1e9))) + Expect(got[2].Sounds[0].End).To(Equal(int64(30.0 * 1e9))) + }) + + It("flushes the scene stream is_last before the final result, then frees it", func() { + begun, freed := liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { + if calls == 2 { + Expect(isLast).To(Equal(int32(1))) + return pool.cstr(`{"speakers":[{"speaker":1,"start":1.0,"end":2.0}],"sounds":[]}`) + } + Expect(isLast).To(Equal(int32(0))) + return pool.cstr(`{"speakers":[],"sounds":[]}`) + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(3)) // ready, speaker from the is_last flush, final + Expect(got[1].Speakers).To(HaveLen(1)) + Expect(got[1].Speakers[0].Speaker).To(Equal("1")) + Expect(got[2].FinalResult).NotTo(BeNil()) + Expect(*begun).To(Equal(1)) + Expect(*freed).To(Equal(1)) + }) + + It("frees and begins the scene stream again on a mid-stream config reset", func() { + streamBegun := 0 + CppStreamBegin = func(ctx uintptr) uintptr { streamBegun++; return uintptr(10 + streamBegun) } + var seenStreams []uintptr + begun, freed := liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { + seenStreams = append(seenStreams, s) + return pool.cstr(`{"speakers":[],"sounds":[]}`) + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + in <- liveConfig("") // reset + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + collectLive(out) + + Expect(*begun).To(Equal(2), "scene stream begun again on reset") + Expect(*freed).To(Equal(2), "old scene stream freed on reset, new one on unwind") + // One scene feed per audio message (pre-reset, post-reset) plus the + // close is_last flush, which runs on the post-reset stream. + Expect(seenStreams).To(HaveLen(3)) + Expect(seenStreams[0]).NotTo(Equal(seenStreams[1]), "post-reset audio must go to the new scene stream handle") + Expect(seenStreams[2]).To(Equal(seenStreams[1]), "the close flush uses the post-reset stream too") + }) + + It("returns without a C call when Free() ran between begin and a scene feed", func() { + feedCalls := 0 + CppSceneOptsDefault = func(o *cSceneOpts) { *o = cSceneOpts{} } + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { return 999 } + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + feedCalls++ + return pool.cstr(`{"speakers":[],"sounds":[]}`) + } + + h := p.sceneBegin() + Expect(h.s).NotTo(BeZero()) + + // Simulate a Free() racing in between the begin and the next feed: it + // zeroes the companion context under engineMu, exactly as the real + // Free() does. + p.diarCtx = 0 + + _, err := p.sceneFeed(h, make([]float32, 10), false) + Expect(grpcerrors.IsModelNotLoaded(err)).To(BeTrue()) + Expect(feedCalls).To(Equal(0), "no C call once the scene stream's contexts were freed") + }) + + It("degrades to ASR-only after a mid-session scene feed failure: freed once, no more scene events, ASR keeps working", func() { + CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { + return pool.cstr(`{"text":"hi ","eou":0,"frame_sec":0.08,` + + `"words":[{"w":"hi","start":0.1,"end":0.3,"conf":0.9}]}`) + } + sceneFeedCalls := 0 + begun, freed := liveSceneStubs(func(calls int, s uintptr, isLast int32) uintptr { + sceneFeedCalls++ + return 0 // fails every call; only the first should ever be reached + }) + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) // scene feed fails here: warn, free, zero the handle + in <- liveAudio(make([]float32, 10)) // ASR-only: no scene C call at all + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + // ready, ASR delta (msg 1), ASR delta (msg 2), final: no scene-only + // response ever appears, before or after the failure. + Expect(got).To(HaveLen(4)) + Expect(got[0].Ready).To(BeTrue()) + Expect(got[1].Delta).To(Equal("hi ")) + Expect(got[2].Delta).To(Equal("hi ")) + for _, r := range got { + Expect(r.Speakers).To(BeEmpty(), "no speaker events once the scene stream has failed") + Expect(r.Sounds).To(BeEmpty(), "no sound events once the scene stream has failed") + } + Expect(got[3].FinalResult).NotTo(BeNil()) + Expect(got[3].FinalResult.Text).To(Equal("hi hi")) + + Expect(*begun).To(Equal(1)) + Expect(sceneFeedCalls).To(Equal(1), "the second audio message must not retry the broken scene stream") + Expect(*freed).To(Equal(1), "the broken scene stream is freed exactly once, not again at RPC unwind") + }) + + It("continues without scene events when scene begin fails", func() { + CppSceneOptsDefault = func(o *cSceneOpts) { *o = cSceneOpts{} } + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { return 0 } + sceneFeedCalled := false + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + sceneFeedCalled = true + return 0 + } + CppSceneStreamFree = func(s uintptr) {} + + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(got).To(HaveLen(2)) // ready, final only: no scene events, no ASR delta this stub sends + Expect(sceneFeedCalled).To(BeFalse(), "no feed call once begin failed") + }) }) var _ = Describe("stripEouMarker", func() { diff --git a/backend/go/parakeet-cpp/main.go b/backend/go/parakeet-cpp/main.go index 9c6466b131e5..865d5b6f1947 100644 --- a/backend/go/parakeet-cpp/main.go +++ b/backend/go/parakeet-cpp/main.go @@ -90,6 +90,34 @@ func main() { purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json") } + // Model roles + diarization/sound (ABI v7-v8): parakeet_capi_model_kind is + // what lets Load tell an ASR/diarization/sound context apart, so it gates + // every other new symbol below (an older libparakeet.so gets none of + // them, and companion model options are rejected in roles.go). Diarization + // itself (diarize_pcm, transcribe_and_diarize_json) predates model_kind + // (ABI v7), so it is probed on its own. + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_pcm"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppDiarizePCM, lib, "parakeet_capi_diarize_pcm") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_transcribe_and_diarize_json"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppTranscribeAndDiarizeJSON, lib, "parakeet_capi_transcribe_and_diarize_json") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_model_kind"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppModelKind, lib, "parakeet_capi_model_kind") + purego.RegisterLibFunc(&CppNumClasses, lib, "parakeet_capi_num_classes") + purego.RegisterLibFunc(&CppSoundOptsDefault, lib, "parakeet_capi_sound_opts_default") + purego.RegisterLibFunc(&CppSoundStreamBegin, lib, "parakeet_capi_sound_stream_begin") + purego.RegisterLibFunc(&CppSoundStreamFeed, lib, "parakeet_capi_sound_stream_feed") + purego.RegisterLibFunc(&CppSoundStreamDrainScoresJSON, lib, "parakeet_capi_sound_stream_drain_scores_json") + purego.RegisterLibFunc(&CppFreeSoundSegments, lib, "parakeet_capi_free_sound_segments") + purego.RegisterLibFunc(&CppSoundStreamFree, lib, "parakeet_capi_sound_stream_free") + purego.RegisterLibFunc(&CppSceneOptsDefault, lib, "parakeet_capi_scene_opts_default") + purego.RegisterLibFunc(&CppSceneStreamBegin, lib, "parakeet_capi_scene_stream_begin") + purego.RegisterLibFunc(&CppSceneStreamFeedJSON, lib, "parakeet_capi_scene_stream_feed_json") + purego.RegisterLibFunc(&CppSceneStreamLastError, lib, "parakeet_capi_scene_stream_last_error") + purego.RegisterLibFunc(&CppSceneStreamFree, lib, "parakeet_capi_scene_stream_free") + } + fmt.Fprintf(os.Stderr, "[parakeet-cpp] ABI=%d\n", CppAbiVersion()) flag.Parse() diff --git a/backend/go/parakeet-cpp/roles.go b/backend/go/parakeet-cpp/roles.go new file mode 100644 index 000000000000..c6a1e912e1c1 --- /dev/null +++ b/backend/go/parakeet-cpp/roles.go @@ -0,0 +1,245 @@ +package main + +import ( + "errors" + "fmt" + "path/filepath" + "strings" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" +) + +// Model kinds returned by parakeet_capi_model_kind (ABI v8; mirrors the +// PARAKEET_MODEL_KIND_* defines in parakeet_capi.h). +const ( + modelKindNone = 0 + modelKindASR = 1 + modelKindDiarization = 2 + modelKindSound = 3 +) + +// Diarization streaming latency modes (mirrors PARAKEET_DIAR_LATENCY_* in +// parakeet_capi.h). diarLatencyLow is the spec's default when +// diarization_latency: is unset. +const ( + diarLatencyModel int32 = 0 + diarLatencyLow int32 = 1 + diarLatencyVeryLow int32 = 2 + diarLatencyUltraLow int32 = 3 +) + +// modelKindName renders a model kind for error messages. +func modelKindName(kind int32) string { + switch kind { + case modelKindASR: + return "ASR" + case modelKindDiarization: + return "diarization" + case modelKindSound: + return "sound" + default: + return "unknown" + } +} + +// optString reads a string model option (key:value form) from ModelOptions, +// returning "" when the key is absent. Same strings.Cut parsing as optInt. +func optString(opts *pb.ModelOptions, key string) string { + for _, o := range opts.GetOptions() { + k, v, ok := strings.Cut(o, ":") + if ok && strings.TrimSpace(k) == key { + return strings.TrimSpace(v) + } + } + return "" +} + +// resolveModelPath resolves a companion model option's path against +// modelPath (opts.ModelPath, the LocalAI models root): an absolute p, or an +// empty modelPath, passes through unchanged; anything else is joined onto +// modelPath. Mirrors vibevoice-cpp's resolvePath for tokenizer=/voice=/etc. +func resolveModelPath(modelPath, p string) string { + if p == "" || filepath.IsAbs(p) || modelPath == "" { + return p + } + return filepath.Join(modelPath, p) +} + +// parseDiarLatency maps the diarization_latency option value to a +// PARAKEET_DIAR_LATENCY_* mode. "" defaults to "low" (the spec's default); +// any other unrecognized value is a Load error. +func parseDiarLatency(s string) (int32, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "": + return diarLatencyLow, nil + case "model": + return diarLatencyModel, nil + case "low": + return diarLatencyLow, nil + case "very_low": + return diarLatencyVeryLow, nil + case "ultra_low": + return diarLatencyUltraLow, nil + default: + return 0, fmt.Errorf("parakeet-cpp: unknown diarization_latency %q (want model|low|very_low|ultra_low)", s) + } +} + +// companionSpec is one asr_model:/diarization_model:/sound_model: option: its +// name (for error messages and path resolution), the raw option value, the +// model kind the loaded companion must report, the ParakeetCpp field it is +// assigned to on success, and a getter for that same field's current value +// (used to reject a companion whose role the primary already occupies). +type companionSpec struct { + optName string + value string + wantKind int32 + assign func(*ParakeetCpp, uintptr) + current func(*ParakeetCpp) uintptr +} + +// indefiniteArticle returns "an" for a word starting with a vowel sound and +// "a" otherwise, for grammatical error messages built from modelKindName. +func indefiniteArticle(word string) string { + if len(word) == 0 { + return "a" + } + switch word[0] { + case 'A', 'E', 'I', 'O', 'U', 'a', 'e', 'i', 'o', 'u': + return "an" + default: + return "a" + } +} + +// loadRoles loads opts.ModelFile as the primary parakeet_ctx, classifies it +// with parakeet_capi_model_kind (ABI v8) into ctxPtr/diarCtx/tagCtx, and +// loads any companion models named in Options[] (asr_model:, +// diarization_model:, sound_model:; paths resolved against opts.ModelPath). +// It also parses diarization_latency: into p.diarLatency. +// +// Against an older libparakeet.so (CppModelKind == nil) the primary is +// treated as ASR — the pre-v8 behavior — and companion model options are +// rejected outright, since there is no way to verify what they loaded. +// +// On any failure every context this call opened (primary and any companions +// loaded before the failure) is freed before the error is returned. +func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { + diarModelOpt := optString(opts, "diarization_model") + asrModelOpt := optString(opts, "asr_model") + soundModelOpt := optString(opts, "sound_model") + hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" + + if hasCompanionOpts && CppModelKind == nil { + return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model options need " + + "parakeet_capi_model_kind (ABI v8) to verify what they load; the loaded libparakeet.so " + + "is too old to report companion model roles") + } + + latency, err := parseDiarLatency(optString(opts, "diarization_latency")) + if err != nil { + return err + } + + primary := CppLoad(opts.ModelFile) + if primary == 0 { + // No ctx to ask for last_error (the C-API's last-error buffer lives on + // the ctx that was never returned). Surface the path so the operator + // at least knows which load failed. + return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for %q", opts.ModelFile) + } + loaded := []uintptr{primary} + // freeLoaded undoes everything loadRoles opened this call: every context + // it freed AND every ParakeetCpp field it may have assigned (the primary + // lands in one of ctxPtr/diarCtx/tagCtx before the companion loop runs, + // and an earlier companion's spec.assign runs before a later one fails). + // Leaving a role field pointing at a freed ctx would double-free it on a + // later Free() call. + freeLoaded := func() { + for _, c := range loaded { + CppFree(c) + } + p.ctxPtr, p.diarCtx, p.tagCtx = 0, 0, 0 + p.companions = nil + } + + primaryKind := int32(modelKindASR) // old-library default: today's behavior + if CppModelKind != nil { + primaryKind = CppModelKind(primary) + if primaryKind == modelKindNone { + xlog.Warn("parakeet-cpp: parakeet_capi_model_kind reported PARAKEET_MODEL_KIND_NONE " + + "for a successfully loaded primary; treating it as an ASR model") + } + } + switch primaryKind { + case modelKindDiarization: + p.diarCtx = primary + case modelKindSound: + p.tagCtx = primary + default: + p.ctxPtr = primary + } + + specs := []companionSpec{ + {"diarization_model", diarModelOpt, modelKindDiarization, + func(pp *ParakeetCpp, c uintptr) { pp.diarCtx = c }, + func(pp *ParakeetCpp) uintptr { return pp.diarCtx }}, + {"asr_model", asrModelOpt, modelKindASR, + func(pp *ParakeetCpp, c uintptr) { pp.ctxPtr = c }, + func(pp *ParakeetCpp) uintptr { return pp.ctxPtr }}, + {"sound_model", soundModelOpt, modelKindSound, + func(pp *ParakeetCpp, c uintptr) { pp.tagCtx = c }, + func(pp *ParakeetCpp) uintptr { return pp.tagCtx }}, + } + for _, spec := range specs { + if spec.value == "" { + continue + } + // A companion whose role the primary already occupies (e.g. asr_model: + // on an already-ASR primary) would overwrite that role field below, + // leaking the primary ctx: Free() only walks ctxPtr/diarCtx/tagCtx, so + // the overwritten pointer is never freed. Reject it before loading. + if spec.current(p) != 0 { + freeLoaded() + return fmt.Errorf("parakeet-cpp: %s is not allowed on %s %s model", + spec.optName, indefiniteArticle(modelKindName(spec.wantKind)), modelKindName(spec.wantKind)) + } + resolved := resolveModelPath(opts.ModelPath, spec.value) + cctx := CppLoad(resolved) + if cctx == 0 { + freeLoaded() + return fmt.Errorf("parakeet-cpp: failed to load %s %q", spec.optName, resolved) + } + loaded = append(loaded, cctx) + if gotKind := CppModelKind(cctx); gotKind != spec.wantKind { + freeLoaded() + return fmt.Errorf("parakeet-cpp: %s %q is a %s model, expected a %s model", + spec.optName, resolved, modelKindName(gotKind), modelKindName(spec.wantKind)) + } + spec.assign(p, cctx) + p.companions = append(p.companions, cctx) + } + + p.diarLatency = latency + return nil +} + +// notASRError reports why AudioTranscription (and the streaming/live RPCs) +// cannot run when p.ctxPtr == 0: a loaded diarization or sound primary with +// no asr_model companion, named explicitly so the caller knows to use the +// right RPC instead of a generic "model not loaded". Returns nil when +// neither role is loaded (genuinely no model), leaving the caller to report +// the ordinary ModelNotLoaded error. +func (p *ParakeetCpp) notASRError() error { + switch { + case p.diarCtx != 0: + return errors.New("parakeet-cpp: loaded model is a diarization model, not ASR " + + "(use Diarize, or load with an asr_model: companion)") + case p.tagCtx != 0: + return errors.New("parakeet-cpp: loaded model is a sound model, not ASR " + + "(use SoundDetection)") + default: + return nil + } +} diff --git a/backend/go/parakeet-cpp/roles_test.go b/backend/go/parakeet-cpp/roles_test.go new file mode 100644 index 000000000000..7b2a892b333d --- /dev/null +++ b/backend/go/parakeet-cpp/roles_test.go @@ -0,0 +1,417 @@ +package main + +import ( + "context" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The role-loading specs drive Load/Free entirely against stubbed +// CppLoad/CppModelKind/CppFree (the same seam live_test.go and +// batcher_test.go use), so they run without libparakeet.so. + +// fakeLib is a tiny in-memory stand-in for libparakeet.so: paths registered +// via withModel resolve to a fresh ctx handle of the given model kind on +// CppLoad, any other path fails the load, CppModelKind reads the kind back +// by ctx, and CppFree records every ctx it was asked to free, in order. +type fakeLib struct { + kinds map[string]int32 + ctxKind map[uintptr]int32 + next uintptr + loadedPaths []string + freed []uintptr +} + +func newFakeLib() *fakeLib { + return &fakeLib{kinds: map[string]int32{}, ctxKind: map[uintptr]int32{}, next: 1} +} + +func (f *fakeLib) withModel(path string, kind int32) *fakeLib { + f.kinds[path] = kind + return f +} + +// install swaps CppLoad/CppModelKind/CppFree for fakes backed by f and +// returns a restore func for AfterEach (mirrors live_test.go's liveStubs). +func (f *fakeLib) install() (restore func()) { + savedLoad, savedKind, savedFree := CppLoad, CppModelKind, CppFree + CppLoad = func(path string) uintptr { + f.loadedPaths = append(f.loadedPaths, path) + kind, ok := f.kinds[path] + if !ok { + return 0 + } + ctx := f.next + f.next++ + f.ctxKind[ctx] = kind + return ctx + } + CppModelKind = func(ctx uintptr) int32 { return f.ctxKind[ctx] } + CppFree = func(ctx uintptr) { f.freed = append(f.freed, ctx) } + return func() { + CppLoad, CppModelKind, CppFree = savedLoad, savedKind, savedFree + } +} + +var _ = Describe("model roles (stubbed C API)", func() { + var restore func() + + AfterEach(func() { + if restore != nil { + restore() + restore = nil + } + }) + + It("loads an ASR primary with no options", func() { + f := newFakeLib().withModel("asr.gguf", modelKindASR) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "asr.gguf"})).To(Succeed()) + + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + }) + + It("loads a diarization primary and rejects AudioTranscription without any further C call", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "diar.gguf"})).To(Succeed()) + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.ctxPtr).To(BeZero()) + + // CppTranscribePathJSON/CppTranscribePcmBatchJSON are left nil (the + // zero value): if AudioTranscription tried to call either, this would + // panic instead of returning cleanly, so a clean typed error here also + // proves no C call was made. + _, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: "x.wav"}) + Expect(err).To(MatchError(ContainSubstring("diarization model"))) + }) + + It("loads a sound primary", func() { + f := newFakeLib().withModel("sound.gguf", modelKindSound) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "sound.gguf"})).To(Succeed()) + + Expect(p.tagCtx).ToNot(BeZero()) + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + }) + + It("loads an ASR primary plus diarization and sound companions, and Free releases all three", func() { + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("/models/x.gguf", modelKindDiarization). + withModel("/abs/y.gguf", modelKindSound) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + ModelPath: "/models", + Options: []string{"diarization_model:x.gguf", "sound_model:/abs/y.gguf"}, + }) + Expect(err).ToNot(HaveOccurred()) + + Expect(f.loadedPaths).To(Equal([]string{"asr.gguf", "/models/x.gguf", "/abs/y.gguf"}), + "diarization_model resolves against ModelPath, sound_model's absolute path passes through") + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.tagCtx).ToNot(BeZero()) + Expect(p.companions).To(HaveLen(2)) + + asrCtx, diarCtx, tagCtx := p.ctxPtr, p.diarCtx, p.tagCtx + Expect(p.Free()).To(Succeed()) + Expect(f.freed).To(ConsistOf(asrCtx, diarCtx, tagCtx)) + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + }) + + It("loads a diarization primary plus an asr_model companion into ctxPtr", func() { + f := newFakeLib(). + withModel("diar.gguf", modelKindDiarization). + withModel("/models/z.gguf", modelKindASR) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "diar.gguf", + ModelPath: "/models", + Options: []string{"asr_model:z.gguf"}, + }) + Expect(err).ToNot(HaveOccurred()) + + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.ctxPtr).ToNot(Equal(p.diarCtx)) + }) + + It("fails a companion of the wrong kind and frees every ctx it opened", func() { + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("/models/wrong.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + ModelPath: "/models", + Options: []string{"sound_model:wrong.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sound_model")) + Expect(err.Error()).To(ContainSubstring("diarization")) + Expect(f.freed).To(HaveLen(2), "the primary and the wrong-kind companion must both be freed") + + // Every role field the failed load may have assigned (the primary + // lands in ctxPtr before the companion loop runs) must be reset, or a + // later Free() would double-free an already-freed context. + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + Expect(p.companions).To(BeEmpty()) + + freedBeforeFree := len(f.freed) + Expect(p.Free()).To(Succeed()) + Expect(f.freed).To(HaveLen(freedBeforeFree), "Free after a failed Load must not free anything again") + }) + + It("rejects an asr_model companion on an already-ASR primary and frees everything it opened", func() { + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("/models/other.gguf", modelKindASR) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + ModelPath: "/models", + Options: []string{"asr_model:other.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(`parakeet-cpp: asr_model is not allowed on an ASR model`)) + Expect(f.loadedPaths).To(Equal([]string{"asr.gguf"})) + Expect(f.freed).To(HaveLen(1), "the primary must be freed too, or it leaks") + + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + Expect(p.companions).To(BeEmpty()) + }) + + It("rejects a diarization_model companion on an already-diarization primary", func() { + f := newFakeLib(). + withModel("diar.gguf", modelKindDiarization). + withModel("/models/other.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "diar.gguf", + ModelPath: "/models", + Options: []string{"diarization_model:other.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(`parakeet-cpp: diarization_model is not allowed on a diarization model`)) + Expect(f.loadedPaths).To(Equal([]string{"diar.gguf"})) + Expect(p.diarCtx).To(BeZero()) + }) + + It("rejects a sound_model companion on an already-sound (CED) primary", func() { + f := newFakeLib(). + withModel("sound.gguf", modelKindSound). + withModel("/models/other.gguf", modelKindSound) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "sound.gguf", + ModelPath: "/models", + Options: []string{"sound_model:other.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(`parakeet-cpp: sound_model is not allowed on a sound model`)) + Expect(f.loadedPaths).To(Equal([]string{"sound.gguf"})) + Expect(p.tagCtx).To(BeZero()) + }) + + It("does not overwrite ctxPtr (and so does not leak the primary) when an asr_model companion "+ + "duplicates an ASR primary loaded alongside other companions", func() { + // Regression for the leak this whole check exists to close: before + // the fix, spec.assign(p, cctx) overwrote p.ctxPtr with the + // companion's ctx, and Free() (which only walks + // ctxPtr/diarCtx/tagCtx) never saw the original primary again. + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("/models/diar.gguf", modelKindDiarization). + withModel("/models/dup.gguf", modelKindASR) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + ModelPath: "/models", + // diarization_model loads first (declared first in loadRoles' + // specs) and succeeds; asr_model then collides with the primary. + Options: []string{"diarization_model:diar.gguf", "asr_model:dup.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(f.freed).To(HaveLen(2), "the primary and the already-loaded diarization companion") + Expect(f.loadedPaths).To(Equal([]string{"asr.gguf", "/models/diar.gguf"}), + "dup.gguf must never be loaded: the role check runs before CppLoad") + + Expect(p.ctxPtr).To(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.companions).To(BeEmpty()) + }) + + It("treats a primary reporting PARAKEET_MODEL_KIND_NONE as ASR", func() { + f := newFakeLib().withModel("model.gguf", modelKindNone) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "model.gguf"})).To(Succeed()) + + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + }) + + It("parses diarization_latency:very_low", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "diar.gguf", + Options: []string{"diarization_latency:very_low"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(p.diarLatency).To(Equal(int32(2))) + }) + + It("defaults diarization_latency to low (1) when unset", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "diar.gguf"})).To(Succeed()) + Expect(p.diarLatency).To(Equal(int32(1))) + }) + + It("rejects an invalid diarization_latency before any C call", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "diar.gguf", + Options: []string{"diarization_latency:bogus"}, + }) + Expect(err).To(HaveOccurred()) + Expect(f.loadedPaths).To(BeEmpty()) + }) + + Context("old library (no parakeet_capi_model_kind)", func() { + It("treats the primary as ASR, matching pre-v8 behavior", func() { + f := newFakeLib().withModel("model.gguf", modelKindDiarization) // kind is never consulted + restore = f.install() + CppModelKind = nil + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "model.gguf"})).To(Succeed()) + + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).To(BeZero()) + Expect(p.tagCtx).To(BeZero()) + }) + + It("rejects companion model options with an error naming the library as too old", func() { + f := newFakeLib().withModel("asr.gguf", modelKindASR) + restore = f.install() + CppModelKind = nil + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "asr.gguf", + Options: []string{"diarization_model:diar.gguf"}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("too old")) + Expect(f.loadedPaths).To(BeEmpty(), "rejected before any C call") + }) + }) + + Context("batcher gating", func() { + It("does not start the batcher for a non-ASR primary with no asr_model companion", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + savedBatch := CppTranscribePcmBatchJSON + defer func() { CppTranscribePcmBatchJSON = savedBatch }() + CppTranscribePcmBatchJSON = func(uintptr, []float32, []int32, int32, int32, int32) uintptr { return 0 } + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "diar.gguf"})).To(Succeed()) + Expect(p.bat).To(BeNil()) + }) + + It("starts the batcher for an ASR ctxPtr", func() { + f := newFakeLib().withModel("asr.gguf", modelKindASR) + restore = f.install() + savedBatch := CppTranscribePcmBatchJSON + defer func() { CppTranscribePcmBatchJSON = savedBatch }() + CppTranscribePcmBatchJSON = func(uintptr, []float32, []int32, int32, int32, int32) uintptr { return 0 } + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "asr.gguf"})).To(Succeed()) + Expect(p.bat).ToNot(BeNil()) + Expect(p.Free()).To(Succeed()) // stop the dispatcher goroutine + }) + }) +}) + +var _ = Describe("resolveModelPath", func() { + It("keeps an absolute path unchanged", func() { + Expect(resolveModelPath("/models", "/abs/y.gguf")).To(Equal("/abs/y.gguf")) + }) + + It("joins a relative path onto modelPath", func() { + Expect(resolveModelPath("/models", "x.gguf")).To(Equal("/models/x.gguf")) + }) + + It("passes a relative path through when modelPath is empty", func() { + Expect(resolveModelPath("", "x.gguf")).To(Equal("x.gguf")) + }) +}) + +var _ = Describe("parseDiarLatency", func() { + It("defaults to low (1) for an empty value", func() { + v, err := parseDiarLatency("") + Expect(err).ToNot(HaveOccurred()) + Expect(v).To(Equal(int32(1))) + }) + + It("maps every named mode", func() { + for s, want := range map[string]int32{ + "model": 0, "low": 1, "very_low": 2, "ultra_low": 3, + } { + v, err := parseDiarLatency(s) + Expect(err).ToNot(HaveOccurred()) + Expect(v).To(Equal(want), "mode %q", s) + } + }) + + It("rejects an unknown value", func() { + _, err := parseDiarLatency("bogus") + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/backend/go/parakeet-cpp/scene.go b/backend/go/parakeet-cpp/scene.go new file mode 100644 index 000000000000..590740f13a05 --- /dev/null +++ b/backend/go/parakeet-cpp/scene.go @@ -0,0 +1,258 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// sceneSpeakerJSON mirrors one element of a scene feed document's "speakers" +// array: {"speaker":0,"start":0.0,"end":0.6}. +type sceneSpeakerJSON struct { + Speaker int `json:"speaker"` + Start float64 `json:"start"` + End float64 `json:"end"` +} + +// sceneSoundJSON mirrors one element of a scene feed document's "sounds" +// array: {"index":99,"label":"Chicken, rooster","start":24.0,"end":30.0,"peak":0.86}. +type sceneSoundJSON struct { + Index int `json:"index"` + Label string `json:"label"` + Start float64 `json:"start"` + End float64 `json:"end"` + Peak float32 `json:"peak"` +} + +// sceneFeedJSON mirrors the subset of the document +// parakeet_capi_scene_stream_feed_json returns (docs/sound.md) that the live +// path consumes: the closed "speakers" and "sounds" arrays. "t", +// "utterances", "words" and "active" belong to an offline scene/SAS +// consumer, not the live path, and are not decoded here. +type sceneFeedJSON struct { + Speakers []sceneSpeakerJSON `json:"speakers"` + Sounds []sceneSoundJSON `json:"sounds"` +} + +// sceneWanted reports whether AudioTranscriptionLive should run a companion +// scene stream beside the ASR streaming session: at least one of the +// diarization/sound companions must be loaded, and the scene C-API symbols +// must be present. In practice the nil checks are defensive rather than +// live: loadRoles only ever sets diarCtx/tagCtx when parakeet_capi_model_kind +// (ABI v8) is present, and main.go registers every scene symbol in the same +// Dlsym-gated block as model_kind, so a companion being loaded already +// guarantees the scene symbols exist. +func (p *ParakeetCpp) sceneWanted() bool { + return (p.diarCtx != 0 || p.tagCtx != 0) && + CppSceneOptsDefault != nil && CppSceneStreamBegin != nil && + CppSceneStreamFeedJSON != nil && CppSceneStreamFree != nil +} + +// sceneStreamHandle bundles the C scene_stream pointer with the diar/tag +// contexts it was begun with. sceneFeed re-checks those against p.diarCtx/ +// p.tagCtx under engineMu before every call, so a Free() racing between the +// begin and a later feed (freeing the very contexts the stream borrows) is +// caught instead of handed to the C side — mirroring streamFeedDoc's re-check +// of p.ctxPtr (see the "Per-C-call engine serialization" comment in +// goparakeetcpp.go). The zero value (s == 0) means "no scene stream". +type sceneStreamHandle struct { + s uintptr + diar uintptr + tag uintptr +} + +// sceneBegin opens a no-ASR scene stream (diarization and/or sound events +// only; the live path's own ASR session already covers transcription) under +// engineMu. Call only when sceneWanted() is true. Refuses to begin with both +// contexts 0 (defensive: sceneWanted() already guards this). A zero handle +// means the C call itself failed; the caller logs a warning and continues +// the live session without speaker/sound events. +func (p *ParakeetCpp) sceneBegin() sceneStreamHandle { + p.engineMu.Lock() + defer p.engineMu.Unlock() + diar, tag := p.diarCtx, p.tagCtx + if diar == 0 && tag == 0 { + return sceneStreamHandle{} + } + var opts cSceneOpts + CppSceneOptsDefault(&opts) + opts.DiarLatency = p.diarLatency + // The live scene path never drains sound scores (unlike the offline + // SoundDetection RPC, see sound.go), so the default top_k of 5 would + // leave the C side's per-window score queue growing for the session's + // whole lifetime. 0 disables per-class score retention; sound EVENTS + // (onset/offset, what the live path actually consumes) are unaffected. + opts.Sound.TopK = 0 + s := CppSceneStreamBegin(0, diar, tag, &opts) + if s == 0 { + return sceneStreamHandle{} + } + return sceneStreamHandle{s: s, diar: diar, tag: tag} +} + +// sceneFree releases a scene stream opened by sceneBegin. A zero handle +// (scene events disabled or never began) is a no-op. Safe to call even after +// the contexts the stream borrowed have been freed: parakeet_scene_stream's +// destructor only releases its own buffers and never dereferences the +// borrowed asr/diar/tagger pointers (verified against +// parakeet.cpp's parakeet_capi_scene_stream_free / SceneStream::~SceneStream +// / DiarPcmStream::~DiarPcmStream, all `= default`), unlike a feed call. +func (p *ParakeetCpp) sceneFree(h sceneStreamHandle) { + if h.s == 0 { + return + } + p.engineMu.Lock() + defer p.engineMu.Unlock() + CppSceneStreamFree(h.s) +} + +// sceneFeed runs one scene-stream feed (or the is_last flush) under +// engineMu and returns the parsed document. Before touching the C side it +// re-checks that p.diarCtx/p.tagCtx still match what the stream was begun +// with: Free() can run between the caller's ASR feed and this call (both +// take engineMu individually, never for a session's lifetime, so nothing +// blocks a concurrent Free()) and free the very model the stream borrows. +// A mismatch returns ModelNotLoaded without making the C call; last_error is +// otherwise stream-scoped (parakeet_capi_scene_stream_last_error), read +// under the same lock as the failing call. +func (p *ParakeetCpp) sceneFeed(h sceneStreamHandle, pcm []float32, isLast bool) (sceneFeedJSON, error) { + p.engineMu.Lock() + defer p.engineMu.Unlock() + + if p.diarCtx != h.diar || p.tagCtx != h.tag { + return sceneFeedJSON{}, grpcerrors.ModelNotLoaded("parakeet-cpp") + } + + var last int32 + if isLast { + last = 1 + } + var ptr *float32 + if len(pcm) > 0 { + ptr = &pcm[0] + } + ret := CppSceneStreamFeedJSON(h.s, ptr, int32(len(pcm)), last) + if ret == 0 { + msg := "" + if CppSceneStreamLastError != nil { + msg = CppSceneStreamLastError(h.s) + } + if msg == "" { + msg = "unknown error" + } + return sceneFeedJSON{}, fmt.Errorf("parakeet-cpp: scene stream feed failed: %s", msg) + } + raw := goStringFromCPtr(ret) + CppFreeString(ret) + var doc sceneFeedJSON + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return sceneFeedJSON{}, fmt.Errorf("parakeet-cpp: decode scene json: %w", err) + } + return doc, nil +} + +// feedSlicesScene mirrors driver.go's feedSlices but also feeds the same pcm +// slice to an optional companion scene stream right after each ASR slice, so +// the live path's speaker/sound events stay time-aligned with the ASR decode +// increments. scene.s == 0 disables scene feeding for this call (no +// companions, or a previous scene feed already disabled it this session). +// +// The ASR result is emitted immediately after the ASR feed — the same +// response contents/timing a no-companion session would produce — before the +// scene feed for that slice runs, so a companion model never adds scene +// compute latency in front of the ASR delta/ that drives realtime turn +// detection. Any closed speakers/sounds from the scene feed are emitted +// afterward as their own response, so a slice with both produces two +// responses, ASR first. +// +// A scene feed failure degrades gracefully rather than aborting live +// transcription over a secondary feature: it frees the broken stream, warns +// once, and zeroes the handle so the caller carries the ASR-only session +// forward. Returns the (possibly now-zeroed) scene handle plus the +// cumulative ASR and scene wall time this call spent in feedChunk/sceneFeed, +// for the caller's lag log line. +func (p *ParakeetCpp) feedSlicesScene(ctx context.Context, stream uintptr, scene sceneStreamHandle, pcm []float32, onFeed func(streamFeedResult, sceneFeedJSON) error) (sceneStreamHandle, time.Duration, time.Duration, error) { + var asrWall, sceneWall time.Duration + for off := 0; off < len(pcm); off += streamChunkSamples { + if ctx != nil { + if err := ctx.Err(); err != nil { + return scene, asrWall, sceneWall, status.Error(codes.Canceled, "transcription cancelled") + } + } + end := min(off+streamChunkSamples, len(pcm)) + chunk := pcm[off:end] + + asrStart := time.Now() + res, err := p.feedChunk(stream, chunk, false) + asrWall += time.Since(asrStart) + if err != nil { + return scene, asrWall, sceneWall, err + } + if err := onFeed(res, sceneFeedJSON{}); err != nil { + return scene, asrWall, sceneWall, err + } + + if scene.s == 0 { + continue + } + sceneStart := time.Now() + sceneDoc, serr := p.sceneFeed(scene, chunk, false) + sceneWall += time.Since(sceneStart) + if serr != nil { + xlog.Warn("parakeet-cpp: live scene feed failed; disabling speaker/sound events for this session", + "err", serr) + p.sceneFree(scene) + scene = sceneStreamHandle{} + continue + } + if err := onFeed(streamFeedResult{}, sceneDoc); err != nil { + return scene, asrWall, sceneWall, err + } + } + return scene, asrWall, sceneWall, nil +} + +// liveSpeakersToProto maps a scene feed document's closed "speakers" into +// TranscriptLiveResponse.speakers (stream-relative nanoseconds). Reuses +// diarize.go's speakerLabel so the live path renders speaker indices the +// same way the offline Diarize RPC does. +func liveSpeakersToProto(speakers []sceneSpeakerJSON) []*pb.LiveSpeakerSegment { + if len(speakers) == 0 { + return nil + } + out := make([]*pb.LiveSpeakerSegment, len(speakers)) + for i, s := range speakers { + out[i] = &pb.LiveSpeakerSegment{ + Speaker: speakerLabel(s.Speaker), + Start: secondsToNanos(s.Start), + End: secondsToNanos(s.End), + } + } + return out +} + +// liveSoundsToProto maps a scene feed document's closed "sounds" into +// TranscriptLiveResponse.sounds (stream-relative nanoseconds). +func liveSoundsToProto(sounds []sceneSoundJSON) []*pb.LiveSoundEvent { + if len(sounds) == 0 { + return nil + } + out := make([]*pb.LiveSoundEvent, len(sounds)) + for i, s := range sounds { + out[i] = &pb.LiveSoundEvent{ + Label: s.Label, + Index: int32(s.Index), + Peak: s.Peak, + Start: secondsToNanos(s.Start), + End: secondsToNanos(s.End), + } + } + return out +} diff --git a/backend/go/parakeet-cpp/scene_test.go b/backend/go/parakeet-cpp/scene_test.go new file mode 100644 index 000000000000..9dbfa22055a2 --- /dev/null +++ b/backend/go/parakeet-cpp/scene_test.go @@ -0,0 +1,42 @@ +package main + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The sceneBegin spec drives it entirely against stubbed +// CppSceneOptsDefault/CppSceneStreamBegin (the same seam live_test.go uses +// for the full live-session scene specs), so it runs without libparakeet.so. + +var _ = Describe("ParakeetCpp.sceneBegin", func() { + It("forces Sound.TopK to 0 and carries p.diarLatency into the begin opts", func() { + savedOptsDefault := CppSceneOptsDefault + savedBegin := CppSceneStreamBegin + defer func() { + CppSceneOptsDefault = savedOptsDefault + CppSceneStreamBegin = savedBegin + }() + + // parakeet_capi_scene_opts_default's real default is top_k = 5 (a + // sound-window score history); simulate that here so the test proves + // sceneBegin overrides it rather than merely never setting it. + CppSceneOptsDefault = func(o *cSceneOpts) { + *o = cSceneOpts{Sound: cSoundOpts{TopK: 5}} + } + var gotOpts cSceneOpts + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { + gotOpts = *o + return 1 + } + + p := &ParakeetCpp{diarCtx: 42, diarLatency: diarLatencyVeryLow} + h := p.sceneBegin() + Expect(h.s).ToNot(BeZero()) + Expect(gotOpts.Sound.TopK).To(Equal(int32(0)), + "the live scene path never drains sound scores (see sound.go's SoundDetection, "+ + "which does); a nonzero top_k leaves the C side's per-window score queue "+ + "growing for the session's lifetime") + Expect(gotOpts.DiarLatency).To(Equal(diarLatencyVeryLow)) + }) +}) diff --git a/backend/go/parakeet-cpp/sound.go b/backend/go/parakeet-cpp/sound.go new file mode 100644 index 000000000000..1a3a358ad09d --- /dev/null +++ b/backend/go/parakeet-cpp/sound.go @@ -0,0 +1,261 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// soundFeedChunkSamples is how much 16 kHz mono PCM soundStreamScores hands +// to sound_stream_feed per call (10 s), matching the window/hop it asks for +// below. The clip is fed in these pieces with is_last set on the final one, +// mirroring the streaming ASR path's chunked feed. +const soundFeedChunkSamples = 10 * 16000 + +// soundTagJSON mirrors one element of a soundWindowJSON's "tags" array. +type soundTagJSON struct { + Index int `json:"index"` + Label string `json:"label"` + Score float32 `json:"score"` +} + +// soundWindowJSON mirrors one element of the array +// parakeet_capi_sound_stream_drain_scores_json returns: +// +// [{"start":0.0,"end":10.0,"tags":[{"index":0,"label":"Speech","score":0.93}, ...]}] +type soundWindowJSON struct { + Start float64 `json:"start"` + End float64 `json:"end"` + Tags []soundTagJSON `json:"tags"` +} + +// classAvg is one class's score averaged across the drained windows, plus +// the label the tagger reported for it. +type classAvg struct { + Index int + Label string + Score float32 +} + +// SoundDetection runs the loaded CED model (p.tagCtx) over the clip at +// req.Src through a one-shot sound stream (window 10 s, hop 10 s, top_k set +// to the tagger's full class count so every window's drain carries a score +// for every class), averages each class's score across the drained windows, +// sorts descending, applies req.Threshold, then req.TopK (0 = all classes). +func (p *ParakeetCpp) SoundDetection(ctx context.Context, req *pb.SoundDetectionRequest) (*pb.SoundDetectionResponse, error) { + if p.tagCtx == 0 { + return nil, status.Error(codes.FailedPrecondition, + "parakeet-cpp: model is not a sound (CED) model") + } + if CppSoundStreamBegin == nil || CppSoundStreamFeed == nil || CppSoundStreamDrainScoresJSON == nil || + CppSoundStreamFree == nil || CppSoundOptsDefault == nil || CppNumClasses == nil { + return nil, status.Error(codes.Unimplemented, + "parakeet-cpp: loaded libparakeet.so has no sound-event detection support "+ + "(parakeet_capi_sound_stream_* missing)") + } + if req.GetSrc() == "" { + return nil, status.Error(codes.InvalidArgument, + "parakeet-cpp: SoundDetectionRequest.src (audio path) is required") + } + + pcm, _, err := decodeWavMono16k(req.GetSrc()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "parakeet-cpp: decode audio: %s", err) + } + + windows, nClasses, err := p.soundStreamScores(ctx, pcm) + if err != nil { + return nil, err + } + + avgs := averageWindowScores(windows, nClasses) + sortSoundDetectionsDesc(avgs) + avgs = filterSoundDetections(avgs, req.GetThreshold(), req.GetTopK()) + + resp := &pb.SoundDetectionResponse{Detections: make([]*pb.SoundClass, 0, len(avgs))} + for _, a := range avgs { + resp.Detections = append(resp.Detections, &pb.SoundClass{ + Label: a.Label, + Score: a.Score, + Index: int32(a.Index), + }) + } + return resp, nil +} + +// soundStreamScores runs pcm through a fresh sound stream and returns the +// drained per-window scores plus the tagger's class count. The C calls run +// under engineMu (see soundStreamDrain); JSON decoding happens after the +// lock is released. +func (p *ParakeetCpp) soundStreamScores(ctx context.Context, pcm []float32) ([]soundWindowJSON, int, error) { + doc, nClasses, err := p.soundStreamDrain(ctx, pcm) + if err != nil { + return nil, nClasses, err + } + + var windows []soundWindowJSON + if err := json.Unmarshal([]byte(doc), &windows); err != nil { + return nil, nClasses, fmt.Errorf("parakeet-cpp: decode sound scores json: %w", err) + } + return windows, nClasses, nil +} + +// soundStreamDrain runs pcm through a fresh sound stream and returns the +// raw JSON document parakeet_capi_sound_stream_drain_scores_json drained, +// plus the tagger's class count. Every C call (opts default, begin, feed, +// free, drain) runs under engineMu; the stream is freed (deferred right +// after a successful begin) even when a later feed or drain call fails, or +// ctx is cancelled mid-feed. Each feed's returned segments array is freed +// with parakeet_capi_free_sound_segments even though SoundDetection has no +// use for the segments themselves (it only reads the drained window +// scores). ctx.Err() is checked before each feed slice, mirroring +// driver.go's feedSlices, so a long clip can be cancelled mid-feed; the +// caller decodes the returned JSON outside the lock. +func (p *ParakeetCpp) soundStreamDrain(ctx context.Context, pcm []float32) (string, int, error) { + p.engineMu.Lock() + defer p.engineMu.Unlock() + + // SoundDetection's own p.tagCtx==0 check runs before this lock is taken; + // re-check here so a Free() racing in between (which zeroes p.tagCtx + // under this same engineMu) is caught instead of handed to the C side, + // mirroring streamFeedDoc's/sceneFeed's re-check. + if p.tagCtx == 0 { + return "", 0, grpcerrors.ModelNotLoaded("parakeet-cpp") + } + + nClasses := int(CppNumClasses(p.tagCtx)) + + var opts cSoundOpts + CppSoundOptsDefault(&opts) + opts.WindowSec = 10 + opts.HopSec = 10 + opts.TopK = int32(nClasses) + + stream := CppSoundStreamBegin(p.tagCtx, &opts) + if stream == 0 { + return "", nClasses, fmt.Errorf("parakeet-cpp: sound_stream_begin failed: %s", soundLastError(p.tagCtx)) + } + defer CppSoundStreamFree(stream) + + offset := 0 + for { + if ctx != nil { + if err := ctx.Err(); err != nil { + return "", nClasses, status.Error(codes.Canceled, "parakeet-cpp: sound detection cancelled") + } + } + + end := offset + soundFeedChunkSamples + isLast := int32(0) + if end >= len(pcm) { + end = len(pcm) + isLast = 1 + } + var samplePtr *float32 + if end > offset { + samplePtr = &pcm[offset] + } + + var segsOut uintptr + var nOut int32 + rc := CppSoundStreamFeed(stream, samplePtr, int32(end-offset), isLast, &segsOut, &nOut) + if segsOut != 0 && CppFreeSoundSegments != nil { + CppFreeSoundSegments(segsOut) + } + if rc != 0 { + return "", nClasses, fmt.Errorf("parakeet-cpp: sound_stream_feed failed: %s", soundLastError(p.tagCtx)) + } + + offset = end + if isLast == 1 { + break + } + } + + raw := CppSoundStreamDrainScoresJSON(stream) + if raw == 0 { + return "", nClasses, fmt.Errorf("parakeet-cpp: sound_stream_drain_scores_json failed: %s", soundLastError(p.tagCtx)) + } + doc := goStringFromCPtr(raw) + CppFreeString(raw) + return doc, nClasses, nil +} + +// soundLastError reads ctx's last_error, substituting a fallback message +// when the C side left it empty. +func soundLastError(ctx uintptr) string { + msg := CppLastError(ctx) + if msg == "" { + msg = "unknown error" + } + return msg +} + +// averageWindowScores averages each class's score across the drained +// per-window scores: CED's own long-clip method, summing a class's score +// over every window and dividing by the window count (a class absent from a +// window's tags counts as 0 in that window). Only classes that appeared in +// at least one window are returned, in no particular order; callers sort and +// filter afterward. A pure function so it is easy to unit test in isolation +// from the C stream. +func averageWindowScores(windows []soundWindowJSON, nClasses int) []classAvg { + if len(windows) == 0 { + return nil + } + + sums := make(map[int]float32) + labels := make(map[int]string) + for _, w := range windows { + for _, t := range w.Tags { + if t.Index < 0 || (nClasses > 0 && t.Index >= nClasses) { + continue + } + sums[t.Index] += t.Score + if _, ok := labels[t.Index]; !ok { + labels[t.Index] = t.Label + } + } + } + + n := float32(len(windows)) + out := make([]classAvg, 0, len(sums)) + for idx, sum := range sums { + out = append(out, classAvg{Index: idx, Label: labels[idx], Score: sum / n}) + } + return out +} + +// sortSoundDetectionsDesc sorts avgs by score descending, breaking ties by +// class index for a deterministic order (map iteration in +// averageWindowScores is otherwise unordered). +func sortSoundDetectionsDesc(avgs []classAvg) { + sort.Slice(avgs, func(i, j int) bool { + if avgs[i].Score != avgs[j].Score { + return avgs[i].Score > avgs[j].Score + } + return avgs[i].Index < avgs[j].Index + }) +} + +// filterSoundDetections drops entries scoring below threshold, then keeps +// only the first topK entries (0 = keep all). avgs is assumed already sorted +// descending by score. +func filterSoundDetections(avgs []classAvg, threshold float32, topK int32) []classAvg { + out := avgs[:0:0] + for _, a := range avgs { + if a.Score < threshold { + continue + } + out = append(out, a) + } + if topK > 0 && int32(len(out)) > topK { + out = out[:topK] + } + return out +} diff --git a/backend/go/parakeet-cpp/sound_test.go b/backend/go/parakeet-cpp/sound_test.go new file mode 100644 index 000000000000..1162a84ac6e2 --- /dev/null +++ b/backend/go/parakeet-cpp/sound_test.go @@ -0,0 +1,383 @@ +package main + +import ( + "context" + "path/filepath" + "sync" + "unsafe" + + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// The SoundDetection specs drive it entirely against stubbed +// CppSoundStreamBegin / CppSoundStreamFeed / CppSoundStreamDrainScoresJSON / +// CppSoundStreamFree / CppFreeSoundSegments / CppSoundOptsDefault / +// CppNumClasses / CppFreeString / CppLastError (the same seam diarize_test.go +// and live_test.go use), so they run without libparakeet.so. + +// soundCstrPool hands out NUL-terminated C-style strings backed by Go memory +// and keeps them alive for the duration of a spec (goStringFromCPtr reads +// through the raw pointer; mirrors diarize_test.go's diarizeCstrPool). +type soundCstrPool struct { + mu sync.Mutex + bufs [][]byte +} + +func (p *soundCstrPool) cstr(s string) uintptr { + p.mu.Lock() + defer p.mu.Unlock() + b := append([]byte(s), 0) + p.bufs = append(p.bufs, b) + return uintptr(unsafe.Pointer(&b[0])) +} + +// soundStubs swaps every C entry point SoundDetection touches and returns a +// restore func for AfterEach (mirrors diarize_test.go's diarizeStubs). +func soundStubs() (restore func()) { + savedBegin := CppSoundStreamBegin + savedFeed := CppSoundStreamFeed + savedDrain := CppSoundStreamDrainScoresJSON + savedFree := CppSoundStreamFree + savedFreeSegs := CppFreeSoundSegments + savedOptsDefault := CppSoundOptsDefault + savedNumClasses := CppNumClasses + savedFreeString := CppFreeString + savedLastError := CppLastError + return func() { + CppSoundStreamBegin = savedBegin + CppSoundStreamFeed = savedFeed + CppSoundStreamDrainScoresJSON = savedDrain + CppSoundStreamFree = savedFree + CppFreeSoundSegments = savedFreeSegs + CppSoundOptsDefault = savedOptsDefault + CppNumClasses = savedNumClasses + CppFreeString = savedFreeString + CppLastError = savedLastError + } +} + +// soundWav writes a silent 16 kHz mono WAV of the given duration (seconds) to +// a fresh temp file and returns its path. decodeWavMono16k reads real audio +// bytes off disk, so SoundDetection needs a file on disk even though the +// stubbed C calls never look at its samples. +func soundWav(seconds float64) string { + GinkgoHelper() + path := filepath.Join(GinkgoT().TempDir(), "sound.wav") + writeMono16kWav(path, int(seconds*16000)) + return path +} + +// noopFeed is a CppSoundStreamFeed stub that always succeeds and returns no +// segments, for specs that only care about the drained window scores. +func noopFeed(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + *out = 0 + *nOut = 0 + return 0 +} + +var _ = Describe("ParakeetCpp.SoundDetection", func() { + var restore func() + var pool *soundCstrPool + + BeforeEach(func() { + restore = soundStubs() + pool = &soundCstrPool{} + CppFreeString = func(uintptr) {} + CppFreeSoundSegments = func(uintptr) {} + CppSoundOptsDefault = func(o *cSoundOpts) { *o = cSoundOpts{} } + }) + AfterEach(func() { restore() }) + + It("fails with FailedPrecondition when no sound model is loaded", func() { + p := &ParakeetCpp{} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.FailedPrecondition)) + Expect(err.Error()).To(ContainSubstring("model is not a sound")) + }) + + It("fails with Unimplemented when the loaded libparakeet.so has no sound_stream_begin symbol", func() { + CppSoundStreamBegin = nil + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + }) + + It("averages two windows per class and sorts descending", func() { + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + return pool.cstr(`[` + + `{"start":0,"end":10,"tags":[{"index":0,"label":"Speech","score":0.8},{"index":1,"label":"Music","score":0.2}]},` + + `{"start":10,"end":20,"tags":[{"index":0,"label":"Speech","score":0.4},{"index":1,"label":"Music","score":0.6}]}` + + `]`) + } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(HaveLen(2)) + Expect(resp.Detections[0].Label).To(Equal("Speech")) + Expect(resp.Detections[0].Score).To(BeNumerically("~", 0.6, 1e-6)) + Expect(resp.Detections[1].Label).To(Equal("Music")) + Expect(resp.Detections[1].Score).To(BeNumerically("~", 0.4, 1e-6)) + }) + + It("drops classes scoring below threshold", func() { + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + return pool.cstr(`[{"start":0,"end":10,"tags":[` + + `{"index":0,"label":"Speech","score":0.8},` + + `{"index":1,"label":"Music","score":0.2}]}]`) + } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{ + Src: soundWav(1), Threshold: 0.5, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(HaveLen(1)) + Expect(resp.Detections[0].Label).To(Equal("Speech")) + }) + + It("keeps only the top_k entries", func() { + CppNumClasses = func(uintptr) int32 { return 4 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + return pool.cstr(`[{"start":0,"end":10,"tags":[` + + `{"index":0,"label":"A","score":0.9},` + + `{"index":1,"label":"B","score":0.7},` + + `{"index":2,"label":"C","score":0.5},` + + `{"index":3,"label":"D","score":0.3}]}]`) + } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{ + Src: soundWav(1), TopK: 3, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(HaveLen(3)) + Expect(resp.Detections[0].Label).To(Equal("A")) + Expect(resp.Detections[1].Label).To(Equal("B")) + Expect(resp.Detections[2].Label).To(Equal("C")) + }) + + It("keeps all classes when top_k is 0", func() { + CppNumClasses = func(uintptr) int32 { return 4 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + return pool.cstr(`[{"start":0,"end":10,"tags":[` + + `{"index":0,"label":"A","score":0.9},` + + `{"index":1,"label":"B","score":0.7},` + + `{"index":2,"label":"C","score":0.5},` + + `{"index":3,"label":"D","score":0.3}]}]`) + } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{ + Src: soundWav(1), TopK: 0, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(HaveLen(4)) + }) + + It("passes window 10s, hop 10s and top_k = the tagger's class count to sound_stream_begin", func() { + CppNumClasses = func(uintptr) int32 { return 527 } + var gotOpts cSoundOpts + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { + gotOpts = *o + return 1 + } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { return pool.cstr(`[]`) } + + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).ToNot(HaveOccurred()) + Expect(gotOpts.WindowSec).To(BeNumerically("==", 10)) + Expect(gotOpts.HopSec).To(BeNumerically("==", 10)) + Expect(gotOpts.TopK).To(Equal(int32(527))) + }) + + It("returns no detections without error for a short clip whose drain is empty", func() { + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = noopFeed + CppSoundStreamFree = func(uintptr) {} + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { return pool.cstr(`[]`) } + + p := &ParakeetCpp{tagCtx: 42} + resp, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(0.1)}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Detections).To(BeEmpty()) + }) + + It("surfaces last_error and still frees the stream when feed fails", func() { + freed := false + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + *out = 0 + *nOut = 0 + return 1 + } + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + Fail("drain_scores_json must not be called when feed failed") + return 0 + } + CppSoundStreamFree = func(uintptr) { freed = true } + CppLastError = func(uintptr) string { return "boom" } + + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: soundWav(1)}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("boom")) + Expect(freed).To(BeTrue()) + }) + + It("returns Canceled without feeding when ctx is already cancelled, and frees the stream", func() { + freed := false + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + Fail("sound_stream_feed must not be called when ctx is already cancelled") + return 0 + } + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + Fail("drain_scores_json must not be called when ctx is already cancelled") + return 0 + } + CppSoundStreamFree = func(uintptr) { freed = true } + + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(ctx, &pb.SoundDetectionRequest{Src: soundWav(15)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Canceled)) + Expect(freed).To(BeTrue()) + }) + + It("stops feeding and frees the stream when ctx is cancelled mid-feed", func() { + freed := false + feedCount := 0 + ctx, cancel := context.WithCancel(context.Background()) + + CppNumClasses = func(uintptr) int32 { return 2 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { return 1 } + CppSoundStreamFeed = func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + feedCount++ + cancel() // cancel after the first feed so a second chunk would exist if not stopped + *out = 0 + *nOut = 0 + return 0 + } + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { + Fail("drain_scores_json must not be called when the feed loop was cancelled") + return 0 + } + CppSoundStreamFree = func(uintptr) { freed = true } + + // Two 10 s chunks, so a second feed call would happen without the cancel. + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(ctx, &pb.SoundDetectionRequest{Src: soundWav(15)}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Canceled)) + Expect(feedCount).To(Equal(1)) + Expect(freed).To(BeTrue()) + }) + + It("wraps a decode failure as InvalidArgument", func() { + // Every required symbol must be non-nil to clear SoundDetection's own + // Unimplemented gate and reach the decode step this spec targets; + // none of them may actually be called. + fail := func(string) { Fail("no C call once the decode itself has failed") } + CppNumClasses = func(uintptr) int32 { fail("num_classes"); return 0 } + CppSoundStreamBegin = func(tagger uintptr, o *cSoundOpts) uintptr { fail("begin"); return 0 } + CppSoundStreamFeed = func(s uintptr, pcm *float32, n int32, isLast int32, out *uintptr, nOut *int32) int32 { + fail("feed") + return 0 + } + CppSoundStreamDrainScoresJSON = func(uintptr) uintptr { fail("drain"); return 0 } + CppSoundStreamFree = func(uintptr) { fail("free") } + + p := &ParakeetCpp{tagCtx: 42} + _, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{ + Src: filepath.Join(GinkgoT().TempDir(), "missing.wav"), + }) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + }) + + It("returns ModelNotLoaded without a C call when tagCtx is zeroed between the entry check and the call", func() { + called := false + CppNumClasses = func(uintptr) int32 { called = true; return 2 } + + p := &ParakeetCpp{tagCtx: 42} + // Simulate a Free() racing between SoundDetection's own tagCtx==0 + // check and soundStreamDrain's lock, exactly as it zeroes tagCtx + // under engineMu. + p.tagCtx = 0 + _, _, err := p.soundStreamDrain(context.Background(), make([]float32, 10)) + Expect(grpcerrors.IsModelNotLoaded(err)).To(BeTrue()) + Expect(called).To(BeFalse(), "no C call once tagCtx was cleared") + }) +}) + +var _ = Describe("averageWindowScores", func() { + It("returns nil for no windows", func() { + Expect(averageWindowScores(nil, 2)).To(BeNil()) + }) + + It("treats a class absent from a window as 0 in that window's contribution", func() { + windows := []soundWindowJSON{ + {Tags: []soundTagJSON{{Index: 0, Label: "Speech", Score: 1.0}}}, + {Tags: []soundTagJSON{}}, // Speech absent this window + } + out := averageWindowScores(windows, 2) + Expect(out).To(HaveLen(1)) + Expect(out[0].Index).To(Equal(0)) + Expect(out[0].Score).To(BeNumerically("~", 0.5, 1e-6)) + }) + + It("ignores an out-of-range class index", func() { + windows := []soundWindowJSON{ + {Tags: []soundTagJSON{{Index: 5, Label: "Bogus", Score: 1.0}}}, + } + Expect(averageWindowScores(windows, 2)).To(BeEmpty()) + }) +}) + +var _ = Describe("filterSoundDetections", func() { + It("keeps everything when top_k is 0 and threshold is 0", func() { + in := []classAvg{{Index: 0, Score: 0.1}, {Index: 1, Score: 0.9}} + Expect(filterSoundDetections(in, 0, 0)).To(HaveLen(2)) + }) + + It("drops entries below threshold before applying top_k", func() { + in := []classAvg{ + {Index: 0, Score: 0.9}, + {Index: 1, Score: 0.4}, + {Index: 2, Score: 0.1}, + } + out := filterSoundDetections(in, 0.3, 5) + Expect(out).To(HaveLen(2)) + }) +}) diff --git a/backend/go/parakeet-cpp/speakers.go b/backend/go/parakeet-cpp/speakers.go new file mode 100644 index 000000000000..ddd2c4292b6e --- /dev/null +++ b/backend/go/parakeet-cpp/speakers.go @@ -0,0 +1,121 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" +) + +// Speaker labels on transcripts. With a diarization_model companion attached +// to an ASR model, unary transcription tags each segment (and, with word +// timestamps, each word) with its speaker, and the stream=true final result +// tags each utterance. Live transcription carries speakers through the scene +// stream instead (scene.go). + +// speakerSnapSeconds mirrors parakeet.cpp's merge_asr_diarization: a word that +// overlaps no speaker segment takes the nearest segment's speaker when that +// segment is this close. ASR word boundaries and diarization boundaries can +// disagree by a frame or two; a word farther than this has no speaker. +const speakerSnapSeconds = 0.5 + +// transcriptSpeaker renders a 0-based speaker for a transcript segment or +// word; -1 (no speaker) is left empty so the field is omitted. +func transcriptSpeaker(spk int) string { + if spk < 0 { + return "" + } + return strconv.Itoa(spk) +} + +// wantSpeakers reports whether a transcription should carry speaker labels: +// a diarization companion is attached, the library can diarize, and the +// request did not turn it off (the OpenAI endpoint sends diarize=true unless +// the client passes diarize=false). +func (p *ParakeetCpp) wantSpeakers(diarize bool) bool { + return diarize && p.ctxPtr != 0 && p.diarCtx != 0 && CppDiarizePCM != nil +} + +// diarizeSegmentsPCM runs the diarization companion over 16 kHz PCM +// (parakeet_capi_diarize_pcm, the checkpoint's own mode, as NeMo's +// diarize()) and returns its segments. +func (p *ParakeetCpp) diarizeSegmentsPCM(pcm []float32) ([]diarizeSegmentJSON, error) { + if len(pcm) == 0 { + return nil, nil + } + raw, err := p.diarizeCall(pcm, false) + if err != nil { + return nil, err + } + var doc diarizePCMDoc + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return nil, fmt.Errorf("parakeet-cpp: decode diarization json: %w", err) + } + return doc.Segments, nil +} + +// assignSpeakers gives each word the speaker whose segments overlap it most, +// falling back to the nearest segment within speakerSnapSeconds; -1 when none. +// Same rule as parakeet.cpp's merge_asr_diarization, so the labels match the +// library's own speaker-attributed ASR. +func assignSpeakers(words []transcriptWord, segs []diarizeSegmentJSON) []int { + out := make([]int, len(words)) + for i, w := range words { + best, bestOverlap := -1, 0.0 + for _, s := range segs { + if ov := min(w.End, s.End) - max(w.Start, s.Start); ov > bestOverlap { + best, bestOverlap = s.Speaker, ov + } + } + if best < 0 { + bestDist := speakerSnapSeconds + for _, s := range segs { + dist := s.Start - w.End + if s.End <= w.Start { + dist = w.Start - s.End + } + if dist >= 0 && dist <= bestDist { + best, bestDist = s.Speaker, dist + } + } + } + out[i] = best + } + return out +} + +// splitAtSpeakerChanges splits each word group wherever the speaker changes, +// so every segment has one speaker. speakers is indexed like the +// concatenation of groups. Returns the new groups and each group's speaker. +func splitAtSpeakerChanges(groups [][]transcriptWord, speakers []int) ([][]transcriptWord, []int) { + var outGroups [][]transcriptWord + var outSpk []int + k := 0 + for _, g := range groups { + start := 0 + for i := 1; i <= len(g); i++ { + if i == len(g) || speakers[k+i] != speakers[k+start] { + outGroups = append(outGroups, g[start:i]) + outSpk = append(outSpk, speakers[k+start]) + start = i + } + } + k += len(g) + } + return outGroups, outSpk +} + +// majoritySpeaker is the speaker covering most of the words' duration, or -1. +func majoritySpeaker(words []transcriptWord, speakers []int) int { + dur := map[int]float64{} + best, bestDur := -1, 0.0 + for i, w := range words { + if speakers[i] < 0 { + continue + } + dur[speakers[i]] += max(w.End-w.Start, 1e-3) + if dur[speakers[i]] > bestDur { + best, bestDur = speakers[i], dur[speakers[i]] + } + } + return best +} diff --git a/backend/go/parakeet-cpp/speakers_test.go b/backend/go/parakeet-cpp/speakers_test.go new file mode 100644 index 000000000000..b302bb6e9356 --- /dev/null +++ b/backend/go/parakeet-cpp/speakers_test.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "os" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func sseg(spk int, start, end float64) diarizeSegmentJSON { + return diarizeSegmentJSON{Speaker: spk, Start: start, End: end} +} + +// speakerTurnsOf collapses consecutive repeats: [0 0 1 0] -> [0 1 0]. +func speakerTurnsOf(labels []string) []string { + var out []string + for _, l := range labels { + if len(out) == 0 || out[len(out)-1] != l { + out = append(out, l) + } + } + return out +} + +var _ = Describe("transcript speaker labels", func() { + Context("assignSpeakers", func() { + It("picks the speaker with the largest overlap", func() { + words := []transcriptWord{tw("a", 0.5, 0.8)} + Expect(assignSpeakers(words, []diarizeSegmentJSON{sseg(0, 0.0, 0.6), sseg(1, 0.55, 2.0)})). + To(Equal([]int{1})) + }) + + It("snaps a word just outside a segment to it, but not a far one", func() { + words := []transcriptWord{tw("well", 19.92, 20.00), tw("far", 25.0, 25.2)} + segs := []diarizeSegmentJSON{sseg(0, 14.78, 18.75), sseg(1, 20.10, 23.60)} + Expect(assignSpeakers(words, segs)).To(Equal([]int{1, -1})) + }) + }) + + It("splits segments at speaker turns and labels segments and words", func() { + doc := transcriptJSON{ + Text: "hi there. hello", + Words: []transcriptWord{tw("hi", 0.0, 0.3), tw("there.", 0.3, 0.6), tw("hello", 1.0, 1.4)}, + } + // Punctuation gives [hi there.] [hello]; the turn after "hi" splits the first. + opts := &pb.TranscriptRequest{TimestampGranularities: []string{"word"}} + res := transcriptResultWithSpeakers(doc, opts, 0, []int{0, 1, 1}) + Expect(res.Segments).To(HaveLen(3)) + Expect([]string{res.Segments[0].Speaker, res.Segments[1].Speaker, res.Segments[2].Speaker}). + To(Equal([]string{"0", "1", "1"})) + Expect(res.Segments[1].Text).To(Equal("there.")) + Expect(res.Segments[2].Words[0].Speaker).To(Equal("1")) + for i, seg := range res.Segments { + Expect(seg.Id).To(Equal(int32(i))) + } + }) + + It("leaves speakers empty without diarization and for unknown words", func() { + doc := transcriptJSON{Text: "hi.", Words: []transcriptWord{tw("hi.", 0, 0.3)}} + Expect(transcriptResultFromDoc(doc, &pb.TranscriptRequest{}, 0).Segments[0].Speaker).To(BeEmpty()) + Expect(transcriptResultWithSpeakers(doc, &pb.TranscriptRequest{}, 0, []int{-1}).Segments[0].Speaker). + To(BeEmpty()) + }) + + It("picks the speaker covering most of an utterance", func() { + words := []transcriptWord{tw("a", 0, 0.2), tw("b", 0.2, 1.5), tw("c", 1.5, 1.6)} + Expect(majoritySpeaker(words, []int{0, 1, 0})).To(Equal(1)) + Expect(majoritySpeaker(words, []int{-1, -1, -1})).To(Equal(-1)) + }) + + It("only diarizes with a companion, a capable library and diarize=true", func() { + saved := CppDiarizePCM + defer func() { CppDiarizePCM = saved }() + CppDiarizePCM = func(uintptr, *float32, int32, int32) uintptr { return 0 } + p := &ParakeetCpp{ctxPtr: 1, diarCtx: 2} + Expect(p.wantSpeakers(true)).To(BeTrue()) + Expect(p.wantSpeakers(false)).To(BeFalse()) + Expect((&ParakeetCpp{ctxPtr: 1}).wantSpeakers(true)).To(BeFalse()) + CppDiarizePCM = nil + Expect(p.wantSpeakers(true)).To(BeFalse()) + }) +}) + +var _ = Describe("ParakeetCpp transcript speakers (real models)", func() { + It("labels unary and stream=true transcripts with a diarization_model companion", func() { + diarModel := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_MODEL") + asrModel := os.Getenv("PARAKEET_BACKEND_TEST_MODEL") + wavPath := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_WAV") + if diarModel == "" || asrModel == "" || wavPath == "" { + Skip("set PARAKEET_BACKEND_TEST_DIAR_MODEL, PARAKEET_BACKEND_TEST_MODEL and " + + "PARAKEET_BACKEND_TEST_DIAR_WAV (a multi-speaker 16 kHz WAV)") + } + ensureLibLoaded() + if CppDiarizePCM == nil || CppModelKind == nil { + Skip("libparakeet.so has no diarization / model-kind C-API") + } + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ + ModelFile: asrModel, + Options: []string{"diarization_model:" + diarModel}, + })).To(Succeed()) + defer func() { _ = p.Free() }() + + res, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath, Diarize: true}) + Expect(err).ToNot(HaveOccurred()) + labels := make([]string, len(res.Segments)) + for i, s := range res.Segments { + Expect(s.Speaker).ToNot(BeEmpty(), "segment %d %q", i, s.Text) + labels[i] = s.Speaker + } + // parakeet.cpp's tests/fixtures/two_speakers.wav alternates A-B-A-B. + Expect(speakerTurnsOf(labels)).To(Equal([]string{"0", "1", "0", "1"})) + + plain, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath}) + Expect(err).ToNot(HaveOccurred()) + for _, s := range plain.Segments { + Expect(s.Speaker).To(BeEmpty()) + } + Expect(plain.Text).To(Equal(res.Text)) + }) +}) + +var _ = Describe("ParakeetCpp scene companions on an offline ASR model (real models)", func() { + It("labels speakers and detects sounds from one model with both companions", func() { + asrModel := os.Getenv("PARAKEET_BACKEND_TEST_SCENE_ASR_MODEL") // e.g. tdt-0.6b-v3 + diarModel := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_MODEL") + soundModel := os.Getenv("PARAKEET_BACKEND_TEST_SOUND_MODEL") + wavPath := os.Getenv("PARAKEET_BACKEND_TEST_SCENE_WAV") // speech + a non-speech sound + wantLabel := os.Getenv("PARAKEET_BACKEND_TEST_SCENE_LABEL") // e.g. "Chicken, rooster" + if asrModel == "" || diarModel == "" || soundModel == "" || wavPath == "" || wantLabel == "" { + Skip("set PARAKEET_BACKEND_TEST_SCENE_ASR_MODEL, _DIAR_MODEL, _SOUND_MODEL, " + + "PARAKEET_BACKEND_TEST_SCENE_WAV and PARAKEET_BACKEND_TEST_SCENE_LABEL") + } + ensureLibLoaded() + if CppDiarizePCM == nil || CppModelKind == nil { + Skip("libparakeet.so has no diarization / model-kind C-API") + } + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ + ModelFile: asrModel, + Options: []string{"diarization_model:" + diarModel, "sound_model:" + soundModel}, + })).To(Succeed()) + defer func() { _ = p.Free() }() + + res, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath, Diarize: true}) + Expect(err).ToNot(HaveOccurred()) + labels := make([]string, 0, len(res.Segments)) + for _, s := range res.Segments { + GinkgoWriter.Printf("[%5.1f-%5.1f] spk %q: %s\n", float64(s.Start)/1e9, float64(s.End)/1e9, s.Speaker, s.Text) + labels = append(labels, s.Speaker) + } + Expect(len(speakerTurnsOf(labels))).To(BeNumerically(">=", 3), "speaker turns: %v", labels) + Expect(labels).To(ContainElements("0", "1")) + + sd, err := p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: wavPath, TopK: 5}) + Expect(err).ToNot(HaveOccurred()) + var got []string + for _, d := range sd.GetDetections() { + GinkgoWriter.Printf("sound %q %.2f\n", d.GetLabel(), d.GetScore()) + got = append(got, d.GetLabel()) + } + Expect(got).To(ContainElement(wantLabel)) + }) +}) diff --git a/core/backend/transcript.go b/core/backend/transcript.go index 0ddee3adddff..27b13bc55316 100644 --- a/core/backend/transcript.go +++ b/core/backend/transcript.go @@ -213,9 +213,10 @@ func transcriptResultFromProto(r *proto.TranscriptResult) *schema.TranscriptionR var words []schema.TranscriptionWord for _, w := range s.Words { var word = schema.TranscriptionWord{ - Start: time.Duration(w.Start), - End: time.Duration(w.End), - Text: w.Text, + Start: time.Duration(w.Start), + End: time.Duration(w.End), + Text: w.Text, + Speaker: w.Speaker, } words = append(words, word) tr.Words = append(tr.Words, word) diff --git a/core/backend/transcript_live.go b/core/backend/transcript_live.go index 0ad6d72e1456..e64e54a578e2 100644 --- a/core/backend/transcript_live.go +++ b/core/backend/transcript_live.go @@ -26,11 +26,33 @@ import ( // backchannel ("uh-huh") ended — callers must NOT treat Eob as a turn // boundary. type LiveTranscriptionEvent struct { - Delta string - Eou bool - Eob bool - Words []schema.TranscriptionWord - Final *schema.TranscriptionResult + Delta string + Eou bool + Eob bool + Words []schema.TranscriptionWord + Speakers []LiveSpeakerSegment + Sounds []LiveSoundEvent + Final *schema.TranscriptionResult +} + +// LiveSpeakerSegment is one closed speaker segment from a companion +// diarization/scene stream running alongside live transcription. Start/End +// are stream-relative seconds (mapped from the backend's nanoseconds). +type LiveSpeakerSegment struct { + Speaker string + Start float64 + End float64 +} + +// LiveSoundEvent is one closed sound event from a companion sound/scene +// stream running alongside live transcription. Start/End are stream-relative +// seconds (mapped from the backend's nanoseconds). +type LiveSoundEvent struct { + Label string + Index int + Peak float32 + Start float64 + End float64 } // LiveTranscriptionSession is a handle on an open live transcription stream. @@ -298,9 +320,26 @@ func liveEventFromProto(r *proto.TranscriptLiveResponse) LiveTranscriptionEvent } for _, w := range r.GetWords() { ev.Words = append(ev.Words, schema.TranscriptionWord{ - Start: time.Duration(w.Start), - End: time.Duration(w.End), - Text: w.Text, + Start: time.Duration(w.Start), + End: time.Duration(w.End), + Text: w.Text, + Speaker: w.Speaker, + }) + } + for _, s := range r.GetSpeakers() { + ev.Speakers = append(ev.Speakers, LiveSpeakerSegment{ + Speaker: s.GetSpeaker(), + Start: time.Duration(s.GetStart()).Seconds(), + End: time.Duration(s.GetEnd()).Seconds(), + }) + } + for _, s := range r.GetSounds() { + ev.Sounds = append(ev.Sounds, LiveSoundEvent{ + Label: s.GetLabel(), + Index: int(s.GetIndex()), + Peak: s.GetPeak(), + Start: time.Duration(s.GetStart()).Seconds(), + End: time.Duration(s.GetEnd()).Seconds(), }) } if r.GetFinalResult() != nil { diff --git a/core/backend/transcript_live_internal_test.go b/core/backend/transcript_live_internal_test.go index 6f6bed6a4b5c..8b2aee91e0c2 100644 --- a/core/backend/transcript_live_internal_test.go +++ b/core/backend/transcript_live_internal_test.go @@ -54,11 +54,53 @@ var _ = Describe("liveEventFromProto", func() { Expect(ev.Final).To(BeNil()) }) + It("carries word speakers and final segment speakers from a diarizing backend", func() { + ev := liveEventFromProto(&proto.TranscriptLiveResponse{ + Words: []*proto.TranscriptWord{{Text: "hi", Speaker: "1"}}, + }) + Expect(ev.Words[0].Speaker).To(Equal("1")) + ev = liveEventFromProto(&proto.TranscriptLiveResponse{ + FinalResult: &proto.TranscriptResult{ + Text: "hi there", + Segments: []*proto.TranscriptSegment{{Text: "hi", Speaker: "0"}, {Text: "there", Speaker: "1"}}, + }, + }) + Expect(ev.Final.Segments[1].Speaker).To(Equal("1")) + }) + It("maps the eob backchannel flag separately from eou", func() { ev := liveEventFromProto(&proto.TranscriptLiveResponse{Delta: "uh-huh", Eob: true}) Expect(ev.Eob).To(BeTrue()) Expect(ev.Eou).To(BeFalse()) }) + + It("maps speaker segments and sound events (ns -> seconds)", func() { + ev := liveEventFromProto(&proto.TranscriptLiveResponse{ + Speakers: []*proto.LiveSpeakerSegment{ + {Speaker: "1", Start: int64(1500 * time.Millisecond), End: int64(3200 * time.Millisecond)}, + }, + Sounds: []*proto.LiveSoundEvent{ + {Label: "Dog bark", Index: 5, Peak: 0.8, Start: int64(500 * time.Millisecond), End: int64(900 * time.Millisecond)}, + }, + }) + Expect(ev.Speakers).To(HaveLen(1)) + Expect(ev.Speakers[0].Speaker).To(Equal("1")) + Expect(ev.Speakers[0].Start).To(BeNumerically("~", 1.5, 1e-9)) + Expect(ev.Speakers[0].End).To(BeNumerically("~", 3.2, 1e-9)) + + Expect(ev.Sounds).To(HaveLen(1)) + Expect(ev.Sounds[0].Label).To(Equal("Dog bark")) + Expect(ev.Sounds[0].Index).To(Equal(5)) + Expect(ev.Sounds[0].Peak).To(BeNumerically("~", 0.8, 1e-6)) + Expect(ev.Sounds[0].Start).To(BeNumerically("~", 0.5, 1e-9)) + Expect(ev.Sounds[0].End).To(BeNumerically("~", 0.9, 1e-9)) + }) + + It("leaves speakers and sounds nil when the proto carries none", func() { + ev := liveEventFromProto(&proto.TranscriptLiveResponse{Delta: "hi"}) + Expect(ev.Speakers).To(BeNil()) + Expect(ev.Sounds).To(BeNil()) + }) }) // liveTraceState is what makes streaming-only pipelines visible on the diff --git a/core/cli/transcript.go b/core/cli/transcript.go index 06764f4dd277..e187453a6fa9 100644 --- a/core/cli/transcript.go +++ b/core/cli/transcript.go @@ -93,18 +93,20 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error { } for _, word := range(tr.Words) { trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } for _, seg := range(tr.Segments) { segWords := []schema.TranscriptionWordSeconds{} for _, word := range(seg.Words) { segWords = append(segWords, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{ diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index 886ddd22a185..b48266ead4df 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -477,11 +477,15 @@ var BackendCapabilities = map[string]BackendCapability{ DefaultUsecases: []string{UsecaseTranscript}, Description: "NVIDIA NeMo speech recognition", }, + // parakeet-cpp loads three model kinds, picked from the GGUF: an ASR model + // transcribes (and labels speakers when a diarization_model companion is + // attached), a Nemotron-3-Diarization model answers Diarize, and a CED model + // answers SoundDetection. PossibleUsecases is their union. "parakeet-cpp": { - GRPCMethods: []GRPCMethod{MethodAudioTranscription}, - PossibleUsecases: []string{UsecaseTranscript}, + GRPCMethods: []GRPCMethod{MethodAudioTranscription, MethodDiarize, MethodSoundDetection}, + PossibleUsecases: []string{UsecaseTranscript, UsecaseDiarization, UsecaseSoundClassification}, DefaultUsecases: []string{UsecaseTranscript}, - Description: "NVIDIA NeMo Parakeet ASR (parakeet.cpp)", + Description: "NVIDIA NeMo Parakeet ASR, Nemotron-3-Diarization speaker diarization and CED sound-event detection (parakeet.cpp)", }, // nemo-speech-cpp is one gRPC server in front of four NeMo-Speech.cpp model // families, picked at load time from the GGUF general.architecture key, so diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index c98514a785d5..dfc2dc61aca4 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -509,6 +509,13 @@ func DefaultRegistry() map[string]FieldMetaOverride { Min: f64(0), Order: 66, }, + "pipeline.diarization": { + Section: "pipeline", + Label: "Speaker Diarization", + Description: "Label speakers on each committed utterance and emit every labelled segment as a conversation.item.input_audio_transcription.segment event. Needs a transcription model that diarizes (e.g. parakeet-cpp with a diarization_model companion). Speaker labels are per turn.", + Component: "toggle", + Order: 67, + }, "pipeline.reasoning_effort": { Section: "pipeline", Label: "Reasoning Effort", diff --git a/core/config/model_config.go b/core/config/model_config.go index 3c8987920149..c8502fae5865 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -833,6 +833,14 @@ type Pipeline struct { SoundDetectionWindowMs int `yaml:"sound_detection_window_ms,omitempty" json:"sound_detection_window_ms,omitempty"` SoundDetectionHopMs int `yaml:"sound_detection_hop_ms,omitempty" json:"sound_detection_hop_ms,omitempty"` + // Diarization asks the transcription model for speaker labels on each + // VAD-committed utterance and emits every labelled segment as a + // conversation.item.input_audio_transcription.segment event. It needs a + // transcription model that diarizes (e.g. parakeet-cpp with a + // diarization_model companion); off by default because some backends fail + // a diarize request they cannot serve. Speaker labels are per turn. + Diarization bool `yaml:"diarization,omitempty" json:"diarization,omitempty"` + // ReasoningEffort sets the reasoning effort (none|minimal|low|medium|high) for // the pipeline's LLM without editing the LLM model config. Overrides the LLM's // own reasoning_effort. Unset leaves the LLM model config in charge. diff --git a/core/gallery/importers/parakeet-cpp.go b/core/gallery/importers/parakeet-cpp.go index aa732aa3e7fc..5b758167b457 100644 --- a/core/gallery/importers/parakeet-cpp.go +++ b/core/gallery/importers/parakeet-cpp.go @@ -108,6 +108,11 @@ func (i *ParakeetCppImporter) Import(details Details) (gallery.ModelConfig, erro uri := downloader.URI(details.URI) directGGUF := isParakeetGGUF(filepath.Base(details.URI)) + // A speaker diarization GGUF is served by the same backend but answers + // /v1/audio/diarization, not transcription. + if directGGUF && isParakeetDiarGGUF(filepath.Base(details.URI)) { + modelConfig.KnownUsecaseStrings = []string{"diarization"} + } switch { case uri.LooksLikeURL() && directGGUF: // Direct file URL (e.g. .../resolve/main/tdt_ctc-110m-f16.gguf). The @@ -128,12 +133,23 @@ func (i *ParakeetCppImporter) Import(details Details) (gallery.ModelConfig, erro // HF repo: collect every parakeet GGUF, pick the preferred quant, and // nest under parakeet-cpp/models// so a multi-quant repo doesn't // collide on disk. - var ggufFiles []hfapi.ModelFile + // Prefer ASR weights: a repo that also ships the diarization model + // (mudler/parakeet-cpp-gguf) imports as a transcription model, and the + // diarization GGUF is imported by its direct URL. A repo with only + // diarization weights imports as a diarization model. + var ggufFiles, diarFiles []hfapi.ModelFile for _, f := range details.HuggingFace.Files { - if isParakeetGGUF(filepath.Base(f.Path)) { + switch base := filepath.Base(f.Path); { + case isParakeetDiarGGUF(base): + diarFiles = append(diarFiles, f) + case isParakeetGGUF(base): ggufFiles = append(ggufFiles, f) } } + if len(ggufFiles) == 0 && len(diarFiles) > 0 { + ggufFiles = diarFiles + modelConfig.KnownUsecaseStrings = []string{"diarization"} + } if chosen, ok := pickPreferredGGMLFile(ggufFiles, quants); ok { target := filepath.Join("parakeet-cpp", "models", name, filepath.Base(chosen.Path)) cfg.Files = append(cfg.Files, gallery.File{ @@ -176,5 +192,14 @@ func isParakeetGGUF(name string) bool { return true } } - return false + return isParakeetDiarGGUF(name) +} + +// isParakeetDiarGGUF reports whether name is the parakeet.cpp speaker +// diarization GGUF (nemotron-3-diarization-.gguf). Matched by its +// published name only, so diarization weights for other backends are not +// claimed. +func isParakeetDiarGGUF(name string) bool { + lower := strings.ToLower(name) + return strings.HasSuffix(lower, ".gguf") && strings.Contains(lower, "nemotron-3-diarization") } diff --git a/core/gallery/importers/parakeet-cpp_test.go b/core/gallery/importers/parakeet-cpp_test.go index 4aa87c4110e6..8636c929de6d 100644 --- a/core/gallery/importers/parakeet-cpp_test.go +++ b/core/gallery/importers/parakeet-cpp_test.go @@ -50,6 +50,11 @@ var _ = Describe("ParakeetCppImporter", func() { Expect(imp.Match(d)).To(BeTrue()) }) + It("matches a direct URL to the diarization GGUF", func() { + d := parakeetDetails("https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf", `{}`) + Expect(imp.Match(d)).To(BeTrue()) + }) + It("does NOT claim a generic llama-style GGUF", func() { d := parakeetDetails("huggingface://someorg/some-llm-gguf", `{}`, hfapi.ModelFile{Path: "llama-3-8b-instruct-q4_k_m.gguf"}, @@ -66,6 +71,43 @@ var _ = Describe("ParakeetCppImporter", func() { }) Context("import (Import)", func() { + It("imports the diarization GGUF as a diarization model", func() { + d := parakeetDetails("https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf", + `{"name":"nemotron-diarization"}`) + cfg, err := imp.Import(d) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.ConfigFile).To(ContainSubstring("backend: parakeet-cpp")) + Expect(cfg.ConfigFile).To(ContainSubstring("diarization")) + Expect(cfg.ConfigFile).ToNot(ContainSubstring("transcript")) + Expect(cfg.Files).To(HaveLen(1)) + Expect(cfg.Files[0].Filename).To(HaveSuffix("nemotron-3-diarization-q8_0.gguf")) + }) + + It("keeps picking ASR weights from a repo that also ships the diarization model", func() { + d := parakeetDetails("huggingface://mudler/parakeet-cpp-gguf", `{"name":"parakeet-110m"}`, + hfapi.ModelFile{Path: "nemotron-3-diarization-f16.gguf", URL: "https://hf/diar-f16", SHA256: "ddd"}, + hfapi.ModelFile{Path: "tdt_ctc-110m-f16.gguf", URL: "https://hf/f16", SHA256: "aaa"}, + hfapi.ModelFile{Path: "nemotron-3-diarization-q8_0.gguf", URL: "https://hf/diar-q8", SHA256: "eee"}, + ) + cfg, err := imp.Import(d) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.Files).To(HaveLen(1)) + Expect(cfg.Files[0].URI).To(Equal("https://hf/f16")) + Expect(cfg.ConfigFile).To(ContainSubstring("transcript")) + }) + + It("imports a diarization-only repo as a diarization model", func() { + d := parakeetDetails("huggingface://someone/diar-gguf", `{"name":"diar"}`, + hfapi.ModelFile{Path: "nemotron-3-diarization-f16.gguf", URL: "https://hf/diar-f16", SHA256: "ddd"}, + hfapi.ModelFile{Path: "nemotron-3-diarization-q8_0.gguf", URL: "https://hf/diar-q8", SHA256: "eee"}, + ) + cfg, err := imp.Import(d) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.Files).To(HaveLen(1)) + Expect(cfg.Files[0].URI).To(Equal("https://hf/diar-q8"), "default quant ladder picks q8_0 before f16") + Expect(cfg.ConfigFile).To(ContainSubstring("diarization")) + }) + It("picks the default quant (q4_k) from a multi-quant HF repo", func() { d := parakeetDetails("huggingface://mudler/parakeet-cpp-gguf", `{"name":"parakeet-110m"}`, hfapi.ModelFile{Path: "tdt_ctc-110m-f16.gguf", URL: "https://hf/f16", SHA256: "aaa"}, diff --git a/core/http/endpoints/openai/realtime_doubles_test.go b/core/http/endpoints/openai/realtime_doubles_test.go index e82a4a13056f..9ddc1299fdfa 100644 --- a/core/http/endpoints/openai/realtime_doubles_test.go +++ b/core/http/endpoints/openai/realtime_doubles_test.go @@ -88,6 +88,7 @@ type fakeModel struct { transcribeDeltas []string transcribeFinal *schema.TranscriptionResult transcribeErr error + lastDiarize bool // diarize flag of the last Transcribe/TranscribeStream call // TranscribeLive scripting: liveErr makes the open fail (degrade path); // liveEvents are delivered to onEvent synchronously at open; @@ -189,7 +190,8 @@ func (m *fakeModel) VAD(_ context.Context, req *schema.VADRequest) (*schema.VADR return &schema.VADResponse{Segments: m.vadSegments}, nil } -func (m *fakeModel) Transcribe(context.Context, string, string, bool, bool, string) (*schema.TranscriptionResult, error) { +func (m *fakeModel) Transcribe(_ context.Context, _, _ string, _, diarize bool, _ string) (*schema.TranscriptionResult, error) { + m.lastDiarize = diarize return m.transcribeFinal, m.transcribeErr } @@ -236,7 +238,8 @@ func (m *fakeModel) TTSStream(_ context.Context, _, _, _ string, onAudio func(pc return nil } -func (m *fakeModel) TranscribeStream(_ context.Context, _, _ string, _, _ bool, _ string, onDelta func(text string)) (*schema.TranscriptionResult, error) { +func (m *fakeModel) TranscribeStream(_ context.Context, _, _ string, _, diarize bool, _ string, onDelta func(text string)) (*schema.TranscriptionResult, error) { + m.lastDiarize = diarize for _, d := range m.transcribeDeltas { onDelta(d) } diff --git a/core/http/endpoints/openai/realtime_semantic_vad.go b/core/http/endpoints/openai/realtime_semantic_vad.go index 75a71ba25a85..df56abf19783 100644 --- a/core/http/endpoints/openai/realtime_semantic_vad.go +++ b/core/http/endpoints/openai/realtime_semantic_vad.go @@ -209,6 +209,35 @@ func (l *liveTurnState) drainEvents(audioSec float64) { if ev.Final != nil && strings.TrimSpace(ev.Final.Text) != "" { l.finalText = ev.Final.Text } + // Speaker and sound events from a companion diarization/scene + // stream: forward each as its own event under the turn's item + // id, same as caption deltas. Text is empty — the event exists + // to carry the speaker/segment boundary, not transcript text. + if l.transport != nil && l.itemID != "" { + for _, seg := range ev.Speakers { + sendEvent(l.transport, types.ConversationItemInputAudioTranscriptionSegmentEvent{ + ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, + ItemID: l.itemID, + ContentIndex: 0, + Speaker: seg.Speaker, + Start: seg.Start, + End: seg.End, + }) + } + for _, sound := range ev.Sounds { + start, end := sound.Start, sound.End + sendEvent(l.transport, types.ConversationItemSoundDetectionEvent{ + ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, + ItemID: l.itemID, + ContentIndex: 0, + Detections: []types.SoundDetectionTag{ + {Label: sound.Label, Score: sound.Peak, Index: sound.Index}, + }, + Start: &start, + End: &end, + }) + } + } default: return } diff --git a/core/http/endpoints/openai/realtime_semantic_vad_test.go b/core/http/endpoints/openai/realtime_semantic_vad_test.go index b1107c1f6442..5a92e3b15000 100644 --- a/core/http/endpoints/openai/realtime_semantic_vad_test.go +++ b/core/http/endpoints/openai/realtime_semantic_vad_test.go @@ -291,6 +291,67 @@ var _ = Describe("liveTurnState", func() { Expect(ftr.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionFailed)).To(Equal(0)) }) }) + + Describe("scene events (speakers and sounds)", func() { + It("emits a segment event per speaker with empty text under the turn's item id", func() { + Expect(lts.openTurn(context.Background(), "item1")).To(BeTrue()) + turnID := lts.itemID + + m.liveSession.onEvent(backend.LiveTranscriptionEvent{ + Speakers: []backend.LiveSpeakerSegment{{Speaker: "1", Start: 1.2, End: 3.4}}, + }) + lts.drainEvents(3.4) + + var got []types.ConversationItemInputAudioTranscriptionSegmentEvent + for _, e := range ftr.events() { + if seg, ok := e.(types.ConversationItemInputAudioTranscriptionSegmentEvent); ok { + got = append(got, seg) + } + } + Expect(got).To(HaveLen(1)) + Expect(got[0].ItemID).To(Equal(turnID)) + Expect(got[0].Speaker).To(Equal("1")) + Expect(got[0].Start).To(BeNumerically("~", 1.2, 1e-9)) + Expect(got[0].End).To(BeNumerically("~", 3.4, 1e-9)) + Expect(got[0].Text).To(BeEmpty()) + }) + + It("emits a sound_detection event per sound with one tag and start/end", func() { + Expect(lts.openTurn(context.Background(), "item1")).To(BeTrue()) + turnID := lts.itemID + + m.liveSession.onEvent(backend.LiveTranscriptionEvent{ + Sounds: []backend.LiveSoundEvent{{Label: "Dog bark", Index: 5, Peak: 0.8, Start: 0.5, End: 0.9}}, + }) + lts.drainEvents(1.0) + + var got []types.ConversationItemSoundDetectionEvent + for _, e := range ftr.events() { + if sd, ok := e.(types.ConversationItemSoundDetectionEvent); ok { + got = append(got, sd) + } + } + Expect(got).To(HaveLen(1)) + Expect(got[0].ItemID).To(Equal(turnID)) + Expect(got[0].Detections).To(HaveLen(1)) + Expect(got[0].Detections[0].Label).To(Equal("Dog bark")) + Expect(got[0].Detections[0].Score).To(BeNumerically("~", 0.8, 1e-6)) + Expect(got[0].Detections[0].Index).To(Equal(5)) + Expect(got[0].Start).NotTo(BeNil()) + Expect(*got[0].Start).To(BeNumerically("~", 0.5, 1e-9)) + Expect(got[0].End).NotTo(BeNil()) + Expect(*got[0].End).To(BeNumerically("~", 0.9, 1e-9)) + }) + + It("sends neither event when a live event carries no speakers or sounds", func() { + Expect(lts.openTurn(context.Background(), "item1")).To(BeTrue()) + m.liveSession.onEvent(backend.LiveTranscriptionEvent{Delta: "hi"}) + lts.drainEvents(1.0) + + Expect(ftr.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionSegment)).To(Equal(0)) + Expect(ftr.countEvents(types.ServerEventTypeConversationItemSoundDetection)).To(Equal(0)) + }) + }) }) // commitUtteranceWithTranscript routes the three transcript sources: the diff --git a/core/http/endpoints/openai/realtime_sound_detection_test.go b/core/http/endpoints/openai/realtime_sound_detection_test.go index 058c74076ec7..94406e7cc6aa 100644 --- a/core/http/endpoints/openai/realtime_sound_detection_test.go +++ b/core/http/endpoints/openai/realtime_sound_detection_test.go @@ -3,6 +3,7 @@ package openai import ( "context" "encoding/binary" + "encoding/json" "errors" "os" @@ -14,6 +15,69 @@ import ( "github.com/mudler/LocalAI/core/schema" ) +// ConversationItemSoundDetectionEvent gained optional Start/End (seconds) +// for the live scene-event path; the unary/windowed paths never set them, +// so existing consumers must see no start/end keys at all. +var _ = Describe("ConversationItemSoundDetectionEvent JSON", func() { + It("omits start and end when nil", func() { + ev := types.ConversationItemSoundDetectionEvent{ + ItemID: "item1", + Detections: []types.SoundDetectionTag{{Label: "Speech", Score: 0.5, Index: 7}}, + } + b, err := json.Marshal(ev) + Expect(err).ToNot(HaveOccurred()) + + var got map[string]any + Expect(json.Unmarshal(b, &got)).To(Succeed()) + _, hasStart := got["start"] + _, hasEnd := got["end"] + Expect(hasStart).To(BeFalse()) + Expect(hasEnd).To(BeFalse()) + }) + + It("includes start and end when set", func() { + start, end := 0.5, 0.9 + ev := types.ConversationItemSoundDetectionEvent{ + ItemID: "item1", + Start: &start, + End: &end, + } + b, err := json.Marshal(ev) + Expect(err).ToNot(HaveOccurred()) + + var got map[string]any + Expect(json.Unmarshal(b, &got)).To(Succeed()) + Expect(got["start"]).To(BeNumerically("~", 0.5, 1e-9)) + Expect(got["end"]).To(BeNumerically("~", 0.9, 1e-9)) + }) +}) + +// ConversationItemInputAudioTranscriptionSegmentEvent.Start/End are plain +// float64 (no omitempty): a speaker segment starting at 0.0s must still +// carry "start" in the JSON, unlike the sound-detection event's optional +// pointer fields above. +var _ = Describe("ConversationItemInputAudioTranscriptionSegmentEvent JSON", func() { + It("marshals start:0 and end:1.5 even when start is the zero value", func() { + ev := types.ConversationItemInputAudioTranscriptionSegmentEvent{ + ItemID: "item1", + Speaker: "1", + Start: 0, + End: 1.5, + } + b, err := json.Marshal(ev) + Expect(err).ToNot(HaveOccurred()) + + var got map[string]any + Expect(json.Unmarshal(b, &got)).To(Succeed()) + _, hasStart := got["start"] + _, hasEnd := got["end"] + Expect(hasStart).To(BeTrue()) + Expect(hasEnd).To(BeTrue()) + Expect(got["start"]).To(BeNumerically("~", 0.0, 1e-9)) + Expect(got["end"]).To(BeNumerically("~", 1.5, 1e-9)) + }) +}) + // emitSoundDetection classifies a committed utterance and emits a single // conversation.item.sound_detection event carrying the scored AudioSet tags. var _ = Describe("emitSoundDetection", func() { diff --git a/core/http/endpoints/openai/realtime_transcription.go b/core/http/endpoints/openai/realtime_transcription.go index 28a5147c17e0..c10535f18809 100644 --- a/core/http/endpoints/openai/realtime_transcription.go +++ b/core/http/endpoints/openai/realtime_transcription.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/mudler/LocalAI/core/http/endpoints/openai/types" + "github.com/mudler/LocalAI/core/schema" ) // emitPrecomputedTranscription emits the transcription events for a turn @@ -42,9 +43,10 @@ func emitPrecomputedTranscription(t Transport, itemID string, deltas []string, t // a single completed event. delta and completed events share itemID. func emitTranscription(ctx context.Context, t Transport, session *Session, itemID, audioPath string) (string, error) { cfg := session.InputAudioTranscription + diarize := session.ModelConfig != nil && session.ModelConfig.Pipeline.Diarization if session.ModelConfig != nil && session.ModelConfig.Pipeline.StreamTranscription() { - final, err := session.ModelInterface.TranscribeStream(ctx, audioPath, cfg.Language, false, false, cfg.Prompt, func(delta string) { + final, err := session.ModelInterface.TranscribeStream(ctx, audioPath, cfg.Language, false, diarize, cfg.Prompt, func(delta string) { _ = t.SendEvent(types.ConversationItemInputAudioTranscriptionDeltaEvent{ ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, ItemID: itemID, @@ -58,6 +60,11 @@ func emitTranscription(ctx context.Context, t Transport, session *Session, itemI transcript := "" if final != nil { transcript = final.Text + if diarize { + if err := emitSpeakerSegments(t, itemID, final); err != nil { + return "", err + } + } } if err := t.SendEvent(types.ConversationItemInputAudioTranscriptionCompletedEvent{ ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, @@ -71,13 +78,18 @@ func emitTranscription(ctx context.Context, t Transport, session *Session, itemI } // Unary fallback: transcribe the whole utterance, emit one completed event. - tr, err := session.ModelInterface.Transcribe(ctx, audioPath, cfg.Language, false, false, cfg.Prompt) + tr, err := session.ModelInterface.Transcribe(ctx, audioPath, cfg.Language, false, diarize, cfg.Prompt) if err != nil { return "", err } if tr == nil { return "", fmt.Errorf("transcribe result is nil") } + if diarize { + if err := emitSpeakerSegments(t, itemID, tr); err != nil { + return "", err + } + } if err := t.SendEvent(types.ConversationItemInputAudioTranscriptionCompletedEvent{ ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, ItemID: itemID, @@ -88,3 +100,29 @@ func emitTranscription(ctx context.Context, t Transport, session *Session, itemI } return tr.Text, nil } + +// emitSpeakerSegments forwards each speaker-labelled segment of a committed +// turn's transcript as a conversation.item.input_audio_transcription.segment +// event (pipeline.diarization), before the turn's completed event. Times are +// relative to the turn's audio and speaker labels are only consistent within +// the turn, as on the live path. Segments without a speaker are skipped. +func emitSpeakerSegments(t Transport, itemID string, tr *schema.TranscriptionResult) error { + for _, seg := range tr.Segments { + if seg.Speaker == "" { + continue + } + if err := t.SendEvent(types.ConversationItemInputAudioTranscriptionSegmentEvent{ + ServerEventBase: types.ServerEventBase{EventID: "event_TODO"}, + ItemID: itemID, + ContentIndex: 0, + ID: fmt.Sprintf("seg_%d", seg.Id), + Speaker: seg.Speaker, + Start: seg.Start.Seconds(), + End: seg.End.Seconds(), + Text: seg.Text, + }); err != nil { + return err + } + } + return nil +} diff --git a/core/http/endpoints/openai/realtime_transcription_test.go b/core/http/endpoints/openai/realtime_transcription_test.go index f3f760fd8a5f..e8ab399fd87b 100644 --- a/core/http/endpoints/openai/realtime_transcription_test.go +++ b/core/http/endpoints/openai/realtime_transcription_test.go @@ -2,6 +2,7 @@ package openai import ( "context" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -51,4 +52,86 @@ var _ = Describe("emitTranscription", func() { Expect(t.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionDelta)).To(Equal(0)) Expect(t.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionCompleted)).To(Equal(1)) }) + + Context("pipeline.diarization", func() { + labelled := &schema.TranscriptionResult{ + Text: "hi there. hello", + Segments: []schema.TranscriptionSegment{ + {Id: 0, Text: "hi there.", Start: 0, End: 600 * time.Millisecond, Speaker: "0"}, + {Id: 1, Text: "hello", Start: time.Second, End: 1400 * time.Millisecond, Speaker: "1"}, + {Id: 2, Text: "unlabelled"}, + }, + } + + segmentEvents := func(t *fakeTransport) []types.ConversationItemInputAudioTranscriptionSegmentEvent { + var out []types.ConversationItemInputAudioTranscriptionSegmentEvent + for _, e := range t.sent { + if seg, ok := e.(types.ConversationItemInputAudioTranscriptionSegmentEvent); ok { + out = append(out, seg) + } + } + return out + } + + It("requests speakers and emits one segment event per labelled segment", func() { + m := &fakeModel{transcribeFinal: labelled} + session := &Session{ + InputAudioTranscription: &types.AudioTranscription{}, + ModelConfig: &config.ModelConfig{Pipeline: config.Pipeline{Diarization: true}}, + ModelInterface: m, + } + t := &fakeTransport{} + + transcript, err := emitTranscription(context.Background(), t, session, "item1", "/tmp/x.wav") + + Expect(err).ToNot(HaveOccurred()) + Expect(transcript).To(Equal("hi there. hello")) + Expect(m.lastDiarize).To(BeTrue()) + segs := segmentEvents(t) + Expect(segs).To(HaveLen(2)) + Expect(segs[0].ItemID).To(Equal("item1")) + Expect(segs[0].Speaker).To(Equal("0")) + Expect(segs[0].Text).To(Equal("hi there.")) + Expect(segs[1].Speaker).To(Equal("1")) + Expect(segs[1].Start).To(BeNumerically("~", 1.0, 1e-9)) + Expect(segs[1].End).To(BeNumerically("~", 1.4, 1e-9)) + Expect(t.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionCompleted)).To(Equal(1)) + }) + + It("also emits segment events on the streaming transcription path", func() { + on := true + m := &fakeModel{transcribeDeltas: []string{"hi"}, transcribeFinal: labelled} + session := &Session{ + InputAudioTranscription: &types.AudioTranscription{}, + ModelConfig: &config.ModelConfig{Pipeline: config.Pipeline{ + Diarization: true, + Streaming: config.PipelineStreaming{Transcription: &on}, + }}, + ModelInterface: m, + } + t := &fakeTransport{} + + _, err := emitTranscription(context.Background(), t, session, "item1", "/tmp/x.wav") + + Expect(err).ToNot(HaveOccurred()) + Expect(m.lastDiarize).To(BeTrue()) + Expect(segmentEvents(t)).To(HaveLen(2)) + }) + + It("neither asks for speakers nor emits segments when off", func() { + m := &fakeModel{transcribeFinal: labelled} + session := &Session{ + InputAudioTranscription: &types.AudioTranscription{}, + ModelConfig: &config.ModelConfig{}, + ModelInterface: m, + } + t := &fakeTransport{} + + _, err := emitTranscription(context.Background(), t, session, "item1", "/tmp/x.wav") + + Expect(err).ToNot(HaveOccurred()) + Expect(m.lastDiarize).To(BeFalse()) + Expect(segmentEvents(t)).To(BeEmpty()) + }) + }) }) diff --git a/core/http/endpoints/openai/transcription.go b/core/http/endpoints/openai/transcription.go index 6c99d7afcaaa..49825c3a13cd 100644 --- a/core/http/endpoints/openai/transcription.go +++ b/core/http/endpoints/openai/transcription.go @@ -210,18 +210,20 @@ func TranscriptEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app } for _, word := range tr.Words { trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } for _, seg := range tr.Segments { segWords := []schema.TranscriptionWordSeconds{} for _, word := range seg.Words { segWords = append(segWords, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{ @@ -338,12 +340,16 @@ func streamTranscription(c echo.Context, req backend.TranscriptionRequest, ml *m if len(finalResult.Segments) > 0 { segs := make([]map[string]any, 0, len(finalResult.Segments)) for _, seg := range finalResult.Segments { - segs = append(segs, map[string]any{ + entry := map[string]any{ "id": seg.Id, "start": seg.Start.Seconds(), "end": seg.End.Seconds(), "text": seg.Text, - }) + } + if seg.Speaker != "" { + entry["speaker"] = seg.Speaker + } + segs = append(segs, entry) } doneEvent["segments"] = segs } diff --git a/core/http/endpoints/openai/types/server_events.go b/core/http/endpoints/openai/types/server_events.go index 114a7065a055..4cab30a61d80 100644 --- a/core/http/endpoints/openai/types/server_events.go +++ b/core/http/endpoints/openai/types/server_events.go @@ -512,6 +512,15 @@ type ConversationItemSoundDetectionEvent struct { // The scored sound-event tags, in score-descending order. Detections []SoundDetectionTag `json:"detections"` + + // The start time of the detection window in seconds, when known. Set by + // the live scene-event path (a companion sound stream alongside live + // transcription); omitted by the unary/windowed sound-detection paths, + // which have no per-event timing. + Start *float64 `json:"start,omitempty"` + + // The end time of the detection window in seconds, when known. + End *float64 `json:"end,omitempty"` } func (m ConversationItemSoundDetectionEvent) ServerEventType() ServerEventType { @@ -586,11 +595,13 @@ type ConversationItemInputAudioTranscriptionSegmentEvent struct { // The speaker label for the segment, if available. Speaker string `json:"speaker,omitempty"` - // The start time of the segment in seconds. - Start float64 `json:"start,omitempty"` + // The start time of the segment in seconds. Always present (not + // omitempty: a segment starting at 0.0s must still carry "start"). + Start float64 `json:"start"` - // The end time of the segment in seconds. - End float64 `json:"end,omitempty"` + // The end time of the segment in seconds. Always present (not + // omitempty: see Start). + End float64 `json:"end"` // The text content of the segment. Text string `json:"text,omitempty"` diff --git a/core/schema/transcription.go b/core/schema/transcription.go index 8414fd0ba805..aab914f704e9 100644 --- a/core/schema/transcription.go +++ b/core/schema/transcription.go @@ -13,9 +13,10 @@ type TranscriptionSegment struct { } type TranscriptionWord struct { - Start time.Duration `json:"start"` - End time.Duration `json:"end"` - Text string `json:"text"` + Start time.Duration `json:"start"` + End time.Duration `json:"end"` + Text string `json:"text"` + Speaker string `json:"speaker,omitempty"` } type TranscriptionResult struct { @@ -42,9 +43,10 @@ type TranscriptionSegmentSeconds struct { } type TranscriptionWordSeconds struct { - Start float64 `json:"start"` - End float64 `json:"end"` - Text string `json:"text"` + Start float64 `json:"start"` + End float64 `json:"end"` + Text string `json:"text"` + Speaker string `json:"speaker,omitempty"` } type TranscriptionResultSeconds struct { diff --git a/docs/content/features/audio-classification.md b/docs/content/features/audio-classification.md index 09759ab72da3..4f9c58fb9d61 100644 --- a/docs/content/features/audio-classification.md +++ b/docs/content/features/audio-classification.md @@ -9,6 +9,8 @@ Sound-event classification (audio tagging) answers the question **"what am I hea LocalAI exposes this through the `/v1/audio/classification` endpoint, modelled after `/v1/audio/transcriptions`. The reference backend is **[ced.cpp](https://github.com/localai-org/ced.cpp)** (CED, a 527-class AudioSet tagger), a small ViT over a log-mel spectrogram ported to ggml with full PyTorch parity. Apache-2.0 weights are redistributable as GGUF. +**[parakeet.cpp](https://github.com/mudler/parakeet.cpp)** can also load a CED model (through `third_party/ced.cpp`) and serve `/v1/audio/classification` from the same backend used for ASR and diarization. It scores the clip in 10 s windows and averages each class's score across the windows before sorting and applying `top_k`/`threshold` - CED's own method for clips longer than one window. Install `parakeet-cpp-ced-tiny` or `parakeet-cpp-ced-base` from the gallery, or point `parameters.model` at a CED GGUF under `backend: parakeet-cpp`. A parakeet-cpp ASR model can also point `sound_model` at a CED GGUF to add live sound events during realtime transcription - see [Realtime API]({{% relref "openai-realtime" %}}). + Because classification is exposed as a regular OpenAI-style endpoint, any HTTP client works - there is no Python dependency on the consumer side. In distributed mode, LocalAI stages uploaded audio and realtime sound-detection @@ -59,6 +61,25 @@ curl http://localhost:8080/v1/audio/classification \ -F top_k=10 ``` +The same request works unchanged against a parakeet-cpp CED model: + +```yaml +name: parakeet-ced-tiny +backend: parakeet-cpp +parameters: + model: ced-tiny-q8_0.gguf +known_usecases: + - sound_classification +``` + +```bash +curl http://localhost:8080/v1/audio/classification \ + -H "Content-Type: multipart/form-data" \ + -F file="@/path/to/clip.wav" \ + -F model="parakeet-ced-tiny" \ + -F top_k=10 +``` + ## See also - [Audio to Text]({{% relref "audio-to-text" %}}) - speech transcription diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md index 83a51343e561..37f8c81596b8 100644 --- a/docs/content/features/audio-diarization.md +++ b/docs/content/features/audio-diarization.md @@ -9,12 +9,13 @@ url = "/features/audio-diarization/" Speaker diarization answers the question **"who spoke when?"** - given an audio clip with multiple speakers, it returns time-stamped segments labelled with a stable speaker ID (`SPEAKER_00`, `SPEAKER_01`, …). -LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Four backends are supported today: +LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Five backends are supported today: - **[sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx)** - pyannote-3.0 segmentation + a speaker-embedding extractor (3D-Speaker, NeMo, WeSpeaker) + fast clustering. Pure diarization - no transcription cost. Recommended when you only need speaker turns. - **[vibevoice.cpp](https://github.com/microsoft/VibeVoice)** - produces speaker-labelled segments as a by-product of its long-form ASR pass, so you can optionally get a transcript per segment for free. - **[NeMo-Speech.cpp](https://github.com/NVIDIA/NeMo-Speech.cpp)** - NVIDIA Sortformer, served standalone by the [NeMo-Speech.cpp backend]({{%relref "features/nemo-speech-cpp" %}}). It is end to end, so the speaker capacity is fixed by the checkpoint and the count hints are ignored. The same backend can instead put speaker tags on a transcript, by attaching a Sortformer model to an ASR one. - **[audio.cpp](https://github.com/0xShug0/audio.cpp)** - the `sortformer_diar` family, served by the multi-modality [audio.cpp backend]({{%relref "features/audio-cpp" %}}). +- **[parakeet.cpp](https://github.com/mudler/parakeet.cpp)** - NVIDIA Nemotron-3-Diarization (Sortformer, up to 8 speakers), served standalone or paired with a Parakeet ASR model for per-segment text. See the [Audio to Text]({{% relref "audio-to-text" %}}) page for the parakeet-cpp option reference. Because diarization is exposed as a regular OpenAI-compatible endpoint, any HTTP client works. There is no Python dependency on pyannote or NeMo on the consumer side. @@ -157,10 +158,38 @@ curl http://localhost:8080/v1/audio/diarization \ -F response_format=verbose_json ``` +## Backend setup - parakeet-cpp (Nemotron-3-Diarization) + +Nemotron-3-Diarization is Sortformer, served standalone or paired with a Parakeet ASR model. Install `parakeet-cpp-nemotron-3-diarization` from the gallery for diarization only, or `parakeet-cpp-nemotron-3-diarization-asr` for the same model paired with `parakeet-cpp-tdt_ctc-110m` through the `asr_model` option: + +```yaml +name: parakeet-diarize +backend: parakeet-cpp +parameters: + model: nemotron-3-diarization-q8_0.gguf +options: + - asr_model:tdt_ctc-110m-f16.gguf +known_usecases: + - diarization +``` + +Getting text on each segment needs both: an `asr_model` companion loaded on the model, and `include_text=true` on the request. With only one of the two, segments carry no text and no error is raised. Sortformer has a fixed speaker capacity and no clustering stage, so `num_speakers`, `min_speakers`, `max_speakers` and `clustering_threshold` are ignored (logged at debug); `min_duration_on` and `min_duration_off` are honored. Speaker labels are the decimal index the model assigned (`"0"`, `"1"`, …), or `"unknown"` when a segment has no diarized speaker. + +```bash +curl http://localhost:8080/v1/audio/diarization \ + -H "Content-Type: multipart/form-data" \ + -F file="@meeting.wav" \ + -F model="parakeet-diarize" \ + -F include_text=true \ + -F response_format=verbose_json +``` + +Sortformer clusters on voice-like characteristics, not on "is this a human". A loud non-speech sound with voice-like pitch and rhythm (a rooster crow, in one test clip) can come back as its own speaker segment alongside the real speakers. This is model behavior, not a bug in the LocalAI integration: treat an unexpected extra speaker as a hint the clip may contain a non-speech sound, and use [Sound Classification]({{% relref "audio-classification" %}}) to confirm what it is. + ## Notes - **Speaker identity across files**: speaker IDs (`SPEAKER_00`, `SPEAKER_01`, …) are local to each request. To track the same person across multiple recordings, combine `/v1/audio/diarization` with `/v1/voice/embed` (speaker embedding) and maintain your own embedding store. -- **Hints vs. forces**: `num_speakers` overrides clustering when set; `min_speakers` / `max_speakers` are advisory and only honored by backends that expose a range hint. vibevoice.cpp ignores them - its model picks the count itself. +- **Hints vs. forces**: `num_speakers` overrides clustering when set; `min_speakers` / `max_speakers` are advisory and only honored by backends that expose a range hint. vibevoice.cpp and parakeet-cpp (Sortformer) ignore them - the model picks the count itself. - **Sample rate**: input is automatically converted to 16 kHz mono via ffmpeg before the backend sees it; sherpa-onnx pyannote-3.0 requires 16 kHz. ## See also diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 5a5e833cf3a3..d01edec4e391 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -12,7 +12,7 @@ The transcription endpoint allows to convert audio files to text. The endpoint s - **moonshine**: Ultra-fast transcription engine optimized for low-end devices - **faster-whisper**: Fast Whisper implementation with CTranslate2 - **WhisperX**: Whisper transcription with word alignment and optional speaker diarization. Set `HF_TOKEN` and pass `diarize=true` to load WhisperX's gated pyannote diarization pipeline. -- **[parakeet-cpp](https://github.com/mudler/parakeet.cpp)**: A C++/ggml port of NVIDIA NeMo Parakeet (FastConformer TDT/CTC/RNNT/hybrid). Runs quantized GGUFs on CPU or GPU, emits word-level timestamps, and supports cache-aware streaming (the `realtime_eou` model surfaces end-of-utterance events). +- **[parakeet-cpp](https://github.com/mudler/parakeet.cpp)**: A C++/ggml port of NVIDIA NeMo Parakeet (FastConformer TDT/CTC/RNNT/hybrid). Runs quantized GGUFs on CPU or GPU, emits word-level timestamps, and supports cache-aware streaming (the `realtime_eou` model surfaces end-of-utterance events). The same backend also loads Nemotron-3-Diarization (`/v1/audio/diarization`) and CED sound models (`/v1/audio/classification`), and can attach either as a companion to a transcription model. - **llama-cpp**: Route transcription to any multimodal-audio GGUF model served by the `llama-cpp` backend (e.g. [Qwen3-ASR](https://huggingface.co/ggml-org/Qwen3-ASR-0.6B-GGUF), Voxtral, Qwen2-Audio). Under the hood the request is converted into a chat completion with the audio attached via the model's audio encoder - the same path the upstream llama.cpp server uses. Set `backend: llama-cpp` in the model YAML and point `mmproj` at the matching audio encoder. - **voxtral**: Voxtral-family models served by a dedicated backend - **[NeMo-Speech.cpp](https://github.com/NVIDIA/NeMo-Speech.cpp)**: NVIDIA's C++/ggml runtime for the Nemotron Speech models. Serves offline, streaming and live transcription, with VAD, punctuation, inverse text normalization and Sortformer speaker tags attached through model options, and covers diarization, speech synthesis and translation from the same backend. See the [NeMo-Speech.cpp backend]({{%relref "features/nemo-speech-cpp" %}}) page for the model options. @@ -190,6 +190,21 @@ curl http://localhost:8080/v1/audio/transcriptions \ For real-time use, load a cache-aware streaming model (e.g. `realtime_eou_120m-v1-*.gguf`) and pass `-F stream=true`. Deltas are emitted as the audio is decoded, with end-of-utterance events closing each segment. +### Diarization and sound classification + +The same backend also serves the `/v1/audio/diarization` and `/v1/audio/classification` endpoints, and can attach a diarization or sound model to a live transcription session. `options:` accepts paths relative to the models directory, or absolute: + +| Option | Allowed on | Used for | +|---|---|---| +| `asr_model:` | a diarization model | `include_text` on `/v1/audio/diarization` | +| `diarization_model:` | an ASR model | a `speaker` on transcript segments (and words), and speaker segments during realtime live transcription | +| `sound_model:` | an ASR model | sound events during realtime live transcription | +| `diarization_latency:` | a model with a diarization companion | latency mode for the live speaker stream; default `low` | + +With a `diarization_model` companion, `/v1/audio/transcriptions` labels each segment with its `speaker` (`"0"`, `"1"`, ... in order of first appearance) and splits segments where the speaker changes; with `timestamp_granularities[]=word` each word carries its speaker too. With `stream=true` the closing `transcript.text.done` event lists the segments with their speakers. Pass `-F diarize=false` to skip diarization for one request. The diarization GGUF can also be imported directly: `local-ai models import https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-f16.gguf`. + +The loader rejects a companion whose role duplicates the primary's own (for example `asr_model:` on an already-ASR primary, or `sound_model:` on a CED primary), and rejects a companion GGUF that does not match the role its option names (for example `sound_model:` pointing at an ASR GGUF fails to load, naming the kind it expected). See [Speaker Diarization]({{% relref "audio-diarization" %}}) for the `Diarize` RPC and [Sound Classification]({{% relref "audio-classification" %}}) for `SoundDetection`, and [Realtime API]({{% relref "openai-realtime" %}}) for the live speaker/sound events emitted during a realtime session. + ### Segment timestamps Transcriptions are split into segments the same way NVIDIA NeMo does: a new segment starts after sentence-ending punctuation (`.`, `?`, `!`), and each segment carries `start`/`end` times. This is the default (NeMo's punctuation-only segmentation) and needs no configuration. While streaming, each end-of-utterance closes a segment, now with timestamps. diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index ac7afb484796..150966cc452a 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -129,6 +129,113 @@ A client `session.update` still overrides `type` and `eagerness` per session. - `false` (default): the transcript accumulated from the live stream is used as-is - the model runs once per utterance and the LLM starts immediately at commit. - `true`: the committed audio is re-transcribed offline. If the batch decode also ends with the end-of-utterance token the turn proceeds (using the batch transcript); if it does **not**, the commit is cancelled and the session keeps listening - treating the streaming token as a false positive. Both transcripts are compared and logged, which makes this mode a useful diagnostic for how well the streaming and batch decodes align, at the cost of one extra decode per turn. +### Live speaker and sound events (parakeet-cpp) + +When the `semantic_vad` transcription model is a parakeet-cpp model loaded with a `diarization_model` and/or `sound_model` companion (see [Audio to Text]({{% relref "audio-to-text" %}})), the realtime session also streams speaker and sound events while a turn is live, alongside the transcript deltas. Nothing needs to change on the client: unrecognized event types are ignored by standard OpenAI Realtime clients. + +The transcription model, with its companions: + +```yaml +name: parakeet-realtime-scene +backend: parakeet-cpp +parameters: + model: realtime_eou_120m-v1-f16.gguf +options: + - diarization_model:nemotron-3-diarization-q8_0.gguf + - sound_model:ced-tiny-q8_0.gguf +``` + +The realtime pipeline that uses it: + +```yaml +name: gpt-realtime +pipeline: + vad: silero-vad-ggml + transcription: parakeet-realtime-scene + llm: qwen3-4b + tts: tts-1 + turn_detection: + type: semantic_vad +``` + +Each closed speaker segment emits a `conversation.item.input_audio_transcription.segment` event under the turn's item id, with an empty `text` (the event exists to carry the speaker boundary, not a transcript - the transcript still comes from the ordinary delta/completed events): + +```json +{ + "type": "conversation.item.input_audio_transcription.segment", + "item_id": "item_abc", + "content_index": 0, + "speaker": "0", + "start": 1.92, + "end": 4.10, + "text": "" +} +``` + +Each sound event emits a `conversation.item.sound_detection` event with one tag and the detection window's `start`/`end`: + +```json +{ + "type": "conversation.item.sound_detection", + "item_id": "item_abc", + "content_index": 0, + "detections": [{"label": "Chicken, rooster", "score": 0.91, "index": 99}], + "start": 24.0, + "end": 30.0 +} +``` + +The `start`/`end` on both event types are seconds measured from the start of the current turn's own audio, not the session or the WebSocket connection - the same base the streamed transcript words use. + +The companion stream is opened fresh for each speech turn, alongside that turn's ASR live session, and closed when the turn commits: whatever it had not yet emitted is drained and sent at that point. Because the diarization model runs a brand new session every turn, its speaker indices are scoped to the turn too - `"speaker": "0"` in one turn and `"speaker": "0"` in the next are not guaranteed to be the same person, even within the same conversation. + +`score` is the peak score seen for that tag while the sound was live, not an average. + +**Limitation**: under `semantic_vad`, live transcription (and so this companion stream) only runs during speech turns - it does not see audio between turns. A sound that happens while nobody is speaking is not detected this way. If you need sound events independent of speech turns, use the pipeline's `sound_detection` model instead (see [Sound Classification]({{% relref "audio-classification" %}})), which classifies each VAD-committed utterance on its own. Use one or the other, not both, on the same session - they overlap in purpose and would emit sound detections twice. + +### Speaker and sound events with an offline model (Parakeet TDT v3) + +The live events above need a cache-aware streaming transcription model. An offline model such as Parakeet TDT 0.6B v3 (25 languages) runs under `server_vad` instead: each VAD-committed turn is transcribed as a whole. The gallery model `parakeet-cpp-realtime-scene-tdt` bundles it with Nemotron-3-Diarization and CED-Tiny, so one parakeet-cpp backend handles transcription, speakers and sounds. Point both `transcription` and `sound_detection` at it and turn on `diarization`: + +```yaml +name: gpt-realtime-scene +pipeline: + vad: silero-vad-ggml + transcription: parakeet-cpp-realtime-scene-tdt + sound_detection: parakeet-cpp-realtime-scene-tdt + diarization: true + llm: qwen3-4b + tts: tts-1 +``` + +`pipeline.diarization` asks the transcription model for speaker labels on each committed turn and emits every labelled segment as a `conversation.item.input_audio_transcription.segment` event before the turn's `completed` event. Unlike the live path, these segments carry their `text`: + +```json +{ + "type": "conversation.item.input_audio_transcription.segment", + "item_id": "item_abc", + "content_index": 0, + "id": "seg_1", + "speaker": "1", + "start": 6.85, + "end": 10.82, + "text": "Well, I don't wish to see it any more, observed Phoebe, turning away her eyes." +} +``` + +`sound_detection` classifies the same committed audio and emits one `conversation.item.sound_detection` event per turn (see [Sound Classification]({{% relref "audio-classification" %}})). As on the live path, times are relative to the turn's audio and speaker labels are only consistent within a turn. `pipeline.diarization` is off by default: it needs a transcription model that diarizes (parakeet-cpp with a `diarization_model` companion), and some other backends fail a diarization request they cannot serve. + +#### Choosing the sound model + +Both scene models ship with CED-Tiny, the cheapest to run all the time. `parakeet-cpp-realtime-scene-base` and `parakeet-cpp-realtime-scene-tdt-base` are the same pipelines with CED-Base (86M, the largest CED), which tags sounds more confidently. Any CED GGUF from [`mudler/ced-gguf`](https://huggingface.co/mudler/ced-gguf) (tiny, mini, small, base) works as `sound_model`. Measured on CPU (Ryzen 9 9950X3D) over a 37 s clip with two speakers and a rooster, as a fraction of real time: + +| | CED-Tiny | CED-Base | +|---|---|---| +| Live scene stream (diarization `low` + sound), EOU path | 0.103 | 0.125 | +| Sound detection per committed turn, TDT path | 0.005 | 0.031 | + +The EOU model's own ASR stream adds 0.016. Diarization dominates the live cost, so CED-Base keeps the live path about 7x faster than real time. + ### Disabling thinking For reasoning models, you can force the pipeline LLM's thinking off without editing the LLM model config: diff --git a/gallery/index.yaml b/gallery/index.yaml index 4e37b0e78aa6..71c572e6bad2 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -53670,6 +53670,389 @@ - filename: parakeet-cpp/nemotron-3.5-asr-streaming-0.6b-q8_0.gguf uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3.5-asr-streaming-0.6b-q8_0.gguf sha256: ba2f13eccd4a5245be728f77e6149bd6a4fdcdd133ff2e08ac6005bcef7a99f1 +- name: parakeet-cpp-nemotron-3-diarization + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://github.com/mudler/parakeet.cpp + description: | + Nemotron-3-Diarization (Sortformer), Q8_0 GGUF for the parakeet-cpp backend + (C++/ggml port of NVIDIA NeMo). Speaker diarization only: served through + /v1/audio/diarization, returns per-segment start, end and speaker label + ("0", "1", ...). It does not transcribe; pair it with an ASR model and set + asr_model to get speaker-attributed text from the same call. num_speakers, + min_speakers, max_speakers and clustering_threshold are not supported by + Sortformer and are ignored. + license: openmdw-1.1 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - diarization + - speaker-diarization + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - diarization + name: parakeet-cpp-nemotron-3-diarization + parameters: + model: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + files: + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 +- name: parakeet-cpp-nemotron-3-diarization-asr + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/nvidia/parakeet-tdt_ctc-110m + - https://github.com/mudler/parakeet.cpp + description: | + Nemotron-3-Diarization (Sortformer) paired with the Parakeet TDT+CTC 110M + ASR model through the asr_model option, both Q8_0/F16 GGUF for the + parakeet-cpp backend (C++/ggml port of NVIDIA NeMo). Served through + /v1/audio/diarization with include_text: each speaker segment comes back + with its transcribed text in one call. Diarization model is + OpenMDW-1.1, ASR model is CC-BY-4.0. + license: openmdw-1.1 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - asr + - diarization + - speaker-diarization + - speech-recognition + - stt + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - diarization + name: parakeet-cpp-nemotron-3-diarization-asr + options: + - asr_model:parakeet-cpp/tdt_ctc-110m-f16.gguf + parameters: + model: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + files: + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/tdt_ctc-110m-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/tdt_ctc-110m-f16.gguf + sha256: 7f9a6376edde6a74592ace48b2ebdc27a1ac972d0be9dfcc29e668d99381faf1 +- name: parakeet-cpp-ced-tiny + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/mispeech/ced-tiny + - https://github.com/mudler/parakeet.cpp + description: | + CED-Tiny sound event tagger, Q8_0 GGUF for the parakeet-cpp backend + (C++/ggml, loaded through third_party/ced.cpp). Served through + /v1/audio/classification: 10 s windows are scored and averaged over the + clip, then sorted by score with threshold and top_k applied. Smallest and + fastest of the CED sizes; use ced-base for higher accuracy. + license: apache-2.0 + tags: + - parakeet-cpp + - ced + - sound-classification + - audio-tagging + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - sound_classification + name: parakeet-cpp-ced-tiny + parameters: + model: parakeet-cpp/ced-tiny-q8_0.gguf + files: + - filename: parakeet-cpp/ced-tiny-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-tiny-q8_0.gguf + sha256: 48bee4e2fc3cc85d7806e03471db24e77fda6c2a2e81ffe9ef67caebaf2bd674 +- name: parakeet-cpp-ced-base + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/mispeech/ced-base + - https://github.com/mudler/parakeet.cpp + description: | + CED-Base sound event tagger, Q8_0 GGUF for the parakeet-cpp backend + (C++/ggml, loaded through third_party/ced.cpp). Served through + /v1/audio/classification: 10 s windows are scored and averaged over the + clip, then sorted by score with threshold and top_k applied. Larger and + more accurate than ced-tiny, still CPU-friendly. + license: apache-2.0 + tags: + - parakeet-cpp + - ced + - sound-classification + - audio-tagging + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - sound_classification + name: parakeet-cpp-ced-base + parameters: + model: parakeet-cpp/ced-base-q8_0.gguf + files: + - filename: parakeet-cpp/ced-base-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-base-q8_0.gguf + sha256: bd34a7710169f0047fea17267965d211f967828ab25ba6fb9d3768481393f6e2 +- name: parakeet-cpp-realtime-scene + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/nvidia/parakeet_realtime_eou_120m-v1 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-tiny + - https://github.com/mudler/parakeet.cpp + description: | + Cache-aware streaming RNNT FastConformer with end-of-utterance (EOU) + detection, 120M, paired with Nemotron-3-Diarization and CED-Tiny through + the diarization_model and sound_model options. F16/Q8_0 GGUF for the + parakeet-cpp backend (C++/ggml port of NVIDIA NeMo). Use with streaming + transcription: while a turn is live, closed speaker segments and sound + events are surfaced alongside the ASR text (realtime + conversation.item.input_audio_transcription.segment and + conversation.item.sound_detection events). Live speaker/sound events only + fire during speech turns under semantic_vad; sounds between turns are not + seen by this path. License per model: transcription model NVIDIA Open + Model License, diarization model OpenMDW-1.1, CED-Tiny Apache-2.0. + license: nvidia-open-model-license + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - asr + - speech-recognition + - diarization + - sound-classification + - streaming + - realtime + - stt + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + name: parakeet-cpp-realtime-scene + options: + - diarization_model:parakeet-cpp/nemotron-3-diarization-q8_0.gguf + - sound_model:parakeet-cpp/ced-tiny-q8_0.gguf + parameters: + model: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + files: + - filename: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/realtime_eou_120m-v1-f16.gguf + sha256: d1a2b12f12b8a096a57499c9111ed13b442a2b786e17a292c168be45088f0edc + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/ced-tiny-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-tiny-q8_0.gguf + sha256: 48bee4e2fc3cc85d7806e03471db24e77fda6c2a2e81ffe9ef67caebaf2bd674 +- name: parakeet-cpp-realtime-scene-tdt + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-tiny + - https://github.com/mudler/parakeet.cpp + description: | + Parakeet TDT 0.6B v3 (multilingual, 25 European languages) paired with + Nemotron-3-Diarization and CED-Tiny through the diarization_model and + sound_model options: one parakeet-cpp backend transcribes, labels speakers + and tags sound events. GGUF for the parakeet-cpp backend (C++/ggml port of + NVIDIA NeMo). TDT is not a streaming model, so in a realtime pipeline use + it with server_vad: set it as both transcription and sound_detection and + turn on pipeline.diarization, and each committed turn gets speaker segments + (conversation.item.input_audio_transcription.segment, with text) and + sound tags (conversation.item.sound_detection). Also labels speakers on + /v1/audio/transcriptions. Speaker labels are per turn. License per model: + transcription model CC-BY-4.0, diarization model OpenMDW-1.1, CED-Tiny + Apache-2.0. + license: cc-by-4.0 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - asr + - speech-recognition + - diarization + - sound-classification + - multilingual + - realtime + - stt + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + - diarization + - sound_classification + name: parakeet-cpp-realtime-scene-tdt + options: + - diarization_model:parakeet-cpp/nemotron-3-diarization-q8_0.gguf + - sound_model:parakeet-cpp/ced-tiny-q8_0.gguf + parameters: + model: parakeet-cpp/tdt-0.6b-v3-f16.gguf + files: + - filename: parakeet-cpp/tdt-0.6b-v3-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/tdt-0.6b-v3-f16.gguf + sha256: 8ba47343e1e919895aca90e099150a01ed203ee0942d8ed31e27295efc5abb22 + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/ced-tiny-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-tiny-q8_0.gguf + sha256: 48bee4e2fc3cc85d7806e03471db24e77fda6c2a2e81ffe9ef67caebaf2bd674 +- name: parakeet-cpp-realtime-scene-base + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/nvidia/parakeet_realtime_eou_120m-v1 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-base + - https://github.com/mudler/parakeet.cpp + description: | + Cache-aware streaming RNNT FastConformer with end-of-utterance (EOU) + detection, 120M, paired with Nemotron-3-Diarization and CED-Base (86M, the largest CED; + more confident sound tags than CED-Tiny at a small extra cost: on CPU the + live diarization + sound stream runs at 0.125 of real time against 0.103 + with CED-Tiny) through + the diarization_model and sound_model options. F16/Q8_0 GGUF for the + parakeet-cpp backend (C++/ggml port of NVIDIA NeMo). Use with streaming + transcription: while a turn is live, closed speaker segments and sound + events are surfaced alongside the ASR text (realtime + conversation.item.input_audio_transcription.segment and + conversation.item.sound_detection events). Live speaker/sound events only + fire during speech turns under semantic_vad; sounds between turns are not + seen by this path. License per model: transcription model NVIDIA Open + Model License, diarization model OpenMDW-1.1, CED-Base Apache-2.0. + license: nvidia-open-model-license + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - asr + - speech-recognition + - diarization + - sound-classification + - streaming + - realtime + - stt + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + name: parakeet-cpp-realtime-scene-base + options: + - diarization_model:parakeet-cpp/nemotron-3-diarization-q8_0.gguf + - sound_model:parakeet-cpp/ced-base-q8_0.gguf + parameters: + model: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + files: + - filename: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/realtime_eou_120m-v1-f16.gguf + sha256: d1a2b12f12b8a096a57499c9111ed13b442a2b786e17a292c168be45088f0edc + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/ced-base-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-base-q8_0.gguf + sha256: bd34a7710169f0047fea17267965d211f967828ab25ba6fb9d3768481393f6e2 +- name: parakeet-cpp-realtime-scene-tdt-base + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-base + - https://github.com/mudler/parakeet.cpp + description: | + Parakeet TDT 0.6B v3 (multilingual, 25 European languages) paired with + Nemotron-3-Diarization and CED-Base (86M, the largest CED; about 0.03 s of + CPU per second of audio per committed turn, against 0.005 for CED-Tiny) + through the diarization_model and + sound_model options: one parakeet-cpp backend transcribes, labels speakers + and tags sound events. GGUF for the parakeet-cpp backend (C++/ggml port of + NVIDIA NeMo). TDT is not a streaming model, so in a realtime pipeline use + it with server_vad: set it as both transcription and sound_detection and + turn on pipeline.diarization, and each committed turn gets speaker segments + (conversation.item.input_audio_transcription.segment, with text) and + sound tags (conversation.item.sound_detection). Also labels speakers on + /v1/audio/transcriptions. Speaker labels are per turn. License per model: + transcription model CC-BY-4.0, diarization model OpenMDW-1.1, CED-Tiny + Apache-2.0. + license: cc-by-4.0 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - asr + - speech-recognition + - diarization + - sound-classification + - multilingual + - realtime + - stt + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + - diarization + - sound_classification + name: parakeet-cpp-realtime-scene-tdt-base + options: + - diarization_model:parakeet-cpp/nemotron-3-diarization-q8_0.gguf + - sound_model:parakeet-cpp/ced-base-q8_0.gguf + parameters: + model: parakeet-cpp/tdt-0.6b-v3-f16.gguf + files: + - filename: parakeet-cpp/tdt-0.6b-v3-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/tdt-0.6b-v3-f16.gguf + sha256: 8ba47343e1e919895aca90e099150a01ed203ee0942d8ed31e27295efc5abb22 + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/ced-base-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-base-q8_0.gguf + sha256: bd34a7710169f0047fea17267965d211f967828ab25ba6fb9d3768481393f6e2 - name: moss-transcribe-cpp-0.9b url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: