diff --git a/backend/backend.proto b/backend/backend.proto index 54255528e5ad..9ec5f862fd33 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -647,6 +647,7 @@ message TranscriptWord { int64 start = 1; int64 end = 2; string text = 3; + string speaker = 4; // backend speaker label when diarizing; empty otherwise } message TranscriptSegment { diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile index e288f6fcc648..4d462d11b46b 100644 --- a/backend/go/parakeet-cpp/Makefile +++ b/backend/go/parakeet-cpp/Makefile @@ -1,6 +1,6 @@ # parakeet-cpp backend Makefile. # -# Upstream pin lives below as PARAKEET_VERSION?=2bf88954dc628b32835734e2e9159550a75a1dc6 +# Upstream pin lives below as PARAKEET_VERSION?=6dea76a81ee0bba83c6dad320dccc2fbeef111e2 # (.github/bump_deps.sh) can find and update it - matches the # whisper.cpp / ds4 / vibevoice-cpp convention. # @@ -15,7 +15,7 @@ # That's what the L0 smoke test uses. The default target below does the # proper clone-at-pin + cmake build so CI doesn't need a side-checkout. -PARAKEET_VERSION?=2bf88954dc628b32835734e2e9159550a75a1dc6 +PARAKEET_VERSION?=6dea76a81ee0bba83c6dad320dccc2fbeef111e2 PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp GOCMD?=go diff --git a/backend/go/parakeet-cpp/diarize.go b/backend/go/parakeet-cpp/diarize.go new file mode 100644 index 000000000000..a1eb4decde09 --- /dev/null +++ b/backend/go/parakeet-cpp/diarize.go @@ -0,0 +1,505 @@ +package main + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "strings" + "unsafe" + + gguf "github.com/gpustack/gguf-parser-go" + "github.com/mudler/LocalAI/pkg/grpc/grpcerrors" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Speaker diarization (nvidia/Nemotron-3-Diarization and compatible Sortformer +// models, parakeet.cpp ABI >= 7). +// +// A parakeet_ctx holds either an ASR model or a diarization model; +// parakeet_capi_load detects which from the GGUF. The backend serves +// diarization in two shapes: +// +// - the model file IS a diarization GGUF: the Diarize RPC works, transcription +// does not (there is no ASR to run); +// - an ASR model with a diarization GGUF attached through the diar_model +// option (the same key nemo-speech-cpp uses): transcripts carry a speaker +// per segment, and Diarize can fill each segment's text (include_text). +// +// Diarization runs the checkpoint's own mode, NeMo's diarize() default, which +// for Nemotron-3-Diarization is cache-aware streaming in 21.12 s chunks. +// Speakers are 0-based and numbered in order of first appearance; the model's +// speaker capacity (8) is fixed by the checkpoint. + +// CppDiarizePcm is parakeet_capi_diarize_pcm: 16 kHz mono float PCM in, a +// malloc'd JSON document out (uintptr, freed via CppFreeString): +// +// {"speakers":8,"segments":[{"speaker":0,"start":0.50,"end":5.52}, ...]} +// +// Present only in libparakeet.so with ABI >= 7; nil disables diarization. +var CppDiarizePcm func(ctx uintptr, samples []float32, nSamples int32, sampleRate int32) uintptr + +// Streaming diarization (parakeet_capi_diarize_stream_*), used by live +// transcription. begin_latency takes a PARAKEET_DIAR_LATENCY_* mode; feed and +// active return a malloc'd parakeet_diar_segment array through out/nOut (free +// with CppFreeDiarSegments) and 0 on success. Present only in newer +// libparakeet.so; nil disables speaker labels on live transcripts. +var ( + CppDiarizeStreamBeginLatency func(ctx uintptr, latency int32) uintptr + CppDiarizeStreamFeed func(s uintptr, pcm []float32, nSamples int32, isLast int32, out unsafe.Pointer, nOut unsafe.Pointer) int32 + CppDiarizeStreamActive func(s uintptr, out unsafe.Pointer, nOut unsafe.Pointer) int32 + CppDiarizeStreamFree func(s uintptr) + CppFreeDiarSegments func(segs uintptr) +) + +// cDiarSegment mirrors parakeet_diar_segment {int speaker; float start, end}. +type cDiarSegment struct { + Speaker int32 + Start float32 + End float32 +} + +// Latency modes of parakeet_capi_diarize_stream_begin_latency +// (PARAKEET_DIAR_LATENCY_*), by diar_latency option value. +var diarLatencyModes = map[string]int32{ + "model": 0, // the checkpoint's configuration (21.12 s for Nemotron-3) + "low": 1, // 1.04 s + "very_low": 2, // 0.64 s + "ultra_low": 3, // 0.32 s +} + +// defaultLiveDiarLatency is used for live transcription when diar_latency is +// unset: the model card's recommended low-latency configuration. +const defaultLiveDiarLatency = "low" + +// liveDiarizationAvailable reports whether the library can diarize live. +func liveDiarizationAvailable() bool { + return CppDiarizeStreamBeginLatency != nil && CppDiarizeStreamFeed != nil && + CppDiarizeStreamActive != nil && CppDiarizeStreamFree != nil && CppFreeDiarSegments != nil +} + +// takeDiarSegments copies a C parakeet_diar_segment array and frees it. +func takeDiarSegments(ptr uintptr, n int32) []diarSegmentDoc { + if ptr == 0 || n <= 0 { + return nil + } + // C-owned malloc'd array, not Go-GC memory; copied out before freeing. + cs := unsafe.Slice((*cDiarSegment)(unsafe.Pointer(ptr)), int(n)) //nolint:govet // see goStringFromCPtr + out := make([]diarSegmentDoc, len(cs)) + for i, c := range cs { + out[i] = diarSegmentDoc{Speaker: int(c.Speaker), Start: float64(c.Start), End: float64(c.End)} + } + CppFreeDiarSegments(ptr) + return out +} + +// liveDiarizer runs a streaming diarization session next to a live ASR +// stream. It keeps every closed segment (a live session is bounded by its +// caller, and segments are small) so the final transcript can be relabelled +// with complete information. +type liveDiarizer struct { + p *ParakeetCpp + stream uintptr + closed []diarSegmentDoc + active []diarSegmentDoc +} + +func (p *ParakeetCpp) newLiveDiarizer(latency int32) (*liveDiarizer, error) { + p.diarMu.Lock() + defer p.diarMu.Unlock() + if p.diarCtx == 0 { + return nil, grpcerrors.ModelNotLoaded("parakeet-cpp") + } + s := CppDiarizeStreamBeginLatency(p.diarCtx, latency) + if s == 0 { + return nil, fmt.Errorf("parakeet-cpp: diarize stream begin failed: %s", CppLastError(p.diarCtx)) + } + return &liveDiarizer{p: p, stream: s}, nil +} + +// feed pushes PCM (and, with last, flushes the stream), recording the segments +// that closed and those still open. +func (d *liveDiarizer) feed(pcm []float32, last bool) error { + d.p.diarMu.Lock() + defer d.p.diarMu.Unlock() + if d.p.diarCtx == 0 { + return grpcerrors.ModelNotLoaded("parakeet-cpp") + } + var ptr uintptr + var n int32 + isLast := int32(0) + if last { + isLast = 1 + } + if CppDiarizeStreamFeed(d.stream, pcm, int32(len(pcm)), isLast, unsafe.Pointer(&ptr), unsafe.Pointer(&n)) != 0 { + return fmt.Errorf("parakeet-cpp: diarize stream feed failed: %s", CppLastError(d.p.diarCtx)) + } + d.closed = append(d.closed, takeDiarSegments(ptr, n)...) + ptr, n = 0, 0 + if CppDiarizeStreamActive(d.stream, unsafe.Pointer(&ptr), unsafe.Pointer(&n)) != 0 { + return fmt.Errorf("parakeet-cpp: diarize stream active failed: %s", CppLastError(d.p.diarCtx)) + } + d.active = takeDiarSegments(ptr, n) + return nil +} + +// speakersFor labels words with what is known so far. Diarization trails the +// audio by its latency, so an open segment is assumed to go on through the +// words it has not reached yet; a word is only mislabelled right at a turn +// that diarization has not seen. +func (d *liveDiarizer) speakersFor(words []transcriptWord) []int { + segs := append([]diarSegmentDoc(nil), d.closed...) + var lastEnd float64 + for _, w := range words { + lastEnd = max(lastEnd, w.End) + } + for _, a := range d.active { + a.End = max(a.End, lastEnd) + segs = append(segs, a) + } + return assignSpeakers(words, segs) +} + +func (d *liveDiarizer) free() { + if d == nil || d.stream == 0 { + return + } + d.p.diarMu.Lock() + defer d.p.diarMu.Unlock() + CppDiarizeStreamFree(d.stream) + d.stream = 0 +} + +// speakerTurns groups labelled words into one segment per speaker turn. +func speakerTurns(words []transcriptWord, speakers []int) []*pb.TranscriptSegment { + var out []*pb.TranscriptSegment + start := 0 + for i := 1; i <= len(words); i++ { + if i < len(words) && speakers[i] == speakers[start] { + continue + } + parts := make([]string, 0, i-start) + for _, w := range words[start:i] { + parts = append(parts, w.W) + } + out = append(out, &pb.TranscriptSegment{ + Id: int32(len(out)), + Start: secondsToNanos(words[start].Start), + End: secondsToNanos(words[i-1].End), + Text: strings.TrimSpace(strings.Join(parts, " ")), + Speaker: speakerLabel(speakers[start]), + }) + start = i + } + return out +} + +// majoritySpeaker is the speaker covering most of the words' duration, or -1. +func majoritySpeaker(words []transcriptWord, speakers []int) int { + dur := map[int]float64{} + best, bestDur := -1, 0.0 + for i, w := range words { + if speakers[i] < 0 { + continue + } + dur[speakers[i]] += max(w.End-w.Start, 1e-3) + if dur[speakers[i]] > bestDur { + best, bestDur = speakers[i], dur[speakers[i]] + } + } + return best +} + +// diarModelArch is the parakeet.arch GGUF value of a diarization model. +const diarModelArch = "diarization" + +// speakerSnapSeconds mirrors parakeet.cpp's merge_asr_diarization: a word that +// overlaps no speaker segment takes the nearest segment's speaker when that +// segment is this close. ASR word boundaries and diarization boundaries can +// disagree by a frame or two; a word farther than this has no speaker. +const speakerSnapSeconds = 0.5 + +type diarJSON struct { + Speakers int `json:"speakers"` + Segments []diarSegmentDoc `json:"segments"` +} + +type diarSegmentDoc struct { + Speaker int `json:"speaker"` + Start float64 `json:"start"` + End float64 `json:"end"` +} + +// parakeetArch reads parakeet.arch ("tdt", "ctc", ..., "diarization") from a +// parakeet.cpp GGUF. general.architecture is "parakeet" for every model, so it +// cannot tell ASR from diarization. +func parakeetArch(path string) (string, error) { + f, err := gguf.ParseGGUFFile(path, gguf.UseMMap(), gguf.SkipLargeMetadata()) + if err != nil { + return "", fmt.Errorf("parakeet-cpp: parse gguf %q: %w", path, err) + } + kv, found := f.Header.MetadataKV.Index([]string{"parakeet.arch"}) + if found == 0 { + return "", fmt.Errorf("parakeet-cpp: %q has no parakeet.arch key", path) + } + arch := kv["parakeet.arch"] + // ValueString panics on a mistyped key; a hand-edited GGUF is exactly where + // that happens, and a load-time check must report, not crash. + if arch.ValueType != gguf.GGUFMetadataValueTypeString { + return "", fmt.Errorf("parakeet-cpp: %q has a non-string parakeet.arch", path) + } + return arch.ValueString(), nil +} + +// optString reads a string model option (key:value form), splitting on the +// first colon so values may contain one. Returns "" when absent. +func optString(opts *pb.ModelOptions, key string) string { + for _, o := range opts.GetOptions() { + k, v, ok := strings.Cut(o, ":") + if ok && strings.TrimSpace(k) == key { + return strings.TrimSpace(v) + } + } + return "" +} + +// resolveModelPath makes a relative option path absolute against the models +// directory, as nemo-speech-cpp does for its auxiliary models. +func resolveModelPath(base, p string) string { + if p == "" || filepath.IsAbs(p) { + return p + } + return filepath.Join(base, p) +} + +// loadDiarModel attaches the diar_model option's GGUF to an ASR model. +func (p *ParakeetCpp) loadDiarModel(path string) error { + if CppDiarizePcm == nil { + return fmt.Errorf("parakeet-cpp: diar_model is set but libparakeet.so has no " + + "diarization C-API (needs parakeet.cpp ABI >= 7)") + } + arch, err := parakeetArch(path) + if err != nil { + return err + } + if arch != diarModelArch { + return fmt.Errorf("parakeet-cpp: diar_model %q is a %q model, not a diarization model", path, arch) + } + ctx := CppLoad(path) + if ctx == 0 { + return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for diar_model %q", path) + } + p.diarCtx = ctx + xlog.Info("parakeet-cpp: speaker diarization attached", "diar_model", path) + return nil +} + +// diarizerCtx is the context that diarizes: the model itself when it is a +// diarization model, else the attached diar_model, else 0. +func (p *ParakeetCpp) diarizerCtx() uintptr { + if p.isDiarModel { + return p.ctxPtr + } + return p.diarCtx +} + +// runDiarization diarizes 16 kHz mono PCM. diarMu serializes calls on the +// diarization context, which is separate from the ASR engine (engineMu). +func (p *ParakeetCpp) runDiarization(pcm []float32) (diarJSON, error) { + p.diarMu.Lock() + defer p.diarMu.Unlock() + ctx := p.diarizerCtx() + if ctx == 0 { + return diarJSON{}, grpcerrors.ModelNotLoaded("parakeet-cpp") + } + if len(pcm) == 0 { + return diarJSON{}, nil + } + cstr := CppDiarizePcm(ctx, pcm, int32(len(pcm)), 16000) + if cstr == 0 { + return diarJSON{}, fmt.Errorf("parakeet-cpp: diarize failed: %s", CppLastError(ctx)) + } + raw := goStringFromCPtr(cstr) + CppFreeString(cstr) + var doc diarJSON + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return diarJSON{}, fmt.Errorf("parakeet-cpp: decode diarization json: %w", err) + } + return doc, nil +} + +// assignSpeakers gives each word the speaker whose segments overlap it most, +// falling back to the nearest segment within speakerSnapSeconds; -1 when none. +// Same rule as parakeet.cpp's merge_asr_diarization, so the backend's speaker +// tags match the library's own speaker-attributed ASR. +func assignSpeakers(words []transcriptWord, segs []diarSegmentDoc) []int { + out := make([]int, len(words)) + for i, w := range words { + best, bestOverlap := -1, 0.0 + for _, s := range segs { + if ov := min(w.End, s.End) - max(w.Start, s.Start); ov > bestOverlap { + best, bestOverlap = s.Speaker, ov + } + } + if best < 0 { + bestDist := speakerSnapSeconds + for _, s := range segs { + dist := s.Start - w.End + if s.End <= w.Start { + dist = w.Start - s.End + } + if dist >= 0 && dist <= bestDist { + best, bestDist = s.Speaker, dist + } + } + } + out[i] = best + } + return out +} + +// speakerLabel is the TranscriptSegment/DiarizeSegment speaker string: the +// 0-based index, or "" for no speaker. +func speakerLabel(spk int) string { + if spk < 0 { + return "" + } + return strconv.Itoa(spk) +} + +// splitAtSpeakerChanges splits each word group wherever the speaker changes, so +// every segment has exactly one speaker. speakers is indexed like the +// concatenation of groups. Returns the new groups and each group's speaker. +func splitAtSpeakerChanges(groups [][]transcriptWord, speakers []int) ([][]transcriptWord, []int) { + var outGroups [][]transcriptWord + var outSpk []int + k := 0 + for _, g := range groups { + start := 0 + for i := 1; i <= len(g); i++ { + if i == len(g) || speakers[k+i] != speakers[k+start] { + outGroups = append(outGroups, g[start:i]) + outSpk = append(outSpk, speakers[k+start]) + start = i + } + } + k += len(g) + } + return outGroups, outSpk +} + +// diarizeSegments maps diarization segments (and, when words are given, the +// transcript words attributed to each speaker) to DiarizeSegments, and counts +// the distinct speakers. A word belongs to the segment of its assigned speaker +// that contains its midpoint. +func diarizeSegments(doc diarJSON, words []transcriptWord) ([]*pb.DiarizeSegment, int32) { + var speakers []int + if len(words) > 0 { + speakers = assignSpeakers(words, doc.Segments) + } + var out []*pb.DiarizeSegment + seen := map[int]bool{} + for i, s := range doc.Segments { + seg := &pb.DiarizeSegment{ + Id: int32(i), + Start: float32(s.Start), + End: float32(s.End), + Speaker: speakerLabel(s.Speaker), + } + if len(words) > 0 { + var parts []string + for j, w := range words { + mid := (w.Start + w.End) / 2 + if speakers[j] == s.Speaker && mid >= s.Start && mid < s.End { + parts = append(parts, w.W) + } + } + seg.Text = strings.Join(parts, " ") + } + out = append(out, seg) + seen[s.Speaker] = true + } + return out, int32(len(seen)) +} + +// unsupportedDiarizeFields names the DiarizeRequest fields Sortformer cannot +// honour, so they are logged rather than silently dropped. The model is +// end-to-end: there is no speaker-count input and no clustering stage, and the +// segment thresholds come from the checkpoint. +func unsupportedDiarizeFields(req *pb.DiarizeRequest) []string { + var out []string + if req.GetNumSpeakers() != 0 { + out = append(out, "num_speakers") + } + if req.GetMinSpeakers() != 0 { + out = append(out, "min_speakers") + } + if req.GetMaxSpeakers() != 0 { + out = append(out, "max_speakers") + } + if req.GetClusteringThreshold() != 0 { + out = append(out, "clustering_threshold") + } + if req.GetMinDurationOn() != 0 { + out = append(out, "min_duration_on") + } + if req.GetMinDurationOff() != 0 { + out = append(out, "min_duration_off") + } + return out +} + +// Diarize answers "who spoke when" for the audio at req.Dst. With +// include_text, and an ASR model loaded alongside the diar_model, each segment +// also carries the words its speaker said in it. +func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error) { + if p.ctxPtr == 0 { + return pb.DiarizeResponse{}, grpcerrors.ModelNotLoaded("parakeet-cpp") + } + if p.diarizerCtx() == 0 || CppDiarizePcm == nil { + return pb.DiarizeResponse{}, status.Error(codes.Unimplemented, + "parakeet-cpp: this model has no diarization; load a diarization GGUF "+ + "or attach one to an ASR model with the diar_model option") + } + if req.GetDst() == "" { + return pb.DiarizeResponse{}, status.Error(codes.InvalidArgument, + "parakeet-cpp: DiarizeRequest.dst (audio path) is required") + } + if f := unsupportedDiarizeFields(req); len(f) > 0 { + xlog.Warn("parakeet-cpp: ignoring diarization options Sortformer has no equivalent for", + "fields", strings.Join(f, ",")) + } + + pcm, duration, err := decodeWavMono16k(req.GetDst()) + if err != nil { + return pb.DiarizeResponse{}, err + } + doc, err := p.runDiarization(pcm) + if err != nil { + return pb.DiarizeResponse{}, err + } + + var words []transcriptWord + if req.GetIncludeText() { + if p.isDiarModel { + xlog.Warn("parakeet-cpp: include_text needs an ASR model with diar_model attached; " + + "returning segments without text") + } else { + tr, err := p.transcribeDoc(pcm, req.GetLanguage()) + if err != nil { + return pb.DiarizeResponse{}, err + } + words = tr.Words + } + } + segs, n := diarizeSegments(doc, words) + return pb.DiarizeResponse{ + Segments: segs, + NumSpeakers: n, + Duration: duration, + Language: req.GetLanguage(), + }, nil +} diff --git a/backend/go/parakeet-cpp/diarize_test.go b/backend/go/parakeet-cpp/diarize_test.go new file mode 100644 index 000000000000..ac0d2729e285 --- /dev/null +++ b/backend/go/parakeet-cpp/diarize_test.go @@ -0,0 +1,362 @@ +package main + +import ( + "context" + "encoding/binary" + "os" + "path/filepath" + "strings" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func ds(spk int, start, end float64) diarSegmentDoc { + return diarSegmentDoc{Speaker: spk, Start: start, End: end} +} + +// writeGGUFWithArch writes a GGUF v3 file with no tensors and a single string +// KV, parakeet.arch=arch: enough for parakeetArch, which reads metadata only. +func writeGGUFWithArch(path, arch string) { + GinkgoHelper() + var b []byte + b = append(b, "GGUF"...) + b = binary.LittleEndian.AppendUint32(b, 3) // version + b = binary.LittleEndian.AppendUint64(b, 0) // tensor count + b = binary.LittleEndian.AppendUint64(b, 1) // kv count + str := func(s string) { + b = binary.LittleEndian.AppendUint64(b, uint64(len(s))) + b = append(b, s...) + } + str("parakeet.arch") + b = binary.LittleEndian.AppendUint32(b, 8) // GGUF string type + str(arch) + Expect(os.WriteFile(path, b, 0o644)).To(Succeed()) +} + +var _ = Describe("speaker diarization helpers", func() { + Context("assignSpeakers", func() { + It("picks the speaker with the largest overlap", func() { + words := []transcriptWord{tw("a", 0.5, 0.8)} + segs := []diarSegmentDoc{ds(0, 0.0, 0.6), ds(1, 0.55, 2.0)} + Expect(assignSpeakers(words, segs)).To(Equal([]int{1})) + }) + + It("snaps a word just outside a segment to it, but not a far one", func() { + words := []transcriptWord{tw("well", 19.92, 20.00), tw("far", 25.0, 25.2)} + segs := []diarSegmentDoc{ds(0, 14.78, 18.75), ds(1, 20.10, 23.60)} + Expect(assignSpeakers(words, segs)).To(Equal([]int{1, -1})) + }) + + It("leaves every word unassigned without segments", func() { + Expect(assignSpeakers([]transcriptWord{tw("a", 0, 1)}, nil)).To(Equal([]int{-1})) + }) + }) + + Context("splitAtSpeakerChanges", func() { + It("splits groups where the speaker changes and keeps the rest intact", func() { + groups := [][]transcriptWord{ + {tw("a", 0, 1), tw("b", 1, 2), tw("c.", 2, 3)}, + {tw("d", 4, 5)}, + } + out, spk := splitAtSpeakerChanges(groups, []int{0, 0, 1, 1}) + Expect(out).To(HaveLen(3)) + Expect(out[0]).To(HaveLen(2)) + Expect(out[1][0].W).To(Equal("c.")) + Expect(out[2][0].W).To(Equal("d")) + Expect(spk).To(Equal([]int{0, 1, 1})) + }) + }) + + Context("transcriptResultWithSpeakers", func() { + It("labels segments with their speaker and splits at speaker turns", func() { + doc := transcriptJSON{ + Text: "hi there. hello", + Words: []transcriptWord{ + tw("hi", 0.0, 0.3), tw("there.", 0.3, 0.6), tw("hello", 1.0, 1.4), + }, + } + // Punctuation gives [hi there.] [hello]; the turn after "hi" splits + // the first group. + res := transcriptResultWithSpeakers(doc, &pb.TranscriptRequest{}, 0, []int{0, 1, 1}) + Expect(res.Segments).To(HaveLen(3)) + Expect(res.Segments[0].Text).To(Equal("hi")) + Expect(res.Segments[0].Speaker).To(Equal("0")) + Expect(res.Segments[1].Text).To(Equal("there.")) + Expect(res.Segments[1].Speaker).To(Equal("1")) + Expect(res.Segments[2].Text).To(Equal("hello")) + Expect(res.Segments[2].Speaker).To(Equal("1")) + + // With word timestamps requested, words carry their speaker too. + res = transcriptResultWithSpeakers(doc, &pb.TranscriptRequest{TimestampGranularities: []string{"word"}}, 0, []int{0, 1, 1}) + Expect(res.Segments[0].Words[0].Speaker).To(Equal("0")) + Expect(res.Segments[2].Words[0].Speaker).To(Equal("1")) + for i, seg := range res.Segments { + Expect(seg.Id).To(Equal(int32(i))) + } + }) + + It("leaves speakers empty without diarization", func() { + doc := transcriptJSON{Text: "hi.", Words: []transcriptWord{tw("hi.", 0, 0.3)}} + res := transcriptResultFromDoc(doc, &pb.TranscriptRequest{}, 0) + Expect(res.Segments).To(HaveLen(1)) + Expect(res.Segments[0].Speaker).To(BeEmpty()) + }) + }) + + Context("diarizeSegments", func() { + It("fills each segment's text with its speaker's words", func() { + doc := diarJSON{Speakers: 8, Segments: []diarSegmentDoc{ds(0, 0, 2), ds(1, 2, 4)}} + words := []transcriptWord{tw("hello", 0.2, 0.6), tw("world", 0.7, 1.1), tw("hi", 2.5, 2.8)} + segs, n := diarizeSegments(doc, words) + Expect(n).To(Equal(int32(2))) + Expect(segs).To(HaveLen(2)) + Expect(segs[0].Speaker).To(Equal("0")) + Expect(segs[0].Text).To(Equal("hello world")) + Expect(segs[1].Text).To(Equal("hi")) + }) + + It("leaves text empty without words", func() { + segs, _ := diarizeSegments(diarJSON{Segments: []diarSegmentDoc{ds(0, 0, 1)}}, nil) + Expect(segs[0].Text).To(BeEmpty()) + }) + }) + + Context("live speaker labels", func() { + It("assumes an open segment continues through words diarization has not reached", func() { + d := &liveDiarizer{ + closed: []diarSegmentDoc{ds(0, 0.0, 2.0)}, + active: []diarSegmentDoc{ds(1, 2.5, 3.0)}, // diarized up to 3.0 s + } + words := []transcriptWord{tw("a", 0.5, 0.9), tw("b", 2.6, 2.9), tw("c", 3.4, 3.9)} + Expect(d.speakersFor(words)).To(Equal([]int{0, 1, 1})) + }) + + It("groups labelled words into speaker turns", func() { + words := []transcriptWord{tw("hi", 0, 0.3), tw("there.", 0.3, 0.6), tw("hello", 1.0, 1.4)} + segs := speakerTurns(words, []int{0, 0, 1}) + Expect(segs).To(HaveLen(2)) + Expect(segs[0].Text).To(Equal("hi there.")) + Expect(segs[0].Speaker).To(Equal("0")) + Expect(segs[1].Text).To(Equal("hello")) + Expect(segs[1].Speaker).To(Equal("1")) + Expect(segs[1].Id).To(Equal(int32(1))) + }) + + It("picks the speaker covering most of an utterance", func() { + words := []transcriptWord{tw("a", 0, 0.2), tw("b", 0.2, 1.5), tw("c", 1.5, 1.6)} + Expect(majoritySpeaker(words, []int{0, 1, 0})).To(Equal(1)) + Expect(majoritySpeaker(words, []int{-1, -1, -1})).To(Equal(-1)) + }) + + It("maps diar_latency option values to the C-API modes", func() { + Expect(diarLatencyModes).To(HaveKeyWithValue("model", int32(0))) + Expect(diarLatencyModes).To(HaveKeyWithValue(defaultLiveDiarLatency, int32(1))) + Expect(diarLatencyModes).To(HaveKeyWithValue("ultra_low", int32(3))) + }) + }) + + Context("parakeetArch", func() { + It("reads parakeet.arch from the GGUF metadata", func() { + path := filepath.Join(GinkgoT().TempDir(), "m.gguf") + writeGGUFWithArch(path, "diarization") + arch, err := parakeetArch(path) + Expect(err).ToNot(HaveOccurred()) + Expect(arch).To(Equal("diarization")) + }) + + It("fails on a file that is not a GGUF", func() { + path := filepath.Join(GinkgoT().TempDir(), "x.gguf") + Expect(os.WriteFile(path, []byte("nope"), 0o644)).To(Succeed()) + _, err := parakeetArch(path) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("options", func() { + It("resolves diar_model against the models directory", func() { + opts := &pb.ModelOptions{Options: []string{"batch_max_size:4", "diar_model: diar/model.gguf"}} + Expect(optString(opts, "diar_model")).To(Equal("diar/model.gguf")) + Expect(resolveModelPath("/models", "diar/model.gguf")).To(Equal("/models/diar/model.gguf")) + Expect(resolveModelPath("/models", "/abs/m.gguf")).To(Equal("/abs/m.gguf")) + }) + + It("names the request fields Sortformer ignores", func() { + Expect(unsupportedDiarizeFields(&pb.DiarizeRequest{NumSpeakers: 2, MinDurationOn: 0.3})). + To(ConsistOf("num_speakers", "min_duration_on")) + Expect(unsupportedDiarizeFields(&pb.DiarizeRequest{IncludeText: true})).To(BeEmpty()) + }) + }) +}) + +// diarFixturesOrSkip returns the diarization model, the ASR model and a +// multi-speaker 16 kHz WAV (parakeet.cpp's tests/fixtures/two_speakers.wav: +// two LibriSpeech speakers alternating A-B-A-B), or skips. +func diarFixturesOrSkip() (diarModel, asrModel, wavPath string) { + diarModel = os.Getenv("PARAKEET_BACKEND_TEST_DIAR_MODEL") + asrModel = os.Getenv("PARAKEET_BACKEND_TEST_MODEL") + wavPath = os.Getenv("PARAKEET_BACKEND_TEST_DIAR_WAV") + if diarModel == "" || asrModel == "" || wavPath == "" { + Skip("set PARAKEET_BACKEND_TEST_DIAR_MODEL, PARAKEET_BACKEND_TEST_MODEL and " + + "PARAKEET_BACKEND_TEST_DIAR_WAV to run this spec") + } + return diarModel, asrModel, wavPath +} + +// turns collapses consecutive repeats: [0 0 1 0] -> [0 1 0]. +func turns(labels []string) []string { + var out []string + for _, l := range labels { + if len(out) == 0 || out[len(out)-1] != l { + out = append(out, l) + } + } + return out +} + +var _ = Describe("ParakeetCpp speaker diarization", func() { + It("diarizes with a diarization model and refuses to transcribe", func() { + diarModel, _, wavPath := diarFixturesOrSkip() + ensureLibLoaded() + if CppDiarizePcm == nil { + Skip("libparakeet.so has no diarization C-API") + } + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: diarModel})).To(Succeed()) + defer func() { _ = p.Free() }() + + res, err := p.Diarize(&pb.DiarizeRequest{Dst: wavPath}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.NumSpeakers).To(Equal(int32(2))) + labels := make([]string, len(res.Segments)) + for i, s := range res.Segments { + labels[i] = s.Speaker + Expect(s.End).To(BeNumerically(">", s.Start)) + } + Expect(turns(labels)).To(Equal([]string{"0", "1", "0", "1"})) + + _, err = p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath}) + Expect(status.Code(err)).To(Equal(codes.FailedPrecondition)) + }) + + It("tags transcript segments with speakers when diar_model is attached", func() { + diarModel, asrModel, wavPath := diarFixturesOrSkip() + ensureLibLoaded() + if CppDiarizePcm == nil { + Skip("libparakeet.so has no diarization C-API") + } + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ + ModelFile: asrModel, + Options: []string{"diar_model:" + diarModel}, + })).To(Succeed()) + defer func() { _ = p.Free() }() + + res, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath, Diarize: true}) + Expect(err).ToNot(HaveOccurred()) + labels := make([]string, len(res.Segments)) + for i, s := range res.Segments { + labels[i] = s.Speaker + Expect(s.Speaker).ToNot(BeEmpty(), "segment %d %q has no speaker", i, s.Text) + } + Expect(turns(labels)).To(Equal([]string{"0", "1", "0", "1"})) + + // diarize=false keeps the plain transcript. + plain, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath}) + Expect(err).ToNot(HaveOccurred()) + for _, s := range plain.Segments { + Expect(s.Speaker).To(BeEmpty()) + } + Expect(plain.Text).To(Equal(res.Text)) + + // Diarize with include_text attributes the words to the segments. + d, err := p.Diarize(&pb.DiarizeRequest{Dst: wavPath, IncludeText: true}) + Expect(err).ToNot(HaveOccurred()) + var all []string + for _, s := range d.Segments { + if s.Text != "" { + all = append(all, s.Text) + } + } + Expect(strings.Join(all, " ")).To(ContainSubstring("Quilter")) + }) + + It("reports diarization as unimplemented on a plain ASR model", func() { + _, asrModel, wavPath := diarFixturesOrSkip() + ensureLibLoaded() + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: asrModel})).To(Succeed()) + defer func() { _ = p.Free() }() + _, err := p.Diarize(&pb.DiarizeRequest{Dst: wavPath}) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + }) +}) + +var _ = Describe("ParakeetCpp live speaker labels", func() { + It("labels live words and returns speaker turns in the final result", func() { + diarModel := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_MODEL") + streamModel := os.Getenv("PARAKEET_BACKEND_TEST_STREAM_MODEL") + wavPath := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_WAV") + if diarModel == "" || streamModel == "" || wavPath == "" { + Skip("set PARAKEET_BACKEND_TEST_DIAR_MODEL, PARAKEET_BACKEND_TEST_STREAM_MODEL " + + "(a cache-aware streaming ASR model) and PARAKEET_BACKEND_TEST_DIAR_WAV") + } + ensureLibLoaded() + if !liveDiarizationAvailable() { + Skip("libparakeet.so has no streaming diarization C-API") + } + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ + ModelFile: streamModel, + Options: []string{"diar_model:" + diarModel}, + })).To(Succeed()) + defer func() { _ = p.Free() }() + + pcm, _, err := decodeWavMono16k(wavPath) + Expect(err).ToNot(HaveOccurred()) + in := make(chan *pb.TranscriptLiveRequest) + out := make(chan *pb.TranscriptLiveResponse, 1024) + errCh := make(chan error, 1) + go func() { errCh <- p.AudioTranscriptionLive(in, out) }() + in <- &pb.TranscriptLiveRequest{Payload: &pb.TranscriptLiveRequest_Config{Config: &pb.TranscriptLiveConfig{}}} + for lo := 0; lo < len(pcm); lo += 1600 { + hi := min(lo+1600, len(pcm)) + in <- &pb.TranscriptLiveRequest{Payload: &pb.TranscriptLiveRequest_Audio{ + Audio: &pb.TranscriptLiveAudio{Pcm: pcm[lo:hi]}}} + } + close(in) + Expect(<-errCh).To(Succeed()) + + var live []string + var final *pb.TranscriptResult + for r := range out { + for _, w := range r.GetWords() { + live = append(live, w.GetSpeaker()) + } + if r.GetFinalResult() != nil { + final = r.GetFinalResult() + } + } + Expect(live).ToNot(BeEmpty()) + labelled := 0 + for _, l := range live { + if l != "" { + labelled++ + } + } + // Words finalized before diarization's first chunk (1.04 s) have no + // speaker yet; nearly all others do. + Expect(labelled).To(BeNumerically(">=", len(live)*9/10), "live labels: %v", live) + Expect(turns(live)).To(ContainElements("0", "1")) + + Expect(final).ToNot(BeNil()) + labels := make([]string, len(final.Segments)) + for i, s := range final.Segments { + labels[i] = s.Speaker + } + Expect(turns(labels)).To(Equal([]string{"0", "1", "0", "1"}), "final turns: %v", labels) + }) +}) diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index 23e5c548ce0c..de5269a8bd20 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -148,6 +148,17 @@ type ParakeetCpp struct { // YAML option, default 0=off). When >0 it adds NeMo's silence-gap split on // top of the punctuation split; converted to seconds via the JSON frame_sec. segmentGapFrames int + + // Speaker diarization (see diarize.go). isDiarModel: ctxPtr itself holds a + // diarization model, so there is no ASR. diarCtx: a diarization model + // attached to an ASR model through the diar_model option. diarMu + // serializes calls on whichever context diarizes. + isDiarModel bool + diarCtx uintptr + diarMu sync.Mutex + // liveDiarLatency is the PARAKEET_DIAR_LATENCY_* mode live transcription + // diarizes with (diar_latency option, default low = 1.04 s). + liveDiarLatency int32 } // Load is the LocalAI gRPC entry point for LoadModel: it calls @@ -158,6 +169,16 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error { return errors.New("parakeet-cpp: ModelFile is required") } + // A diarization GGUF loads into the same kind of context but serves only + // Diarize: no ASR, so none of the transcription setup below applies. + if arch, err := parakeetArch(opts.ModelFile); err == nil && arch == diarModelArch { + if CppDiarizePcm == nil { + return fmt.Errorf("parakeet-cpp: %q is a diarization model but libparakeet.so has "+ + "no diarization C-API (needs parakeet.cpp ABI >= 7)", opts.ModelFile) + } + p.isDiarModel = true + } + ctx := CppLoad(opts.ModelFile) if ctx == 0 { // No ctx to ask for last_error (the C-API's last-error buffer @@ -166,6 +187,28 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error { return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for %q", opts.ModelFile) } p.ctxPtr = ctx + if p.isDiarModel { + xlog.Info("parakeet-cpp: loaded a speaker diarization model (Diarize only)") + return nil + } + + // Optional speaker diarization for transcripts (diarize.go). + lat := optString(opts, "diar_latency") + if lat == "" { + lat = defaultLiveDiarLatency + } + mode, ok := diarLatencyModes[lat] + if !ok { + return fmt.Errorf("parakeet-cpp: unknown diar_latency %q (model, low, very_low, ultra_low)", lat) + } + p.liveDiarLatency = mode + if dm := optString(opts, "diar_model"); dm != "" { + if err := p.loadDiarModel(resolveModelPath(opts.ModelPath, dm)); err != nil { + CppFree(p.ctxPtr) + p.ctxPtr = 0 + return err + } + } // Dynamic batching knobs (model YAML options:, key:value form). Batching is // OFF by default (batch_max_size:1): each request runs on its own. On GPU, @@ -287,8 +330,9 @@ func (p *ParakeetCpp) runBatch(reqs []*batchRequest) { // OpenAI API, whose default is segment-level); token ids always populate // Segment.Tokens. // -// translate/diarize/prompt/temperature/threads are not applicable to parakeet -// and are ignored; language is honored on the batched + streaming paths (see +// With a diar_model attached, diarize=true tags each segment with its speaker +// (diarize.go). translate/prompt/temperature/threads are not applicable to +// parakeet and are ignored; language is honored on the batched + streaming paths (see // opts.GetLanguage() below); streaming is handled by AudioTranscriptionStream // (L2). func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.TranscriptRequest) (pb.TranscriptResult, error) { @@ -298,6 +342,9 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip if opts.Dst == "" { return pb.TranscriptResult{}, errors.New("parakeet-cpp: TranscriptRequest.dst (audio path) is required") } + if p.isDiarModel { + return pb.TranscriptResult{}, errDiarModelNoASR + } // Fallback when the batched C-API is unavailable: transcribe from a file // path (original behavior, no batching). The C library's audio loader only @@ -331,26 +378,63 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip if err != nil { return pb.TranscriptResult{}, err } + doc, err := p.transcribeDocCtx(ctx, pcm, opts.GetLanguage()) + if err != nil { + return pb.TranscriptResult{}, err + } + + // With a diar_model attached, tag each segment with its speaker. The + // request's diarize flag can turn it off (the OpenAI endpoint sends true + // unless the client passes diarize=false). + var speakers []int + if p.diarCtx != 0 && opts.GetDiarize() && len(doc.Words) > 0 { + dd, err := p.runDiarization(pcm) + if err != nil { + return pb.TranscriptResult{}, err + } + speakers = assignSpeakers(doc.Words, dd.Segments) + } + return transcriptResultWithSpeakers(doc, opts, p.segmentGapFrames, speakers), nil +} + +// errDiarModelNoASR is returned by the transcription RPCs on a diarization +// model: it answers Diarize, it cannot transcribe. +var errDiarModelNoASR = status.Error(codes.FailedPrecondition, + "parakeet-cpp: this is a speaker diarization model; use the diarization endpoint, "+ + "or attach it to an ASR model with the diar_model option") + +// transcribeDocCtx runs one clip through the batcher and returns its decoded +// transcript document. Honours ctx cancellation on both channel operations. +func (p *ParakeetCpp) transcribeDocCtx(ctx context.Context, pcm []float32, lang string) (transcriptJSON, error) { + if p.bat == nil { + return transcriptJSON{}, errors.New("parakeet-cpp: libparakeet.so has no batched C-API") + } rep := make(chan batchReply, 1) select { - case p.bat.submit <- &batchRequest{pcm: pcm, decoder: 0, language: opts.GetLanguage(), reply: rep}: + case p.bat.submit <- &batchRequest{pcm: pcm, decoder: 0, language: lang, reply: rep}: case <-ctx.Done(): - return pb.TranscriptResult{}, status.Error(codes.Canceled, "transcription cancelled") + return transcriptJSON{}, status.Error(codes.Canceled, "transcription cancelled") } var res batchReply select { case res = <-rep: case <-ctx.Done(): - return pb.TranscriptResult{}, status.Error(codes.Canceled, "transcription cancelled") + return transcriptJSON{}, status.Error(codes.Canceled, "transcription cancelled") } if res.err != nil { - return pb.TranscriptResult{}, res.err + return transcriptJSON{}, res.err } var doc transcriptJSON if err := json.Unmarshal([]byte(res.json), &doc); err != nil { - return pb.TranscriptResult{}, fmt.Errorf("parakeet-cpp: decode transcript json: %w", err) + return transcriptJSON{}, fmt.Errorf("parakeet-cpp: decode transcript json: %w", err) } - return transcriptResultFromDoc(doc, opts, p.segmentGapFrames), nil + return doc, nil +} + +// transcribeDoc is transcribeDocCtx for callers without a request context +// (the Diarize RPC). +func (p *ParakeetCpp) transcribeDoc(pcm []float32, lang string) (transcriptJSON, error) { + return p.transcribeDocCtx(context.Background(), pcm, lang) } // segmentSeparators is NeMo's default segment_seperators (sentence-ending @@ -365,6 +449,13 @@ var segmentSeparators = []rune{'.', '?', '!'} // the caller requested word granularity; token ids populate each segment's // Tokens by time-window membership. Shared by the batched and direct paths. func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gapFrames int) pb.TranscriptResult { + return transcriptResultWithSpeakers(doc, opts, gapFrames, nil) +} + +// transcriptResultWithSpeakers is transcriptResultFromDoc plus optional +// per-word speakers (indexed like doc.Words, -1 = none): segments additionally +// split wherever the speaker changes and carry the speaker's label. +func transcriptResultWithSpeakers(doc transcriptJSON, opts *pb.TranscriptRequest, gapFrames int, speakers []int) pb.TranscriptResult { text, eou := stripEouMarker(strings.TrimSpace(doc.Text)) // Frame-unit gap threshold -> seconds (NeMo segment_gap_threshold). 0 = off. @@ -388,6 +479,11 @@ func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gap } } + var groupSpeakers []int + if len(speakers) == len(doc.Words) && speakers != nil { + groups, groupSpeakers = splitAtSpeakerChanges(groups, speakers) + } + wantWords := wordsRequested(opts.TimestampGranularities) segments := make([]*pb.TranscriptSegment, 0, len(groups)) for id, group := range groups { @@ -402,10 +498,14 @@ func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gap Text: strings.TrimSpace(strings.Join(parts, " ")), Tokens: tokensInWindow(doc.Tokens, group[0].Start, group[len(group)-1].End), } + if groupSpeakers != nil { + seg.Speaker = speakerLabel(groupSpeakers[id]) + } if wantWords { ws := make([]*pb.TranscriptWord, len(group)) for i, gw := range group { - ws[i] = &pb.TranscriptWord{Start: secondsToNanos(gw.Start), End: secondsToNanos(gw.End), Text: gw.W} + ws[i] = &pb.TranscriptWord{Start: secondsToNanos(gw.Start), End: secondsToNanos(gw.End), Text: gw.W, + Speaker: seg.Speaker} } seg.Words = ws } @@ -503,7 +603,8 @@ func tokensInWindow(tokens []transcriptToken, start, end float64) []int32 { // text-only library (no words) it falls back to segmenting the delta text, so // the same assembler serves both paths. type streamSegmenter struct { - segs []*pb.TranscriptSegment + segs []*pb.TranscriptSegment + segWords [][]transcriptWord // words of each segment (nil for text-only ones) cur []transcriptWord // words for the open segment (ABI v4 JSON path) curText []string // delta text for the open segment (text-only path) nextID int32 @@ -534,12 +635,14 @@ func (s *streamSegmenter) flush() { End: secondsToNanos(s.cur[len(s.cur)-1].End), Text: strings.TrimSpace(strings.Join(parts, " ")), }) + s.segWords = append(s.segWords, s.cur) s.nextID++ case len(s.curText) > 0: // No words this segment: emit a text-only segment (no timestamps), // skipping a purely-whitespace one as the legacy text path did. if t := strings.TrimSpace(strings.Join(s.curText, "")); t != "" { s.segs = append(s.segs, &pb.TranscriptSegment{Id: s.nextID, Text: t}) + s.segWords = append(s.segWords, nil) s.nextID++ } } @@ -695,6 +798,9 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra return status.Error(codes.Canceled, "transcription cancelled") } + if p.isDiarModel { + return errDiarModelNoASR + } stream, err := p.streamBegin(opts.GetLanguage()) if err != nil { return err @@ -753,6 +859,28 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra // The single-segment fallback stays trimmed. fullText := full.String() segments := seg.segments() + + // With a diar_model attached, label each utterance with the speaker who + // said most of it. The whole file is available, so this runs the same + // diarization as the unary path rather than a low-latency stream. + if p.diarCtx != 0 && opts.GetDiarize() && len(seg.segWords) == len(segments) { + var all []transcriptWord + for _, ws := range seg.segWords { + all = append(all, ws...) + } + if len(all) > 0 { + dd, err := p.runDiarization(data) + if err != nil { + return err + } + speakers := assignSpeakers(all, dd.Segments) + k := 0 + for i, ws := range seg.segWords { + segments[i].Speaker = speakerLabel(majoritySpeaker(ws, speakers[k:k+len(ws)])) + k += len(ws) + } + } + } if trimmed := strings.TrimSpace(fullText); len(segments) == 0 && trimmed != "" { segments = append(segments, &pb.TranscriptSegment{Id: 0, Text: trimmed}) } @@ -834,6 +962,12 @@ func (p *ParakeetCpp) Free() error { CppFree(p.ctxPtr) p.ctxPtr = 0 } + p.diarMu.Lock() + defer p.diarMu.Unlock() + if p.diarCtx != 0 { + CppFree(p.diarCtx) + p.diarCtx = 0 + } return nil } diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go index a6f6af1f04a0..2a96091696b1 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp_test.go +++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go @@ -59,6 +59,16 @@ func ensureLibLoaded() { purego.RegisterLibFunc(&CppStreamFeedJSON, lib, "parakeet_capi_stream_feed_json") purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json") } + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_pcm"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppDiarizePcm, lib, "parakeet_capi_diarize_pcm") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_stream_begin_latency"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppDiarizeStreamBeginLatency, lib, "parakeet_capi_diarize_stream_begin_latency") + purego.RegisterLibFunc(&CppDiarizeStreamFeed, lib, "parakeet_capi_diarize_stream_feed") + purego.RegisterLibFunc(&CppDiarizeStreamActive, lib, "parakeet_capi_diarize_stream_active") + purego.RegisterLibFunc(&CppDiarizeStreamFree, lib, "parakeet_capi_diarize_stream_free") + purego.RegisterLibFunc(&CppFreeDiarSegments, lib, "parakeet_capi_free_diar_segments") + } purego.RegisterLibFunc(&CppFreeString, lib, "parakeet_capi_free_string") purego.RegisterLibFunc(&CppLastError, lib, "parakeet_capi_last_error") }) diff --git a/backend/go/parakeet-cpp/live.go b/backend/go/parakeet-cpp/live.go index 3d68a2914be1..0bb4cc204e61 100644 --- a/backend/go/parakeet-cpp/live.go +++ b/backend/go/parakeet-cpp/live.go @@ -43,6 +43,9 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest if p.ctxPtr == 0 { return grpcerrors.ModelNotLoaded("parakeet-cpp") } + if p.isDiarModel { + return errDiarModelNoASR + } first, ok := <-in if !ok { @@ -68,6 +71,26 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest // current when the RPC unwinds. defer func() { p.streamFree(stream) }() + // Speaker labels (diarize.go): a low-latency diarization stream runs next + // to the ASR stream when a diar_model is attached, unless the session + // opts out with params diarize=false. + var diar *liveDiarizer + var allWords []transcriptWord + startDiar := func(c *pb.TranscriptLiveConfig) error { + diar.free() + diar, allWords = nil, nil + if p.diarCtx == 0 || !liveDiarizationAvailable() || c.GetParams()["diarize"] == "false" { + return nil + } + var err error + diar, err = p.newLiveDiarizer(p.liveDiarLatency) + return err + } + if err := startDiar(cfg); err != nil { + return err + } + defer func() { diar.free() }() + out <- &pb.TranscriptLiveResponse{Ready: true} var ( @@ -94,11 +117,18 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest full.WriteString(r.Delta) } if r.Delta != "" || r.Eou || r.Eob || len(r.Words) > 0 { + words := liveWordsToProto(r.Words) + if diar != nil && len(r.Words) > 0 { + allWords = append(allWords, r.Words...) + for i, spk := range diar.speakersFor(r.Words) { + words[i].Speaker = speakerLabel(spk) + } + } out <- &pb.TranscriptLiveResponse{ Delta: r.Delta, Eou: r.Eou, Eob: r.Eob, - Words: liveWordsToProto(r.Words), + Words: words, } } return nil @@ -122,11 +152,20 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest } full.Reset() fedSecs = 0 + if err := startDiar(payload.Config); err != nil { + return err + } case *pb.TranscriptLiveRequest_Audio: pcm := payload.Audio.GetPcm() audioSec := float64(len(pcm)) / liveSampleRate fedSecs += audioSec start := time.Now() + // Diarize first so the words this feed finalizes can be labelled. + if diar != nil { + if err := diar.feed(pcm, false); err != nil { + return err + } + } // nil ctx: a live session is bounded by this request channel, not a // context — cancellation is the caller closing the stream. if err := p.feedSlices(nil, stream, pcm, emit); err != nil { @@ -150,15 +189,23 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest } // Send side closed: flush the streaming tail and emit the final transcript. - // The live FinalResult carries only Text — the authoritative full-turn - // transcript the realtime core commits. Per-utterance segments, duration, - // and the terminal flag are not produced on the live path. + // The live FinalResult carries Text — the authoritative full-turn + // transcript the realtime core commits — plus, when diarizing, one segment + // per speaker turn. Duration and the terminal flag are not produced + // on the live path. if err := p.flushTail(stream, emit); err != nil { return err } - out <- &pb.TranscriptLiveResponse{ - FinalResult: &pb.TranscriptResult{Text: strings.TrimSpace(full.String())}, + final := &pb.TranscriptResult{Text: strings.TrimSpace(full.String())} + // With speakers, the final result also carries one segment per speaker + // turn, relabelled now that diarization has seen the whole session. + if diar != nil && len(allWords) > 0 { + if err := diar.feed(nil, true); err != nil { + return err + } + final.Segments = speakerTurns(allWords, assignSpeakers(allWords, diar.closed)) } + out <- &pb.TranscriptLiveResponse{FinalResult: final} return nil } diff --git a/backend/go/parakeet-cpp/main.go b/backend/go/parakeet-cpp/main.go index 9c6466b131e5..fb2c38fd3300 100644 --- a/backend/go/parakeet-cpp/main.go +++ b/backend/go/parakeet-cpp/main.go @@ -90,6 +90,23 @@ func main() { purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json") } + // Speaker diarization (ABI v7): a diarization GGUF, or one attached to an + // ASR model with the diar_model option. Same probe pattern; absent in older + // libparakeet.so, where diarization is reported as unavailable. + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_pcm"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppDiarizePcm, lib, "parakeet_capi_diarize_pcm") + } + + // Streaming diarization with latency modes, for speaker labels on live + // transcripts. All-or-nothing: live diarization needs every entry point. + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_stream_begin_latency"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppDiarizeStreamBeginLatency, lib, "parakeet_capi_diarize_stream_begin_latency") + purego.RegisterLibFunc(&CppDiarizeStreamFeed, lib, "parakeet_capi_diarize_stream_feed") + purego.RegisterLibFunc(&CppDiarizeStreamActive, lib, "parakeet_capi_diarize_stream_active") + purego.RegisterLibFunc(&CppDiarizeStreamFree, lib, "parakeet_capi_diarize_stream_free") + purego.RegisterLibFunc(&CppFreeDiarSegments, lib, "parakeet_capi_free_diar_segments") + } + fmt.Fprintf(os.Stderr, "[parakeet-cpp] ABI=%d\n", CppAbiVersion()) flag.Parse() diff --git a/core/backend/transcript.go b/core/backend/transcript.go index 0ddee3adddff..27b13bc55316 100644 --- a/core/backend/transcript.go +++ b/core/backend/transcript.go @@ -213,9 +213,10 @@ func transcriptResultFromProto(r *proto.TranscriptResult) *schema.TranscriptionR var words []schema.TranscriptionWord for _, w := range s.Words { var word = schema.TranscriptionWord{ - Start: time.Duration(w.Start), - End: time.Duration(w.End), - Text: w.Text, + Start: time.Duration(w.Start), + End: time.Duration(w.End), + Text: w.Text, + Speaker: w.Speaker, } words = append(words, word) tr.Words = append(tr.Words, word) diff --git a/core/backend/transcript_live.go b/core/backend/transcript_live.go index 0ad6d72e1456..d0dc59d871bf 100644 --- a/core/backend/transcript_live.go +++ b/core/backend/transcript_live.go @@ -298,9 +298,10 @@ func liveEventFromProto(r *proto.TranscriptLiveResponse) LiveTranscriptionEvent } for _, w := range r.GetWords() { ev.Words = append(ev.Words, schema.TranscriptionWord{ - Start: time.Duration(w.Start), - End: time.Duration(w.End), - Text: w.Text, + Start: time.Duration(w.Start), + End: time.Duration(w.End), + Text: w.Text, + Speaker: w.Speaker, }) } if r.GetFinalResult() != nil { diff --git a/core/backend/transcript_live_internal_test.go b/core/backend/transcript_live_internal_test.go index 6f6bed6a4b5c..975ac844bcc0 100644 --- a/core/backend/transcript_live_internal_test.go +++ b/core/backend/transcript_live_internal_test.go @@ -54,6 +54,20 @@ var _ = Describe("liveEventFromProto", func() { Expect(ev.Final).To(BeNil()) }) + It("carries word speakers and final segment speakers from a diarizing backend", func() { + ev := liveEventFromProto(&proto.TranscriptLiveResponse{ + Words: []*proto.TranscriptWord{{Text: "hi", Speaker: "1"}}, + }) + Expect(ev.Words[0].Speaker).To(Equal("1")) + ev = liveEventFromProto(&proto.TranscriptLiveResponse{ + FinalResult: &proto.TranscriptResult{ + Text: "hi there", + Segments: []*proto.TranscriptSegment{{Text: "hi", Speaker: "0"}, {Text: "there", Speaker: "1"}}, + }, + }) + Expect(ev.Final.Segments[1].Speaker).To(Equal("1")) + }) + It("maps the eob backchannel flag separately from eou", func() { ev := liveEventFromProto(&proto.TranscriptLiveResponse{Delta: "uh-huh", Eob: true}) Expect(ev.Eob).To(BeTrue()) diff --git a/core/cli/transcript.go b/core/cli/transcript.go index 06764f4dd277..e187453a6fa9 100644 --- a/core/cli/transcript.go +++ b/core/cli/transcript.go @@ -93,18 +93,20 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error { } for _, word := range(tr.Words) { trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } for _, seg := range(tr.Segments) { segWords := []schema.TranscriptionWordSeconds{} for _, word := range(seg.Words) { segWords = append(segWords, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{ diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index 886ddd22a185..de54b65e3614 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -477,11 +477,14 @@ var BackendCapabilities = map[string]BackendCapability{ DefaultUsecases: []string{UsecaseTranscript}, Description: "NVIDIA NeMo speech recognition", }, + // parakeet-cpp: an ASR GGUF transcribes (and tags speakers when a + // diarization GGUF is attached with the diar_model option); a diarization + // GGUF (Nemotron-3-Diarization) only answers Diarize. "parakeet-cpp": { - GRPCMethods: []GRPCMethod{MethodAudioTranscription}, - PossibleUsecases: []string{UsecaseTranscript}, + GRPCMethods: []GRPCMethod{MethodAudioTranscription, MethodDiarize}, + PossibleUsecases: []string{UsecaseTranscript, UsecaseDiarization}, DefaultUsecases: []string{UsecaseTranscript}, - Description: "NVIDIA NeMo Parakeet ASR (parakeet.cpp)", + Description: "NVIDIA NeMo Parakeet ASR and Nemotron-3-Diarization speaker diarization (parakeet.cpp)", }, // nemo-speech-cpp is one gRPC server in front of four NeMo-Speech.cpp model // families, picked at load time from the GGUF general.architecture key, so diff --git a/core/gallery/importers/parakeet-cpp.go b/core/gallery/importers/parakeet-cpp.go index aa732aa3e7fc..5b758167b457 100644 --- a/core/gallery/importers/parakeet-cpp.go +++ b/core/gallery/importers/parakeet-cpp.go @@ -108,6 +108,11 @@ func (i *ParakeetCppImporter) Import(details Details) (gallery.ModelConfig, erro uri := downloader.URI(details.URI) directGGUF := isParakeetGGUF(filepath.Base(details.URI)) + // A speaker diarization GGUF is served by the same backend but answers + // /v1/audio/diarization, not transcription. + if directGGUF && isParakeetDiarGGUF(filepath.Base(details.URI)) { + modelConfig.KnownUsecaseStrings = []string{"diarization"} + } switch { case uri.LooksLikeURL() && directGGUF: // Direct file URL (e.g. .../resolve/main/tdt_ctc-110m-f16.gguf). The @@ -128,12 +133,23 @@ func (i *ParakeetCppImporter) Import(details Details) (gallery.ModelConfig, erro // HF repo: collect every parakeet GGUF, pick the preferred quant, and // nest under parakeet-cpp/models// so a multi-quant repo doesn't // collide on disk. - var ggufFiles []hfapi.ModelFile + // Prefer ASR weights: a repo that also ships the diarization model + // (mudler/parakeet-cpp-gguf) imports as a transcription model, and the + // diarization GGUF is imported by its direct URL. A repo with only + // diarization weights imports as a diarization model. + var ggufFiles, diarFiles []hfapi.ModelFile for _, f := range details.HuggingFace.Files { - if isParakeetGGUF(filepath.Base(f.Path)) { + switch base := filepath.Base(f.Path); { + case isParakeetDiarGGUF(base): + diarFiles = append(diarFiles, f) + case isParakeetGGUF(base): ggufFiles = append(ggufFiles, f) } } + if len(ggufFiles) == 0 && len(diarFiles) > 0 { + ggufFiles = diarFiles + modelConfig.KnownUsecaseStrings = []string{"diarization"} + } if chosen, ok := pickPreferredGGMLFile(ggufFiles, quants); ok { target := filepath.Join("parakeet-cpp", "models", name, filepath.Base(chosen.Path)) cfg.Files = append(cfg.Files, gallery.File{ @@ -176,5 +192,14 @@ func isParakeetGGUF(name string) bool { return true } } - return false + return isParakeetDiarGGUF(name) +} + +// isParakeetDiarGGUF reports whether name is the parakeet.cpp speaker +// diarization GGUF (nemotron-3-diarization-.gguf). Matched by its +// published name only, so diarization weights for other backends are not +// claimed. +func isParakeetDiarGGUF(name string) bool { + lower := strings.ToLower(name) + return strings.HasSuffix(lower, ".gguf") && strings.Contains(lower, "nemotron-3-diarization") } diff --git a/core/gallery/importers/parakeet-cpp_test.go b/core/gallery/importers/parakeet-cpp_test.go index 4aa87c4110e6..8636c929de6d 100644 --- a/core/gallery/importers/parakeet-cpp_test.go +++ b/core/gallery/importers/parakeet-cpp_test.go @@ -50,6 +50,11 @@ var _ = Describe("ParakeetCppImporter", func() { Expect(imp.Match(d)).To(BeTrue()) }) + It("matches a direct URL to the diarization GGUF", func() { + d := parakeetDetails("https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf", `{}`) + Expect(imp.Match(d)).To(BeTrue()) + }) + It("does NOT claim a generic llama-style GGUF", func() { d := parakeetDetails("huggingface://someorg/some-llm-gguf", `{}`, hfapi.ModelFile{Path: "llama-3-8b-instruct-q4_k_m.gguf"}, @@ -66,6 +71,43 @@ var _ = Describe("ParakeetCppImporter", func() { }) Context("import (Import)", func() { + It("imports the diarization GGUF as a diarization model", func() { + d := parakeetDetails("https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf", + `{"name":"nemotron-diarization"}`) + cfg, err := imp.Import(d) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.ConfigFile).To(ContainSubstring("backend: parakeet-cpp")) + Expect(cfg.ConfigFile).To(ContainSubstring("diarization")) + Expect(cfg.ConfigFile).ToNot(ContainSubstring("transcript")) + Expect(cfg.Files).To(HaveLen(1)) + Expect(cfg.Files[0].Filename).To(HaveSuffix("nemotron-3-diarization-q8_0.gguf")) + }) + + It("keeps picking ASR weights from a repo that also ships the diarization model", func() { + d := parakeetDetails("huggingface://mudler/parakeet-cpp-gguf", `{"name":"parakeet-110m"}`, + hfapi.ModelFile{Path: "nemotron-3-diarization-f16.gguf", URL: "https://hf/diar-f16", SHA256: "ddd"}, + hfapi.ModelFile{Path: "tdt_ctc-110m-f16.gguf", URL: "https://hf/f16", SHA256: "aaa"}, + hfapi.ModelFile{Path: "nemotron-3-diarization-q8_0.gguf", URL: "https://hf/diar-q8", SHA256: "eee"}, + ) + cfg, err := imp.Import(d) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.Files).To(HaveLen(1)) + Expect(cfg.Files[0].URI).To(Equal("https://hf/f16")) + Expect(cfg.ConfigFile).To(ContainSubstring("transcript")) + }) + + It("imports a diarization-only repo as a diarization model", func() { + d := parakeetDetails("huggingface://someone/diar-gguf", `{"name":"diar"}`, + hfapi.ModelFile{Path: "nemotron-3-diarization-f16.gguf", URL: "https://hf/diar-f16", SHA256: "ddd"}, + hfapi.ModelFile{Path: "nemotron-3-diarization-q8_0.gguf", URL: "https://hf/diar-q8", SHA256: "eee"}, + ) + cfg, err := imp.Import(d) + Expect(err).ToNot(HaveOccurred()) + Expect(cfg.Files).To(HaveLen(1)) + Expect(cfg.Files[0].URI).To(Equal("https://hf/diar-q8"), "default quant ladder picks q8_0 before f16") + Expect(cfg.ConfigFile).To(ContainSubstring("diarization")) + }) + It("picks the default quant (q4_k) from a multi-quant HF repo", func() { d := parakeetDetails("huggingface://mudler/parakeet-cpp-gguf", `{"name":"parakeet-110m"}`, hfapi.ModelFile{Path: "tdt_ctc-110m-f16.gguf", URL: "https://hf/f16", SHA256: "aaa"}, diff --git a/core/http/endpoints/openai/transcription.go b/core/http/endpoints/openai/transcription.go index 920094f84e08..a7e01ba950de 100644 --- a/core/http/endpoints/openai/transcription.go +++ b/core/http/endpoints/openai/transcription.go @@ -191,18 +191,20 @@ func TranscriptEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app } for _, word := range tr.Words { trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } for _, seg := range tr.Segments { segWords := []schema.TranscriptionWordSeconds{} for _, word := range seg.Words { segWords = append(segWords, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{ @@ -309,12 +311,16 @@ func streamTranscription(c echo.Context, req backend.TranscriptionRequest, ml *m if len(finalResult.Segments) > 0 { segs := make([]map[string]any, 0, len(finalResult.Segments)) for _, seg := range finalResult.Segments { - segs = append(segs, map[string]any{ + entry := map[string]any{ "id": seg.Id, "start": seg.Start.Seconds(), "end": seg.End.Seconds(), "text": seg.Text, - }) + } + if seg.Speaker != "" { + entry["speaker"] = seg.Speaker + } + segs = append(segs, entry) } doneEvent["segments"] = segs } diff --git a/core/schema/transcription.go b/core/schema/transcription.go index 8414fd0ba805..aab914f704e9 100644 --- a/core/schema/transcription.go +++ b/core/schema/transcription.go @@ -13,9 +13,10 @@ type TranscriptionSegment struct { } type TranscriptionWord struct { - Start time.Duration `json:"start"` - End time.Duration `json:"end"` - Text string `json:"text"` + Start time.Duration `json:"start"` + End time.Duration `json:"end"` + Text string `json:"text"` + Speaker string `json:"speaker,omitempty"` } type TranscriptionResult struct { @@ -42,9 +43,10 @@ type TranscriptionSegmentSeconds struct { } type TranscriptionWordSeconds struct { - Start float64 `json:"start"` - End float64 `json:"end"` - Text string `json:"text"` + Start float64 `json:"start"` + End float64 `json:"end"` + Text string `json:"text"` + Speaker string `json:"speaker,omitempty"` } type TranscriptionResultSeconds struct { diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md index 83a51343e561..3bfde1aef164 100644 --- a/docs/content/features/audio-diarization.md +++ b/docs/content/features/audio-diarization.md @@ -9,11 +9,12 @@ url = "/features/audio-diarization/" Speaker diarization answers the question **"who spoke when?"** - given an audio clip with multiple speakers, it returns time-stamped segments labelled with a stable speaker ID (`SPEAKER_00`, `SPEAKER_01`, …). -LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Four backends are supported today: +LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Five backends are supported today: - **[sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx)** - pyannote-3.0 segmentation + a speaker-embedding extractor (3D-Speaker, NeMo, WeSpeaker) + fast clustering. Pure diarization - no transcription cost. Recommended when you only need speaker turns. - **[vibevoice.cpp](https://github.com/microsoft/VibeVoice)** - produces speaker-labelled segments as a by-product of its long-form ASR pass, so you can optionally get a transcript per segment for free. - **[NeMo-Speech.cpp](https://github.com/NVIDIA/NeMo-Speech.cpp)** - NVIDIA Sortformer, served standalone by the [NeMo-Speech.cpp backend]({{%relref "features/nemo-speech-cpp" %}}). It is end to end, so the speaker capacity is fixed by the checkpoint and the count hints are ignored. The same backend can instead put speaker tags on a transcript, by attaching a Sortformer model to an ASR one. +- **[parakeet.cpp](https://github.com/mudler/parakeet.cpp)** - NVIDIA Nemotron-3-Diarization (Sortformer, up to 8 speakers), matched against NeMo. A diarization model runs on its own, or attaches to a Parakeet ASR model to put speaker labels on transcripts and text on diarization segments. - **[audio.cpp](https://github.com/0xShug0/audio.cpp)** - the `sortformer_diar` family, served by the multi-modality [audio.cpp backend]({{%relref "features/audio-cpp" %}}). Because diarization is exposed as a regular OpenAI-compatible endpoint, any HTTP client works. There is no Python dependency on pyannote or NeMo on the consumer side. @@ -105,7 +106,7 @@ curl http://localhost:8080/v1/audio/diarization \ -F num_speakers=3 ``` -The sections below show how to configure the two supported backends by hand when you want full control over the segmentation and embedding models. +The sections below show how to configure some of the supported backends by hand when you want full control over the segmentation and embedding models. ## Backend setup - sherpa-onnx (pure diarization) @@ -157,6 +158,41 @@ curl http://localhost:8080/v1/audio/diarization \ -F response_format=verbose_json ``` +## Backend setup - parakeet.cpp (Nemotron-3-Diarization) + +The [parakeet-cpp backend]({{%relref "features/audio-to-text#using-the-parakeet-cpp-backend" %}}) runs [nvidia/Nemotron-3-Diarization](https://huggingface.co/nvidia/Nemotron-3-Diarization). GGUF weights are published in [`mudler/parakeet-cpp-gguf`](https://huggingface.co/mudler/parakeet-cpp-gguf): F16 (191 MB) and Q8_0 (104 MB) both give the same segments as NeMo. Import one directly: + +```bash +local-ai models import https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf +``` + +Or configure it by hand, as a diarization-only model: + +```yaml +name: nemotron-diarization +backend: parakeet-cpp +parameters: + model: nemotron-3-diarization-q8_0.gguf +known_usecases: + - FLAG_DIARIZATION +``` + +Attach the same GGUF to a Parakeet ASR model with `diar_model` to get both: transcripts gain a speaker per segment, and `include_text=true` on `/v1/audio/diarization` fills each segment with the words its speaker said. + +```yaml +name: parakeet-speakers +backend: parakeet-cpp +parameters: + model: tdt-0.6b-v3-f16.gguf +options: + - diar_model:nemotron-3-diarization-q8_0.gguf +known_usecases: + - FLAG_TRANSCRIPT + - FLAG_DIARIZATION +``` + +Sortformer is end to end: the speaker count, clustering and segment-duration fields of the request have no equivalent and are ignored (the backend logs which ones). It processes audio the way NeMo's `diarize()` does for this checkpoint, in 21 s chunks with a speaker cache, so long recordings keep consistent speaker IDs: a 12 minute, 3 speaker recording diarizes in about 7 s on a desktop CPU. + ## Notes - **Speaker identity across files**: speaker IDs (`SPEAKER_00`, `SPEAKER_01`, …) are local to each request. To track the same person across multiple recordings, combine `/v1/audio/diarization` with `/v1/voice/embed` (speaker embedding) and maintain your own embedding store. diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 5a5e833cf3a3..aa67edc82824 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -205,6 +205,43 @@ options: - segment_gap_threshold:12 # split on silence > 12 encoder frames (default 0 = off, punctuation-only) ``` +### Speaker labels + +Attach a speaker diarization model with the `diar_model` option and each transcript segment carries a `speaker` (`"0"`, `"1"`, ... in order of first appearance). Segments split wherever the speaker changes, on top of the punctuation split. The model is NVIDIA Nemotron-3-Diarization, published as `nemotron-3-diarization-f16.gguf` and `nemotron-3-diarization-q8_0.gguf` in [`mudler/parakeet-cpp-gguf`](https://huggingface.co/mudler/parakeet-cpp-gguf); see [Speaker Diarization]({{< relref "audio-diarization.md" >}}) for the conversion and for the `/v1/audio/diarization` endpoint the same model serves. + +```yaml +name: parakeet-speakers +backend: parakeet-cpp +parameters: + model: tdt-0.6b-v3-f16.gguf +options: +- diar_model:nemotron-3-diarization-q8_0.gguf # relative to the models directory +``` + +The `diarize` form field turns it off per request (`-F diarize=false`). With word timestamps (`timestamp_granularities[]=word`) each word carries its speaker too. + +Streaming works as well: + +- `stream=true`: the closing `transcript.text.done` event lists the segments with their `speaker`. Speakers are assigned once the whole file is transcribed, with the same diarization as a non-streaming request. +- Live transcription (a cache-aware streaming model such as `realtime_eou_120m-v1`, driven through the live transcription API): a low-latency diarization stream runs alongside the recognizer, each finalized word carries a `speaker`, and the final result lists one segment per speaker turn. Pass the live config parameter `diarize=false` to turn it off for a session. + +Live labels trail the audio by the diarization latency, set with `diar_latency` (the Nemotron-3-Diarization model card's modes): + +| `diar_latency` | Latency | | +|---|---|---| +| `low` (default) | 1.04 s | the model card's recommended configuration | +| `very_low` | 0.64 s | | +| `ultra_low` | 0.32 s | | +| `model` | 21.12 s | the checkpoint's own configuration, what non-live requests use | + +A word said before diarization has caught up takes the speaker who is talking at that moment, so right after a speaker change the first word or two can carry the previous speaker; the final result is relabelled once the whole session has been diarized. + +```yaml +options: +- diar_model:nemotron-3-diarization-q8_0.gguf +- diar_latency:very_low +``` + ### Dynamic batching The backend can coalesce concurrent transcription requests into a single batched engine call, which improves throughput on GPU when many requests arrive at once. Batching is **off by default** (`batch_max_size:1`, one request at a time); raise it to opt in. Two `options:` knobs control it: diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md index 40c6ae4f34e5..69e1cfdc35b1 100644 --- a/docs/content/reference/compatibility-table.md +++ b/docs/content/reference/compatibility-table.md @@ -43,7 +43,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [faster-whisper](https://github.com/SYSTRAN/faster-whisper) | Fast Whisper with CTranslate2 | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal, Jetson L4T | | [WhisperX](https://github.com/m-bain/whisperX) | Word-level timestamps and speaker diarization | CPU, CUDA 12/13, Metal, Jetson L4T | | [moonshine](https://github.com/moonshine-ai/moonshine) | Ultra-fast transcription for low-end devices (ONNX) | CPU, CUDA 12/13, Metal | -| [parakeet.cpp](https://github.com/mudler/parakeet.cpp) | C++/GGML port of NVIDIA NeMo Parakeet (tdt/ctc/rnnt/hybrid), with cache-aware streaming | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | +| [parakeet.cpp](https://github.com/mudler/parakeet.cpp) | C++/GGML port of NVIDIA NeMo Parakeet (tdt/ctc/rnnt/hybrid), with cache-aware streaming and Nemotron-3-Diarization speaker diarization | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | | [CrispASR](https://github.com/CrispStrobe/CrispASR) | Unified speech engine (whisper.cpp fork) supporting Parakeet, Canary, and many ASR architectures, plus TTS | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | | [voxtral](https://github.com/antirez/voxtral.c) | Voxtral Realtime 4B speech-to-text in pure C | CPU, Metal | | [Qwen3-ASR](https://github.com/QwenLM/Qwen3-ASR) | Qwen3 automatic speech recognition | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal, Jetson L4T |