From e7535900d9562be8d9fd1c63b9d6e4ac30f765ef Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:20:53 +0000 Subject: [PATCH 01/13] feat(voice): list registered voices and record which encoder made them The voice registry could register, identify and forget but not list, and it did not remember which speaker encoder produced an embedding. Add Metadata.Model and Registry.List, answered from the index the store registry already keeps for Forget. Needed so a backend can be given the registered voices that match its own speaker encoder. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- .../openai/realtime_voicegate_test.go | 3 + core/services/voicerecognition/registry.go | 17 +++ .../voicerecognition/store_registry.go | 31 +++-- .../voicerecognition/store_registry_test.go | 110 ++++++++++++++++++ .../voicerecognition_suite_test.go | 13 +++ 5 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 core/services/voicerecognition/store_registry_test.go create mode 100644 core/services/voicerecognition/voicerecognition_suite_test.go diff --git a/core/http/endpoints/openai/realtime_voicegate_test.go b/core/http/endpoints/openai/realtime_voicegate_test.go index 3d9b458e132f..a4c900d46722 100644 --- a/core/http/endpoints/openai/realtime_voicegate_test.go +++ b/core/http/endpoints/openai/realtime_voicegate_test.go @@ -116,6 +116,9 @@ func (f *fakeRegistry) Identify(ctx context.Context, probe []float32, topK int) return f.matches, f.err } func (f *fakeRegistry) Forget(ctx context.Context, id string) error { return nil } +func (f *fakeRegistry) List(ctx context.Context) ([]voicerecognition.Entry, error) { + return nil, nil +} var _ = Describe("voiceGate identify mode", func() { stubEmbed := func(emb []float32, err error) func(context.Context, string) ([]float32, error) { diff --git a/core/services/voicerecognition/registry.go b/core/services/voicerecognition/registry.go index 85ed9e3b7a90..b76c0a16c9ec 100644 --- a/core/services/voicerecognition/registry.go +++ b/core/services/voicerecognition/registry.go @@ -32,6 +32,19 @@ type Registry interface { // Forget removes a previously-registered embedding by ID. // Returns ErrNotFound if the ID is unknown. Forget(ctx context.Context, id string) error + + // List returns every registered voice with its embedding, oldest first + // (ties broken by ID). The embeddings are copies. The store registry + // answers from its in-process index, so it lists what this process + // registered since it started, which is also everything the in-memory + // store holds. + List(ctx context.Context) ([]Entry, error) +} + +// Entry is a registered voice together with its embedding, as returned by List. +type Entry struct { + Metadata Metadata + Embedding []float32 } // Metadata is the user-supplied payload stored alongside a speaker embedding. @@ -41,6 +54,10 @@ type Metadata struct { Name string `json:"name"` Labels map[string]string `json:"labels,omitempty"` RegisteredAt time.Time `json:"registered_at"` + // Model names the speaker encoder that produced the embedding (the voice + // backend's model name, by default the GGUF file name). Empty for voices + // registered before this field existed. + Model string `json:"model,omitempty"` } // Match is a single result from Identify, ranked by similarity. diff --git a/core/services/voicerecognition/store_registry.go b/core/services/voicerecognition/store_registry.go index 39df946190fc..94e2897f6cda 100644 --- a/core/services/voicerecognition/store_registry.go +++ b/core/services/voicerecognition/store_registry.go @@ -41,11 +41,11 @@ type storeRegistry struct { dim int // TODO(postgres): the local-store gRPC surface keys by embedding - // vector and exposes no "list all" method, so we cannot delete by - // ID without remembering the embedding. This in-memory index is - // rebuilt on every Register and lost on restart — acceptable while - // the only implementation is itself in-memory. - idIndex sync.Map // map[string][]float32 + // vector and exposes no "list all" method, so we cannot delete by ID + // or list voices without remembering them. This in-memory index holds + // every registration with its metadata. It is rebuilt on every Register + // and lost on restart, which matches the lifetime of the in-memory store. + idIndex sync.Map // map[string]Entry } func (r *storeRegistry) Register(ctx context.Context, embedding []float32, meta Metadata) (Metadata, error) { @@ -76,7 +76,7 @@ func (r *storeRegistry) Register(ctx context.Context, embedding []float32, meta } embCopy := append([]float32(nil), embedding...) - r.idIndex.Store(meta.ID, embCopy) + r.idIndex.Store(meta.ID, Entry{Metadata: meta, Embedding: embCopy}) return meta, nil } @@ -124,7 +124,7 @@ func (r *storeRegistry) Forget(ctx context.Context, id string) error { if !ok { return ErrNotFound } - embedding := raw.([]float32) + embedding := raw.(Entry).Embedding backend, err := r.resolve(ctx, r.storeName) if err != nil { @@ -136,3 +136,20 @@ func (r *storeRegistry) Forget(ctx context.Context, id string) error { r.idIndex.Delete(id) return nil } + +func (r *storeRegistry) List(ctx context.Context) ([]Entry, error) { + var out []Entry + r.idIndex.Range(func(_, v any) bool { + e := v.(Entry) + e.Embedding = append([]float32(nil), e.Embedding...) + out = append(out, e) + return true + }) + sort.SliceStable(out, func(i, j int) bool { + if !out[i].Metadata.RegisteredAt.Equal(out[j].Metadata.RegisteredAt) { + return out[i].Metadata.RegisteredAt.Before(out[j].Metadata.RegisteredAt) + } + return out[i].Metadata.ID < out[j].Metadata.ID + }) + return out, nil +} diff --git a/core/services/voicerecognition/store_registry_test.go b/core/services/voicerecognition/store_registry_test.go new file mode 100644 index 000000000000..bd35598887e8 --- /dev/null +++ b/core/services/voicerecognition/store_registry_test.go @@ -0,0 +1,110 @@ +package voicerecognition_test + +import ( + "context" + "sync" + + "github.com/mudler/LocalAI/core/services/voicerecognition" + "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ggrpc "google.golang.org/grpc" +) + +// fakeStore records Set and Delete calls; everything else panics (nil embedded interface). +type fakeStore struct { + grpc.Backend + mu sync.Mutex + sets int + deletes int + values [][]byte +} + +func (f *fakeStore) StoresSet(ctx context.Context, in *pb.StoresSetOptions, opts ...ggrpc.CallOption) (*pb.Result, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.sets++ + for _, v := range in.Values { + f.values = append(f.values, v.Bytes) + } + return &pb.Result{Success: true}, nil +} + +func (f *fakeStore) StoresDelete(ctx context.Context, in *pb.StoresDeleteOptions, opts ...ggrpc.CallOption) (*pb.Result, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.deletes++ + return &pb.Result{Success: true}, nil +} + +// StoresFind returns every stored value as a perfect match. +func (f *fakeStore) StoresFind(ctx context.Context, in *pb.StoresFindOptions, opts ...ggrpc.CallOption) (*pb.StoresFindResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + res := &pb.StoresFindResult{} + for _, v := range f.values { + res.Keys = append(res.Keys, &pb.StoresKey{Floats: in.Key.Floats}) + res.Values = append(res.Values, &pb.StoresValue{Bytes: v}) + res.Similarities = append(res.Similarities, 1) + } + return res, nil +} + +var _ = Describe("storeRegistry List", func() { + var ( + fs *fakeStore + reg voicerecognition.Registry + ctx = context.Background() + ) + BeforeEach(func() { + fs = &fakeStore{} + reg = voicerecognition.NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return fs, nil }, "t", 0) + }) + + It("lists what was registered, with the encoder tag and a copy of the embedding", func() { + a, err := reg.Register(ctx, []float32{1, 0, 0}, voicerecognition.Metadata{Name: "ada", Model: "voice-detect-wespeaker-resnet34.gguf"}) + Expect(err).ToNot(HaveOccurred()) + _, err = reg.Register(ctx, []float32{0, 1, 0}, voicerecognition.Metadata{Name: "ben"}) + Expect(err).ToNot(HaveOccurred()) + + got, err := reg.List(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(2)) + Expect(got[0].Metadata.ID).To(Equal(a.ID)) // oldest first + Expect(got[0].Metadata.Name).To(Equal("ada")) + Expect(got[0].Metadata.Model).To(Equal("voice-detect-wespeaker-resnet34.gguf")) + Expect(got[0].Embedding).To(Equal([]float32{1, 0, 0})) + Expect(got[1].Metadata.Model).To(BeEmpty()) // an untagged (legacy style) registration + + got[0].Embedding[0] = 99 // mutating the result must not touch the registry + again, _ := reg.List(ctx) + Expect(again[0].Embedding[0]).To(Equal(float32(1))) + }) + + It("forgets a voice from the list", func() { + a, _ := reg.Register(ctx, []float32{1, 0}, voicerecognition.Metadata{Name: "ada"}) + _, _ = reg.Register(ctx, []float32{0, 1}, voicerecognition.Metadata{Name: "ben"}) + Expect(reg.Forget(ctx, a.ID)).To(Succeed()) + got, err := reg.List(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(1)) + Expect(got[0].Metadata.Name).To(Equal("ben")) + Expect(fs.deletes).To(Equal(1)) + }) + + It("is empty for a fresh registry", func() { + got, err := reg.List(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeEmpty()) + }) + + It("returns the encoder tag from Identify", func() { + _, err := reg.Register(ctx, []float32{1, 0}, voicerecognition.Metadata{Name: "ada", Model: "enc.gguf"}) + Expect(err).ToNot(HaveOccurred()) + matches, err := reg.Identify(ctx, []float32{1, 0}, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(matches).To(HaveLen(1)) + Expect(matches[0].Metadata.Model).To(Equal("enc.gguf")) + }) +}) diff --git a/core/services/voicerecognition/voicerecognition_suite_test.go b/core/services/voicerecognition/voicerecognition_suite_test.go new file mode 100644 index 000000000000..3ebd43584d3a --- /dev/null +++ b/core/services/voicerecognition/voicerecognition_suite_test.go @@ -0,0 +1,13 @@ +package voicerecognition_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestVoiceRecognition(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VoiceRecognition Suite") +} From 5e39642d7534ac5367cd805db4edb0a54d8bc34c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:23:01 +0000 Subject: [PATCH 02/13] feat(voice): store the encoder model with a registered voice Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- core/http/endpoints/localai/voice_register.go | 12 ++++++++---- .../endpoints/localai/voice_register_test.go | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 core/http/endpoints/localai/voice_register_test.go diff --git a/core/http/endpoints/localai/voice_register.go b/core/http/endpoints/localai/voice_register.go index d8d97d619b94..4a7e3680682e 100644 --- a/core/http/endpoints/localai/voice_register.go +++ b/core/http/endpoints/localai/voice_register.go @@ -45,10 +45,7 @@ func VoiceRegisterEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, return mapBackendError(err) } - stored, err := registry.Register(c.Request().Context(), res.GetEmbedding(), voicerecognition.Metadata{ - Name: input.Name, - Labels: input.Labels, - }) + stored, err := registry.Register(c.Request().Context(), res.GetEmbedding(), voiceMetadata(input.Name, input.Labels, res.GetModel())) if err != nil { return err } @@ -59,3 +56,10 @@ func VoiceRegisterEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, }) } } + +// voiceMetadata is what a registration stores next to the embedding. Model is +// the speaker encoder that produced it, so a consumer with a different encoder +// can tell the vectors are not comparable. +func voiceMetadata(name string, labels map[string]string, embedderModel string) voicerecognition.Metadata { + return voicerecognition.Metadata{Name: name, Labels: labels, Model: embedderModel} +} diff --git a/core/http/endpoints/localai/voice_register_test.go b/core/http/endpoints/localai/voice_register_test.go new file mode 100644 index 000000000000..b66d0e48ded3 --- /dev/null +++ b/core/http/endpoints/localai/voice_register_test.go @@ -0,0 +1,18 @@ +package localai + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("voiceMetadata", func() { + It("carries the name, the labels and the encoder that embedded the voice", func() { + m := voiceMetadata("ada", map[string]string{"team": "a"}, "voice-detect-wespeaker-resnet34.gguf") + Expect(m.Name).To(Equal("ada")) + Expect(m.Labels).To(Equal(map[string]string{"team": "a"})) + Expect(m.Model).To(Equal("voice-detect-wespeaker-resnet34.gguf")) + }) + It("leaves the tag empty when the backend did not say", func() { + Expect(voiceMetadata("ada", nil, "").Model).To(BeEmpty()) + }) +}) From d209ba316f8be90f53553d7aef4db520ed8c6773 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:23:32 +0000 Subject: [PATCH 03/13] feat(voice): pick the registered voices that match a speaker model Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- .../services/voicerecognition/known_voices.go | 88 ++++++++++++ .../voicerecognition/known_voices_test.go | 125 ++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 core/services/voicerecognition/known_voices.go create mode 100644 core/services/voicerecognition/known_voices_test.go diff --git a/core/services/voicerecognition/known_voices.go b/core/services/voicerecognition/known_voices.go new file mode 100644 index 000000000000..b70cf33f9c2e --- /dev/null +++ b/core/services/voicerecognition/known_voices.go @@ -0,0 +1,88 @@ +package voicerecognition + +import ( + "context" + "path" + "path/filepath" + "strings" +) + +// KnownVoice is one registered voice as it is sent to a backend that matches +// speakers itself. +type KnownVoice struct { + Name string + Embedding []float32 + Model string +} + +// SpeakerModelFromOptions returns the value of a speaker_model: entry in +// a model config's options, or "" when there is none. +func SpeakerModelFromOptions(options []string) string { + for _, o := range options { + k, v, ok := strings.Cut(o, ":") + if ok && strings.TrimSpace(k) == "speaker_model" { + return strings.TrimSpace(v) + } + } + return "" +} + +// EncoderTag is how an encoder is identified: the lowercased base name of its +// model file. A voice registered through the voice-detect backend carries the +// backend's model name, which defaults to that base name. +func EncoderTag(modelPath string) string { + return strings.ToLower(path.Base(filepath.ToSlash(modelPath))) +} + +// KnownVoiceSelection is the result of SelectKnownVoices. +type KnownVoiceSelection struct { + Voices []KnownVoice + OtherEncoder int // voices skipped because another encoder made them + Untagged int // voices with no encoder tag that were included +} + +// SelectKnownVoices picks, from everything registered, the voices that can be +// compared with embeddings from the speaker model at speakerModelPath: those +// with the same encoder tag, then the untagged voices whose size matches the +// tagged ones (all untagged voices when none matched). Voices from another +// encoder are counted and skipped, as are voices without a name or embedding. +// The input is not modified. +func SelectKnownVoices(entries []Entry, speakerModelPath string) KnownVoiceSelection { + tag := EncoderTag(speakerModelPath) + var sel KnownVoiceSelection + matchedDim := 0 + var untagged []Entry + for _, e := range entries { + if e.Metadata.Name == "" || len(e.Embedding) == 0 { + continue + } + switch { + case e.Metadata.Model == "": + untagged = append(untagged, e) + case EncoderTag(e.Metadata.Model) == tag: + if matchedDim == 0 { + matchedDim = len(e.Embedding) + } + sel.Voices = append(sel.Voices, KnownVoice{Name: e.Metadata.Name, Embedding: e.Embedding, Model: e.Metadata.Model}) + default: + sel.OtherEncoder++ + } + } + for _, e := range untagged { + if matchedDim != 0 && len(e.Embedding) != matchedDim { + continue + } + sel.Untagged++ + sel.Voices = append(sel.Voices, KnownVoice{Name: e.Metadata.Name, Embedding: e.Embedding}) + } + return sel +} + +// KnownVoicesFor lists the registry and selects the voices for a speaker model. +func KnownVoicesFor(ctx context.Context, reg Registry, speakerModelPath string) (KnownVoiceSelection, error) { + entries, err := reg.List(ctx) + if err != nil { + return KnownVoiceSelection{}, err + } + return SelectKnownVoices(entries, speakerModelPath), nil +} diff --git a/core/services/voicerecognition/known_voices_test.go b/core/services/voicerecognition/known_voices_test.go new file mode 100644 index 000000000000..ef4818ea9ffe --- /dev/null +++ b/core/services/voicerecognition/known_voices_test.go @@ -0,0 +1,125 @@ +package voicerecognition_test + +import ( + "context" + "errors" + + "github.com/mudler/LocalAI/core/services/voicerecognition" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func entry(name, model string, emb ...float32) voicerecognition.Entry { + return voicerecognition.Entry{ + Metadata: voicerecognition.Metadata{ID: name, Name: name, Model: model}, + Embedding: emb, + } +} + +var _ = Describe("SpeakerModelFromOptions", func() { + It("reads the speaker_model option", func() { + Expect(voicerecognition.SpeakerModelFromOptions([]string{"diarization_model:d.gguf", "speaker_model: voice-detect-wespeaker-resnet34.gguf "})). + To(Equal("voice-detect-wespeaker-resnet34.gguf")) + }) + It("is empty without the option", func() { + Expect(voicerecognition.SpeakerModelFromOptions([]string{"diarization_model:d.gguf"})).To(BeEmpty()) + Expect(voicerecognition.SpeakerModelFromOptions(nil)).To(BeEmpty()) + }) +}) + +var _ = Describe("EncoderTag", func() { + It("is the lowercased basename", func() { + Expect(voicerecognition.EncoderTag("models/Voice-Detect-WeSpeaker.GGUF")).To(Equal("voice-detect-wespeaker.gguf")) + Expect(voicerecognition.EncoderTag("voice-detect-ecapa-tdnn-voxceleb.gguf")).To(Equal("voice-detect-ecapa-tdnn-voxceleb.gguf")) + }) +}) + +var _ = Describe("SelectKnownVoices", func() { + const wespeaker = "voice-detect-wespeaker-resnet34.gguf" + It("keeps the voices made by the speaker model's encoder", func() { + sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{ + entry("ada", wespeaker, 1, 0), + entry("ben", "voice-detect-ecapa-tdnn-voxceleb.gguf", 0, 1, 0), + entry("cy", "Voice-Detect-WeSpeaker-ResNet34.gguf", 0, 1), + }, "models/"+wespeaker) + Expect(sel.Voices).To(HaveLen(2)) + Expect(sel.Voices[0].Name).To(Equal("ada")) + Expect(sel.Voices[1].Name).To(Equal("cy")) + Expect(sel.OtherEncoder).To(Equal(1)) + }) + It("matches a speaker_model value that has a directory and upper-case letters", func() { + sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{entry("ada", wespeaker, 1, 0)}, "Some/Dir/Voice-Detect-WeSpeaker-ResNet34.GGUF") + Expect(sel.Voices).To(HaveLen(1)) + }) + It("sends nothing, and says why, when every voice is from another encoder", func() { + sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{entry("ben", "voice-detect-ecapa-tdnn-voxceleb.gguf", 0, 1, 0)}, wespeaker) + Expect(sel.Voices).To(BeEmpty()) + Expect(sel.OtherEncoder).To(Equal(1)) + }) + It("includes an untagged voice only when its size matches the tagged voices", func() { + sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{ + entry("ada", wespeaker, 1, 0), + entry("old_same", "", 0, 1), + entry("old_other", "", 0, 1, 0), + }, wespeaker) + Expect(sel.Voices).To(HaveLen(2)) + Expect(sel.Voices[1].Name).To(Equal("old_same")) + Expect(sel.Untagged).To(Equal(1)) + }) + It("puts tagged voices first even when an untagged one was registered earlier", func() { + sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{ + entry("old", "", 0, 1), + entry("ada", wespeaker, 1, 0), + }, wespeaker) + Expect(sel.Voices).To(HaveLen(2)) + Expect(sel.Voices[0].Name).To(Equal("ada")) + Expect(sel.Voices[1].Name).To(Equal("old")) + }) + It("includes every untagged voice when no voice is tagged for this encoder", func() { + sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{entry("old", "", 1, 0)}, wespeaker) + Expect(sel.Voices).To(HaveLen(1)) + Expect(sel.Untagged).To(Equal(1)) + }) + It("skips voices without a name or an embedding", func() { + sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{entry("", wespeaker, 1, 0), entry("x", wespeaker)}, wespeaker) + Expect(sel.Voices).To(BeEmpty()) + }) + It("is empty for an empty registry", func() { + Expect(voicerecognition.SelectKnownVoices(nil, wespeaker).Voices).To(BeEmpty()) + }) + It("handles an empty speaker model path without matching tagged voices", func() { + sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{entry("ada", wespeaker, 1, 0), entry("old", "", 0, 1)}, "") + Expect(sel.OtherEncoder).To(Equal(1)) + Expect(sel.Voices).To(HaveLen(1)) + Expect(sel.Voices[0].Name).To(Equal("old")) + }) + It("does not modify the input entries", func() { + in := []voicerecognition.Entry{entry("old", "", 0, 1), entry("ada", wespeaker, 1, 0)} + voicerecognition.SelectKnownVoices(in, wespeaker) + Expect(in[0].Metadata.Name).To(Equal("old")) + Expect(in[1].Metadata.Name).To(Equal("ada")) + }) +}) + +type listRegistry struct { + voicerecognition.Registry + entries []voicerecognition.Entry + err error +} + +func (r listRegistry) List(context.Context) ([]voicerecognition.Entry, error) { + return r.entries, r.err +} + +var _ = Describe("KnownVoicesFor", func() { + It("selects from the registry listing", func() { + sel, err := voicerecognition.KnownVoicesFor(context.Background(), listRegistry{entries: []voicerecognition.Entry{entry("ada", "m.gguf", 1)}}, "m.gguf") + Expect(err).ToNot(HaveOccurred()) + Expect(sel.Voices).To(HaveLen(1)) + }) + It("propagates a List error", func() { + boom := errors.New("boom") + _, err := voicerecognition.KnownVoicesFor(context.Background(), listRegistry{err: boom}, "m.gguf") + Expect(err).To(MatchError(boom)) + }) +}) From 3667b42b5f89db53d85921ddfad2c00462841958 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:26:42 +0000 Subject: [PATCH 04/13] feat(proto): carry known voices and speaker names on diarize and live messages Assisted-by: Claude:claude-haiku-4-5 [Claude Code] --- backend/backend.proto | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/backend.proto b/backend/backend.proto index 6a09b98eb39b..eda5ff6c8f9f 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -628,6 +628,7 @@ message TranscriptLiveConfig { string language = 1; // "" => model default int32 sample_rate = 2; // 0 => 16000; backends may reject others map params = 3; // backend-specific tuning + repeated KnownVoice known_voices = 4; // see DiarizeRequest.known_voices } message TranscriptLiveAudio { @@ -649,6 +650,7 @@ message LiveSpeakerSegment { string speaker = 1; // decimal speaker index int64 start = 2; // stream-relative nanoseconds int64 end = 3; + string name = 4; // registered speaker name when the backend identified the speaker, else empty } message LiveSoundEvent { @@ -825,6 +827,9 @@ message DiarizeRequest { // PredictOptions.ModelIdentity for the full rationale. Empty means "no // identity supplied" and backends MUST skip the check. string ModelIdentity = 11; + // Registered voices the backend may use to name speakers. Only backends that + // identify speakers themselves read this; others ignore it. + repeated KnownVoice known_voices = 12; } message DiarizeSegment { @@ -833,6 +838,17 @@ message DiarizeSegment { float end = 3; // seconds string speaker = 4; // backend-emitted speaker label (e.g. "0", "SPEAKER_00") string text = 5; // optional per-segment transcript (empty unless include_text and supported) + string name = 6; // registered speaker name, empty when unknown or not identified + float name_score = 7; // match score of that name (cosine similarity), 0 when unnamed +} + +// KnownVoice is one registered voice: a name and its speaker embedding. `model` +// names the encoder that produced it, so a backend with a different encoder can +// refuse vectors that are not comparable. +message KnownVoice { + string name = 1; + repeated float embedding = 2; + string model = 3; } message DiarizeResponse { From fa4b80bd6306b58186b1eba3f40c0aca35ee029b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:28:32 +0000 Subject: [PATCH 05/13] feat(diarization): name speakers from the voice registry When a diarization model has a speaker_model option, the endpoint sends the registered voices made by that encoder to the backend. The backend's name and name_score come back as extra fields next to the normalized SPEAKER_NN speaker, and the speakers summary carries the first name seen for each speaker. RTTM output and results without names are unchanged. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- core/backend/diarization.go | 28 ++++++++-- core/backend/diarization_names_test.go | 54 ++++++++++++++++++ .../endpoints/openai/audio_upload_test.go | 2 +- core/http/endpoints/openai/diarization.go | 26 ++++++++- .../http/endpoints/openai/diarization_test.go | 56 +++++++++++++++++++ core/http/routes/openai.go | 2 +- core/schema/diarization.go | 6 ++ swagger/docs.go | 10 ++++ swagger/swagger.json | 10 ++++ swagger/swagger.yaml | 10 ++++ 10 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 core/backend/diarization_names_test.go diff --git a/core/backend/diarization.go b/core/backend/diarization.go index 241d1b20cf0f..77c6ad22d2bb 100644 --- a/core/backend/diarization.go +++ b/core/backend/diarization.go @@ -7,6 +7,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/voicerecognition" grpcPkg "github.com/mudler/LocalAI/pkg/grpc" "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -28,10 +29,17 @@ type DiarizationRequest struct { MinDurationOn float32 MinDurationOff float32 IncludeText bool + // KnownVoices are registered voices a speaker-identifying backend may use + // to name the speakers. Empty for every other backend and model. + KnownVoices []voicerecognition.KnownVoice } // modelIdentity: see the note on TranscriptionRequest.toProto. func (r *DiarizationRequest) toProto(threads uint32, modelIdentity string) *proto.DiarizeRequest { + known := make([]*proto.KnownVoice, 0, len(r.KnownVoices)) + for _, v := range r.KnownVoices { + known = append(known, &proto.KnownVoice{Name: v.Name, Embedding: v.Embedding, Model: v.Model}) + } return &proto.DiarizeRequest{ ModelIdentity: modelIdentity, Dst: r.Audio, @@ -44,6 +52,7 @@ func (r *DiarizationRequest) toProto(threads uint32, modelIdentity string) *prot MinDurationOn: r.MinDurationOn, MinDurationOff: r.MinDurationOff, IncludeText: r.IncludeText, + KnownVoices: known, } } @@ -103,6 +112,7 @@ func diarizationResultFromProto(r *proto.DiarizeResponse) *schema.DiarizationRes idx int duration float64 segments int + name string } stats := map[string]*speakerStats{} order := []string{} @@ -126,14 +136,19 @@ func diarizationResultFromProto(r *proto.DiarizeResponse) *schema.DiarizationRes st.duration += dur } st.segments++ + if st.name == "" { + st.name = s.Name + } out.Segments = append(out.Segments, schema.DiarizationSegment{ - Id: i, - Speaker: fmt.Sprintf("SPEAKER_%02d", st.idx), - Label: raw, - Start: float64(s.Start), - End: float64(s.End), - Text: s.Text, + Id: i, + Speaker: fmt.Sprintf("SPEAKER_%02d", st.idx), + Label: raw, + Start: float64(s.Start), + End: float64(s.End), + Text: s.Text, + Name: s.Name, + NameScore: s.NameScore, }) } @@ -148,6 +163,7 @@ func diarizationResultFromProto(r *proto.DiarizeResponse) *schema.DiarizationRes out.Speakers = append(out.Speakers, schema.DiarizationSpeaker{ Id: fmt.Sprintf("SPEAKER_%02d", st.idx), Label: raw, + Name: st.name, TotalSpeechDuration: st.duration, SegmentCount: st.segments, }) diff --git a/core/backend/diarization_names_test.go b/core/backend/diarization_names_test.go new file mode 100644 index 000000000000..f01506256637 --- /dev/null +++ b/core/backend/diarization_names_test.go @@ -0,0 +1,54 @@ +package backend + +import ( + "encoding/json" + + "github.com/mudler/LocalAI/core/services/voicerecognition" + "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("diarization names", func() { + It("adds name and name_score next to the normalized speaker, and keeps SPEAKER_NN", func() { + res := diarizationResultFromProto(&proto.DiarizeResponse{ + Duration: 10, + Segments: []*proto.DiarizeSegment{ + {Speaker: "0", Start: 0, End: 4, Name: "Ada", NameScore: 0.93}, + {Speaker: "1", Start: 4, End: 8}, + {Speaker: "0", Start: 8, End: 10, Name: "Ada", NameScore: 0.93}, + }, + }) + Expect(res.Segments[0].Speaker).To(Equal("SPEAKER_00")) + Expect(res.Segments[0].Name).To(Equal("Ada")) + Expect(res.Segments[0].NameScore).To(BeNumerically("~", 0.93, 1e-6)) + Expect(res.Segments[1].Speaker).To(Equal("SPEAKER_01")) + Expect(res.Segments[1].Name).To(BeEmpty()) + Expect(res.Speakers[0].Name).To(Equal("Ada")) + Expect(res.Speakers[1].Name).To(BeEmpty()) + }) + + It("marshals a result with no names byte for byte as before", func() { + res := diarizationResultFromProto(&proto.DiarizeResponse{ + Duration: 10, + Segments: []*proto.DiarizeSegment{{Speaker: "0", Start: 0, End: 1}}, + }) + b, err := json.Marshal(res) + Expect(err).ToNot(HaveOccurred()) + Expect(string(b)).To(Equal(`{"task":"diarize","duration":10,"num_speakers":1,` + + `"segments":[{"id":0,"speaker":"SPEAKER_00","label":"0","start":0,"end":1}],` + + `"speakers":[{"id":"SPEAKER_00","label":"0","total_speech_duration":1,"segment_count":1}]}`)) + }) + + It("sends the known voices to the backend", func() { + req := DiarizationRequest{Audio: "a.wav", KnownVoices: []voicerecognition.KnownVoice{ + {Name: "Ada", Embedding: []float32{1, 0}, Model: "m.gguf"}, + }} + p := req.toProto(0, "x") + Expect(p.KnownVoices).To(HaveLen(1)) + Expect(p.KnownVoices[0].Name).To(Equal("Ada")) + Expect(p.KnownVoices[0].Embedding).To(Equal([]float32{1, 0})) + Expect(p.KnownVoices[0].Model).To(Equal("m.gguf")) + Expect((&DiarizationRequest{Audio: "a.wav"}).toProto(0, "x").KnownVoices).To(BeEmpty()) + }) +}) diff --git a/core/http/endpoints/openai/audio_upload_test.go b/core/http/endpoints/openai/audio_upload_test.go index ee788a4fd4d0..028bb39b1e27 100644 --- a/core/http/endpoints/openai/audio_upload_test.go +++ b/core/http/endpoints/openai/audio_upload_test.go @@ -27,7 +27,7 @@ var _ = Describe("audio upload endpoints reject bad uploads as client errors", f return TranscriptEndpoint(nil, nil, config.NewApplicationConfig()) }}, "diarization": {"/v1/audio/diarization", func() echo.HandlerFunc { - return DiarizationEndpoint(nil, nil, config.NewApplicationConfig()) + return DiarizationEndpoint(nil, nil, config.NewApplicationConfig(), nil) }}, "sound classification": {"/v1/audio/classifications", func() echo.HandlerFunc { return SoundClassificationEndpoint(nil, nil, config.NewApplicationConfig()) diff --git a/core/http/endpoints/openai/diarization.go b/core/http/endpoints/openai/diarization.go index b39935e8b6b4..3dd1fd4d24b0 100644 --- a/core/http/endpoints/openai/diarization.go +++ b/core/http/endpoints/openai/diarization.go @@ -1,6 +1,7 @@ package openai import ( + "context" "fmt" "io" "net/http" @@ -15,6 +16,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/voicerecognition" model "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/xlog" @@ -47,7 +49,7 @@ import ( // @Param response_format formData string false "json (default), verbose_json, or rttm" // @Success 200 {object} schema.DiarizationResult // @Router /v1/audio/diarization [post] -func DiarizationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { +func DiarizationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, registry voicerecognition.Registry) echo.HandlerFunc { return func(c echo.Context) error { input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest) if !ok || input.Model == "" { @@ -69,6 +71,7 @@ func DiarizationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, ap req.ClusteringThreshold = float32(parseFormFloat(c, "clustering_threshold", 0)) req.MinDurationOn = float32(parseFormFloat(c, "min_duration_on", 0)) req.MinDurationOff = float32(parseFormFloat(c, "min_duration_off", 0)) + attachKnownVoices(c.Request().Context(), &req, modelConfig.Options, registry) responseFormat := schema.DiarizationResponseFormatType(strings.ToLower(c.FormValue("response_format"))) if responseFormat == "" { @@ -137,6 +140,27 @@ func DiarizationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, ap } } +// attachKnownVoices names the speakers from the voice registry when the model +// has a speaker model. Only voices made by that model's encoder are sent. A +// missing registry or speaker model, or a registry read error, leaves the +// request unnamed. +func attachKnownVoices(ctx context.Context, req *backend.DiarizationRequest, options []string, registry voicerecognition.Registry) { + sm := voicerecognition.SpeakerModelFromOptions(options) + if sm == "" || registry == nil { + return + } + sel, err := voicerecognition.KnownVoicesFor(ctx, registry, sm) + if err != nil { + xlog.Warn("diarization: could not read the voice registry; speakers stay unnamed", "error", err) + return + } + req.KnownVoices = sel.Voices + if len(sel.Voices) == 0 && sel.OtherEncoder > 0 { + xlog.Warn("diarization: registered voices were made with a different encoder than this model's speaker_model; speakers stay unnamed", + "speaker_model", sm, "voices_from_other_encoder", sel.OtherEncoder) + } +} + // renderRTTM emits NIST RTTM rows. Each row: // SPEAKER 1 // Field separators are spaces; one row per segment. diff --git a/core/http/endpoints/openai/diarization_test.go b/core/http/endpoints/openai/diarization_test.go index 9cba206a304b..743548262167 100644 --- a/core/http/endpoints/openai/diarization_test.go +++ b/core/http/endpoints/openai/diarization_test.go @@ -1,9 +1,13 @@ package openai import ( + "context" + "errors" "strings" + "github.com/mudler/LocalAI/core/backend" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/voicerecognition" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -49,3 +53,55 @@ var _ = Describe("renderRTTM", func() { Expect(out).To(HavePrefix("SPEAKER audio 1 ")) }) }) + +type fakeVoiceRegistry struct { + voicerecognition.Registry + entries []voicerecognition.Entry + err error +} + +func (f fakeVoiceRegistry) List(context.Context) ([]voicerecognition.Entry, error) { + return f.entries, f.err +} + +var _ = Describe("attachKnownVoices", func() { + ada := voicerecognition.Entry{ + Metadata: voicerecognition.Metadata{Name: "Ada", Model: "enc.gguf"}, + Embedding: []float32{1, 0}, + } + + It("sends the voices made by the speaker model's encoder", func() { + req := backend.DiarizationRequest{} + attachKnownVoices(context.Background(), &req, []string{"speaker_model:/models/enc.gguf"}, + fakeVoiceRegistry{entries: []voicerecognition.Entry{ada}}) + Expect(req.KnownVoices).To(HaveLen(1)) + Expect(req.KnownVoices[0].Name).To(Equal("Ada")) + }) + + It("leaves the request alone without a speaker_model option", func() { + req := backend.DiarizationRequest{} + attachKnownVoices(context.Background(), &req, []string{"other:x"}, + fakeVoiceRegistry{entries: []voicerecognition.Entry{ada}}) + Expect(req.KnownVoices).To(BeEmpty()) + }) + + It("leaves the request alone without a registry", func() { + req := backend.DiarizationRequest{} + attachKnownVoices(context.Background(), &req, []string{"speaker_model:enc.gguf"}, nil) + Expect(req.KnownVoices).To(BeEmpty()) + }) + + It("leaves the request unnamed when the registry cannot be read", func() { + req := backend.DiarizationRequest{} + attachKnownVoices(context.Background(), &req, []string{"speaker_model:enc.gguf"}, + fakeVoiceRegistry{err: errors.New("boom")}) + Expect(req.KnownVoices).To(BeEmpty()) + }) + + It("skips voices made by another encoder", func() { + req := backend.DiarizationRequest{} + attachKnownVoices(context.Background(), &req, []string{"speaker_model:other.gguf"}, + fakeVoiceRegistry{entries: []voicerecognition.Entry{ada}}) + Expect(req.KnownVoices).To(BeEmpty()) + }) +}) diff --git a/core/http/routes/openai.go b/core/http/routes/openai.go index 16b74804af30..604adacf7228 100644 --- a/core/http/routes/openai.go +++ b/core/http/routes/openai.go @@ -194,7 +194,7 @@ func RegisterOpenAIRoutes(app *echo.Echo, app.POST("/v1/audio/transcriptions", audioHandler, audioMiddleware...) app.POST("/audio/transcriptions", audioHandler, audioMiddleware...) - diarizationHandler := openai.DiarizationEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig()) + diarizationHandler := openai.DiarizationEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.VoiceRegistry()) diarizationMiddleware := []echo.MiddlewareFunc{ traceMiddleware, re.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_DIARIZATION)), diff --git a/core/schema/diarization.go b/core/schema/diarization.go index cbac34f5d746..8f8561ab3f68 100644 --- a/core/schema/diarization.go +++ b/core/schema/diarization.go @@ -12,6 +12,11 @@ type DiarizationSegment struct { Start float64 `json:"start"` End float64 `json:"end"` Text string `json:"text,omitempty"` + // Name is the registered speaker this segment was matched to, and NameScore + // the cosine similarity of the match. Both are omitted when the backend did + // not identify the speaker. Speaker stays the normalized SPEAKER_NN label. + Name string `json:"name,omitempty"` + NameScore float32 `json:"name_score,omitempty"` } // DiarizationSpeaker summarizes one speaker across the whole audio so @@ -20,6 +25,7 @@ type DiarizationSegment struct { type DiarizationSpeaker struct { Id string `json:"id"` Label string `json:"label,omitempty"` + Name string `json:"name,omitempty"` TotalSpeechDuration float64 `json:"total_speech_duration"` SegmentCount int `json:"segment_count"` } diff --git a/swagger/docs.go b/swagger/docs.go index c7cd3c8f3d9d..628a577966b2 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -5851,6 +5851,13 @@ const docTemplate = `{ "label": { "type": "string" }, + "name": { + "description": "Name is the registered speaker this segment was matched to, and NameScore\nthe cosine similarity of the match. Both are omitted when the backend did\nnot identify the speaker. Speaker stays the normalized SPEAKER_NN label.", + "type": "string" + }, + "name_score": { + "type": "number" + }, "speaker": { "type": "string" }, @@ -5871,6 +5878,9 @@ const docTemplate = `{ "label": { "type": "string" }, + "name": { + "type": "string" + }, "segment_count": { "type": "integer" }, diff --git a/swagger/swagger.json b/swagger/swagger.json index 3f374d87c556..fbaa509b55b9 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -5848,6 +5848,13 @@ "label": { "type": "string" }, + "name": { + "description": "Name is the registered speaker this segment was matched to, and NameScore\nthe cosine similarity of the match. Both are omitted when the backend did\nnot identify the speaker. Speaker stays the normalized SPEAKER_NN label.", + "type": "string" + }, + "name_score": { + "type": "number" + }, "speaker": { "type": "string" }, @@ -5868,6 +5875,9 @@ "label": { "type": "string" }, + "name": { + "type": "string" + }, "segment_count": { "type": "integer" }, diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 7eeb216fd77c..728d0398bf95 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -1076,6 +1076,14 @@ definitions: type: integer label: type: string + name: + description: |- + Name is the registered speaker this segment was matched to, and NameScore + the cosine similarity of the match. Both are omitted when the backend did + not identify the speaker. Speaker stays the normalized SPEAKER_NN label. + type: string + name_score: + type: number speaker: type: string start: @@ -1089,6 +1097,8 @@ definitions: type: string label: type: string + name: + type: string segment_count: type: integer total_speech_duration: From 03eb7e3438cefac70ba7ad3cd1121cb0e33e5ce8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:31:30 +0000 Subject: [PATCH 06/13] feat(live): pass registered voices to a live session and surface speaker names Live sessions now send the registered voices that match the model's speaker_model to the backend, and each speaker segment carries the name the backend matched. The realtime segment event gains an optional speaker_name field. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- core/backend/transcript_live.go | 43 ++++++++++++++++--- core/backend/transcript_live_names_test.go | 36 ++++++++++++++++ core/http/endpoints/openai/diarization.go | 18 +++++--- core/http/endpoints/openai/realtime.go | 7 ++- .../openai/realtime_live_voices_test.go | 41 ++++++++++++++++++ core/http/endpoints/openai/realtime_model.go | 26 ++++++++++- .../endpoints/openai/realtime_semantic_vad.go | 1 + .../endpoints/openai/types/server_events.go | 4 ++ 8 files changed, 161 insertions(+), 15 deletions(-) create mode 100644 core/backend/transcript_live_names_test.go create mode 100644 core/http/endpoints/openai/realtime_live_voices_test.go diff --git a/core/backend/transcript_live.go b/core/backend/transcript_live.go index e64e54a578e2..49b69cd2785d 100644 --- a/core/backend/transcript_live.go +++ b/core/backend/transcript_live.go @@ -11,6 +11,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/voicerecognition" "github.com/mudler/LocalAI/core/trace" grpcPkg "github.com/mudler/LocalAI/pkg/grpc" "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -40,8 +41,11 @@ type LiveTranscriptionEvent struct { // are stream-relative seconds (mapped from the backend's nanoseconds). type LiveSpeakerSegment struct { Speaker string - Start float64 - End float64 + // Name is the registered speaker name the backend matched, empty when the + // speaker is unknown. + Name string + Start float64 + End float64 } // LiveSoundEvent is one closed sound event from a companion sound/scene @@ -222,6 +226,28 @@ func (ts *liveTraceState) record(closeErr error) { trace.RecordBackendTrace(bt) } +// LiveOption tunes a live transcription session. +type LiveOption func(*liveOptions) + +type liveOptions struct { + knownVoices []voicerecognition.KnownVoice +} + +// WithKnownVoices gives the backend the registered voices it may use to name +// the speakers it detects. Backends without speaker identification ignore them. +func WithKnownVoices(v []voicerecognition.KnownVoice) LiveOption { + return func(o *liveOptions) { o.knownVoices = v } +} + +// liveConfigProto builds the first message of a live session. +func liveConfigProto(language string, o liveOptions) *proto.TranscriptLiveConfig { + cfg := &proto.TranscriptLiveConfig{Language: language, SampleRate: liveSampleRate} + for _, v := range o.knownVoices { + cfg.KnownVoices = append(cfg.KnownVoices, &proto.KnownVoice{Name: v.Name, Embedding: v.Embedding, Model: v.Model}) + } + return cfg +} + // ModelTranscriptionLive loads the transcription backend, opens the // bidirectional AudioTranscriptionLive RPC, sends the session config, and // BLOCKS until the backend's ready ack. A grpcerrors. @@ -232,7 +258,12 @@ func (ts *liveTraceState) record(closeErr error) { // the backend streams, ending with the Final event triggered by Close. func ModelTranscriptionLive(ctx context.Context, language string, ml *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig, - onEvent func(LiveTranscriptionEvent)) (LiveTranscriptionSession, error) { + onEvent func(LiveTranscriptionEvent), opts ...LiveOption) (LiveTranscriptionSession, error) { + + lo := liveOptions{} + for _, f := range opts { + f(&lo) + } transcriptionModel, err := loadTranscriptionModel(ctx, ml, modelConfig, appConfig) if err != nil { @@ -262,10 +293,7 @@ func ModelTranscriptionLive(ctx context.Context, language string, } if err := stream.Send(&proto.TranscriptLiveRequest{ - Payload: &proto.TranscriptLiveRequest_Config{Config: &proto.TranscriptLiveConfig{ - Language: language, - SampleRate: liveSampleRate, - }}, + Payload: &proto.TranscriptLiveRequest_Config{Config: liveConfigProto(language, lo)}, }); err != nil { return fail(err) } @@ -329,6 +357,7 @@ func liveEventFromProto(r *proto.TranscriptLiveResponse) LiveTranscriptionEvent for _, s := range r.GetSpeakers() { ev.Speakers = append(ev.Speakers, LiveSpeakerSegment{ Speaker: s.GetSpeaker(), + Name: s.GetName(), Start: time.Duration(s.GetStart()).Seconds(), End: time.Duration(s.GetEnd()).Seconds(), }) diff --git a/core/backend/transcript_live_names_test.go b/core/backend/transcript_live_names_test.go new file mode 100644 index 000000000000..34a76ff1c9ed --- /dev/null +++ b/core/backend/transcript_live_names_test.go @@ -0,0 +1,36 @@ +package backend + +import ( + "github.com/mudler/LocalAI/core/services/voicerecognition" + "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("live speaker names", func() { + It("puts the known voices in the session config", func() { + o := liveOptions{} + WithKnownVoices([]voicerecognition.KnownVoice{{Name: "Ada", Embedding: []float32{1, 0}, Model: "m.gguf"}})(&o) + cfg := liveConfigProto("en", o) + Expect(cfg.Language).To(Equal("en")) + Expect(cfg.SampleRate).To(Equal(int32(liveSampleRate))) + Expect(cfg.KnownVoices).To(HaveLen(1)) + Expect(cfg.KnownVoices[0].Name).To(Equal("Ada")) + Expect(cfg.KnownVoices[0].Model).To(Equal("m.gguf")) + }) + It("sends no known voices by default", func() { + cfg := liveConfigProto("en", liveOptions{}) + Expect(cfg.KnownVoices).To(BeEmpty()) + Expect(cfg.Language).To(Equal("en")) + Expect(cfg.SampleRate).To(Equal(int32(liveSampleRate))) + }) + It("carries the speaker name from the backend event", func() { + ev := liveEventFromProto(&proto.TranscriptLiveResponse{ + Speakers: []*proto.LiveSpeakerSegment{{Speaker: "0", Name: "Ada", Start: 1e9, End: 2e9}, {Speaker: "1", Start: 2e9, End: 3e9}}, + }) + Expect(ev.Speakers).To(HaveLen(2)) + Expect(ev.Speakers[0].Name).To(Equal("Ada")) + Expect(ev.Speakers[0].Speaker).To(Equal("0")) + Expect(ev.Speakers[1].Name).To(BeEmpty()) + }) +}) diff --git a/core/http/endpoints/openai/diarization.go b/core/http/endpoints/openai/diarization.go index 3dd1fd4d24b0..24328b177c18 100644 --- a/core/http/endpoints/openai/diarization.go +++ b/core/http/endpoints/openai/diarization.go @@ -145,20 +145,28 @@ func DiarizationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, ap // missing registry or speaker model, or a registry read error, leaves the // request unnamed. func attachKnownVoices(ctx context.Context, req *backend.DiarizationRequest, options []string, registry voicerecognition.Registry) { + req.KnownVoices = selectKnownVoices(ctx, "diarization", options, registry) +} + +// selectKnownVoices returns the registered voices a backend may use to name +// speakers, or nil when the model has no speaker_model, there is no registry, +// or the registry cannot be read. It never fails the caller: unnamed speakers +// are the fallback. feature only prefixes the log messages. +func selectKnownVoices(ctx context.Context, feature string, options []string, registry voicerecognition.Registry) []voicerecognition.KnownVoice { sm := voicerecognition.SpeakerModelFromOptions(options) if sm == "" || registry == nil { - return + return nil } sel, err := voicerecognition.KnownVoicesFor(ctx, registry, sm) if err != nil { - xlog.Warn("diarization: could not read the voice registry; speakers stay unnamed", "error", err) - return + xlog.Warn(feature+": could not read the voice registry; speakers stay unnamed", "error", err) + return nil } - req.KnownVoices = sel.Voices if len(sel.Voices) == 0 && sel.OtherEncoder > 0 { - xlog.Warn("diarization: registered voices were made with a different encoder than this model's speaker_model; speakers stay unnamed", + xlog.Warn(feature+": registered voices were made with a different encoder than this model's speaker_model; speakers stay unnamed", "speaker_model", sm, "voices_from_other_encoder", sel.OtherEncoder) } + return sel.Voices } // renderRTTM emits NIST RTTM rows. Each row: diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index 11d1653cb04a..c152d2c22a61 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -35,6 +35,7 @@ import ( "github.com/mudler/LocalAI/core/services/failover" "github.com/mudler/LocalAI/core/services/routing/router" "github.com/mudler/LocalAI/core/services/voiceprofile" + "github.com/mudler/LocalAI/core/services/voicerecognition" "github.com/mudler/LocalAI/core/templates" laudio "github.com/mudler/LocalAI/pkg/audio" "github.com/mudler/LocalAI/pkg/functions" @@ -841,6 +842,7 @@ func runRealtimeSession(application *application.Application, t Transport, model application.ModelLoader(), application.ApplicationConfig(), application.FailoverManager(), + application.VoiceRegistry(), ); err != nil { xlog.Error("failed to update session", "error", err) // The cause is validation feedback on the client's own @@ -1158,7 +1160,7 @@ func sendTestTone(t Transport) { } } -func updateTransSession(session *Session, update *types.SessionUnion, cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, fm *failover.Manager) error { +func updateTransSession(session *Session, update *types.SessionUnion, cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, fm *failover.Manager, voices voicerecognition.Registry) error { sessionLock.Lock() defer sessionLock.Unlock() @@ -1186,6 +1188,9 @@ func updateTransSession(session *Session, update *types.SessionUnion, cl *config return err } + if tm, ok := m.(*transcriptOnlyModel); ok { + tm.voiceRegistry = voices + } session.ModelInterface = m session.ModelConfig = cfg session.SoundDetectionEnabled = cfg.Pipeline.SoundDetection != "" diff --git a/core/http/endpoints/openai/realtime_live_voices_test.go b/core/http/endpoints/openai/realtime_live_voices_test.go new file mode 100644 index 000000000000..c30679a69171 --- /dev/null +++ b/core/http/endpoints/openai/realtime_live_voices_test.go @@ -0,0 +1,41 @@ +package openai + +import ( + "context" + "encoding/json" + "errors" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/endpoints/openai/types" + "github.com/mudler/LocalAI/core/services/voicerecognition" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("liveVoiceOptions", func() { + ada := voicerecognition.Entry{ + Metadata: voicerecognition.Metadata{Name: "Ada", Model: "enc.gguf"}, + Embedding: []float32{1, 0}, + } + cfgWith := func(opts ...string) *config.ModelConfig { return &config.ModelConfig{Options: opts} } + + It("opens the session with the registered voices", func() { + opts := liveVoiceOptions(context.Background(), fakeVoiceRegistry{entries: []voicerecognition.Entry{ada}}, cfgWith("speaker_model:enc.gguf")) + Expect(opts).To(HaveLen(1)) + }) + It("adds nothing without a speaker_model, a registry, or when the registry fails", func() { + reg := fakeVoiceRegistry{entries: []voicerecognition.Entry{ada}} + Expect(liveVoiceOptions(context.Background(), reg, cfgWith("x:y"))).To(BeEmpty()) + Expect(liveVoiceOptions(context.Background(), nil, cfgWith("speaker_model:enc.gguf"))).To(BeEmpty()) + Expect(liveVoiceOptions(context.Background(), fakeVoiceRegistry{err: errors.New("boom")}, cfgWith("speaker_model:enc.gguf"))).To(BeEmpty()) + }) +}) + +var _ = Describe("transcription segment event", func() { + It("carries speaker_name only when set", func() { + named, _ := json.Marshal(types.ConversationItemInputAudioTranscriptionSegmentEvent{Speaker: "0", SpeakerName: "Ada"}) + Expect(string(named)).To(ContainSubstring(`"speaker_name":"Ada"`)) + plain, _ := json.Marshal(types.ConversationItemInputAudioTranscriptionSegmentEvent{Speaker: "0"}) + Expect(string(plain)).ToNot(ContainSubstring("speaker_name")) + }) +}) diff --git a/core/http/endpoints/openai/realtime_model.go b/core/http/endpoints/openai/realtime_model.go index 678a57a0df6a..dfc49112af58 100644 --- a/core/http/endpoints/openai/realtime_model.go +++ b/core/http/endpoints/openai/realtime_model.go @@ -22,6 +22,7 @@ import ( "github.com/mudler/LocalAI/core/services/failover" "github.com/mudler/LocalAI/core/services/routing/router" "github.com/mudler/LocalAI/core/services/voiceprofile" + "github.com/mudler/LocalAI/core/services/voicerecognition" "github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -86,6 +87,9 @@ type wrappedModel struct { routerSessionID string routerUserID string + // voiceRegistry names live speakers from registered voices; nil disables it. + voiceRegistry voicerecognition.Registry + stageRouter // tuneLLM applies the pipeline's LLM overrides (reasoning effort, // disable_thinking) to a chain target loaded per call. @@ -113,6 +117,9 @@ type transcriptOnlyModel struct { modelLoader *model.ModelLoader confLoader *config.ModelConfigLoader + // voiceRegistry names live speakers from registered voices; nil disables it. + voiceRegistry voicerecognition.Registry + stageRouter } @@ -187,7 +194,7 @@ func (m *transcriptOnlyModel) TranscribeLive(ctx context.Context, language strin // Only opening the live session can move to the next target. err := m.stageCall(ctx, config.PipelineStageTranscription, m.TranscriptionConfig, func(cfg *config.ModelConfig, _ func()) error { var err error - live, err = backend.ModelTranscriptionLive(ctx, language, m.modelLoader, *cfg, m.appConfig, onEvent) + live, err = backend.ModelTranscriptionLive(ctx, language, m.modelLoader, *cfg, m.appConfig, onEvent, liveVoiceOptions(ctx, m.voiceRegistry, cfg)...) return err }) return live, err @@ -579,7 +586,7 @@ func (m *wrappedModel) TranscribeLive(ctx context.Context, language string, onEv // open, events flow to the client for the rest of the utterance. err := m.stageCall(ctx, config.PipelineStageTranscription, m.TranscriptionConfig, func(cfg *config.ModelConfig, _ func()) error { var err error - live, err = backend.ModelTranscriptionLive(ctx, language, m.modelLoader, *cfg, m.appConfig, onEvent) + live, err = backend.ModelTranscriptionLive(ctx, language, m.modelLoader, *cfg, m.appConfig, onEvent, liveVoiceOptions(ctx, m.voiceRegistry, cfg)...) return err }) return live, err @@ -1018,6 +1025,8 @@ type RealtimeRoutingContext struct { UserID string // Failover resolves pipeline stages that name a failover chain. Failover *failover.Manager + // VoiceRegistry holds the voices registered through /v1/voice/register. + VoiceRegistry voicerecognition.Registry } // buildRealtimeRoutingContext assembles the routing dependencies the @@ -1040,6 +1049,8 @@ func buildRealtimeRoutingContext(a *application.Application, sessionID string) * SessionID: sessionID, UserID: userID, Failover: a.FailoverManager(), + + VoiceRegistry: a.VoiceRegistry(), } } @@ -1203,6 +1214,17 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model wm.routerStore = routing.Store wm.routerSessionID = routing.SessionID wm.routerUserID = routing.UserID + wm.voiceRegistry = routing.VoiceRegistry } return wm, nil } + +// liveVoiceOptions selects the registered voices a live session may name speakers +// with. It stays empty without a speaker_model or a voice registry. +func liveVoiceOptions(ctx context.Context, registry voicerecognition.Registry, cfg *config.ModelConfig) []backend.LiveOption { + voices := selectKnownVoices(ctx, "live transcription", cfg.Options, registry) + if len(voices) == 0 { + return nil + } + return []backend.LiveOption{backend.WithKnownVoices(voices)} +} diff --git a/core/http/endpoints/openai/realtime_semantic_vad.go b/core/http/endpoints/openai/realtime_semantic_vad.go index df56abf19783..fa1ed07139af 100644 --- a/core/http/endpoints/openai/realtime_semantic_vad.go +++ b/core/http/endpoints/openai/realtime_semantic_vad.go @@ -220,6 +220,7 @@ func (l *liveTurnState) drainEvents(audioSec float64) { ItemID: l.itemID, ContentIndex: 0, Speaker: seg.Speaker, + SpeakerName: seg.Name, Start: seg.Start, End: seg.End, }) diff --git a/core/http/endpoints/openai/types/server_events.go b/core/http/endpoints/openai/types/server_events.go index 4cab30a61d80..cdf88bf84b22 100644 --- a/core/http/endpoints/openai/types/server_events.go +++ b/core/http/endpoints/openai/types/server_events.go @@ -595,6 +595,10 @@ type ConversationItemInputAudioTranscriptionSegmentEvent struct { // The speaker label for the segment, if available. Speaker string `json:"speaker,omitempty"` + // The registered name of the speaker, when the backend recognised a + // voice registered through /v1/voice/register. + SpeakerName string `json:"speaker_name,omitempty"` + // The start time of the segment in seconds. Always present (not // omitempty: a segment starting at 0.0s must still carry "start"). Start float64 `json:"start"` From dc1712370de6df097f622ab60b606748086a1708 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:33:50 +0000 Subject: [PATCH 07/13] feat(parakeet-cpp): load a speaker model and build per-request voice registries Adds the speaker bindings (ABI v9 and v10, probed separately), the speaker_model, speaker_threshold and speaker_margin options, and a per-request registry builder over the known voices. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- backend/go/parakeet-cpp/goparakeetcpp.go | 31 ++++- backend/go/parakeet-cpp/main.go | 14 ++ backend/go/parakeet-cpp/roles.go | 37 +++++- backend/go/parakeet-cpp/roles_test.go | 125 ++++++++++++++++++ backend/go/parakeet-cpp/speaker_registry.go | 104 +++++++++++++++ .../go/parakeet-cpp/speaker_registry_test.go | 106 +++++++++++++++ 6 files changed, 413 insertions(+), 4 deletions(-) create mode 100644 backend/go/parakeet-cpp/speaker_registry.go create mode 100644 backend/go/parakeet-cpp/speaker_registry_test.go diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index e8ed1c5253f7..59b74979c8cd 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -104,6 +104,21 @@ var ( CppSceneStreamFeedJSON func(s uintptr, pcm *float32, n int32, isLast int32) uintptr CppSceneStreamLastError func(s uintptr) string CppSceneStreamFree func(s uintptr) + + // Speaker identification. CppSpeakerDim, the registry, CppSceneStreamBeginSpeaker and + // CppTranscribeAndDiarizeNamedJSON are ABI v9; CppSpeakerRegistryAddEmbedding and + // CppDiarizeNamedPCMJSON are ABI v10. All are nil on an older libparakeet.so, and + // Load refuses speaker_model: unless the v10 ones are present. + CppSpeakerDim func(ctx uintptr) int32 + CppSpeakerRegistryNew func() uintptr + CppSpeakerRegistryFree func(reg uintptr) + CppSpeakerRegistryAddEmbedding func(reg uintptr, name string, emb *float32, dim int32) int32 + CppSpeakerRegistryLastError func(reg uintptr) string + CppSceneStreamBeginSpeaker func(asr, diar, tagger, speaker, reg uintptr, o *cSceneOpts) uintptr + // CppDiarizeNamedPCMJSON takes two float32 arguments (acceptThreshold, margin), which + // purego passes in floating-point registers. Not exercised without the real library. + CppDiarizeNamedPCMJSON func(diar, speaker, reg uintptr, samples *float32, n, sampleRate int32, acceptThreshold, margin float32) uintptr + CppTranscribeAndDiarizeNamedJSON func(asr, diar, speaker, reg uintptr, samples *float32, n, sampleRate int32) uintptr ) // cSoundOpts and cSceneOpts mirror parakeet_sound_opts / parakeet_scene_opts @@ -121,6 +136,14 @@ type cSceneOpts struct { DiarLatency int32 Sound cSoundOpts Flags int32 + // Speaker identification (parakeet_scene_opts, ABI v9). The C side reads + // these only when Size covers them, so a Go struct built against a v9 + // library and run against a v8 one is still valid. + SpeakerAcceptThreshold float32 + SpeakerMargin float32 + SpeakerMinVoiceSec float32 + SpeakerRefreshSec float32 + SpeakerMaxVoiceSec float32 } // streamChunkSamples is how much 16 kHz mono PCM we hand to stream_feed per @@ -193,6 +216,12 @@ type ParakeetCpp struct { // companion. See roles.go. diarCtx uintptr tagCtx uintptr + // spkCtx is the speaker encoder context (speaker_model: companion); 0 when speaker + // naming is off. speakerAccept is the cosine acceptance threshold and speakerMargin + // the runner-up margin, both from the model options. + spkCtx uintptr + speakerAccept float32 + speakerMargin float32 // diarLatency is the PARAKEET_DIAR_LATENCY_* mode for diarization // streaming (diarization_latency: option, default "low"). Unused until // the diarization/scene streaming paths land. @@ -953,7 +982,7 @@ func (p *ParakeetCpp) Free() error { // re-checks ctxPtr under the lock) can never feed into a freed ctx. p.engineMu.Lock() defer p.engineMu.Unlock() - for _, ctxField := range [...]*uintptr{&p.ctxPtr, &p.diarCtx, &p.tagCtx} { + for _, ctxField := range [...]*uintptr{&p.ctxPtr, &p.diarCtx, &p.tagCtx, &p.spkCtx} { if *ctxField != 0 { CppFree(*ctxField) *ctxField = 0 diff --git a/backend/go/parakeet-cpp/main.go b/backend/go/parakeet-cpp/main.go index 865d5b6f1947..20dd1358ae3c 100644 --- a/backend/go/parakeet-cpp/main.go +++ b/backend/go/parakeet-cpp/main.go @@ -117,6 +117,20 @@ func main() { purego.RegisterLibFunc(&CppSceneStreamLastError, lib, "parakeet_capi_scene_stream_last_error") purego.RegisterLibFunc(&CppSceneStreamFree, lib, "parakeet_capi_scene_stream_free") } + // Speaker identification (ABI v9 and v10). Probed separately from model_kind so an older + // libparakeet.so still loads; speaker_model: is refused in roles.go unless the v10 symbols exist. + if sym, err := purego.Dlsym(lib, "parakeet_capi_scene_stream_begin_speaker"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppSpeakerDim, lib, "parakeet_capi_speaker_dim") + purego.RegisterLibFunc(&CppSpeakerRegistryNew, lib, "parakeet_capi_speaker_registry_new") + purego.RegisterLibFunc(&CppSpeakerRegistryFree, lib, "parakeet_capi_speaker_registry_free") + purego.RegisterLibFunc(&CppSpeakerRegistryLastError, lib, "parakeet_capi_speaker_registry_last_error") + purego.RegisterLibFunc(&CppSceneStreamBeginSpeaker, lib, "parakeet_capi_scene_stream_begin_speaker") + purego.RegisterLibFunc(&CppTranscribeAndDiarizeNamedJSON, lib, "parakeet_capi_transcribe_and_diarize_named_json") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_named_pcm_json"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppSpeakerRegistryAddEmbedding, lib, "parakeet_capi_speaker_registry_add_embedding") + purego.RegisterLibFunc(&CppDiarizeNamedPCMJSON, lib, "parakeet_capi_diarize_named_pcm_json") + } fmt.Fprintf(os.Stderr, "[parakeet-cpp] ABI=%d\n", CppAbiVersion()) diff --git a/backend/go/parakeet-cpp/roles.go b/backend/go/parakeet-cpp/roles.go index c6a1e912e1c1..0ce944f910ff 100644 --- a/backend/go/parakeet-cpp/roles.go +++ b/backend/go/parakeet-cpp/roles.go @@ -17,6 +17,7 @@ const ( modelKindASR = 1 modelKindDiarization = 2 modelKindSound = 3 + modelKindSpeaker = 4 ) // Diarization streaming latency modes (mirrors PARAKEET_DIAR_LATENCY_* in @@ -38,6 +39,8 @@ func modelKindName(kind int32) string { return "diarization" case modelKindSound: return "sound" + case modelKindSpeaker: + return "speaker" default: return "unknown" } @@ -129,14 +132,30 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { diarModelOpt := optString(opts, "diarization_model") asrModelOpt := optString(opts, "asr_model") soundModelOpt := optString(opts, "sound_model") - hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" + speakerModelOpt := optString(opts, "speaker_model") + hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" || speakerModelOpt != "" if hasCompanionOpts && CppModelKind == nil { - return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model options need " + + return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model/speaker_model options need " + "parakeet_capi_model_kind (ABI v8) to verify what they load; the loaded libparakeet.so " + "is too old to report companion model roles") } + if speakerModelOpt != "" { + if CppSpeakerRegistryAddEmbedding == nil || CppSpeakerDim == nil || CppSceneStreamBeginSpeaker == nil { + return errors.New("parakeet-cpp: speaker_model needs libparakeet.so ABI 10 " + + "(parakeet_capi_speaker_registry_add_embedding); the loaded library is older") + } + } + accept, err := parseSpeakerThreshold(optString(opts, "speaker_threshold")) + if err != nil { + return err + } + margin, err := parseSpeakerMargin(optString(opts, "speaker_margin")) + if err != nil { + return err + } + latency, err := parseDiarLatency(optString(opts, "diarization_latency")) if err != nil { return err @@ -160,7 +179,7 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { for _, c := range loaded { CppFree(c) } - p.ctxPtr, p.diarCtx, p.tagCtx = 0, 0, 0 + p.ctxPtr, p.diarCtx, p.tagCtx, p.spkCtx = 0, 0, 0, 0 p.companions = nil } @@ -177,6 +196,10 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { p.diarCtx = primary case modelKindSound: p.tagCtx = primary + case modelKindSpeaker: + freeLoaded() + return errors.New("parakeet-cpp: a speaker model cannot be the primary model; " + + "use it as speaker_model: next to a diarization model") default: p.ctxPtr = primary } @@ -191,6 +214,9 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { {"sound_model", soundModelOpt, modelKindSound, func(pp *ParakeetCpp, c uintptr) { pp.tagCtx = c }, func(pp *ParakeetCpp) uintptr { return pp.tagCtx }}, + {"speaker_model", speakerModelOpt, modelKindSpeaker, + func(pp *ParakeetCpp, c uintptr) { pp.spkCtx = c }, + func(pp *ParakeetCpp) uintptr { return pp.spkCtx }}, } for _, spec := range specs { if spec.value == "" { @@ -221,6 +247,11 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { p.companions = append(p.companions, cctx) } + if p.spkCtx != 0 && p.diarCtx == 0 { + freeLoaded() + return errors.New("parakeet-cpp: speaker_model needs a diarization model (the primary or diarization_model:)") + } + p.speakerAccept, p.speakerMargin = accept, margin p.diarLatency = latency return nil } diff --git a/backend/go/parakeet-cpp/roles_test.go b/backend/go/parakeet-cpp/roles_test.go index 7b2a892b333d..9b72087f0edd 100644 --- a/backend/go/parakeet-cpp/roles_test.go +++ b/backend/go/parakeet-cpp/roles_test.go @@ -185,6 +185,131 @@ var _ = Describe("model roles (stubbed C API)", func() { Expect(f.freed).To(HaveLen(freedBeforeFree), "Free after a failed Load must not free anything again") }) + Describe("speaker_model", func() { + var savedAdd func(uintptr, string, *float32, int32) int32 + var savedDim func(uintptr) int32 + var savedBegin func(asr, diar, tagger, speaker, reg uintptr, o *cSceneOpts) uintptr + var setNew func(on bool) + setNew = func(on bool) { + if on { + CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 0 } + CppSpeakerDim = func(uintptr) int32 { return 3 } + CppSceneStreamBeginSpeaker = func(_, _, _, _, _ uintptr, _ *cSceneOpts) uintptr { return 0 } + } else { + CppSpeakerRegistryAddEmbedding, CppSpeakerDim, CppSceneStreamBeginSpeaker = nil, nil, nil + } + } + BeforeEach(func() { + savedAdd, savedDim, savedBegin = CppSpeakerRegistryAddEmbedding, CppSpeakerDim, CppSceneStreamBeginSpeaker + setNew(true) + }) + AfterEach(func() { + CppSpeakerRegistryAddEmbedding, CppSpeakerDim, CppSceneStreamBeginSpeaker = savedAdd, savedDim, savedBegin + }) + + It("loads a kind-4 companion into spkCtx and Free releases it once", func() { + f := newFakeLib(). + withModel("diar.gguf", modelKindDiarization). + withModel("/models/spk.gguf", modelKindSpeaker) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ + ModelFile: "diar.gguf", + ModelPath: "/models", + Options: []string{"speaker_model:spk.gguf", "speaker_threshold:0.3", "speaker_margin:0.1"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(p.spkCtx).ToNot(BeZero()) + Expect(p.speakerAccept).To(BeNumerically("~", 0.7, 1e-6)) + Expect(p.speakerMargin).To(BeNumerically("~", 0.1, 1e-6)) + + diar, spk := p.diarCtx, p.spkCtx + Expect(p.Free()).To(Succeed()) + Expect(f.freed).To(ConsistOf(diar, spk)) + Expect(p.spkCtx).To(BeZero()) + Expect(p.Free()).To(Succeed()) + Expect(f.freed).To(HaveLen(2), "a second Free frees nothing") + }) + + It("is rejected when the library lacks ABI 10, and nothing is loaded", func() { + setNew(false) + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ModelFile: "diar.gguf", Options: []string{"speaker_model:spk.gguf"}}) + Expect(err).To(MatchError(ContainSubstring("ABI 10"))) + Expect(f.loadedPaths).To(BeEmpty()) + }) + + It("is rejected without a diarization model and frees every context", func() { + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("spk.gguf", modelKindSpeaker) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ModelFile: "asr.gguf", Options: []string{"speaker_model:spk.gguf"}}) + Expect(err).To(MatchError(ContainSubstring("diarization"))) + Expect(f.freed).To(HaveLen(2)) + Expect(p.ctxPtr).To(BeZero()) + Expect(p.spkCtx).To(BeZero()) + Expect(p.Free()).To(Succeed()) + Expect(f.freed).To(HaveLen(2)) + }) + + It("rejects a companion of the wrong kind", func() { + f := newFakeLib(). + withModel("diar.gguf", modelKindDiarization). + withModel("wrong.gguf", modelKindSound) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ModelFile: "diar.gguf", Options: []string{"speaker_model:wrong.gguf"}}) + Expect(err).To(MatchError(ContainSubstring("is a sound model, expected a speaker model"))) + Expect(f.freed).To(HaveLen(2)) + Expect(p.spkCtx).To(BeZero()) + }) + + It("fails Load on an invalid speaker_threshold or speaker_margin", func() { + for _, bad := range []string{"speaker_threshold:abc", "speaker_threshold:2", "speaker_margin:1", "speaker_margin:-1"} { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization) + r := f.install() + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "diar.gguf", Options: []string{bad}})).ToNot(Succeed(), bad) + r() + } + }) + + It("rejects the speaker kind as the primary model and frees it", func() { + f := newFakeLib().withModel("spk.gguf", modelKindSpeaker) + restore = f.install() + + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ModelFile: "spk.gguf"}) + Expect(err).To(MatchError(ContainSubstring("cannot be the primary model"))) + Expect(f.freed).To(HaveLen(1)) + Expect(p.spkCtx).To(BeZero()) + Expect(p.ctxPtr).To(BeZero()) + }) + + It("loads exactly as before without speaker_model on a library with none of the new symbols", func() { + setNew(false) + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel("diar.gguf", modelKindDiarization) + restore = f.install() + + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "asr.gguf", Options: []string{"diarization_model:diar.gguf"}})).To(Succeed()) + Expect(p.spkCtx).To(BeZero()) + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.speakerAccept).To(BeNumerically("~", 0.5, 1e-6)) + }) + }) + It("rejects an asr_model companion on an already-ASR primary and frees everything it opened", func() { f := newFakeLib(). withModel("asr.gguf", modelKindASR). diff --git a/backend/go/parakeet-cpp/speaker_registry.go b/backend/go/parakeet-cpp/speaker_registry.go new file mode 100644 index 000000000000..c79d6328e73e --- /dev/null +++ b/backend/go/parakeet-cpp/speaker_registry.go @@ -0,0 +1,104 @@ +package main + +import ( + "fmt" + "math" + "strconv" + "strings" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + defaultSpeakerDistance = 0.5 // cosine 0.5, parakeet.cpp's default accept_threshold + defaultSpeakerMargin = 0.05 // parakeet.cpp's default runner-up margin +) + +// parseSpeakerThreshold reads speaker_threshold, a distance (1 minus cosine, the unit +// /v1/voice/identify uses), and returns the cosine acceptance threshold the C-API takes. +// Empty means the default. A distance outside (0, 2) is an error. +func parseSpeakerThreshold(s string) (float32, error) { + d := defaultSpeakerDistance + if strings.TrimSpace(s) != "" { + v, err := strconv.ParseFloat(strings.TrimSpace(s), 64) + if err != nil || math.IsNaN(v) || v <= 0 || v >= 2 { + return 0, fmt.Errorf("parakeet-cpp: speaker_threshold %q must be a distance in (0, 2) (1 minus cosine similarity)", s) + } + d = v + } + return float32(1 - d), nil +} + +// parseSpeakerMargin reads speaker_margin, the runner-up margin in [0, 1). +func parseSpeakerMargin(s string) (float32, error) { + if strings.TrimSpace(s) == "" { + return defaultSpeakerMargin, nil + } + v, err := strconv.ParseFloat(strings.TrimSpace(s), 64) + if err != nil || math.IsNaN(v) || v < 0 || v >= 1 { + return 0, fmt.Errorf("parakeet-cpp: speaker_margin %q must be a number in [0, 1)", s) + } + return float32(v), nil +} + +// buildSpeakerRegistryLocked makes a parakeet_speaker_registry from the registered voices +// of one request or stream. Caller holds engineMu. It returns 0 (and no error) when there is +// nothing to build: no speaker model loaded or no usable voices. A voice whose embedding size +// differs from the speaker model's is an error naming the voice and both sizes. The caller +// frees a non-zero result with freeSpeakerRegistry. +func (p *ParakeetCpp) buildSpeakerRegistryLocked(voices []*pb.KnownVoice) (uintptr, error) { + if p.spkCtx == 0 || CppSpeakerRegistryNew == nil || CppSpeakerRegistryAddEmbedding == nil || len(voices) == 0 { + return 0, nil + } + dim := 0 + if CppSpeakerDim != nil { + dim = int(CppSpeakerDim(p.spkCtx)) + } + reg := CppSpeakerRegistryNew() + if reg == 0 { + return 0, status.Error(codes.Internal, "parakeet-cpp: could not create a speaker registry") + } + added := 0 + for _, v := range voices { + emb := v.GetEmbedding() + if v.GetName() == "" || len(emb) == 0 { + xlog.Warn("parakeet-cpp: skipping a known voice with no name or embedding") + continue + } + if dim > 0 && len(emb) != dim { + CppSpeakerRegistryFree(reg) + return 0, status.Errorf(codes.InvalidArgument, + "parakeet-cpp: known voice %q has a %d-value embedding but the speaker model produces %d; "+ + "register the voices again with the same speaker encoder", v.GetName(), len(emb), dim) + } + if rc := CppSpeakerRegistryAddEmbedding(reg, v.GetName(), &emb[0], int32(len(emb))); rc != 0 { + msg := CppSpeakerRegistryLastError(reg) + CppSpeakerRegistryFree(reg) + return 0, status.Errorf(codes.InvalidArgument, "parakeet-cpp: known voice %q was refused: %s", v.GetName(), msg) + } + added++ + } + if added == 0 { + CppSpeakerRegistryFree(reg) + return 0, nil + } + return reg, nil +} + +// buildSpeakerRegistry is buildSpeakerRegistryLocked under engineMu. +func (p *ParakeetCpp) buildSpeakerRegistry(voices []*pb.KnownVoice) (uintptr, error) { + p.engineMu.Lock() + defer p.engineMu.Unlock() + return p.buildSpeakerRegistryLocked(voices) +} + +// freeSpeakerRegistry releases a registry built above. A zero handle is a no-op. +func (p *ParakeetCpp) freeSpeakerRegistry(reg uintptr) { + if reg == 0 || CppSpeakerRegistryFree == nil { + return + } + CppSpeakerRegistryFree(reg) +} diff --git a/backend/go/parakeet-cpp/speaker_registry_test.go b/backend/go/parakeet-cpp/speaker_registry_test.go new file mode 100644 index 000000000000..057f12ed4268 --- /dev/null +++ b/backend/go/parakeet-cpp/speaker_registry_test.go @@ -0,0 +1,106 @@ +package main + +import ( + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("speaker options", func() { + It("turns a distance threshold into the cosine the C side takes", func() { + a, err := parseSpeakerThreshold("") + Expect(err).ToNot(HaveOccurred()) + Expect(a).To(BeNumerically("~", 0.5, 1e-6)) // default distance 0.5 + a, err = parseSpeakerThreshold("0.3") + Expect(err).ToNot(HaveOccurred()) + Expect(a).To(BeNumerically("~", 0.7, 1e-6)) + }) + It("rejects a threshold that is not a distance in (0, 2)", func() { + for _, bad := range []string{"abc", "0", "-0.1", "2", "2.5", "NaN"} { + _, err := parseSpeakerThreshold(bad) + Expect(err).To(HaveOccurred(), bad) + } + }) + It("parses the margin, default 0.05, within [0, 1)", func() { + m, err := parseSpeakerMargin("") + Expect(err).ToNot(HaveOccurred()) + Expect(m).To(BeNumerically("~", 0.05, 1e-6)) + _, err = parseSpeakerMargin("-1") + Expect(err).To(HaveOccurred()) + _, err = parseSpeakerMargin("1") + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("buildSpeakerRegistry", func() { + var restore func() + var added []string + var freed []uintptr + BeforeEach(func() { + sNew, sFree, sAdd, sDim, sErr := CppSpeakerRegistryNew, CppSpeakerRegistryFree, CppSpeakerRegistryAddEmbedding, CppSpeakerDim, CppSpeakerRegistryLastError + restore = func() { + CppSpeakerRegistryNew, CppSpeakerRegistryFree, CppSpeakerRegistryAddEmbedding, CppSpeakerDim, CppSpeakerRegistryLastError = sNew, sFree, sAdd, sDim, sErr + } + added, freed = nil, nil + CppSpeakerDim = func(uintptr) int32 { return 3 } + CppSpeakerRegistryNew = func() uintptr { return 77 } + CppSpeakerRegistryFree = func(r uintptr) { freed = append(freed, r) } + CppSpeakerRegistryAddEmbedding = func(r uintptr, name string, emb *float32, dim int32) int32 { + added = append(added, name) + return 0 + } + CppSpeakerRegistryLastError = func(uintptr) string { return "stub error" } + }) + AfterEach(func() { restore() }) + + voice := func(name string, n int) *pb.KnownVoice { + return &pb.KnownVoice{Name: name, Embedding: make([]float32, n)} + } + + It("adds every known voice, in order", func() { + p := &ParakeetCpp{spkCtx: 5} + reg, err := p.buildSpeakerRegistry([]*pb.KnownVoice{ + {Name: "ada", Embedding: []float32{1, 0, 0}}, {Name: "ben", Embedding: []float32{0, 1, 0}}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(reg).To(Equal(uintptr(77))) + Expect(added).To(Equal([]string{"ada", "ben"})) + Expect(freed).To(BeEmpty()) + }) + It("refuses a voice of the wrong size with both sizes in the message, and frees the registry", func() { + p := &ParakeetCpp{spkCtx: 5} + reg, err := p.buildSpeakerRegistry([]*pb.KnownVoice{{Name: "ada", Embedding: []float32{1, 0, 0}}, voice("cy", 5)}) + Expect(err).To(HaveOccurred()) + Expect(reg).To(Equal(uintptr(0))) + Expect(err.Error()).To(ContainSubstring(`"cy"`)) + Expect(err.Error()).To(ContainSubstring("5")) + Expect(err.Error()).To(ContainSubstring("3")) + Expect(freed).To(Equal([]uintptr{77})) + }) + It("reports the C error when a voice is refused, and frees the registry", func() { + CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 1 } + p := &ParakeetCpp{spkCtx: 5} + _, err := p.buildSpeakerRegistry([]*pb.KnownVoice{{Name: "ada", Embedding: []float32{0, 0, 0}}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("stub error")) + Expect(freed).To(Equal([]uintptr{77})) + }) + It("returns no registry when no speaker model is loaded", func() { + p := &ParakeetCpp{} + reg, err := p.buildSpeakerRegistry([]*pb.KnownVoice{voice("ada", 3)}) + Expect(err).ToNot(HaveOccurred()) + Expect(reg).To(Equal(uintptr(0))) + }) + It("skips a voice with no name or embedding instead of failing the request", func() { + p := &ParakeetCpp{spkCtx: 5} + _, err := p.buildSpeakerRegistry([]*pb.KnownVoice{{Name: "", Embedding: []float32{1, 0, 0}}, {Name: "x"}, {Name: "ada", Embedding: []float32{1, 0, 0}}}) + Expect(err).ToNot(HaveOccurred()) + Expect(added).To(Equal([]string{"ada"})) + }) + It("frees a registry and ignores a zero handle", func() { + p := &ParakeetCpp{} + p.freeSpeakerRegistry(0) + p.freeSpeakerRegistry(9) + Expect(freed).To(Equal([]uintptr{9})) + }) +}) From cac63f45ea87b44565a82f0e9cd94be711fc6d1a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:36:45 +0000 Subject: [PATCH 08/13] feat(parakeet-cpp): name the speakers in Diarize from the known voices Diarize builds a per-request speaker registry from the known voices when a speaker model is loaded, calls the named C functions, and puts each slot's registered name and score on the segments. The registry is freed on every path. A library without ABI 10 reports Unimplemented instead of dropping the names. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- backend/go/parakeet-cpp/diarize.go | 89 ++++++++++++--- backend/go/parakeet-cpp/diarize_test.go | 146 +++++++++++++++++++++++- backend/go/parakeet-cpp/speakers.go | 2 +- 3 files changed, 219 insertions(+), 18 deletions(-) diff --git a/backend/go/parakeet-cpp/diarize.go b/backend/go/parakeet-cpp/diarize.go index 1853aa666314..c36cfbdcd288 100644 --- a/backend/go/parakeet-cpp/diarize.go +++ b/backend/go/parakeet-cpp/diarize.go @@ -27,7 +27,28 @@ type diarizeSegmentJSON struct { // not the count of speakers actually present, so it is not read here; the // response's num_speakers is computed from distinct segment labels instead. type diarizePCMDoc struct { - Segments []diarizeSegmentJSON `json:"segments"` + Segments []diarizeSegmentJSON `json:"segments"` + Names map[string]speakerNameJSON `json:"names"` +} + +// speakerNameJSON mirrors one value of the "names" map the named C-API functions add: +// {"0":{"name":"Ada","score":0.93}}. +type speakerNameJSON struct { + Name string `json:"name"` + Score float32 `json:"score"` +} + +// nameFor returns the registered name of a diarization slot, or "" for an unknown slot, a slot +// with no matching voice, or speaker -1 (no diarized speaker). +func nameFor(names map[string]speakerNameJSON, speaker int) (string, float32) { + if speaker < 0 || len(names) == 0 { + return "", 0 + } + n, ok := names[strconv.Itoa(speaker)] + if !ok || n.Name == "" { + return "", 0 + } + return n.Name, n.Score } // diarizeUtteranceJSON mirrors one element of @@ -45,7 +66,8 @@ type diarizeUtteranceJSON struct { // consumed here; the per-word "words" detail belongs to a speaker-attributed // transcript RPC, not Diarize. type transcribeAndDiarizeDoc struct { - Utterances []diarizeUtteranceJSON `json:"utterances"` + Utterances []diarizeUtteranceJSON `json:"utterances"` + Names map[string]speakerNameJSON `json:"names"` } // speakerLabel renders a 0-based speaker index as the decimal string @@ -115,7 +137,16 @@ func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error wantText := req.GetIncludeText() && p.ctxPtr != 0 && CppTranscribeAndDiarizeJSON != nil - raw, err := p.diarizeCall(pcm, wantText) + var reg uintptr + if len(req.GetKnownVoices()) > 0 && p.spkCtx != 0 { + reg, err = p.buildSpeakerRegistry(req.GetKnownVoices()) + if err != nil { + return pb.DiarizeResponse{}, err + } + defer p.freeSpeakerRegistry(reg) + } + + raw, err := p.diarizeCall(pcm, wantText, reg) if err != nil { return pb.DiarizeResponse{}, err } @@ -142,22 +173,37 @@ func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error // those fields under the same engineMu) would otherwise reach the C side with // a freed context. last_error is ctx-shared, so it is read under the same // lock as the failing call. -func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool) (string, error) { +func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool, reg uintptr) (string, error) { p.engineMu.Lock() defer p.engineMu.Unlock() - if p.diarCtx == 0 || (wantText && p.ctxPtr == 0) { + if p.diarCtx == 0 || (wantText && p.ctxPtr == 0) || (reg != 0 && p.spkCtx == 0) { return "", grpcerrors.ModelNotLoaded("parakeet-cpp") } var cstr uintptr - if wantText { + switch { + case reg != 0 && wantText: + if CppTranscribeAndDiarizeNamedJSON == nil { + return "", status.Error(codes.Unimplemented, + "parakeet-cpp: naming speakers needs libparakeet.so ABI 10 (parakeet_capi_transcribe_and_diarize_named_json)") + } + // This C function takes no threshold or margin, so the text path uses the C side's + // defaults rather than speaker_threshold / speaker_margin. + cstr = CppTranscribeAndDiarizeNamedJSON(p.ctxPtr, p.diarCtx, p.spkCtx, reg, &pcm[0], int32(len(pcm)), 16000) + case reg != 0: + if CppDiarizeNamedPCMJSON == nil { + return "", status.Error(codes.Unimplemented, + "parakeet-cpp: naming speakers needs libparakeet.so ABI 10 (parakeet_capi_diarize_named_pcm_json)") + } + cstr = CppDiarizeNamedPCMJSON(p.diarCtx, p.spkCtx, reg, &pcm[0], int32(len(pcm)), 16000, p.speakerAccept, p.speakerMargin) + case wantText: cstr = CppTranscribeAndDiarizeJSON(p.ctxPtr, p.diarCtx, &pcm[0], int32(len(pcm)), 16000) - } else { + default: cstr = CppDiarizePCM(p.diarCtx, &pcm[0], int32(len(pcm)), 16000) } if cstr == 0 { - return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", diarizeLastError(p, wantText)) + return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", diarizeLastError(p, wantText, reg != 0)) } raw := goStringFromCPtr(cstr) CppFreeString(cstr) @@ -169,7 +215,7 @@ func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool) (string, error) // and either side of the pairing may be the one that set it — then joins // whichever came back non-empty. Called under the same engineMu as the // failing call (last_error is ctx-shared state). -func diarizeLastError(p *ParakeetCpp, wantText bool) string { +func diarizeLastError(p *ParakeetCpp, wantText, named bool) string { var msgs []string if m := CppLastError(p.diarCtx); m != "" { msgs = append(msgs, m) @@ -179,6 +225,11 @@ func diarizeLastError(p *ParakeetCpp, wantText bool) string { msgs = append(msgs, m) } } + if named { + if m := CppLastError(p.spkCtx); m != "" { + msgs = append(msgs, m) + } + } if len(msgs) == 0 { return "unknown error" } @@ -196,11 +247,14 @@ func parseDiarizeDoc(raw string, wantText bool) ([]*pb.DiarizeSegment, error) { } segs := make([]*pb.DiarizeSegment, 0, len(doc.Utterances)) for _, u := range doc.Utterances { + name, score := nameFor(doc.Names, u.Speaker) segs = append(segs, &pb.DiarizeSegment{ - Start: float32(u.Start), - End: float32(u.End), - Speaker: speakerLabel(u.Speaker), - Text: u.Text, + Start: float32(u.Start), + End: float32(u.End), + Speaker: speakerLabel(u.Speaker), + Text: u.Text, + Name: name, + NameScore: score, }) } return segs, nil @@ -212,10 +266,13 @@ func parseDiarizeDoc(raw string, wantText bool) ([]*pb.DiarizeSegment, error) { } segs := make([]*pb.DiarizeSegment, 0, len(doc.Segments)) for _, s := range doc.Segments { + name, score := nameFor(doc.Names, s.Speaker) segs = append(segs, &pb.DiarizeSegment{ - Start: float32(s.Start), - End: float32(s.End), - Speaker: speakerLabel(s.Speaker), + Start: float32(s.Start), + End: float32(s.End), + Speaker: speakerLabel(s.Speaker), + Name: name, + NameScore: score, }) } return segs, nil diff --git a/backend/go/parakeet-cpp/diarize_test.go b/backend/go/parakeet-cpp/diarize_test.go index 1db076771d62..9bced8442b69 100644 --- a/backend/go/parakeet-cpp/diarize_test.go +++ b/backend/go/parakeet-cpp/diarize_test.go @@ -40,7 +40,21 @@ func diarizeStubs() (restore func()) { savedTranscribeAndDiarize := CppTranscribeAndDiarizeJSON savedFreeString := CppFreeString savedLastError := CppLastError + savedNamedDiarize := CppDiarizeNamedPCMJSON + savedNamedText := CppTranscribeAndDiarizeNamedJSON + savedRegNew := CppSpeakerRegistryNew + savedRegFree := CppSpeakerRegistryFree + savedRegAdd := CppSpeakerRegistryAddEmbedding + savedRegLastError := CppSpeakerRegistryLastError + savedSpeakerDim := CppSpeakerDim return func() { + CppDiarizeNamedPCMJSON = savedNamedDiarize + CppTranscribeAndDiarizeNamedJSON = savedNamedText + CppSpeakerRegistryNew = savedRegNew + CppSpeakerRegistryFree = savedRegFree + CppSpeakerRegistryAddEmbedding = savedRegAdd + CppSpeakerRegistryLastError = savedRegLastError + CppSpeakerDim = savedSpeakerDim CppDiarizePCM = savedDiarize CppTranscribeAndDiarizeJSON = savedTranscribeAndDiarize CppFreeString = savedFreeString @@ -245,7 +259,7 @@ var _ = Describe("ParakeetCpp.Diarize", func() { // Simulate a Free() racing between Diarize's own diarCtx==0 check and // diarizeCall's lock, exactly as it zeroes diarCtx under engineMu. p.diarCtx = 0 - _, err := p.diarizeCall(make([]float32, 10), false) + _, err := p.diarizeCall(make([]float32, 10), false, 0) Expect(grpcerrors.IsModelNotLoaded(err)).To(BeTrue()) Expect(called).To(BeFalse(), "no C call once diarCtx was cleared") }) @@ -272,4 +286,134 @@ var _ = Describe("ParakeetCpp.Diarize", func() { Expect(resp.Segments[1].Start).To(BeNumerically("~", 1.05, 0.001)) Expect(resp.Segments[1].End).To(BeNumerically("~", 1.20, 0.001)) }) + Describe("with known voices", func() { + var freed []uintptr + var used string + ada := []*pb.KnownVoice{{Name: "Ada", Embedding: []float32{1, 0}}} + BeforeEach(func() { + freed, used = nil, "" + CppFreeString = func(uintptr) {} + CppSpeakerDim = func(uintptr) int32 { return 2 } + CppSpeakerRegistryNew = func() uintptr { return 9 } + CppSpeakerRegistryFree = func(r uintptr) { freed = append(freed, r) } + CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 0 } + CppDiarizePCM = func(uintptr, *float32, int32, int32) uintptr { + used = "plain" + return pool.cstr(`{"speakers":8,"segments":[{"speaker":0,"start":0.5,"end":2}]}`) + } + CppDiarizeNamedPCMJSON = func(diar, spk, reg uintptr, s *float32, n, sr int32, accept, margin float32) uintptr { + used = "named" + Expect(reg).To(Equal(uintptr(9))) + Expect(accept).To(BeNumerically("~", 0.7, 1e-6)) + Expect(margin).To(BeNumerically("~", 0.05, 1e-6)) + return pool.cstr(`{"speakers":8,"segments":[{"speaker":0,"start":0.5,"end":2.0},{"speaker":1,"start":2.5,"end":4.0}],` + + `"names":{"0":{"name":"Ada","score":0.93},"1":{"name":"","score":0.2}}}`) + } + }) + + It("puts the registered names on the segments and frees the registry", func() { + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, speakerAccept: 0.7, speakerMargin: 0.05} + res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada}) + Expect(err).ToNot(HaveOccurred()) + Expect(used).To(Equal("named")) + Expect(res.Segments).To(HaveLen(2)) + Expect(res.Segments[0].Name).To(Equal("Ada")) + Expect(res.Segments[0].NameScore).To(BeNumerically("~", 0.93, 1e-6)) + Expect(res.Segments[1].Name).To(BeEmpty()) + Expect(freed).To(Equal([]uintptr{9})) + }) + It("uses the plain path when the request has no known voices", func() { + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} + res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5)}) + Expect(err).ToNot(HaveOccurred()) + Expect(used).To(Equal("plain")) + Expect(res.Segments[0].Name).To(BeEmpty()) + Expect(freed).To(BeEmpty()) + }) + It("uses the plain path when no speaker model is loaded, even with known voices", func() { + p := &ParakeetCpp{diarCtx: 1} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada}) + Expect(err).ToNot(HaveOccurred()) + Expect(used).To(Equal("plain")) + }) + It("uses the plain path, without a registry, when no voice has an embedding", func() { + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), + KnownVoices: []*pb.KnownVoice{{Name: "Ada"}, {Embedding: []float32{1, 0}}}}) + Expect(err).ToNot(HaveOccurred()) + Expect(used).To(Equal("plain")) + Expect(freed).To(Equal([]uintptr{9})) // the empty registry built for it is released, once + }) + It("fails clearly on a voice of the wrong size and frees the registry", func() { + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), + KnownVoices: []*pb.KnownVoice{{Name: "Ada", Embedding: []float32{1, 0, 0}}}}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + Expect(err.Error()).To(ContainSubstring("Ada")) + Expect(freed).To(Equal([]uintptr{9})) + }) + It("reports a missing v10 symbol instead of silently dropping the names", func() { + CppDiarizeNamedPCMJSON = nil + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + Expect(freed).To(Equal([]uintptr{9})) + }) + It("reports a missing named transcribe symbol on the include_text path", func() { + CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, s *float32, n, sr int32) uintptr { return 0 } + CppTranscribeAndDiarizeNamedJSON = nil + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, ctxPtr: 3} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), IncludeText: true, KnownVoices: ada}) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + Expect(freed).To(Equal([]uintptr{9})) + }) + It("names utterances on the include_text path", func() { + // wantText also requires the plain text symbol, present in any library that has the named one. + CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, s *float32, n, sr int32) uintptr { return 0 } + CppTranscribeAndDiarizeNamedJSON = func(asr, diar, spk, reg uintptr, s *float32, n, sr int32) uintptr { + used = "named-text" + return pool.cstr(`{"speakers":8,"names":{"0":{"name":"Ada","score":0.9}},"utterances":[{"speaker":0,"name":"Ada","text":"hello","start":0.5,"end":2.0,"conf":0.9},{"speaker":-1,"text":"hm","start":2.5,"end":3.0,"conf":0.5}],"words":[]}`) + } + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, ctxPtr: 3} + res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), IncludeText: true, KnownVoices: ada}) + Expect(err).ToNot(HaveOccurred()) + Expect(used).To(Equal("named-text")) + Expect(res.Segments[0].Name).To(Equal("Ada")) + Expect(res.Segments[0].Text).To(Equal("hello")) + Expect(res.Segments[1].Name).To(BeEmpty()) // speaker -1 has no name + Expect(freed).To(Equal([]uintptr{9})) + }) + It("keeps the name when close segments of one speaker are merged", func() { + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, speakerAccept: 0.7, speakerMargin: 0.05} + CppDiarizeNamedPCMJSON = func(diar, spk, reg uintptr, s *float32, n, sr int32, a, m float32) uintptr { + return pool.cstr(`{"speakers":8,"segments":[{"speaker":0,"start":0.5,"end":2.0},{"speaker":0,"start":2.1,"end":3.0}],"names":{"0":{"name":"Ada","score":0.9}}}`) + } + res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), MinDurationOff: 0.5, KnownVoices: ada}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Segments).To(HaveLen(1)) + Expect(res.Segments[0].Name).To(Equal("Ada")) + Expect(res.Segments[0].NameScore).To(BeNumerically("~", 0.9, 1e-6)) + }) + It("includes the speaker context message when the named call fails", func() { + CppDiarizeNamedPCMJSON = func(diar, spk, reg uintptr, s *float32, n, sr int32, a, m float32) uintptr { return 0 } + CppLastError = func(ctx uintptr) string { + switch ctx { + case 1: + return "diar side broke" + case 2: + return "speaker side broke" + } + return "" + } + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("diar side broke")) + Expect(err.Error()).To(ContainSubstring("speaker side broke")) + Expect(freed).To(Equal([]uintptr{9})) + }) + }) }) diff --git a/backend/go/parakeet-cpp/speakers.go b/backend/go/parakeet-cpp/speakers.go index ddd2c4292b6e..31e9111cef77 100644 --- a/backend/go/parakeet-cpp/speakers.go +++ b/backend/go/parakeet-cpp/speakers.go @@ -42,7 +42,7 @@ func (p *ParakeetCpp) diarizeSegmentsPCM(pcm []float32) ([]diarizeSegmentJSON, e if len(pcm) == 0 { return nil, nil } - raw, err := p.diarizeCall(pcm, false) + raw, err := p.diarizeCall(pcm, false, 0) if err != nil { return nil, err } From 28c8bdfbf22c9e1f7e5bcb952815c149a06f1df5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:39:01 +0000 Subject: [PATCH 09/13] feat(parakeet-cpp): name speakers in the live scene stream The live scene stream now begins with a known-voice registry when a speaker model is loaded and the live config carries voices, and each closed speaker segment takes its slot's current name from the feed's names map. A segment that closes before its slot is identified has an empty name. The registry is freed after the stream, on every path. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- backend/go/parakeet-cpp/live.go | 6 +- backend/go/parakeet-cpp/live_test.go | 144 +++++++++++++++++++++++++- backend/go/parakeet-cpp/scene.go | 49 ++++++++- backend/go/parakeet-cpp/scene_test.go | 107 ++++++++++++++++++- 4 files changed, 298 insertions(+), 8 deletions(-) diff --git a/backend/go/parakeet-cpp/live.go b/backend/go/parakeet-cpp/live.go index 6497779d54ee..782c3e8a78a5 100644 --- a/backend/go/parakeet-cpp/live.go +++ b/backend/go/parakeet-cpp/live.go @@ -81,7 +81,7 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest // scene error. var scene sceneStreamHandle if p.sceneWanted() { - scene = p.sceneBegin() + scene = p.sceneBegin(cfg.GetKnownVoices()) if scene.s == 0 { xlog.Warn("parakeet-cpp: scene stream begin failed; live continues without speaker/sound events") } @@ -118,7 +118,7 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest if r.Delta != "" { full.WriteString(r.Delta) } - speakers := liveSpeakersToProto(sceneDoc.Speakers) + speakers := liveSpeakersToProto(sceneDoc.Speakers, sceneDoc.Names) sounds := liveSoundsToProto(sceneDoc.Sounds) if r.Delta != "" || r.Eou || r.Eob || len(r.Words) > 0 || len(speakers) > 0 || len(sounds) > 0 { out <- &pb.TranscriptLiveResponse{ @@ -154,7 +154,7 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest p.sceneFree(scene) scene = sceneStreamHandle{} if p.sceneWanted() { - scene = p.sceneBegin() + scene = p.sceneBegin(payload.Config.GetKnownVoices()) if scene.s == 0 { xlog.Warn("parakeet-cpp: scene stream begin failed; live continues without speaker/sound events") } diff --git a/backend/go/parakeet-cpp/live_test.go b/backend/go/parakeet-cpp/live_test.go index 11dc496b62d4..4ee4ee61ecd0 100644 --- a/backend/go/parakeet-cpp/live_test.go +++ b/backend/go/parakeet-cpp/live_test.go @@ -47,7 +47,13 @@ func liveStubs() (restore func()) { savedSceneFeedJSON := CppSceneStreamFeedJSON savedSceneLastError := CppSceneStreamLastError savedSceneFree := CppSceneStreamFree + savedSceneBeginSpk := CppSceneStreamBeginSpeaker + savedRegNew, savedRegFree := CppSpeakerRegistryNew, CppSpeakerRegistryFree + savedRegAdd, savedSpkDim := CppSpeakerRegistryAddEmbedding, CppSpeakerDim return func() { + CppSceneStreamBeginSpeaker = savedSceneBeginSpk + CppSpeakerRegistryNew, CppSpeakerRegistryFree = savedRegNew, savedRegFree + CppSpeakerRegistryAddEmbedding, CppSpeakerDim = savedRegAdd, savedSpkDim CppStreamBegin, CppStreamBeginLang = savedBegin, savedBeginLang CppStreamFeed, CppStreamFeedJSON = savedFeed, savedFeedJSON CppStreamFinalize, CppStreamFinalizeJSON = savedFinalize, savedFinalizeJSON @@ -600,7 +606,7 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { return pool.cstr(`{"speakers":[],"sounds":[]}`) } - h := p.sceneBegin() + h := p.sceneBegin(nil) Expect(h.s).NotTo(BeZero()) // Simulate a Free() racing in between the begin and the next feed: it @@ -672,6 +678,142 @@ var _ = Describe("AudioTranscriptionLive scene events (stubbed C API)", func() { }) }) +var _ = Describe("AudioTranscriptionLive named speakers (stubbed C API)", func() { + var ( + pool *liveCstrPool + restore func() + p *ParakeetCpp + regs []uintptr + plain int + spkBeg int + gotReg uintptr + order []string + ) + + liveVoicesConfig := func(voices ...*pb.KnownVoice) *pb.TranscriptLiveRequest { + return &pb.TranscriptLiveRequest{ + Payload: &pb.TranscriptLiveRequest_Config{Config: &pb.TranscriptLiveConfig{KnownVoices: voices}}, + } + } + ada := &pb.KnownVoice{Name: "Ada", Embedding: []float32{1, 0}} + + BeforeEach(func() { + pool = &liveCstrPool{} + restore = liveStubs() + p = &ParakeetCpp{ctxPtr: 1, diarCtx: 2, spkCtx: 3, speakerAccept: 0.7, speakerMargin: 0.05} + regs, plain, spkBeg, gotReg, order = nil, 0, 0, 0, nil + + CppStreamBeginLang = nil + CppStreamBegin = func(ctx uintptr) uintptr { return 7 } + CppStreamFree = func(s uintptr) {} + CppFreeString = func(s uintptr) {} + CppLastError = func(ctx uintptr) string { return "stub error" } + CppStreamFeed = nil + CppStreamFeedJSON = func(s uintptr, pcm []float32, n int32) uintptr { + return pool.cstr(`{"text":"","eou":0,"frame_sec":0.08,"words":[]}`) + } + CppStreamFinalize = nil + CppStreamFinalizeJSON = func(s uintptr) uintptr { + return pool.cstr(`{"text":"","eou":0,"frame_sec":0.08,"words":[]}`) + } + + CppSceneOptsDefault = func(o *cSceneOpts) { *o = cSceneOpts{} } + CppSceneStreamBegin = func(asr, diar, tagger uintptr, o *cSceneOpts) uintptr { plain++; return 100 } + CppSceneStreamBeginSpeaker = func(asr, diar, tag, spk, reg uintptr, o *cSceneOpts) uintptr { + spkBeg++ + gotReg = reg + return 200 + } + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + return pool.cstr(`{"speakers":[{"speaker":0,"start":0.1,"end":0.6}],"sounds":[],` + + `"names":{"0":{"name":"Ada","score":0.9}}}`) + } + CppSceneStreamLastError = func(s uintptr) string { return "" } + CppSceneStreamFree = func(s uintptr) { order = append(order, "stream") } + CppSpeakerDim = func(uintptr) int32 { return 2 } + CppSpeakerRegistryNew = func() uintptr { return 9 } + CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 0 } + CppSpeakerRegistryFree = func(r uintptr) { regs = append(regs, r); order = append(order, "registry") } + }) + + AfterEach(func() { restore() }) + + It("begins a speaker scene stream and emits the slot name on the closed segment", func() { + in, out, errCh := runLive(p) + in <- liveVoicesConfig(ada) + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(spkBeg).To(Equal(1)) + Expect(plain).To(Equal(0)) + Expect(gotReg).To(Equal(uintptr(9))) + var named *pb.LiveSpeakerSegment + for _, r := range got { + if len(r.Speakers) > 0 { + named = r.Speakers[0] + } + } + Expect(named).NotTo(BeNil()) + Expect(named.Speaker).To(Equal("0")) + Expect(named.Name).To(Equal("Ada")) + Expect(regs).To(Equal([]uintptr{9}), "the registry is freed once when the session ends") + Expect(order).To(Equal([]string{"stream", "registry"})) + }) + + It("uses the plain scene begin and emits empty names without known voices", func() { + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { + return pool.cstr(`{"speakers":[{"speaker":0,"start":0.1,"end":0.6}],"sounds":[]}`) + } + in, out, errCh := runLive(p) + in <- liveConfig("") + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + + got := collectLive(out) + Expect(plain).To(Equal(1)) + Expect(spkBeg).To(Equal(0)) + Expect(regs).To(BeEmpty()) + found := false + for _, r := range got { + for _, s := range r.Speakers { + found = true + Expect(s.Name).To(BeEmpty()) + } + } + Expect(found).To(BeTrue()) + }) + + It("names the restarted scene stream after a config reset and frees every registry once", func() { + in, out, errCh := runLive(p) + in <- liveVoicesConfig(ada) + in <- liveAudio(make([]float32, 10)) + in <- liveVoicesConfig(ada) // reset + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + collectLive(out) + + Expect(spkBeg).To(Equal(2)) + Expect(regs).To(HaveLen(2), "one registry per scene stream, each freed exactly once") + }) + + It("frees the registry once when a feed failure disables the scene stream", func() { + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n int32, isLast int32) uintptr { return 0 } + in, out, errCh := runLive(p) + in <- liveVoicesConfig(ada) + in <- liveAudio(make([]float32, 10)) + in <- liveAudio(make([]float32, 10)) + close(in) + Expect(<-errCh).NotTo(HaveOccurred()) + collectLive(out) + + Expect(regs).To(Equal([]uintptr{9})) + }) +}) + var _ = Describe("stripEouMarker", func() { It("strips a trailing and reports it", func() { text, eou := stripEouMarker("it is certainly very like the old portrait") diff --git a/backend/go/parakeet-cpp/scene.go b/backend/go/parakeet-cpp/scene.go index 590740f13a05..e104e724ac0a 100644 --- a/backend/go/parakeet-cpp/scene.go +++ b/backend/go/parakeet-cpp/scene.go @@ -39,6 +39,11 @@ type sceneSoundJSON struct { type sceneFeedJSON struct { Speakers []sceneSpeakerJSON `json:"speakers"` Sounds []sceneSoundJSON `json:"sounds"` + // Names is the CURRENT name of each speaker slot (keyed by the slot + // index as a string) at the time of this feed. Each closed segment takes + // its slot's current name, so a segment that closes before its slot is + // identified carries an empty name. Absent without a speaker model. + Names map[string]speakerNameJSON `json:"names"` } // sceneWanted reports whether AudioTranscriptionLive should run a companion @@ -62,10 +67,16 @@ func (p *ParakeetCpp) sceneWanted() bool { // caught instead of handed to the C side — mirroring streamFeedDoc's re-check // of p.ctxPtr (see the "Per-C-call engine serialization" comment in // goparakeetcpp.go). The zero value (s == 0) means "no scene stream". +// +// spk and reg are the speaker model and the known-voice registry of a +// speaker-named stream (0 for a plain one). The stream borrows both: sceneFree +// frees the stream first and then the registry, which this handle owns. type sceneStreamHandle struct { s uintptr diar uintptr tag uintptr + spk uintptr + reg uintptr } // sceneBegin opens a no-ASR scene stream (diarization and/or sound events @@ -74,7 +85,7 @@ type sceneStreamHandle struct { // contexts 0 (defensive: sceneWanted() already guards this). A zero handle // means the C call itself failed; the caller logs a warning and continues // the live session without speaker/sound events. -func (p *ParakeetCpp) sceneBegin() sceneStreamHandle { +func (p *ParakeetCpp) sceneBegin(voices []*pb.KnownVoice) sceneStreamHandle { p.engineMu.Lock() defer p.engineMu.Unlock() diar, tag := p.diarCtx, p.tagCtx @@ -90,6 +101,29 @@ func (p *ParakeetCpp) sceneBegin() sceneStreamHandle { // whole lifetime. 0 disables per-class score retention; sound EVENTS // (onset/offset, what the live path actually consumes) are unaffected. opts.Sound.TopK = 0 + + // With a speaker model and at least one usable known voice the stream + // names speakers through a registry it borrows. engineMu is already + // held, so use the Locked builder. Any failure keeps the plain stream. + var reg uintptr + if diar != 0 && p.spkCtx != 0 && CppSceneStreamBeginSpeaker != nil { + r, err := p.buildSpeakerRegistryLocked(voices) + if err != nil { + xlog.Warn("parakeet-cpp: could not build the speaker registry for a live session; speakers stay unnamed", "err", err) + } else { + reg = r + } + } + if reg != 0 { + opts.SpeakerAcceptThreshold = p.speakerAccept + opts.SpeakerMargin = p.speakerMargin + s := CppSceneStreamBeginSpeaker(0, diar, tag, p.spkCtx, reg, &opts) + if s == 0 { + p.freeSpeakerRegistry(reg) + return sceneStreamHandle{} + } + return sceneStreamHandle{s: s, diar: diar, tag: tag, spk: p.spkCtx, reg: reg} + } s := CppSceneStreamBegin(0, diar, tag, &opts) if s == 0 { return sceneStreamHandle{} @@ -111,6 +145,8 @@ func (p *ParakeetCpp) sceneFree(h sceneStreamHandle) { p.engineMu.Lock() defer p.engineMu.Unlock() CppSceneStreamFree(h.s) + // The stream borrowed the registry: free it only after the stream. + p.freeSpeakerRegistry(h.reg) } // sceneFeed runs one scene-stream feed (or the is_last flush) under @@ -126,7 +162,9 @@ func (p *ParakeetCpp) sceneFeed(h sceneStreamHandle, pcm []float32, isLast bool) p.engineMu.Lock() defer p.engineMu.Unlock() - if p.diarCtx != h.diar || p.tagCtx != h.tag { + if p.diarCtx != h.diar || p.tagCtx != h.tag || (h.spk != 0 && p.spkCtx != h.spk) { + // A plain stream (h.spk == 0) never borrows the speaker model, so a + // speaker model loaded or freed meanwhile does not concern it. return sceneFeedJSON{}, grpcerrors.ModelNotLoaded("parakeet-cpp") } @@ -223,14 +261,19 @@ func (p *ParakeetCpp) feedSlicesScene(ctx context.Context, stream uintptr, scene // TranscriptLiveResponse.speakers (stream-relative nanoseconds). Reuses // diarize.go's speakerLabel so the live path renders speaker indices the // same way the offline Diarize RPC does. -func liveSpeakersToProto(speakers []sceneSpeakerJSON) []*pb.LiveSpeakerSegment { +// +// names is the feed document's "names" map; each segment takes its slot's +// current name, empty if the slot was not yet identified when it closed. +func liveSpeakersToProto(speakers []sceneSpeakerJSON, names map[string]speakerNameJSON) []*pb.LiveSpeakerSegment { if len(speakers) == 0 { return nil } out := make([]*pb.LiveSpeakerSegment, len(speakers)) for i, s := range speakers { + name, _ := nameFor(names, s.Speaker) out[i] = &pb.LiveSpeakerSegment{ Speaker: speakerLabel(s.Speaker), + Name: name, Start: secondsToNanos(s.Start), End: secondsToNanos(s.End), } diff --git a/backend/go/parakeet-cpp/scene_test.go b/backend/go/parakeet-cpp/scene_test.go index 9dbfa22055a2..76a87a9be03c 100644 --- a/backend/go/parakeet-cpp/scene_test.go +++ b/backend/go/parakeet-cpp/scene_test.go @@ -1,6 +1,9 @@ package main import ( + "unsafe" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -31,7 +34,7 @@ var _ = Describe("ParakeetCpp.sceneBegin", func() { } p := &ParakeetCpp{diarCtx: 42, diarLatency: diarLatencyVeryLow} - h := p.sceneBegin() + h := p.sceneBegin(nil) Expect(h.s).ToNot(BeZero()) Expect(gotOpts.Sound.TopK).To(Equal(int32(0)), "the live scene path never drains sound scores (see sound.go's SoundDetection, "+ @@ -40,3 +43,105 @@ var _ = Describe("ParakeetCpp.sceneBegin", func() { Expect(gotOpts.DiarLatency).To(Equal(diarLatencyVeryLow)) }) }) + +var _ = Describe("scene stream with speaker names", func() { + var ( + restore func() + pool *liveCstrPool + freedRegs []uintptr + ) + BeforeEach(func() { + sOpts, sBegin, sBeginSpk, sFeed, sFree := CppSceneOptsDefault, CppSceneStreamBegin, CppSceneStreamBeginSpeaker, CppSceneStreamFeedJSON, CppSceneStreamFree + sNew, sRegFree, sAdd, sDim, sStr := CppSpeakerRegistryNew, CppSpeakerRegistryFree, CppSpeakerRegistryAddEmbedding, CppSpeakerDim, CppFreeString + restore = func() { + CppSceneOptsDefault, CppSceneStreamBegin, CppSceneStreamBeginSpeaker, CppSceneStreamFeedJSON, CppSceneStreamFree = sOpts, sBegin, sBeginSpk, sFeed, sFree + CppSpeakerRegistryNew, CppSpeakerRegistryFree, CppSpeakerRegistryAddEmbedding, CppSpeakerDim, CppFreeString = sNew, sRegFree, sAdd, sDim, sStr + } + pool = &liveCstrPool{} + freedRegs = nil + CppSceneOptsDefault = func(o *cSceneOpts) { o.Size = int32(unsafe.Sizeof(*o)) } + CppSpeakerDim = func(uintptr) int32 { return 2 } + CppSpeakerRegistryNew = func() uintptr { return 9 } + CppSpeakerRegistryFree = func(r uintptr) { freedRegs = append(freedRegs, r) } + CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 0 } + CppFreeString = func(uintptr) {} + }) + AfterEach(func() { restore() }) + + It("begins a speaker scene stream with the threshold and margin when voices are given", func() { + var gotOpts cSceneOpts + var gotReg, gotSpk uintptr + CppSceneStreamBeginSpeaker = func(asr, diar, tag, spk, reg uintptr, o *cSceneOpts) uintptr { + gotOpts, gotReg, gotSpk = *o, reg, spk + return 55 + } + CppSceneStreamBegin = func(asr, diar, tag uintptr, o *cSceneOpts) uintptr { + Fail("plain begin must not be used") + return 0 + } + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, speakerAccept: 0.7, speakerMargin: 0.05} + h := p.sceneBegin([]*pb.KnownVoice{{Name: "Ada", Embedding: []float32{1, 0}}}) + Expect(h.s).To(Equal(uintptr(55))) + Expect(h.reg).To(Equal(uintptr(9))) + Expect(gotReg).To(Equal(uintptr(9))) + Expect(gotSpk).To(Equal(uintptr(2))) + Expect(gotOpts.SpeakerAcceptThreshold).To(BeNumerically("~", 0.7, 1e-6)) + Expect(gotOpts.SpeakerMargin).To(BeNumerically("~", 0.05, 1e-6)) + }) + + It("uses the plain scene begin without voices or without a speaker model", func() { + plain := 0 + CppSceneStreamBegin = func(asr, diar, tag uintptr, o *cSceneOpts) uintptr { plain++; return 56 } + CppSceneStreamBeginSpeaker = func(asr, diar, tag, spk, reg uintptr, o *cSceneOpts) uintptr { + Fail("speaker begin must not be used") + return 0 + } + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} + Expect(p.sceneBegin(nil).s).To(Equal(uintptr(56))) + p2 := &ParakeetCpp{diarCtx: 1} + Expect(p2.sceneBegin([]*pb.KnownVoice{{Name: "Ada", Embedding: []float32{1, 0}}}).s).To(Equal(uintptr(56))) + Expect(plain).To(Equal(2)) + }) + + It("frees the registry when the speaker begin fails, and degrades to no scene stream", func() { + CppSceneStreamBeginSpeaker = func(asr, diar, tag, spk, reg uintptr, o *cSceneOpts) uintptr { return 0 } + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} + h := p.sceneBegin([]*pb.KnownVoice{{Name: "Ada", Embedding: []float32{1, 0}}}) + Expect(h.s).To(Equal(uintptr(0))) + Expect(freedRegs).To(Equal([]uintptr{9})) + }) + + It("frees the stream before its registry", func() { + var order []string + CppSceneStreamFree = func(uintptr) { order = append(order, "stream") } + CppSpeakerRegistryFree = func(uintptr) { order = append(order, "registry") } + (&ParakeetCpp{}).sceneFree(sceneStreamHandle{s: 55, reg: 9}) + Expect(order).To(Equal([]string{"stream", "registry"})) + }) + + It("turns the names of the feed document into live speaker segment names", func() { + segs := liveSpeakersToProto( + []sceneSpeakerJSON{{Speaker: 0, Start: 0, End: 1}, {Speaker: 1, Start: 1, End: 2}}, + map[string]speakerNameJSON{"0": {Name: "Ada", Score: 0.9}, "1": {Name: ""}}) + Expect(segs[0].Name).To(Equal("Ada")) + Expect(segs[0].Speaker).To(Equal("0")) + Expect(segs[1].Name).To(BeEmpty()) + Expect(liveSpeakersToProto([]sceneSpeakerJSON{{Speaker: 0}}, nil)[0].Name).To(BeEmpty()) + }) + + It("decodes the names map of a scene feed", func() { + CppSceneStreamFeedJSON = func(s uintptr, pcm *float32, n, last int32) uintptr { + return pool.cstr(`{"speakers":[{"speaker":0,"start":0.0,"end":1.0}],"sounds":[],"names":{"0":{"name":"Ada","score":0.9}}}`) + } + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} + doc, err := p.sceneFeed(sceneStreamHandle{s: 55, diar: 1, spk: 2}, []float32{0}, false) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Names["0"].Name).To(Equal("Ada")) + }) + + It("refuses to feed a stream whose speaker model was freed", func() { + p := &ParakeetCpp{diarCtx: 1} + _, err := p.sceneFeed(sceneStreamHandle{s: 55, diar: 1, spk: 2}, []float32{0}, false) + Expect(err).To(HaveOccurred()) + }) +}) From 8d4d960561ab1c27b845282324345510f97694bc Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:42:43 +0000 Subject: [PATCH 10/13] feat(gallery): speaker naming entries and docs for parakeet-cpp Add three gallery entries that load the WeSpeaker ResNet34 speaker model next to the diarization or realtime scene models, and document speaker names in the voice recognition, diarization, audio to text and realtime pages. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- docs/content/features/audio-diarization.md | 20 +++ docs/content/features/audio-to-text.md | 5 + docs/content/features/openai-realtime.md | 3 + docs/content/features/voice-recognition.md | 72 +++++++++ gallery/index.yaml | 161 +++++++++++++++++++++ 5 files changed, 261 insertions(+) diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md index 37f8c81596b8..e052f0b655b0 100644 --- a/docs/content/features/audio-diarization.md +++ b/docs/content/features/audio-diarization.md @@ -79,6 +79,26 @@ Adds per-speaker totals and (when the backend supports it and `include_text=true } ``` +### Speaker names + +With a parakeet-cpp model that has a `speaker_model:` and voices registered through `/v1/voice/register`, segments whose speaker matches a registered voice gain `name` and `name_score` (the cosine similarity of the match), and the matching `speakers` entry gains `name`. Both fields are omitted for a speaker that was not identified, so an unnamed response looks exactly as before. `speaker` stays `SPEAKER_NN`, and RTTM output still uses `SPEAKER_NN`. See [Voice Recognition]({{% relref "voice-recognition" %}}#naming-speakers-in-diarization-and-live-transcription) for the setup and the limits. + +```json +{ + "task": "diarize", + "duration": 12.34, + "num_speakers": 2, + "segments": [ + {"id": 0, "speaker": "SPEAKER_00", "label": "0", "start": 0.00, "end": 2.34, "text": "Hello, world.", "name": "Alice", "name_score": 0.82}, + {"id": 1, "speaker": "SPEAKER_01", "label": "1", "start": 2.34, "end": 4.10, "text": "How are you?"} + ], + "speakers": [ + {"id": "SPEAKER_00", "label": "0", "name": "Alice", "total_speech_duration": 5.6, "segment_count": 3}, + {"id": "SPEAKER_01", "label": "1", "total_speech_duration": 1.76, "segment_count": 1} + ] +} +``` + ### Response - `rttm` NIST RTTM, the standard interchange format used by `pyannote.metrics` / `dscore`: diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index d01edec4e391..0a4b6ee654e5 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -200,9 +200,14 @@ The same backend also serves the `/v1/audio/diarization` and `/v1/audio/classifi | `diarization_model:` | an ASR model | a `speaker` on transcript segments (and words), and speaker segments during realtime live transcription | | `sound_model:` | an ASR model | sound events during realtime live transcription | | `diarization_latency:` | a model with a diarization companion | latency mode for the live speaker stream; default `low` | +| `speaker_model:` | a model with a diarization model | names registered speakers (see [Voice Recognition]({{% relref "voice-recognition" %}}#naming-speakers-in-diarization-and-live-transcription)) | +| `speaker_threshold:` | a model with `speaker_model` | distance (1 minus cosine similarity) under which a speaker is named, in (0, 2); default `0.5` | +| `speaker_margin:` | a model with `speaker_model` | how much the best match must beat the runner-up, in [0, 1); default `0.05` | With a `diarization_model` companion, `/v1/audio/transcriptions` labels each segment with its `speaker` (`"0"`, `"1"`, ... in order of first appearance) and splits segments where the speaker changes; with `timestamp_granularities[]=word` each word carries its speaker too. With `stream=true` the closing `transcript.text.done` event lists the segments with their speakers. Pass `-F diarize=false` to skip diarization for one request. The diarization GGUF can also be imported directly: `local-ai models import https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-f16.gguf`. +`speaker_model:` needs libparakeet with C-API v10. A wrong setup fails at load time with one of these errors: `parakeet-cpp: speaker_model needs libparakeet.so ABI 10 (parakeet_capi_speaker_registry_add_embedding); the loaded library is older`, `parakeet-cpp: speaker_model needs a diarization model (the primary or diarization_model:)`, `parakeet-cpp: a speaker model cannot be the primary model; use it as speaker_model: next to a diarization model`, `parakeet-cpp: speaker_model "" is a model, expected a speaker model` (the file is not a speaker encoder GGUF), or `parakeet-cpp: speaker_threshold "" must be a distance in (0, 2) (1 minus cosine similarity)` / `parakeet-cpp: speaker_margin "" must be a number in [0, 1)` for a bad number. + The loader rejects a companion whose role duplicates the primary's own (for example `asr_model:` on an already-ASR primary, or `sound_model:` on a CED primary), and rejects a companion GGUF that does not match the role its option names (for example `sound_model:` pointing at an ASR GGUF fails to load, naming the kind it expected). See [Speaker Diarization]({{% relref "audio-diarization" %}}) for the `Diarize` RPC and [Sound Classification]({{% relref "audio-classification" %}}) for `SoundDetection`, and [Realtime API]({{% relref "openai-realtime" %}}) for the live speaker/sound events emitted during a realtime session. ### Segment timestamps diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 150966cc452a..f66f498f2aa5 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -166,12 +166,15 @@ Each closed speaker segment emits a `conversation.item.input_audio_transcription "item_id": "item_abc", "content_index": 0, "speaker": "0", + "speaker_name": "Alice", "start": 1.92, "end": 4.10, "text": "" } ``` +`speaker_name` is the name of a voice registered through `/v1/voice/register`, and is present only when the model has a `speaker_model:` and the speaker was identified. A segment that closes before its speaker is identified has none, and later segments of the same speaker do. See [Voice Recognition]({{% relref "voice-recognition" %}}#naming-speakers-in-diarization-and-live-transcription). The segments of the offline path below carry no `speaker_name`. + Each sound event emits a `conversation.item.sound_detection` event with one tag and the detection window's `start`/`end`: ```json diff --git a/docs/content/features/voice-recognition.md b/docs/content/features/voice-recognition.md index 23bb16bd67e8..f74d9a3888d9 100644 --- a/docs/content/features/voice-recognition.md +++ b/docs/content/features/voice-recognition.md @@ -185,6 +185,78 @@ recognition - the voice-recognition HTTP API is designed to swap the backing store without changing the wire format. {{% /notice %}} +## Naming speakers in diarization and live transcription + +The parakeet-cpp backend can put the names of registered voices on +diarization results and on live transcription speaker segments. Without +this, speakers only carry labels such as `SPEAKER_00`. + +1. Register each voice with the WeSpeaker encoder. Install the model with + `local-ai models install voice-detect-wespeaker-resnet34`, then call + `/v1/voice/register` with `"model": "voice-detect-wespeaker-resnet34"` + (see the [1:N workflow](#1n-identification-workflow-register--identify--forget)). +2. Install one of the gallery models that loads the same encoder: + `parakeet-cpp-nemotron-3-diarization-speakers` (diarization), + `parakeet-cpp-nemotron-3-diarization-asr-speakers` (diarization with + `include_text`) or `parakeet-cpp-realtime-scene-speakers` (live + transcription). Each one adds + `speaker_model:voice-detect-wespeaker-resnet34.gguf` to a + parakeet-cpp model config. +3. Call `/v1/audio/diarization` with that model. Matched segments gain a + `name` and a `name_score`, and the matching entry in `speakers` gains a + `name`. `speaker` stays `SPEAKER_NN`, and RTTM output is unchanged. See + [Speaker Diarization]({{% relref "audio-diarization" %}}) for the + response. + +### Which voices are used + +LocalAI sends the backend only the registered voices made by the same +encoder as the model's `speaker_model:` file. Each registered voice is +tagged with the name of the voice-detect model that made it, which by +default is the GGUF file name (`voice-detect-wespeaker-resnet34.gguf` for the +gallery entry). The tag must equal the base name of the `speaker_model:` +file. Voices made with another encoder are ignored, and LocalAI logs a +warning when that leaves no usable voice. Voices registered before the tag +existed have no tag: they are used when their embedding size matches the +tagged ones (or all of them, when no voice carries a matching tag). + +{{% notice warning %}} +Do not set a `model_name:` option on the voice-detect model config. It +replaces the default name, the voices are then tagged with it, and they no +longer match the `speaker_model:` file. Keep the default name. +{{% /notice %}} + +### Options + +These go in the `options:` list of the parakeet-cpp model config (see +[Audio to Text]({{% relref "audio-to-text" %}}) for the other parakeet-cpp +options). + +| Option | Default | Meaning | +|---|---|---| +| `speaker_model:` | none | speaker encoder GGUF; needs a diarization model (the primary one, or `diarization_model:`) | +| `speaker_threshold:` | `0.5` | largest distance (1 minus cosine similarity, the unit `/v1/voice/identify` reports) at which a speaker is named; must be in (0, 2) | +| `speaker_margin:` | `0.05` | the best match must beat the runner-up by this much, otherwise the speaker stays unnamed; must be in [0, 1) | + +parakeet.cpp's measured starting values for `speaker_threshold` are 0.5 for +WeSpeaker ResNet34 and CAM++, and 0.3 for ECAPA. A lower value names fewer +speakers and makes fewer mistakes. + +### Limits + +- The voice registry is in memory and global. Registered names disappear when + LocalAI restarts, and every user of the instance shares them. +- With `include_text=true` the names use the default threshold and margin: + `speaker_threshold` and `speaker_margin` only apply to diarization without + text. +- In live transcription, a speaker segment that closes before its speaker + is identified has no name. Later segments of that speaker do. +- Overlapping speech is not resolved. +- Accuracy was measured on one fixture (two read-speech voices). Check the + threshold on your own audio. +- The backend needs a libparakeet with C-API v10. With an older library a + model config that sets `speaker_model:` fails to load. + ## API reference ### `POST /v1/voice/verify` (1:1) diff --git a/gallery/index.yaml b/gallery/index.yaml index 104585e0118a..b766f88f5ea4 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -53768,6 +53768,167 @@ - filename: parakeet-cpp/tdt_ctc-110m-f16.gguf uri: huggingface://mudler/parakeet-cpp-gguf/tdt_ctc-110m-f16.gguf sha256: 7f9a6376edde6a74592ace48b2ebdc27a1ac972d0be9dfcc29e668d99381faf1 +- name: parakeet-cpp-nemotron-3-diarization-speakers + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/voice-detect-gguf + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://github.com/mudler/parakeet.cpp + description: | + Nemotron-3-Diarization (Sortformer) with WeSpeaker ResNet34 speaker + identification, for the parakeet-cpp backend. Speakers you register with + /v1/voice/register (using the voice-detect-wespeaker-resnet34 model) come back + by name in /v1/audio/diarization, next to the SPEAKER_NN label. Speakers that + are not registered keep only their SPEAKER_NN label. The diarization model is + OpenMDW-1.1, the speaker model is CC-BY-4.0. Naming was measured on one + two-voice fixture only; check the threshold on your own audio. + license: openmdw-1.1 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - diarization + - speaker-diarization + - speaker-recognition + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - diarization + name: parakeet-cpp-nemotron-3-diarization-speakers + options: + - speaker_model:voice-detect-wespeaker-resnet34.gguf + parameters: + model: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + files: + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: voice-detect-wespeaker-resnet34.gguf + uri: https://huggingface.co/mudler/voice-detect-gguf/resolve/main/wespeaker-resnet34-voxceleb.gguf + sha256: 72040372494eafec299836bc1977cfc13c603cb486674ed59b0f4c03758d29da +- name: parakeet-cpp-nemotron-3-diarization-asr-speakers + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/voice-detect-gguf + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/nvidia/parakeet-tdt_ctc-110m + - https://github.com/mudler/parakeet.cpp + description: | + Nemotron-3-Diarization (Sortformer) paired with the Parakeet TDT+CTC 110M + ASR model through the asr_model option, both Q8_0/F16 GGUF for the + parakeet-cpp backend (C++/ggml port of NVIDIA NeMo). Served through + /v1/audio/diarization with include_text: each speaker segment comes back + with its transcribed text in one call. Diarization model is + OpenMDW-1.1, ASR model is CC-BY-4.0. Also loads WeSpeaker ResNet34 + (CC-BY-4.0) through the speaker_model option: speakers registered with + /v1/voice/register (voice-detect-wespeaker-resnet34 model) come back by + name, next to the SPEAKER_NN label. + license: openmdw-1.1 + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - asr + - diarization + - speaker-diarization + - speech-recognition + - stt + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - diarization + name: parakeet-cpp-nemotron-3-diarization-asr-speakers + options: + - asr_model:parakeet-cpp/tdt_ctc-110m-f16.gguf + - speaker_model:voice-detect-wespeaker-resnet34.gguf + parameters: + model: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + files: + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/tdt_ctc-110m-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/tdt_ctc-110m-f16.gguf + sha256: 7f9a6376edde6a74592ace48b2ebdc27a1ac972d0be9dfcc29e668d99381faf1 + - filename: voice-detect-wespeaker-resnet34.gguf + uri: https://huggingface.co/mudler/voice-detect-gguf/resolve/main/wespeaker-resnet34-voxceleb.gguf + sha256: 72040372494eafec299836bc1977cfc13c603cb486674ed59b0f4c03758d29da +- name: parakeet-cpp-realtime-scene-speakers + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://huggingface.co/mudler/ced-gguf + - https://huggingface.co/mudler/voice-detect-gguf + - https://huggingface.co/nvidia/parakeet_realtime_eou_120m-v1 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-tiny + - https://github.com/mudler/parakeet.cpp + description: | + Cache-aware streaming RNNT FastConformer with end-of-utterance (EOU) + detection, 120M, paired with Nemotron-3-Diarization and CED-Tiny through + the diarization_model and sound_model options. F16/Q8_0 GGUF for the + parakeet-cpp backend (C++/ggml port of NVIDIA NeMo). Use with streaming + transcription: while a turn is live, closed speaker segments and sound + events are surfaced alongside the ASR text (realtime + conversation.item.input_audio_transcription.segment and + conversation.item.sound_detection events). Live speaker/sound events only + fire during speech turns under semantic_vad; sounds between turns are not + seen by this path. License per model: transcription model NVIDIA Open + Model License, diarization model OpenMDW-1.1, CED-Tiny Apache-2.0, + WeSpeaker ResNet34 CC-BY-4.0. Also loads WeSpeaker ResNet34 through the + speaker_model option, so live speaker segments carry the name of a voice + registered with /v1/voice/register (voice-detect-wespeaker-resnet34 model) + once the speaker is identified. + license: nvidia-open-model-license + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - asr + - speech-recognition + - diarization + - sound-classification + - streaming + - realtime + - stt + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + name: parakeet-cpp-realtime-scene-speakers + options: + - diarization_model:parakeet-cpp/nemotron-3-diarization-q8_0.gguf + - sound_model:parakeet-cpp/ced-tiny-q8_0.gguf + - speaker_model:voice-detect-wespeaker-resnet34.gguf + parameters: + model: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + files: + - filename: parakeet-cpp/realtime_eou_120m-v1-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/realtime_eou_120m-v1-f16.gguf + sha256: d1a2b12f12b8a096a57499c9111ed13b442a2b786e17a292c168be45088f0edc + - filename: parakeet-cpp/nemotron-3-diarization-q8_0.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3-diarization-q8_0.gguf + sha256: 76c5bb1fb20d82706142ad32769b7ab496d2458489473a000fd7074c52ceec22 + - filename: parakeet-cpp/ced-tiny-q8_0.gguf + uri: huggingface://mudler/ced-gguf/ced-tiny-q8_0.gguf + sha256: 48bee4e2fc3cc85d7806e03471db24e77fda6c2a2e81ffe9ef67caebaf2bd674 + - filename: voice-detect-wespeaker-resnet34.gguf + uri: https://huggingface.co/mudler/voice-detect-gguf/resolve/main/wespeaker-resnet34-voxceleb.gguf + sha256: 72040372494eafec299836bc1977cfc13c603cb486674ed59b0f4c03758d29da - name: parakeet-cpp-ced-tiny url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: From 76d1c6a7bdb48672ea3d532bd20e0775aa86927e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:49:24 +0000 Subject: [PATCH 11/13] fix(parakeet-cpp): skip an unusable registered voice instead of failing the request A registered voice with the wrong embedding size, or one the C side refused, failed the whole diarization request, so one legacy voice broke the model for every user. Skip such voices with a warning that does not carry the voice name, and take the plain path when none is left. Also map an exact 0 speaker threshold or margin to a tiny positive value, since the C side reads 0 as "use the default", and fix a stale comment about which contexts Free() walks. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- backend/go/parakeet-cpp/diarize_test.go | 19 ++++++--- backend/go/parakeet-cpp/roles.go | 2 +- backend/go/parakeet-cpp/speaker_registry.go | 40 +++++++++++++------ .../go/parakeet-cpp/speaker_registry_test.go | 38 +++++++++++++----- .../services/voicerecognition/known_voices.go | 2 + .../voicerecognition/known_voices_test.go | 5 +++ 6 files changed, 78 insertions(+), 28 deletions(-) diff --git a/backend/go/parakeet-cpp/diarize_test.go b/backend/go/parakeet-cpp/diarize_test.go index 9bced8442b69..72d63766d3ef 100644 --- a/backend/go/parakeet-cpp/diarize_test.go +++ b/backend/go/parakeet-cpp/diarize_test.go @@ -344,13 +344,22 @@ var _ = Describe("ParakeetCpp.Diarize", func() { Expect(used).To(Equal("plain")) Expect(freed).To(Equal([]uintptr{9})) // the empty registry built for it is released, once }) - It("fails clearly on a voice of the wrong size and frees the registry", func() { + It("takes the plain path, without failing, when the only voice has the wrong size", func() { p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} - _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), + res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: []*pb.KnownVoice{{Name: "Ada", Embedding: []float32{1, 0, 0}}}}) - Expect(err).To(HaveOccurred()) - Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) - Expect(err.Error()).To(ContainSubstring("Ada")) + Expect(err).ToNot(HaveOccurred()) + Expect(used).To(Equal("plain")) + Expect(res.Segments[0].Name).To(BeEmpty()) + Expect(freed).To(Equal([]uintptr{9})) + }) + It("takes the plain path when the C side refuses the only voice", func() { + CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 1 } + CppSpeakerRegistryLastError = func(uintptr) string { return "nope" } + p := &ParakeetCpp{diarCtx: 1, spkCtx: 2} + _, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada}) + Expect(err).ToNot(HaveOccurred()) + Expect(used).To(Equal("plain")) Expect(freed).To(Equal([]uintptr{9})) }) It("reports a missing v10 symbol instead of silently dropping the names", func() { diff --git a/backend/go/parakeet-cpp/roles.go b/backend/go/parakeet-cpp/roles.go index 0ce944f910ff..6bb0e8c27218 100644 --- a/backend/go/parakeet-cpp/roles.go +++ b/backend/go/parakeet-cpp/roles.go @@ -224,7 +224,7 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { } // A companion whose role the primary already occupies (e.g. asr_model: // on an already-ASR primary) would overwrite that role field below, - // leaking the primary ctx: Free() only walks ctxPtr/diarCtx/tagCtx, so + // leaking the primary ctx: Free() walks ctxPtr/diarCtx/tagCtx/spkCtx, so // the overwritten pointer is never freed. Reject it before loading. if spec.current(p) != 0 { freeLoaded() diff --git a/backend/go/parakeet-cpp/speaker_registry.go b/backend/go/parakeet-cpp/speaker_registry.go index c79d6328e73e..18b0df99e6bf 100644 --- a/backend/go/parakeet-cpp/speaker_registry.go +++ b/backend/go/parakeet-cpp/speaker_registry.go @@ -29,7 +29,19 @@ func parseSpeakerThreshold(s string) (float32, error) { } d = v } - return float32(1 - d), nil + return nonZero(float32(1 - d)), nil +} + +// minPositive stands in for an exact 0 threshold or margin. The C side reads 0 as +// "use the default", so a distance of 1 (cosine 0) or a margin of 0 would silently +// become 0.5 or 0.05. +const minPositive = float32(1e-6) + +func nonZero(v float32) float32 { + if v == 0 { + return minPositive + } + return v } // parseSpeakerMargin reads speaker_margin, the runner-up margin in [0, 1). @@ -41,14 +53,15 @@ func parseSpeakerMargin(s string) (float32, error) { if err != nil || math.IsNaN(v) || v < 0 || v >= 1 { return 0, fmt.Errorf("parakeet-cpp: speaker_margin %q must be a number in [0, 1)", s) } - return float32(v), nil + return nonZero(float32(v)), nil } // buildSpeakerRegistryLocked makes a parakeet_speaker_registry from the registered voices // of one request or stream. Caller holds engineMu. It returns 0 (and no error) when there is // nothing to build: no speaker model loaded or no usable voices. A voice whose embedding size -// differs from the speaker model's is an error naming the voice and both sizes. The caller -// frees a non-zero result with freeSpeakerRegistry. +// differs from the speaker model's, or that the C side refuses, is skipped with a warning +// (without its name: the log is not for the caller who may not see voice names) so one bad +// voice cannot fail every request. The caller frees a non-zero result with freeSpeakerRegistry. func (p *ParakeetCpp) buildSpeakerRegistryLocked(voices []*pb.KnownVoice) (uintptr, error) { if p.spkCtx == 0 || CppSpeakerRegistryNew == nil || CppSpeakerRegistryAddEmbedding == nil || len(voices) == 0 { return 0, nil @@ -61,7 +74,7 @@ func (p *ParakeetCpp) buildSpeakerRegistryLocked(voices []*pb.KnownVoice) (uintp if reg == 0 { return 0, status.Error(codes.Internal, "parakeet-cpp: could not create a speaker registry") } - added := 0 + added, skipped := 0, 0 for _, v := range voices { emb := v.GetEmbedding() if v.GetName() == "" || len(emb) == 0 { @@ -69,19 +82,22 @@ func (p *ParakeetCpp) buildSpeakerRegistryLocked(voices []*pb.KnownVoice) (uintp continue } if dim > 0 && len(emb) != dim { - CppSpeakerRegistryFree(reg) - return 0, status.Errorf(codes.InvalidArgument, - "parakeet-cpp: known voice %q has a %d-value embedding but the speaker model produces %d; "+ - "register the voices again with the same speaker encoder", v.GetName(), len(emb), dim) + xlog.Warn("parakeet-cpp: skipped a registered voice: embedding size does not match the speaker model's", + "voice_size", len(emb), "speaker_model_size", dim) + skipped++ + continue } if rc := CppSpeakerRegistryAddEmbedding(reg, v.GetName(), &emb[0], int32(len(emb))); rc != 0 { - msg := CppSpeakerRegistryLastError(reg) - CppSpeakerRegistryFree(reg) - return 0, status.Errorf(codes.InvalidArgument, "parakeet-cpp: known voice %q was refused: %s", v.GetName(), msg) + xlog.Warn("parakeet-cpp: skipped a registered voice the speaker registry refused", "error", CppSpeakerRegistryLastError(reg)) + skipped++ + continue } added++ } if added == 0 { + if skipped > 0 { + xlog.Warn("parakeet-cpp: no registered voice is usable with this speaker model; speakers stay unnamed", "skipped", skipped) + } CppSpeakerRegistryFree(reg) return 0, nil } diff --git a/backend/go/parakeet-cpp/speaker_registry_test.go b/backend/go/parakeet-cpp/speaker_registry_test.go index 057f12ed4268..aa046ca1c435 100644 --- a/backend/go/parakeet-cpp/speaker_registry_test.go +++ b/backend/go/parakeet-cpp/speaker_registry_test.go @@ -21,6 +21,18 @@ var _ = Describe("speaker options", func() { Expect(err).To(HaveOccurred(), bad) } }) + It("never hands the C side an exact zero, which it reads as use the default", func() { + a, err := parseSpeakerThreshold("1") // distance 1 is cosine 0 + Expect(err).ToNot(HaveOccurred()) + Expect(a).To(Equal(float32(1e-6))) + m, err := parseSpeakerMargin("0") + Expect(err).ToNot(HaveOccurred()) + Expect(m).To(Equal(float32(1e-6))) + a, _ = parseSpeakerThreshold("") + Expect(a).To(BeNumerically("~", 0.5, 1e-6)) + m, _ = parseSpeakerMargin("") + Expect(m).To(BeNumerically("~", 0.05, 1e-6)) + }) It("parses the margin, default 0.05, within [0, 1)", func() { m, err := parseSpeakerMargin("") Expect(err).ToNot(HaveOccurred()) @@ -67,22 +79,28 @@ var _ = Describe("buildSpeakerRegistry", func() { Expect(added).To(Equal([]string{"ada", "ben"})) Expect(freed).To(BeEmpty()) }) - It("refuses a voice of the wrong size with both sizes in the message, and frees the registry", func() { + It("skips a voice of the wrong size without failing, and frees the registry when none is left", func() { p := &ParakeetCpp{spkCtx: 5} - reg, err := p.buildSpeakerRegistry([]*pb.KnownVoice{{Name: "ada", Embedding: []float32{1, 0, 0}}, voice("cy", 5)}) - Expect(err).To(HaveOccurred()) + reg, err := p.buildSpeakerRegistry([]*pb.KnownVoice{voice("cy", 5)}) + Expect(err).ToNot(HaveOccurred()) Expect(reg).To(Equal(uintptr(0))) - Expect(err.Error()).To(ContainSubstring(`"cy"`)) - Expect(err.Error()).To(ContainSubstring("5")) - Expect(err.Error()).To(ContainSubstring("3")) + Expect(added).To(BeEmpty()) Expect(freed).To(Equal([]uintptr{77})) }) - It("reports the C error when a voice is refused, and frees the registry", func() { + It("keeps the usable voices when another one has the wrong size", func() { + p := &ParakeetCpp{spkCtx: 5} + reg, err := p.buildSpeakerRegistry([]*pb.KnownVoice{voice("cy", 5), {Name: "ada", Embedding: []float32{1, 0, 0}}}) + Expect(err).ToNot(HaveOccurred()) + Expect(reg).To(Equal(uintptr(77))) + Expect(added).To(Equal([]string{"ada"})) + Expect(freed).To(BeEmpty()) + }) + It("skips a voice the C side refuses, and frees the registry when none is left", func() { CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 1 } p := &ParakeetCpp{spkCtx: 5} - _, err := p.buildSpeakerRegistry([]*pb.KnownVoice{{Name: "ada", Embedding: []float32{0, 0, 0}}}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("stub error")) + reg, err := p.buildSpeakerRegistry([]*pb.KnownVoice{{Name: "ada", Embedding: []float32{0, 0, 0}}}) + Expect(err).ToNot(HaveOccurred()) + Expect(reg).To(Equal(uintptr(0))) Expect(freed).To(Equal([]uintptr{77})) }) It("returns no registry when no speaker model is loaded", func() { diff --git a/core/services/voicerecognition/known_voices.go b/core/services/voicerecognition/known_voices.go index b70cf33f9c2e..823f4b1500a4 100644 --- a/core/services/voicerecognition/known_voices.go +++ b/core/services/voicerecognition/known_voices.go @@ -68,6 +68,8 @@ func SelectKnownVoices(entries []Entry, speakerModelPath string) KnownVoiceSelec sel.OtherEncoder++ } } + // With no tagged match every untagged voice is included whatever its size; the + // backend skips the voices whose size differs from its speaker model's. for _, e := range untagged { if matchedDim != 0 && len(e.Embedding) != matchedDim { continue diff --git a/core/services/voicerecognition/known_voices_test.go b/core/services/voicerecognition/known_voices_test.go index ef4818ea9ffe..8d2491ea26ed 100644 --- a/core/services/voicerecognition/known_voices_test.go +++ b/core/services/voicerecognition/known_voices_test.go @@ -80,6 +80,11 @@ var _ = Describe("SelectKnownVoices", func() { Expect(sel.Voices).To(HaveLen(1)) Expect(sel.Untagged).To(Equal(1)) }) + It("includes untagged voices of different sizes when no voice is tagged for this encoder", func() { + sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{entry("old", "", 1, 0), entry("older", "", 1, 0, 0)}, wespeaker) + Expect(sel.Voices).To(HaveLen(2)) // the backend skips the ones whose size does not match + Expect(sel.Untagged).To(Equal(2)) + }) It("skips voices without a name or an embedding", func() { sel := voicerecognition.SelectKnownVoices([]voicerecognition.Entry{entry("", wespeaker, 1, 0), entry("x", wespeaker)}, wespeaker) Expect(sel.Voices).To(BeEmpty()) From 31d44d4141acea95eba4bc230be327b2bef59caf Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 17:49:24 +0000 Subject: [PATCH 12/13] fix(diarization): warn once per model about voices from another encoder; document the privacy limit The different-encoder warning fired on every request. Log it once per feature and speaker model, then at debug level. Document that the global voice registry lets any caller of a speaker_model model learn matching names, and that skipped wrong-sized voices are logged. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- core/http/endpoints/openai/diarization.go | 18 ++++++++++++++++-- core/http/endpoints/openai/diarization_test.go | 5 +++++ docs/content/features/voice-recognition.md | 9 ++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/core/http/endpoints/openai/diarization.go b/core/http/endpoints/openai/diarization.go index 24328b177c18..99dbf3ce4873 100644 --- a/core/http/endpoints/openai/diarization.go +++ b/core/http/endpoints/openai/diarization.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/backend" @@ -148,6 +149,15 @@ func attachKnownVoices(ctx context.Context, req *backend.DiarizationRequest, opt req.KnownVoices = selectKnownVoices(ctx, "diarization", options, registry) } +// warned remembers the (feature, speaker model) pairs already warned about. +var warned sync.Map + +// warnOnce reports true the first time it sees key, false afterwards. +func warnOnce(key string) bool { + _, loaded := warned.LoadOrStore(key, struct{}{}) + return !loaded +} + // selectKnownVoices returns the registered voices a backend may use to name // speakers, or nil when the model has no speaker_model, there is no registry, // or the registry cannot be read. It never fails the caller: unnamed speakers @@ -163,8 +173,12 @@ func selectKnownVoices(ctx context.Context, feature string, options []string, re return nil } if len(sel.Voices) == 0 && sel.OtherEncoder > 0 { - xlog.Warn(feature+": registered voices were made with a different encoder than this model's speaker_model; speakers stay unnamed", - "speaker_model", sm, "voices_from_other_encoder", sel.OtherEncoder) + msg := feature + ": registered voices were made with a different encoder than this model's speaker_model; speakers stay unnamed" + if warnOnce(feature + "|" + sm) { + xlog.Warn(msg, "speaker_model", sm, "voices_from_other_encoder", sel.OtherEncoder) + } else { + xlog.Debug(msg, "speaker_model", sm, "voices_from_other_encoder", sel.OtherEncoder) + } } return sel.Voices } diff --git a/core/http/endpoints/openai/diarization_test.go b/core/http/endpoints/openai/diarization_test.go index 743548262167..6a27a04d5a47 100644 --- a/core/http/endpoints/openai/diarization_test.go +++ b/core/http/endpoints/openai/diarization_test.go @@ -78,6 +78,11 @@ var _ = Describe("attachKnownVoices", func() { Expect(req.KnownVoices[0].Name).To(Equal("Ada")) }) + It("warns once per key", func() { + Expect(warnOnce("diarization|warn-once-test.gguf")).To(BeTrue()) + Expect(warnOnce("diarization|warn-once-test.gguf")).To(BeFalse()) + Expect(warnOnce("live|warn-once-test.gguf")).To(BeTrue()) + }) It("leaves the request alone without a speaker_model option", func() { req := backend.DiarizationRequest{} attachKnownVoices(context.Background(), &req, []string{"other:x"}, diff --git a/docs/content/features/voice-recognition.md b/docs/content/features/voice-recognition.md index f74d9a3888d9..0c2938a1ce5b 100644 --- a/docs/content/features/voice-recognition.md +++ b/docs/content/features/voice-recognition.md @@ -218,7 +218,10 @@ gallery entry). The tag must equal the base name of the `speaker_model:` file. Voices made with another encoder are ignored, and LocalAI logs a warning when that leaves no usable voice. Voices registered before the tag existed have no tag: they are used when their embedding size matches the -tagged ones (or all of them, when no voice carries a matching tag). +tagged ones (or all of them, when no voice carries a matching tag). The +backend skips a voice whose embedding size does not match the speaker model's, +with a warning in the LocalAI log. Naming then falls back to the remaining +voices, or to no names. {{% notice warning %}} Do not set a `model_name:` option on the voice-detect model config. It @@ -246,6 +249,10 @@ speakers and makes fewer mistakes. - The voice registry is in memory and global. Registered names disappear when LocalAI restarts, and every user of the instance shares them. +- Anyone who is allowed to call a model with `speaker_model:` can learn which + registered names match their audio, and their audio is matched against voices + registered by any user, because the voice registry is global. Restrict such + models with the per-user model allowlist. - With `include_text=true` the names use the default threshold and margin: `speaker_threshold` and `speaker_margin` only apply to diarization without text. From 9ca076605a17833a95a55f706a478168dd1967cf Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 30 Sep 2026 19:15:08 +0000 Subject: [PATCH 13/13] chore(parakeet-cpp): bump parakeet.cpp to 8c8cec0 (C-API v10) and check speaker naming against the real library The pin moves from 623a968 to 8c8cec0, which brings in everything merged in parakeet.cpp since: the voice identification change (C-API v9, #78) and raw-embedding enroll plus diarize-only speaker naming (C-API v10, #79). New real-library specs (gated on PARAKEET_BACKEND_TEST_SPEAKER_MODEL, _DIAR_MODEL, _WAV and, for the live path, _STREAM_MODEL) name the two speakers of two_speakers.wav from a committed pair of WeSpeaker embeddings, with the voices passed in reversed order. They also check that the float32 threshold reaches C through purego. The shared test loader now registers the v9/v10 and scene symbols as main.go does. The rebase onto origin/master had no conflicts. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- backend/go/parakeet-cpp/Makefile | 4 +- backend/go/parakeet-cpp/goparakeetcpp_test.go | 18 ++ backend/go/parakeet-cpp/speaker_real_test.go | 184 ++++++++++++++++++ .../testdata/two_speakers_embeddings.json | 1 + 4 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 backend/go/parakeet-cpp/speaker_real_test.go create mode 100644 backend/go/parakeet-cpp/testdata/two_speakers_embeddings.json diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile index 59fe567170ca..77386dc925ab 100644 --- a/backend/go/parakeet-cpp/Makefile +++ b/backend/go/parakeet-cpp/Makefile @@ -1,6 +1,6 @@ # parakeet-cpp backend Makefile. # -# Upstream pin lives below as PARAKEET_VERSION?=623a968bccbd2214588df398fcce687cd4218dea +# Upstream pin lives below as PARAKEET_VERSION?=8c8cec0c4564610a0a4b30a8a6f2ead15d1a76fb # (.github/bump_deps.sh) can find and update it - matches the # whisper.cpp / ds4 / vibevoice-cpp convention. # @@ -15,7 +15,7 @@ # That's what the L0 smoke test uses. The default target below does the # proper clone-at-pin + cmake build so CI doesn't need a side-checkout. -PARAKEET_VERSION?=623a968bccbd2214588df398fcce687cd4218dea +PARAKEET_VERSION?=8c8cec0c4564610a0a4b30a8a6f2ead15d1a76fb PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp GOCMD?=go diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go index eda5eaa36ce3..a1364b15dc96 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp_test.go +++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go @@ -75,6 +75,24 @@ func ensureLibLoaded() { purego.RegisterLibFunc(&CppSoundStreamDrainScoresJSON, lib, "parakeet_capi_sound_stream_drain_scores_json") purego.RegisterLibFunc(&CppFreeSoundSegments, lib, "parakeet_capi_free_sound_segments") purego.RegisterLibFunc(&CppSoundStreamFree, lib, "parakeet_capi_sound_stream_free") + purego.RegisterLibFunc(&CppSceneOptsDefault, lib, "parakeet_capi_scene_opts_default") + purego.RegisterLibFunc(&CppSceneStreamBegin, lib, "parakeet_capi_scene_stream_begin") + purego.RegisterLibFunc(&CppSceneStreamFeedJSON, lib, "parakeet_capi_scene_stream_feed_json") + purego.RegisterLibFunc(&CppSceneStreamLastError, lib, "parakeet_capi_scene_stream_last_error") + purego.RegisterLibFunc(&CppSceneStreamFree, lib, "parakeet_capi_scene_stream_free") + } + // Speaker identification (ABI v9 and v10), registered exactly as main.go does. + if sym, err := purego.Dlsym(lib, "parakeet_capi_scene_stream_begin_speaker"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppSpeakerDim, lib, "parakeet_capi_speaker_dim") + purego.RegisterLibFunc(&CppSpeakerRegistryNew, lib, "parakeet_capi_speaker_registry_new") + purego.RegisterLibFunc(&CppSpeakerRegistryFree, lib, "parakeet_capi_speaker_registry_free") + purego.RegisterLibFunc(&CppSpeakerRegistryLastError, lib, "parakeet_capi_speaker_registry_last_error") + purego.RegisterLibFunc(&CppSceneStreamBeginSpeaker, lib, "parakeet_capi_scene_stream_begin_speaker") + purego.RegisterLibFunc(&CppTranscribeAndDiarizeNamedJSON, lib, "parakeet_capi_transcribe_and_diarize_named_json") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_named_pcm_json"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppSpeakerRegistryAddEmbedding, lib, "parakeet_capi_speaker_registry_add_embedding") + purego.RegisterLibFunc(&CppDiarizeNamedPCMJSON, lib, "parakeet_capi_diarize_named_pcm_json") } purego.RegisterLibFunc(&CppFreeString, lib, "parakeet_capi_free_string") purego.RegisterLibFunc(&CppLastError, lib, "parakeet_capi_last_error") diff --git a/backend/go/parakeet-cpp/speaker_real_test.go b/backend/go/parakeet-cpp/speaker_real_test.go new file mode 100644 index 000000000000..c652ca15cb64 --- /dev/null +++ b/backend/go/parakeet-cpp/speaker_real_test.go @@ -0,0 +1,184 @@ +package main + +import ( + "encoding/json" + "os" + "sort" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// fixtureVoices reads testdata/two_speakers_embeddings.json: WeSpeaker ResNet34 +// embeddings of voice A (two_speakers.wav 0.6-4.6 s) and voice B (6.9-10.9 s). +func fixtureVoices() (a, b *pb.KnownVoice) { + raw, err := os.ReadFile("testdata/two_speakers_embeddings.json") + Expect(err).ToNot(HaveOccurred()) + var doc struct { + Dim int `json:"dim"` + Voices []struct { + Name string `json:"name"` + Embedding []float32 `json:"embedding"` + } `json:"voices"` + } + Expect(json.Unmarshal(raw, &doc)).To(Succeed()) + Expect(doc.Voices).To(HaveLen(2)) + Expect(doc.Voices[0].Embedding).To(HaveLen(doc.Dim)) + mk := func(i int) *pb.KnownVoice { + return &pb.KnownVoice{Name: doc.Voices[i].Name, Embedding: doc.Voices[i].Embedding} + } + return mk(0), mk(1) +} + +func speakerFixturesOrSkip() (diarModel, speakerModel, wav string) { + diarModel = os.Getenv("PARAKEET_BACKEND_TEST_DIAR_MODEL") + speakerModel = os.Getenv("PARAKEET_BACKEND_TEST_SPEAKER_MODEL") + wav = os.Getenv("PARAKEET_BACKEND_TEST_WAV") + if diarModel == "" || speakerModel == "" || wav == "" { + Skip("set PARAKEET_BACKEND_TEST_DIAR_MODEL, PARAKEET_BACKEND_TEST_SPEAKER_MODEL and " + + "PARAKEET_BACKEND_TEST_WAV (parakeet.cpp tests/fixtures/two_speakers.wav)") + } + ensureLibLoaded() + if CppDiarizeNamedPCMJSON == nil { + Skip("libparakeet.so has no ABI 10 speaker naming (parakeet_capi_diarize_named_pcm_json)") + } + return +} + +// namesBySlot maps a speaker label to the set of names its segments carry. +func namesBySlot(segs []*pb.DiarizeSegment) map[string][]string { + out := map[string][]string{} + for _, s := range segs { + out[s.Speaker] = append(out[s.Speaker], s.Name) + } + return out +} + +var _ = Describe("ParakeetCpp speaker names (real libparakeet.so, ABI 10)", func() { + load := func(diarModel, speakerModel string, extra ...string) *ParakeetCpp { + p := &ParakeetCpp{} + opts := append([]string{"speaker_model:" + speakerModel}, extra...) + Expect(p.Load(&pb.ModelOptions{ModelFile: diarModel, Options: opts})).To(Succeed()) + return p + } + + // two_speakers.wav: voice A speaks 0.5-5.5 s and 14.8-18.7 s, voice B 6.9-13.5 s and + // 20.1-23.6 s; diarization slot 0 is voice A and slot 1 is voice B. + It("names diarized segments from registered voices, whatever order they arrive in", func() { + diarModel, speakerModel, wav := speakerFixturesOrSkip() + a, b := fixtureVoices() + p := load(diarModel, speakerModel) + defer func() { _ = p.Free() }() + + // Reversed on purpose: naming by arrival order would swap the speakers. + res, err := p.Diarize(&pb.DiarizeRequest{Dst: wav, KnownVoices: []*pb.KnownVoice{b, a}}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Segments).To(HaveLen(5)) + for _, s := range res.Segments { + GinkgoWriter.Printf("[%5.1f-%5.1f] slot %s name %q score %.3f\n", s.Start, s.End, s.Speaker, s.Name, s.NameScore) + want := map[string]string{"0": "voice_a", "1": "voice_b"}[s.Speaker] + Expect(want).ToNot(BeEmpty(), "unexpected slot %q", s.Speaker) + Expect(s.Name).To(Equal(want)) + Expect(s.NameScore).To(BeNumerically(">", 0.5)) + } + }) + + It("leaves a slot unnamed when its voice is not registered", func() { + diarModel, speakerModel, wav := speakerFixturesOrSkip() + _, b := fixtureVoices() + // Distance 0.3 means cosine 0.7; a different speaker scores near 0 with WeSpeaker. + p := load(diarModel, speakerModel, "speaker_threshold:0.3") + defer func() { _ = p.Free() }() + + res, err := p.Diarize(&pb.DiarizeRequest{Dst: wav, KnownVoices: []*pb.KnownVoice{b}}) + Expect(err).ToNot(HaveOccurred()) + names := namesBySlot(res.Segments) + Expect(names).To(HaveKey("0")) + Expect(names).To(HaveKey("1")) + for _, n := range names["0"] { + Expect(n).To(BeEmpty()) + } + for _, n := range names["1"] { + Expect(n).To(Equal("voice_b")) + } + }) + + // A distance of 0.01 asks for cosine 0.99, above any genuine score. If purego did not + // hand the float32 to C, the C side would read 0 and fall back to its 0.5 default, + // which names both slots, so an unnamed result proves the argument arrives. + It("passes the float32 accept threshold through purego (a 0.99 cosine names nobody)", func() { + diarModel, speakerModel, wav := speakerFixturesOrSkip() + a, b := fixtureVoices() + p := load(diarModel, speakerModel, "speaker_threshold:0.01") + defer func() { _ = p.Free() }() + + res, err := p.Diarize(&pb.DiarizeRequest{Dst: wav, KnownVoices: []*pb.KnownVoice{a, b}}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Segments).ToNot(BeEmpty()) + for _, s := range res.Segments { + Expect(s.Name).To(BeEmpty(), "slot %s was named at cosine 0.99", s.Speaker) + } + }) + + It("names live speaker segments from the registered voices", func() { + diarModel, speakerModel, wav := speakerFixturesOrSkip() + streamModel := os.Getenv("PARAKEET_BACKEND_TEST_STREAM_MODEL") + if streamModel == "" { + Skip("set PARAKEET_BACKEND_TEST_STREAM_MODEL (cache-aware streaming model) for the live path") + } + if CppSceneStreamBeginSpeaker == nil { + Skip("libparakeet.so has no scene_stream_begin_speaker") + } + a, b := fixtureVoices() + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ + ModelFile: streamModel, + Options: []string{"diarization_model:" + diarModel, "speaker_model:" + speakerModel}, + })).To(Succeed()) + defer func() { _ = p.Free() }() + + pcm, _, err := decodeWavMono16k(wav) + Expect(err).ToNot(HaveOccurred()) + + in := make(chan *pb.TranscriptLiveRequest, 8) + out := make(chan *pb.TranscriptLiveResponse, 256) + errCh := make(chan error, 1) + go func() { errCh <- p.AudioTranscriptionLive(in, out) }() + in <- &pb.TranscriptLiveRequest{Payload: &pb.TranscriptLiveRequest_Config{ + Config: &pb.TranscriptLiveConfig{KnownVoices: []*pb.KnownVoice{b, a}}, + }} + go func() { + const chunk = 8000 // 0.5 s + for i := 0; i < len(pcm); i += chunk { + end := min(i+chunk, len(pcm)) + in <- liveAudio(pcm[i:end]) + } + close(in) + }() + + var segs []*pb.LiveSpeakerSegment + for r := range out { + segs = append(segs, r.GetSpeakers()...) + } + Expect(<-errCh).ToNot(HaveOccurred()) + + named := 0 + seen := map[string]bool{} + for _, s := range segs { + GinkgoWriter.Printf("live slot %s [%d-%d] name %q\n", s.Speaker, s.Start, s.End, s.Name) + seen[s.Speaker] = true + if s.Name == "" { + continue + } + named++ + Expect(s.Name).To(Equal(map[string]string{"0": "voice_a", "1": "voice_b"}[s.Speaker])) + } + keys := make([]string, 0, len(seen)) + for k := range seen { + keys = append(keys, k) + } + sort.Strings(keys) + Expect(named).To(BeNumerically(">", 0), "no live speaker segment was named; slots seen: %v", keys) + }) +}) diff --git a/backend/go/parakeet-cpp/testdata/two_speakers_embeddings.json b/backend/go/parakeet-cpp/testdata/two_speakers_embeddings.json new file mode 100644 index 000000000000..6288242cbac1 --- /dev/null +++ b/backend/go/parakeet-cpp/testdata/two_speakers_embeddings.json @@ -0,0 +1 @@ +{"model":"voice-detect-wespeaker-resnet34.gguf","dim":256,"voices":[{"name":"voice_a","clip":"two_speakers.wav 0.6-4.6 s","embedding":[0.011315,0.055604,-0.071566,0.00961,0.040386,-0.016421,-0.033611,-0.019875,0.012314,-0.048251,0.025119,-0.035261,-0.051,-0.065269,0.044619,-0.032789,-0.077872,-0.059979,0.077165,0.034743,-0.034322,0.051458,0.119578,-0.060482,-0.032685,-0.066351,-0.043352,0.014827,-0.008393,0.079046,0.075357,0.177104,-0.002038,0.020706,-0.018464,0.004545,0.026542,0.00754,0.126408,0.011555,-0.083333,0.046795,0.029405,0.039988,-0.051861,0.034671,0.063438,0.010646,0.089453,0.009784,0.161807,-0.060077,0.080398,0.0026,-0.179952,-0.092161,0.008115,-0.043436,0.032408,-0.009307,0.04841,0.029929,-0.066182,0.069376,0.017271,0.076435,-0.013476,-0.091337,0.027061,0.070812,0.039052,-0.025303,0.062032,-0.129511,-0.035932,0.123741,0.059323,0.084191,0.017091,-0.071467,0.00336,0.109306,-0.032808,0.071844,0.013484,0.01657,-0.04768,-0.087006,0.07233,-0.004725,-0.058554,-0.061997,0.088107,0.069736,0.008408,-0.104471,0.049595,0.070141,0.071505,0.03694,0.030285,-0.103312,-0.028439,0.053189,0.132217,0.061057,-0.115166,0.04954,-0.04504,-0.044936,0.041552,0.07976,0.005178,0.027334,0.036235,-0.096866,-0.087907,-0.095822,0.10443,0.041865,-0.034412,0.065312,0.008648,-0.075287,0.125829,0.046572,-0.006843,0.106429,0.024523,0.013693,0.011473,-0.053549,0.078732,0.094865,-0.06333,0.03013,-0.090245,-0.013319,-0.090633,0.04209,-0.012844,-0.055208,-0.096489,-0.093557,0.038501,0.033988,0.032093,-0.015028,-0.067271,-0.100653,-0.00632,0.055982,-0.026291,-0.088451,0.019825,0.18585,-0.013341,-0.016392,0.05616,0.022352,0.006584,-0.097643,-0.02696,-0.019335,0.089462,-0.082992,0.02947,-0.014762,-0.055757,-0.011026,0.046157,-0.01546,0.029421,0.005409,-0.080892,-0.009961,0.029963,0.013844,-0.021823,-0.036825,0.001308,0.044549,0.011387,0.058622,0.047924,-0.034883,-0.038361,0.092906,0.006849,-0.076708,0.080052,0.011746,0.168146,-0.017433,-0.017833,0.010594,0.05945,-0.034766,-0.008519,0.02914,0.062197,-0.032722,-0.029624,0.103967,-0.07483,-0.052394,0.022729,-0.036227,0.060867,0.144592,-0.054725,-0.009984,0.056824,-0.076816,-0.005553,-0.054548,-0.047348,0.011424,0.091265,-0.012273,0.013987,-0.019944,0.054455,0.050702,0.004049,0.013685,0.022356,0.003249,-0.081608,-0.06926,-0.077244,-0.048076,0.059592,0.009222,0.007737,0.014863,-0.070568,-0.070963,0.056541,-0.067229,-0.034265,0.072919,0.032005,-0.06045,-0.148719,0.050373,-0.06677,0.020233,-0.031594,-0.084966,0.033239,-0.020293,-0.049292,0.120206,-0.056133,0.090007]},{"name":"voice_b","clip":"two_speakers.wav 6.9-10.9 s","embedding":[-0.106236,0.026909,0.032522,-0.040702,0.06418,0.058867,-0.043462,0.029255,-0.069755,0.02625,-0.012982,0.011717,0.037221,0.004934,-0.148783,0.073643,0.065623,0.176729,0.000483,0.109778,-0.105427,0.02092,0.038877,-0.020431,0.06949,-0.027571,-0.023125,-0.036426,0.050819,-0.047508,-0.000816,-0.073777,0.060473,0.023495,-0.035937,-0.064082,-0.085108,-0.045895,0.029146,0.007555,0.003923,-0.081975,0.038573,0.072211,-0.063173,0.006889,-0.011189,-0.018597,-0.027534,-0.001236,0.056054,-0.055961,-0.016422,0.064765,-0.114341,-0.028348,-0.016785,-0.001291,0.00695,-0.073718,0.146327,-0.030716,-0.029492,0.066045,-0.049192,0.055391,0.105924,0.033385,0.049666,0.055198,-0.0327,-0.051139,-0.009429,0.113493,0.065922,-0.027484,-0.062549,-0.027209,0.006105,-0.016623,0.012859,-0.042844,0.046082,-0.029788,-0.132396,-0.005784,-0.020116,-0.042434,0.045863,-0.00114,0.022023,-0.117683,0.002397,0.187938,0.067782,-0.053011,0.018449,0.043227,-0.007033,0.034628,-0.036276,0.044694,-0.105351,-0.113562,0.051461,-0.040255,0.088879,0.101067,0.110482,-0.011473,0.030959,0.028627,0.042808,0.106958,-0.020069,-0.048371,0.002144,-0.063363,0.065032,-0.032523,0.025647,0.12314,0.026488,0.054077,0.026475,0.088169,-0.112363,-0.03501,-0.061824,-0.085917,0.027558,0.010396,-0.092906,-0.024477,0.027928,-0.012543,0.069895,0.01406,0.196947,0.006955,-0.012518,-0.048722,0.035787,-0.051044,0.007964,-0.009535,-0.045574,0.013616,-0.023598,-0.027236,0.023735,-0.023832,-0.053049,0.023769,0.080018,-0.039285,0.009429,-0.046602,0.023376,-0.011104,-0.118228,-0.089595,0.047087,-0.041417,-0.030876,-0.015441,-0.004501,-0.038595,0.070107,-0.078626,-0.128752,0.012564,-0.034697,-0.069278,0.058026,0.059071,0.082451,0.041674,0.031116,-0.007566,-0.007439,-0.079061,-0.059093,0.014351,-0.006355,-0.010857,-0.050468,0.059146,-0.047707,-0.08106,0.07763,0.065864,0.061567,-0.126551,-0.119901,-0.002173,0.076158,0.066434,0.053258,-0.065215,0.012435,0.015349,-0.00617,-0.033529,-0.024387,0.001493,0.113286,0.039088,0.051456,0.166047,-0.020073,-0.07846,-0.029372,0.022419,-0.06691,0.097686,-0.060915,-0.032454,-0.095959,-0.026277,0.026242,-0.123124,0.092298,-0.036004,0.031126,0.119571,0.09179,-0.060572,0.028013,0.016614,0.09969,-0.113456,-0.094285,-0.058706,-0.071713,0.131803,-0.066991,0.048721,-0.050628,-0.030655,0.010101,-0.046536,-0.024472,-2.4e-05,-0.04725,0.046721,0.036714,-0.082382,0.010479,0.053549,0.068698,0.006237,0.020804,-0.081368,-0.024734,0.051612]}]} \ No newline at end of file