feat(parakeet-cpp): name speakers from the shared voice registry

backend/backend.proto

@@ -628,6 +628,7 @@ message TranscriptLiveConfig {
   string language = 1;             // "" => model default
   int32 sample_rate = 2;           // 0 => 16000; backends may reject others
   map<string, string> params = 3;  // backend-specific tuning
+  repeated KnownVoice known_voices = 4;   // see DiarizeRequest.known_voices
 }
 
 message TranscriptLiveAudio {
@@ -649,6 +650,7 @@ message LiveSpeakerSegment {
   string speaker = 1;   // decimal speaker index
   int64 start = 2;      // stream-relative nanoseconds
   int64 end = 3;
+  string name = 4;      // registered speaker name when the backend identified the speaker, else empty
 }
 
 message LiveSoundEvent {
@@ -825,6 +827,9 @@ message DiarizeRequest {
   // PredictOptions.ModelIdentity for the full rationale. Empty means "no
   // identity supplied" and backends MUST skip the check.
   string ModelIdentity = 11;
+  // Registered voices the backend may use to name speakers. Only backends that
+  // identify speakers themselves read this; others ignore it.
+  repeated KnownVoice known_voices = 12;
 }
 
 message DiarizeSegment {
@@ -833,6 +838,17 @@ message DiarizeSegment {
   float  end = 3;                      // seconds
   string speaker = 4;                  // backend-emitted speaker label (e.g. "0", "SPEAKER_00")
   string text = 5;                     // optional per-segment transcript (empty unless include_text and supported)
+  string name = 6;                     // registered speaker name, empty when unknown or not identified
+  float  name_score = 7;               // match score of that name (cosine similarity), 0 when unnamed
+}
+
+// KnownVoice is one registered voice: a name and its speaker embedding. `model`
+// names the encoder that produced it, so a backend with a different encoder can
+// refuse vectors that are not comparable.
+message KnownVoice {
+  string name = 1;
+  repeated float embedding = 2;
+  string model = 3;
 }
 
 message DiarizeResponse {

backend/go/parakeet-cpp/Makefile

@@ -1,6 +1,6 @@
 # parakeet-cpp backend Makefile.
 #
-# Upstream pin lives below as PARAKEET_VERSION?=623a968bccbd2214588df398fcce687cd4218dea
+# Upstream pin lives below as PARAKEET_VERSION?=8c8cec0c4564610a0a4b30a8a6f2ead15d1a76fb
 # (.github/bump_deps.sh) can find and update it - matches the
 # whisper.cpp / ds4 / vibevoice-cpp convention.
 #
@@ -15,7 +15,7 @@
 # That's what the L0 smoke test uses. The default target below does the
 # proper clone-at-pin + cmake build so CI doesn't need a side-checkout.
 
-PARAKEET_VERSION?=623a968bccbd2214588df398fcce687cd4218dea
+PARAKEET_VERSION?=8c8cec0c4564610a0a4b30a8a6f2ead15d1a76fb
 PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp
 
 GOCMD?=go

backend/go/parakeet-cpp/diarize.go

@@ -27,7 +27,28 @@ type diarizeSegmentJSON struct {
 // not the count of speakers actually present, so it is not read here; the
 // response's num_speakers is computed from distinct segment labels instead.
 type diarizePCMDoc struct {
-	Segments []diarizeSegmentJSON `json:"segments"`
+	Segments []diarizeSegmentJSON       `json:"segments"`
+	Names    map[string]speakerNameJSON `json:"names"`
+}
+
+// speakerNameJSON mirrors one value of the "names" map the named C-API functions add:
+// {"0":{"name":"Ada","score":0.93}}.
+type speakerNameJSON struct {
+	Name  string  `json:"name"`
+	Score float32 `json:"score"`
+}
+
+// nameFor returns the registered name of a diarization slot, or "" for an unknown slot, a slot
+// with no matching voice, or speaker -1 (no diarized speaker).
+func nameFor(names map[string]speakerNameJSON, speaker int) (string, float32) {
+	if speaker < 0 || len(names) == 0 {
+		return "", 0
+	}
+	n, ok := names[strconv.Itoa(speaker)]
+	if !ok || n.Name == "" {
+		return "", 0
+	}
+	return n.Name, n.Score
 }
 
 // diarizeUtteranceJSON mirrors one element of
@@ -45,7 +66,8 @@ type diarizeUtteranceJSON struct {
 // consumed here; the per-word "words" detail belongs to a speaker-attributed
 // transcript RPC, not Diarize.
 type transcribeAndDiarizeDoc struct {
-	Utterances []diarizeUtteranceJSON `json:"utterances"`
+	Utterances []diarizeUtteranceJSON     `json:"utterances"`
+	Names      map[string]speakerNameJSON `json:"names"`
 }
 
 // speakerLabel renders a 0-based speaker index as the decimal string
@@ -115,7 +137,16 @@ func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error
 
 	wantText := req.GetIncludeText() && p.ctxPtr != 0 && CppTranscribeAndDiarizeJSON != nil
 
-	raw, err := p.diarizeCall(pcm, wantText)
+	var reg uintptr
+	if len(req.GetKnownVoices()) > 0 && p.spkCtx != 0 {
+		reg, err = p.buildSpeakerRegistry(req.GetKnownVoices())
+		if err != nil {
+			return pb.DiarizeResponse{}, err
+		}
+		defer p.freeSpeakerRegistry(reg)
+	}
+
+	raw, err := p.diarizeCall(pcm, wantText, reg)
 	if err != nil {
 		return pb.DiarizeResponse{}, err
 	}
@@ -142,22 +173,37 @@ func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error
 // those fields under the same engineMu) would otherwise reach the C side with
 // a freed context. last_error is ctx-shared, so it is read under the same
 // lock as the failing call.
-func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool) (string, error) {
+func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool, reg uintptr) (string, error) {
 	p.engineMu.Lock()
 	defer p.engineMu.Unlock()
 
-	if p.diarCtx == 0 || (wantText && p.ctxPtr == 0) {
+	if p.diarCtx == 0 || (wantText && p.ctxPtr == 0) || (reg != 0 && p.spkCtx == 0) {
 		return "", grpcerrors.ModelNotLoaded("parakeet-cpp")
 	}
 
 	var cstr uintptr
-	if wantText {
+	switch {
+	case reg != 0 && wantText:
+		if CppTranscribeAndDiarizeNamedJSON == nil {
+			return "", status.Error(codes.Unimplemented,
+				"parakeet-cpp: naming speakers needs libparakeet.so ABI 10 (parakeet_capi_transcribe_and_diarize_named_json)")
+		}
+		// This C function takes no threshold or margin, so the text path uses the C side's
+		// defaults rather than speaker_threshold / speaker_margin.
+		cstr = CppTranscribeAndDiarizeNamedJSON(p.ctxPtr, p.diarCtx, p.spkCtx, reg, &pcm[0], int32(len(pcm)), 16000)
+	case reg != 0:
+		if CppDiarizeNamedPCMJSON == nil {
+			return "", status.Error(codes.Unimplemented,
+				"parakeet-cpp: naming speakers needs libparakeet.so ABI 10 (parakeet_capi_diarize_named_pcm_json)")
+		}
+		cstr = CppDiarizeNamedPCMJSON(p.diarCtx, p.spkCtx, reg, &pcm[0], int32(len(pcm)), 16000, p.speakerAccept, p.speakerMargin)
+	case wantText:
 		cstr = CppTranscribeAndDiarizeJSON(p.ctxPtr, p.diarCtx, &pcm[0], int32(len(pcm)), 16000)
-	} else {
+	default:
 		cstr = CppDiarizePCM(p.diarCtx, &pcm[0], int32(len(pcm)), 16000)
 	}
 	if cstr == 0 {
-		return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", diarizeLastError(p, wantText))
+		return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", diarizeLastError(p, wantText, reg != 0))
 	}
 	raw := goStringFromCPtr(cstr)
 	CppFreeString(cstr)
@@ -169,7 +215,7 @@ func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool) (string, error)
 // and either side of the pairing may be the one that set it — then joins
 // whichever came back non-empty. Called under the same engineMu as the
 // failing call (last_error is ctx-shared state).
-func diarizeLastError(p *ParakeetCpp, wantText bool) string {
+func diarizeLastError(p *ParakeetCpp, wantText, named bool) string {
 	var msgs []string
 	if m := CppLastError(p.diarCtx); m != "" {
 		msgs = append(msgs, m)
@@ -179,6 +225,11 @@ func diarizeLastError(p *ParakeetCpp, wantText bool) string {
 			msgs = append(msgs, m)
 		}
 	}
+	if named {
+		if m := CppLastError(p.spkCtx); m != "" {
+			msgs = append(msgs, m)
+		}
+	}
 	if len(msgs) == 0 {
 		return "unknown error"
 	}
@@ -196,11 +247,14 @@ func parseDiarizeDoc(raw string, wantText bool) ([]*pb.DiarizeSegment, error) {
 		}
 		segs := make([]*pb.DiarizeSegment, 0, len(doc.Utterances))
 		for _, u := range doc.Utterances {
+			name, score := nameFor(doc.Names, u.Speaker)
 			segs = append(segs, &pb.DiarizeSegment{
-				Start:   float32(u.Start),
-				End:     float32(u.End),
-				Speaker: speakerLabel(u.Speaker),
-				Text:    u.Text,
+				Start:     float32(u.Start),
+				End:       float32(u.End),
+				Speaker:   speakerLabel(u.Speaker),
+				Text:      u.Text,
+				Name:      name,
+				NameScore: score,
 			})
 		}
 		return segs, nil
@@ -212,10 +266,13 @@ func parseDiarizeDoc(raw string, wantText bool) ([]*pb.DiarizeSegment, error) {
 	}
 	segs := make([]*pb.DiarizeSegment, 0, len(doc.Segments))
 	for _, s := range doc.Segments {
+		name, score := nameFor(doc.Names, s.Speaker)
 		segs = append(segs, &pb.DiarizeSegment{
-			Start:   float32(s.Start),
-			End:     float32(s.End),
-			Speaker: speakerLabel(s.Speaker),
+			Start:     float32(s.Start),
+			End:       float32(s.End),
+			Speaker:   speakerLabel(s.Speaker),
+			Name:      name,
+			NameScore: score,
 		})
 	}
 	return segs, nil

backend/go/parakeet-cpp/diarize_test.go

@@ -40,7 +40,21 @@ func diarizeStubs() (restore func()) {
 	savedTranscribeAndDiarize := CppTranscribeAndDiarizeJSON
 	savedFreeString := CppFreeString
 	savedLastError := CppLastError
+	savedNamedDiarize := CppDiarizeNamedPCMJSON
+	savedNamedText := CppTranscribeAndDiarizeNamedJSON
+	savedRegNew := CppSpeakerRegistryNew
+	savedRegFree := CppSpeakerRegistryFree
+	savedRegAdd := CppSpeakerRegistryAddEmbedding
+	savedRegLastError := CppSpeakerRegistryLastError
+	savedSpeakerDim := CppSpeakerDim
 	return func() {
+		CppDiarizeNamedPCMJSON = savedNamedDiarize
+		CppTranscribeAndDiarizeNamedJSON = savedNamedText
+		CppSpeakerRegistryNew = savedRegNew
+		CppSpeakerRegistryFree = savedRegFree
+		CppSpeakerRegistryAddEmbedding = savedRegAdd
+		CppSpeakerRegistryLastError = savedRegLastError
+		CppSpeakerDim = savedSpeakerDim
 		CppDiarizePCM = savedDiarize
 		CppTranscribeAndDiarizeJSON = savedTranscribeAndDiarize
 		CppFreeString = savedFreeString
@@ -245,7 +259,7 @@ var _ = Describe("ParakeetCpp.Diarize", func() {
 		// Simulate a Free() racing between Diarize's own diarCtx==0 check and
 		// diarizeCall's lock, exactly as it zeroes diarCtx under engineMu.
 		p.diarCtx = 0
-		_, err := p.diarizeCall(make([]float32, 10), false)
+		_, err := p.diarizeCall(make([]float32, 10), false, 0)
 		Expect(grpcerrors.IsModelNotLoaded(err)).To(BeTrue())
 		Expect(called).To(BeFalse(), "no C call once diarCtx was cleared")
 	})
@@ -272,4 +286,143 @@ var _ = Describe("ParakeetCpp.Diarize", func() {
 		Expect(resp.Segments[1].Start).To(BeNumerically("~", 1.05, 0.001))
 		Expect(resp.Segments[1].End).To(BeNumerically("~", 1.20, 0.001))
 	})
+	Describe("with known voices", func() {
+		var freed []uintptr
+		var used string
+		ada := []*pb.KnownVoice{{Name: "Ada", Embedding: []float32{1, 0}}}
+		BeforeEach(func() {
+			freed, used = nil, ""
+			CppFreeString = func(uintptr) {}
+			CppSpeakerDim = func(uintptr) int32 { return 2 }
+			CppSpeakerRegistryNew = func() uintptr { return 9 }
+			CppSpeakerRegistryFree = func(r uintptr) { freed = append(freed, r) }
+			CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 0 }
+			CppDiarizePCM = func(uintptr, *float32, int32, int32) uintptr {
+				used = "plain"
+				return pool.cstr(`{"speakers":8,"segments":[{"speaker":0,"start":0.5,"end":2}]}`)
+			}
+			CppDiarizeNamedPCMJSON = func(diar, spk, reg uintptr, s *float32, n, sr int32, accept, margin float32) uintptr {
+				used = "named"
+				Expect(reg).To(Equal(uintptr(9)))
+				Expect(accept).To(BeNumerically("~", 0.7, 1e-6))
+				Expect(margin).To(BeNumerically("~", 0.05, 1e-6))
+				return pool.cstr(`{"speakers":8,"segments":[{"speaker":0,"start":0.5,"end":2.0},{"speaker":1,"start":2.5,"end":4.0}],` +
+					`"names":{"0":{"name":"Ada","score":0.93},"1":{"name":"","score":0.2}}}`)
+			}
+		})
+
+		It("puts the registered names on the segments and frees the registry", func() {
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, speakerAccept: 0.7, speakerMargin: 0.05}
+			res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(used).To(Equal("named"))
+			Expect(res.Segments).To(HaveLen(2))
+			Expect(res.Segments[0].Name).To(Equal("Ada"))
+			Expect(res.Segments[0].NameScore).To(BeNumerically("~", 0.93, 1e-6))
+			Expect(res.Segments[1].Name).To(BeEmpty())
+			Expect(freed).To(Equal([]uintptr{9}))
+		})
+		It("uses the plain path when the request has no known voices", func() {
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2}
+			res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5)})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(used).To(Equal("plain"))
+			Expect(res.Segments[0].Name).To(BeEmpty())
+			Expect(freed).To(BeEmpty())
+		})
+		It("uses the plain path when no speaker model is loaded, even with known voices", func() {
+			p := &ParakeetCpp{diarCtx: 1}
+			_, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(used).To(Equal("plain"))
+		})
+		It("uses the plain path, without a registry, when no voice has an embedding", func() {
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2}
+			_, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5),
+				KnownVoices: []*pb.KnownVoice{{Name: "Ada"}, {Embedding: []float32{1, 0}}}})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(used).To(Equal("plain"))
+			Expect(freed).To(Equal([]uintptr{9})) // the empty registry built for it is released, once
+		})
+		It("takes the plain path, without failing, when the only voice has the wrong size", func() {
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2}
+			res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5),
+				KnownVoices: []*pb.KnownVoice{{Name: "Ada", Embedding: []float32{1, 0, 0}}}})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(used).To(Equal("plain"))
+			Expect(res.Segments[0].Name).To(BeEmpty())
+			Expect(freed).To(Equal([]uintptr{9}))
+		})
+		It("takes the plain path when the C side refuses the only voice", func() {
+			CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 1 }
+			CppSpeakerRegistryLastError = func(uintptr) string { return "nope" }
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2}
+			_, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(used).To(Equal("plain"))
+			Expect(freed).To(Equal([]uintptr{9}))
+		})
+		It("reports a missing v10 symbol instead of silently dropping the names", func() {
+			CppDiarizeNamedPCMJSON = nil
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2}
+			_, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada})
+			Expect(err).To(HaveOccurred())
+			Expect(status.Code(err)).To(Equal(codes.Unimplemented))
+			Expect(freed).To(Equal([]uintptr{9}))
+		})
+		It("reports a missing named transcribe symbol on the include_text path", func() {
+			CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, s *float32, n, sr int32) uintptr { return 0 }
+			CppTranscribeAndDiarizeNamedJSON = nil
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, ctxPtr: 3}
+			_, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), IncludeText: true, KnownVoices: ada})
+			Expect(err).To(HaveOccurred())
+			Expect(status.Code(err)).To(Equal(codes.Unimplemented))
+			Expect(freed).To(Equal([]uintptr{9}))
+		})
+		It("names utterances on the include_text path", func() {
+			// wantText also requires the plain text symbol, present in any library that has the named one.
+			CppTranscribeAndDiarizeJSON = func(asr, diar uintptr, s *float32, n, sr int32) uintptr { return 0 }
+			CppTranscribeAndDiarizeNamedJSON = func(asr, diar, spk, reg uintptr, s *float32, n, sr int32) uintptr {
+				used = "named-text"
+				return pool.cstr(`{"speakers":8,"names":{"0":{"name":"Ada","score":0.9}},"utterances":[{"speaker":0,"name":"Ada","text":"hello","start":0.5,"end":2.0,"conf":0.9},{"speaker":-1,"text":"hm","start":2.5,"end":3.0,"conf":0.5}],"words":[]}`)
+			}
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, ctxPtr: 3}
+			res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), IncludeText: true, KnownVoices: ada})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(used).To(Equal("named-text"))
+			Expect(res.Segments[0].Name).To(Equal("Ada"))
+			Expect(res.Segments[0].Text).To(Equal("hello"))
+			Expect(res.Segments[1].Name).To(BeEmpty()) // speaker -1 has no name
+			Expect(freed).To(Equal([]uintptr{9}))
+		})
+		It("keeps the name when close segments of one speaker are merged", func() {
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, speakerAccept: 0.7, speakerMargin: 0.05}
+			CppDiarizeNamedPCMJSON = func(diar, spk, reg uintptr, s *float32, n, sr int32, a, m float32) uintptr {
+				return pool.cstr(`{"speakers":8,"segments":[{"speaker":0,"start":0.5,"end":2.0},{"speaker":0,"start":2.1,"end":3.0}],"names":{"0":{"name":"Ada","score":0.9}}}`)
+			}
+			res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), MinDurationOff: 0.5, KnownVoices: ada})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Segments).To(HaveLen(1))
+			Expect(res.Segments[0].Name).To(Equal("Ada"))
+			Expect(res.Segments[0].NameScore).To(BeNumerically("~", 0.9, 1e-6))
+		})
+		It("includes the speaker context message when the named call fails", func() {
+			CppDiarizeNamedPCMJSON = func(diar, spk, reg uintptr, s *float32, n, sr int32, a, m float32) uintptr { return 0 }
+			CppLastError = func(ctx uintptr) string {
+				switch ctx {
+				case 1:
+					return "diar side broke"
+				case 2:
+					return "speaker side broke"
+				}
+				return ""
+			}
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2}
+			_, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada})
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("diar side broke"))
+			Expect(err.Error()).To(ContainSubstring("speaker side broke"))
+			Expect(freed).To(Equal([]uintptr{9}))
+		})
+	})
 })