From 86565706c2704464b104137690d8670aeefddc89 Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Mon, 28 Sep 2026 06:51:19 +0000
Subject: [PATCH 1/5] feat(parakeet-cpp): add speaker diarization with
 Nemotron-3-Diarization

parakeet.cpp now runs NVIDIA Nemotron-3-Diarization (Sortformer, up to 8
speakers), matched against NeMo. This wires it into the backend:

- A diarization GGUF loads as its own model and answers Diarize. The
  transcription RPCs return FailedPrecondition on it.
- An ASR model takes a diarization GGUF through the diar_model option,
  the same key nemo-speech-cpp uses. Transcript segments then carry
  their speaker and split at speaker turns; the request's diarize flag
  turns it off. Diarize with include_text fills each segment with the
  words its speaker said.

The backend tells the two apart from the GGUF's parakeet.arch key. Word
to speaker assignment follows parakeet.cpp's own merge rule (largest
overlap, else the nearest segment within 0.5 s), so the labels match the
library's speaker-attributed ASR. Speaker count, clustering and
segment-duration request fields have no Sortformer equivalent and are
logged as ignored.

parakeet_capi_diarize_pcm is probed with Dlsym like the other optional
entry points, so an older libparakeet.so still loads and reports
diarization as unavailable. The pin moves to parakeet.cpp 238057c,
which includes the diarization C-API (ABI 7).

Assisted-by: Claude:claude-opus-5-5 [Claude Code]
---
 backend/go/parakeet-cpp/Makefile              |   4 +-
 backend/go/parakeet-cpp/diarize.go            | 333 ++++++++++++++++++
 backend/go/parakeet-cpp/diarize_test.go       | 257 ++++++++++++++
 backend/go/parakeet-cpp/goparakeetcpp.go      | 112 +++++-
 backend/go/parakeet-cpp/goparakeetcpp_test.go |   3 +
 backend/go/parakeet-cpp/live.go               |   3 +
 backend/go/parakeet-cpp/main.go               |   7 +
 core/config/backend_capabilities.go           |   9 +-
 docs/content/features/audio-diarization.md    |  42 ++-
 docs/content/features/audio-to-text.md        |  15 +
 docs/content/reference/compatibility-table.md |   2 +-
 11 files changed, 771 insertions(+), 16 deletions(-)
 create mode 100644 backend/go/parakeet-cpp/diarize.go
 create mode 100644 backend/go/parakeet-cpp/diarize_test.go

diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile
index e288f6fcc648..f14e507715f3 100644
--- a/backend/go/parakeet-cpp/Makefile
+++ b/backend/go/parakeet-cpp/Makefile
@@ -1,6 +1,6 @@
 # parakeet-cpp backend Makefile.
 #
-# Upstream pin lives below as PARAKEET_VERSION?=2bf88954dc628b32835734e2e9159550a75a1dc6
+# Upstream pin lives below as PARAKEET_VERSION?=238057cb707fdb6c4b185c853da095328930b539
 # (.github/bump_deps.sh) can find and update it - matches the
 # whisper.cpp / ds4 / vibevoice-cpp convention.
 #
@@ -15,7 +15,7 @@
 # That's what the L0 smoke test uses. The default target below does the
 # proper clone-at-pin + cmake build so CI doesn't need a side-checkout.
 
-PARAKEET_VERSION?=2bf88954dc628b32835734e2e9159550a75a1dc6
+PARAKEET_VERSION?=238057cb707fdb6c4b185c853da095328930b539
 PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp
 
 GOCMD?=go
diff --git a/backend/go/parakeet-cpp/diarize.go b/backend/go/parakeet-cpp/diarize.go
new file mode 100644
index 000000000000..9511846cd353
--- /dev/null
+++ b/backend/go/parakeet-cpp/diarize.go
@@ -0,0 +1,333 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	gguf "github.com/gpustack/gguf-parser-go"
+	"github.com/mudler/LocalAI/pkg/grpc/grpcerrors"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/xlog"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// Speaker diarization (nvidia/Nemotron-3-Diarization and compatible Sortformer
+// models, parakeet.cpp ABI >= 7).
+//
+// A parakeet_ctx holds either an ASR model or a diarization model;
+// parakeet_capi_load detects which from the GGUF. The backend serves
+// diarization in two shapes:
+//
+//   - the model file IS a diarization GGUF: the Diarize RPC works, transcription
+//     does not (there is no ASR to run);
+//   - an ASR model with a diarization GGUF attached through the diar_model
+//     option (the same key nemo-speech-cpp uses): transcripts carry a speaker
+//     per segment, and Diarize can fill each segment's text (include_text).
+//
+// Diarization runs the checkpoint's own mode, NeMo's diarize() default, which
+// for Nemotron-3-Diarization is cache-aware streaming in 21.12 s chunks.
+// Speakers are 0-based and numbered in order of first appearance; the model's
+// speaker capacity (8) is fixed by the checkpoint.
+
+// CppDiarizePcm is parakeet_capi_diarize_pcm: 16 kHz mono float PCM in, a
+// malloc'd JSON document out (uintptr, freed via CppFreeString):
+//
+//	{"speakers":8,"segments":[{"speaker":0,"start":0.50,"end":5.52}, ...]}
+//
+// Present only in libparakeet.so with ABI >= 7; nil disables diarization.
+var CppDiarizePcm func(ctx uintptr, samples []float32, nSamples int32, sampleRate int32) uintptr
+
+// diarModelArch is the parakeet.arch GGUF value of a diarization model.
+const diarModelArch = "diarization"
+
+// speakerSnapSeconds mirrors parakeet.cpp's merge_asr_diarization: a word that
+// overlaps no speaker segment takes the nearest segment's speaker when that
+// segment is this close. ASR word boundaries and diarization boundaries can
+// disagree by a frame or two; a word farther than this has no speaker.
+const speakerSnapSeconds = 0.5
+
+type diarJSON struct {
+	Speakers int              `json:"speakers"`
+	Segments []diarSegmentDoc `json:"segments"`
+}
+
+type diarSegmentDoc struct {
+	Speaker int     `json:"speaker"`
+	Start   float64 `json:"start"`
+	End     float64 `json:"end"`
+}
+
+// parakeetArch reads parakeet.arch ("tdt", "ctc", ..., "diarization") from a
+// parakeet.cpp GGUF. general.architecture is "parakeet" for every model, so it
+// cannot tell ASR from diarization.
+func parakeetArch(path string) (string, error) {
+	f, err := gguf.ParseGGUFFile(path, gguf.UseMMap(), gguf.SkipLargeMetadata())
+	if err != nil {
+		return "", fmt.Errorf("parakeet-cpp: parse gguf %q: %w", path, err)
+	}
+	kv, found := f.Header.MetadataKV.Index([]string{"parakeet.arch"})
+	if found == 0 {
+		return "", fmt.Errorf("parakeet-cpp: %q has no parakeet.arch key", path)
+	}
+	arch := kv["parakeet.arch"]
+	// ValueString panics on a mistyped key; a hand-edited GGUF is exactly where
+	// that happens, and a load-time check must report, not crash.
+	if arch.ValueType != gguf.GGUFMetadataValueTypeString {
+		return "", fmt.Errorf("parakeet-cpp: %q has a non-string parakeet.arch", path)
+	}
+	return arch.ValueString(), nil
+}
+
+// optString reads a string model option (key:value form), splitting on the
+// first colon so values may contain one. Returns "" when absent.
+func optString(opts *pb.ModelOptions, key string) string {
+	for _, o := range opts.GetOptions() {
+		k, v, ok := strings.Cut(o, ":")
+		if ok && strings.TrimSpace(k) == key {
+			return strings.TrimSpace(v)
+		}
+	}
+	return ""
+}
+
+// resolveModelPath makes a relative option path absolute against the models
+// directory, as nemo-speech-cpp does for its auxiliary models.
+func resolveModelPath(base, p string) string {
+	if p == "" || filepath.IsAbs(p) {
+		return p
+	}
+	return filepath.Join(base, p)
+}
+
+// loadDiarModel attaches the diar_model option's GGUF to an ASR model.
+func (p *ParakeetCpp) loadDiarModel(path string) error {
+	if CppDiarizePcm == nil {
+		return fmt.Errorf("parakeet-cpp: diar_model is set but libparakeet.so has no " +
+			"diarization C-API (needs parakeet.cpp ABI >= 7)")
+	}
+	arch, err := parakeetArch(path)
+	if err != nil {
+		return err
+	}
+	if arch != diarModelArch {
+		return fmt.Errorf("parakeet-cpp: diar_model %q is a %q model, not a diarization model", path, arch)
+	}
+	ctx := CppLoad(path)
+	if ctx == 0 {
+		return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for diar_model %q", path)
+	}
+	p.diarCtx = ctx
+	xlog.Info("parakeet-cpp: speaker diarization attached", "diar_model", path)
+	return nil
+}
+
+// diarizerCtx is the context that diarizes: the model itself when it is a
+// diarization model, else the attached diar_model, else 0.
+func (p *ParakeetCpp) diarizerCtx() uintptr {
+	if p.isDiarModel {
+		return p.ctxPtr
+	}
+	return p.diarCtx
+}
+
+// runDiarization diarizes 16 kHz mono PCM. diarMu serializes calls on the
+// diarization context, which is separate from the ASR engine (engineMu).
+func (p *ParakeetCpp) runDiarization(pcm []float32) (diarJSON, error) {
+	p.diarMu.Lock()
+	defer p.diarMu.Unlock()
+	ctx := p.diarizerCtx()
+	if ctx == 0 {
+		return diarJSON{}, grpcerrors.ModelNotLoaded("parakeet-cpp")
+	}
+	if len(pcm) == 0 {
+		return diarJSON{}, nil
+	}
+	cstr := CppDiarizePcm(ctx, pcm, int32(len(pcm)), 16000)
+	if cstr == 0 {
+		return diarJSON{}, fmt.Errorf("parakeet-cpp: diarize failed: %s", CppLastError(ctx))
+	}
+	raw := goStringFromCPtr(cstr)
+	CppFreeString(cstr)
+	var doc diarJSON
+	if err := json.Unmarshal([]byte(raw), &doc); err != nil {
+		return diarJSON{}, fmt.Errorf("parakeet-cpp: decode diarization json: %w", err)
+	}
+	return doc, nil
+}
+
+// assignSpeakers gives each word the speaker whose segments overlap it most,
+// falling back to the nearest segment within speakerSnapSeconds; -1 when none.
+// Same rule as parakeet.cpp's merge_asr_diarization, so the backend's speaker
+// tags match the library's own speaker-attributed ASR.
+func assignSpeakers(words []transcriptWord, segs []diarSegmentDoc) []int {
+	out := make([]int, len(words))
+	for i, w := range words {
+		best, bestOverlap := -1, 0.0
+		for _, s := range segs {
+			if ov := min(w.End, s.End) - max(w.Start, s.Start); ov > bestOverlap {
+				best, bestOverlap = s.Speaker, ov
+			}
+		}
+		if best < 0 {
+			bestDist := speakerSnapSeconds
+			for _, s := range segs {
+				dist := s.Start - w.End
+				if s.End <= w.Start {
+					dist = w.Start - s.End
+				}
+				if dist >= 0 && dist <= bestDist {
+					best, bestDist = s.Speaker, dist
+				}
+			}
+		}
+		out[i] = best
+	}
+	return out
+}
+
+// speakerLabel is the TranscriptSegment/DiarizeSegment speaker string: the
+// 0-based index, or "" for no speaker.
+func speakerLabel(spk int) string {
+	if spk < 0 {
+		return ""
+	}
+	return strconv.Itoa(spk)
+}
+
+// splitAtSpeakerChanges splits each word group wherever the speaker changes, so
+// every segment has exactly one speaker. speakers is indexed like the
+// concatenation of groups. Returns the new groups and each group's speaker.
+func splitAtSpeakerChanges(groups [][]transcriptWord, speakers []int) ([][]transcriptWord, []int) {
+	var outGroups [][]transcriptWord
+	var outSpk []int
+	k := 0
+	for _, g := range groups {
+		start := 0
+		for i := 1; i <= len(g); i++ {
+			if i == len(g) || speakers[k+i] != speakers[k+start] {
+				outGroups = append(outGroups, g[start:i])
+				outSpk = append(outSpk, speakers[k+start])
+				start = i
+			}
+		}
+		k += len(g)
+	}
+	return outGroups, outSpk
+}
+
+// diarizeSegments maps diarization segments (and, when words are given, the
+// transcript words attributed to each speaker) to DiarizeSegments, and counts
+// the distinct speakers. A word belongs to the segment of its assigned speaker
+// that contains its midpoint.
+func diarizeSegments(doc diarJSON, words []transcriptWord) ([]*pb.DiarizeSegment, int32) {
+	var speakers []int
+	if len(words) > 0 {
+		speakers = assignSpeakers(words, doc.Segments)
+	}
+	var out []*pb.DiarizeSegment
+	seen := map[int]bool{}
+	for i, s := range doc.Segments {
+		seg := &pb.DiarizeSegment{
+			Id:      int32(i),
+			Start:   float32(s.Start),
+			End:     float32(s.End),
+			Speaker: speakerLabel(s.Speaker),
+		}
+		if len(words) > 0 {
+			var parts []string
+			for j, w := range words {
+				mid := (w.Start + w.End) / 2
+				if speakers[j] == s.Speaker && mid >= s.Start && mid < s.End {
+					parts = append(parts, w.W)
+				}
+			}
+			seg.Text = strings.Join(parts, " ")
+		}
+		out = append(out, seg)
+		seen[s.Speaker] = true
+	}
+	return out, int32(len(seen))
+}
+
+// unsupportedDiarizeFields names the DiarizeRequest fields Sortformer cannot
+// honour, so they are logged rather than silently dropped. The model is
+// end-to-end: there is no speaker-count input and no clustering stage, and the
+// segment thresholds come from the checkpoint.
+func unsupportedDiarizeFields(req *pb.DiarizeRequest) []string {
+	var out []string
+	if req.GetNumSpeakers() != 0 {
+		out = append(out, "num_speakers")
+	}
+	if req.GetMinSpeakers() != 0 {
+		out = append(out, "min_speakers")
+	}
+	if req.GetMaxSpeakers() != 0 {
+		out = append(out, "max_speakers")
+	}
+	if req.GetClusteringThreshold() != 0 {
+		out = append(out, "clustering_threshold")
+	}
+	if req.GetMinDurationOn() != 0 {
+		out = append(out, "min_duration_on")
+	}
+	if req.GetMinDurationOff() != 0 {
+		out = append(out, "min_duration_off")
+	}
+	return out
+}
+
+// Diarize answers "who spoke when" for the audio at req.Dst. With
+// include_text, and an ASR model loaded alongside the diar_model, each segment
+// also carries the words its speaker said in it.
+func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error) {
+	if p.ctxPtr == 0 {
+		return pb.DiarizeResponse{}, grpcerrors.ModelNotLoaded("parakeet-cpp")
+	}
+	if p.diarizerCtx() == 0 || CppDiarizePcm == nil {
+		return pb.DiarizeResponse{}, status.Error(codes.Unimplemented,
+			"parakeet-cpp: this model has no diarization; load a diarization GGUF "+
+				"or attach one to an ASR model with the diar_model option")
+	}
+	if req.GetDst() == "" {
+		return pb.DiarizeResponse{}, status.Error(codes.InvalidArgument,
+			"parakeet-cpp: DiarizeRequest.dst (audio path) is required")
+	}
+	if f := unsupportedDiarizeFields(req); len(f) > 0 {
+		xlog.Warn("parakeet-cpp: ignoring diarization options Sortformer has no equivalent for",
+			"fields", strings.Join(f, ","))
+	}
+
+	pcm, duration, err := decodeWavMono16k(req.GetDst())
+	if err != nil {
+		return pb.DiarizeResponse{}, err
+	}
+	doc, err := p.runDiarization(pcm)
+	if err != nil {
+		return pb.DiarizeResponse{}, err
+	}
+
+	var words []transcriptWord
+	if req.GetIncludeText() {
+		if p.isDiarModel {
+			xlog.Warn("parakeet-cpp: include_text needs an ASR model with diar_model attached; " +
+				"returning segments without text")
+		} else {
+			tr, err := p.transcribeDoc(pcm, req.GetLanguage())
+			if err != nil {
+				return pb.DiarizeResponse{}, err
+			}
+			words = tr.Words
+		}
+	}
+	segs, n := diarizeSegments(doc, words)
+	return pb.DiarizeResponse{
+		Segments:    segs,
+		NumSpeakers: n,
+		Duration:    duration,
+		Language:    req.GetLanguage(),
+	}, nil
+}
diff --git a/backend/go/parakeet-cpp/diarize_test.go b/backend/go/parakeet-cpp/diarize_test.go
new file mode 100644
index 000000000000..10581fe1acd4
--- /dev/null
+++ b/backend/go/parakeet-cpp/diarize_test.go
@@ -0,0 +1,257 @@
+package main
+
+import (
+	"context"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"strings"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func ds(spk int, start, end float64) diarSegmentDoc {
+	return diarSegmentDoc{Speaker: spk, Start: start, End: end}
+}
+
+// writeGGUFWithArch writes a GGUF v3 file with no tensors and a single string
+// KV, parakeet.arch=arch: enough for parakeetArch, which reads metadata only.
+func writeGGUFWithArch(path, arch string) {
+	GinkgoHelper()
+	var b []byte
+	b = append(b, "GGUF"...)
+	b = binary.LittleEndian.AppendUint32(b, 3) // version
+	b = binary.LittleEndian.AppendUint64(b, 0) // tensor count
+	b = binary.LittleEndian.AppendUint64(b, 1) // kv count
+	str := func(s string) {
+		b = binary.LittleEndian.AppendUint64(b, uint64(len(s)))
+		b = append(b, s...)
+	}
+	str("parakeet.arch")
+	b = binary.LittleEndian.AppendUint32(b, 8) // GGUF string type
+	str(arch)
+	Expect(os.WriteFile(path, b, 0o644)).To(Succeed())
+}
+
+var _ = Describe("speaker diarization helpers", func() {
+	Context("assignSpeakers", func() {
+		It("picks the speaker with the largest overlap", func() {
+			words := []transcriptWord{tw("a", 0.5, 0.8)}
+			segs := []diarSegmentDoc{ds(0, 0.0, 0.6), ds(1, 0.55, 2.0)}
+			Expect(assignSpeakers(words, segs)).To(Equal([]int{1}))
+		})
+
+		It("snaps a word just outside a segment to it, but not a far one", func() {
+			words := []transcriptWord{tw("well", 19.92, 20.00), tw("far", 25.0, 25.2)}
+			segs := []diarSegmentDoc{ds(0, 14.78, 18.75), ds(1, 20.10, 23.60)}
+			Expect(assignSpeakers(words, segs)).To(Equal([]int{1, -1}))
+		})
+
+		It("leaves every word unassigned without segments", func() {
+			Expect(assignSpeakers([]transcriptWord{tw("a", 0, 1)}, nil)).To(Equal([]int{-1}))
+		})
+	})
+
+	Context("splitAtSpeakerChanges", func() {
+		It("splits groups where the speaker changes and keeps the rest intact", func() {
+			groups := [][]transcriptWord{
+				{tw("a", 0, 1), tw("b", 1, 2), tw("c.", 2, 3)},
+				{tw("d", 4, 5)},
+			}
+			out, spk := splitAtSpeakerChanges(groups, []int{0, 0, 1, 1})
+			Expect(out).To(HaveLen(3))
+			Expect(out[0]).To(HaveLen(2))
+			Expect(out[1][0].W).To(Equal("c."))
+			Expect(out[2][0].W).To(Equal("d"))
+			Expect(spk).To(Equal([]int{0, 1, 1}))
+		})
+	})
+
+	Context("transcriptResultWithSpeakers", func() {
+		It("labels segments with their speaker and splits at speaker turns", func() {
+			doc := transcriptJSON{
+				Text: "hi there. hello",
+				Words: []transcriptWord{
+					tw("hi", 0.0, 0.3), tw("there.", 0.3, 0.6), tw("hello", 1.0, 1.4),
+				},
+			}
+			// Punctuation gives [hi there.] [hello]; the turn after "hi" splits
+			// the first group.
+			res := transcriptResultWithSpeakers(doc, &pb.TranscriptRequest{}, 0, []int{0, 1, 1})
+			Expect(res.Segments).To(HaveLen(3))
+			Expect(res.Segments[0].Text).To(Equal("hi"))
+			Expect(res.Segments[0].Speaker).To(Equal("0"))
+			Expect(res.Segments[1].Text).To(Equal("there."))
+			Expect(res.Segments[1].Speaker).To(Equal("1"))
+			Expect(res.Segments[2].Text).To(Equal("hello"))
+			Expect(res.Segments[2].Speaker).To(Equal("1"))
+			for i, seg := range res.Segments {
+				Expect(seg.Id).To(Equal(int32(i)))
+			}
+		})
+
+		It("leaves speakers empty without diarization", func() {
+			doc := transcriptJSON{Text: "hi.", Words: []transcriptWord{tw("hi.", 0, 0.3)}}
+			res := transcriptResultFromDoc(doc, &pb.TranscriptRequest{}, 0)
+			Expect(res.Segments).To(HaveLen(1))
+			Expect(res.Segments[0].Speaker).To(BeEmpty())
+		})
+	})
+
+	Context("diarizeSegments", func() {
+		It("fills each segment's text with its speaker's words", func() {
+			doc := diarJSON{Speakers: 8, Segments: []diarSegmentDoc{ds(0, 0, 2), ds(1, 2, 4)}}
+			words := []transcriptWord{tw("hello", 0.2, 0.6), tw("world", 0.7, 1.1), tw("hi", 2.5, 2.8)}
+			segs, n := diarizeSegments(doc, words)
+			Expect(n).To(Equal(int32(2)))
+			Expect(segs).To(HaveLen(2))
+			Expect(segs[0].Speaker).To(Equal("0"))
+			Expect(segs[0].Text).To(Equal("hello world"))
+			Expect(segs[1].Text).To(Equal("hi"))
+		})
+
+		It("leaves text empty without words", func() {
+			segs, _ := diarizeSegments(diarJSON{Segments: []diarSegmentDoc{ds(0, 0, 1)}}, nil)
+			Expect(segs[0].Text).To(BeEmpty())
+		})
+	})
+
+	Context("parakeetArch", func() {
+		It("reads parakeet.arch from the GGUF metadata", func() {
+			path := filepath.Join(GinkgoT().TempDir(), "m.gguf")
+			writeGGUFWithArch(path, "diarization")
+			arch, err := parakeetArch(path)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(arch).To(Equal("diarization"))
+		})
+
+		It("fails on a file that is not a GGUF", func() {
+			path := filepath.Join(GinkgoT().TempDir(), "x.gguf")
+			Expect(os.WriteFile(path, []byte("nope"), 0o644)).To(Succeed())
+			_, err := parakeetArch(path)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Context("options", func() {
+		It("resolves diar_model against the models directory", func() {
+			opts := &pb.ModelOptions{Options: []string{"batch_max_size:4", "diar_model: diar/model.gguf"}}
+			Expect(optString(opts, "diar_model")).To(Equal("diar/model.gguf"))
+			Expect(resolveModelPath("/models", "diar/model.gguf")).To(Equal("/models/diar/model.gguf"))
+			Expect(resolveModelPath("/models", "/abs/m.gguf")).To(Equal("/abs/m.gguf"))
+		})
+
+		It("names the request fields Sortformer ignores", func() {
+			Expect(unsupportedDiarizeFields(&pb.DiarizeRequest{NumSpeakers: 2, MinDurationOn: 0.3})).
+				To(ConsistOf("num_speakers", "min_duration_on"))
+			Expect(unsupportedDiarizeFields(&pb.DiarizeRequest{IncludeText: true})).To(BeEmpty())
+		})
+	})
+})
+
+// diarFixturesOrSkip returns the diarization model, the ASR model and a
+// multi-speaker 16 kHz WAV (parakeet.cpp's tests/fixtures/two_speakers.wav:
+// two LibriSpeech speakers alternating A-B-A-B), or skips.
+func diarFixturesOrSkip() (diarModel, asrModel, wavPath string) {
+	diarModel = os.Getenv("PARAKEET_BACKEND_TEST_DIAR_MODEL")
+	asrModel = os.Getenv("PARAKEET_BACKEND_TEST_MODEL")
+	wavPath = os.Getenv("PARAKEET_BACKEND_TEST_DIAR_WAV")
+	if diarModel == "" || asrModel == "" || wavPath == "" {
+		Skip("set PARAKEET_BACKEND_TEST_DIAR_MODEL, PARAKEET_BACKEND_TEST_MODEL and " +
+			"PARAKEET_BACKEND_TEST_DIAR_WAV to run this spec")
+	}
+	return diarModel, asrModel, wavPath
+}
+
+// turns collapses consecutive repeats: [0 0 1 0] -> [0 1 0].
+func turns(labels []string) []string {
+	var out []string
+	for _, l := range labels {
+		if len(out) == 0 || out[len(out)-1] != l {
+			out = append(out, l)
+		}
+	}
+	return out
+}
+
+var _ = Describe("ParakeetCpp speaker diarization", func() {
+	It("diarizes with a diarization model and refuses to transcribe", func() {
+		diarModel, _, wavPath := diarFixturesOrSkip()
+		ensureLibLoaded()
+		if CppDiarizePcm == nil {
+			Skip("libparakeet.so has no diarization C-API")
+		}
+		p := &ParakeetCpp{}
+		Expect(p.Load(&pb.ModelOptions{ModelFile: diarModel})).To(Succeed())
+		defer func() { _ = p.Free() }()
+
+		res, err := p.Diarize(&pb.DiarizeRequest{Dst: wavPath})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(res.NumSpeakers).To(Equal(int32(2)))
+		labels := make([]string, len(res.Segments))
+		for i, s := range res.Segments {
+			labels[i] = s.Speaker
+			Expect(s.End).To(BeNumerically(">", s.Start))
+		}
+		Expect(turns(labels)).To(Equal([]string{"0", "1", "0", "1"}))
+
+		_, err = p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath})
+		Expect(status.Code(err)).To(Equal(codes.FailedPrecondition))
+	})
+
+	It("tags transcript segments with speakers when diar_model is attached", func() {
+		diarModel, asrModel, wavPath := diarFixturesOrSkip()
+		ensureLibLoaded()
+		if CppDiarizePcm == nil {
+			Skip("libparakeet.so has no diarization C-API")
+		}
+		p := &ParakeetCpp{}
+		Expect(p.Load(&pb.ModelOptions{
+			ModelFile: asrModel,
+			Options:   []string{"diar_model:" + diarModel},
+		})).To(Succeed())
+		defer func() { _ = p.Free() }()
+
+		res, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath, Diarize: true})
+		Expect(err).ToNot(HaveOccurred())
+		labels := make([]string, len(res.Segments))
+		for i, s := range res.Segments {
+			labels[i] = s.Speaker
+			Expect(s.Speaker).ToNot(BeEmpty(), "segment %d %q has no speaker", i, s.Text)
+		}
+		Expect(turns(labels)).To(Equal([]string{"0", "1", "0", "1"}))
+
+		// diarize=false keeps the plain transcript.
+		plain, err := p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: wavPath})
+		Expect(err).ToNot(HaveOccurred())
+		for _, s := range plain.Segments {
+			Expect(s.Speaker).To(BeEmpty())
+		}
+		Expect(plain.Text).To(Equal(res.Text))
+
+		// Diarize with include_text attributes the words to the segments.
+		d, err := p.Diarize(&pb.DiarizeRequest{Dst: wavPath, IncludeText: true})
+		Expect(err).ToNot(HaveOccurred())
+		var all []string
+		for _, s := range d.Segments {
+			if s.Text != "" {
+				all = append(all, s.Text)
+			}
+		}
+		Expect(strings.Join(all, " ")).To(ContainSubstring("Quilter"))
+	})
+
+	It("reports diarization as unimplemented on a plain ASR model", func() {
+		_, asrModel, wavPath := diarFixturesOrSkip()
+		ensureLibLoaded()
+		p := &ParakeetCpp{}
+		Expect(p.Load(&pb.ModelOptions{ModelFile: asrModel})).To(Succeed())
+		defer func() { _ = p.Free() }()
+		_, err := p.Diarize(&pb.DiarizeRequest{Dst: wavPath})
+		Expect(status.Code(err)).To(Equal(codes.Unimplemented))
+	})
+})
diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go
index 23e5c548ce0c..95621e1c1be2 100644
--- a/backend/go/parakeet-cpp/goparakeetcpp.go
+++ b/backend/go/parakeet-cpp/goparakeetcpp.go
@@ -148,6 +148,14 @@ type ParakeetCpp struct {
 	// YAML option, default 0=off). When >0 it adds NeMo's silence-gap split on
 	// top of the punctuation split; converted to seconds via the JSON frame_sec.
 	segmentGapFrames int
+
+	// Speaker diarization (see diarize.go). isDiarModel: ctxPtr itself holds a
+	// diarization model, so there is no ASR. diarCtx: a diarization model
+	// attached to an ASR model through the diar_model option. diarMu
+	// serializes calls on whichever context diarizes.
+	isDiarModel bool
+	diarCtx     uintptr
+	diarMu      sync.Mutex
 }
 
 // Load is the LocalAI gRPC entry point for LoadModel: it calls
@@ -158,6 +166,16 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error {
 		return errors.New("parakeet-cpp: ModelFile is required")
 	}
 
+	// A diarization GGUF loads into the same kind of context but serves only
+	// Diarize: no ASR, so none of the transcription setup below applies.
+	if arch, err := parakeetArch(opts.ModelFile); err == nil && arch == diarModelArch {
+		if CppDiarizePcm == nil {
+			return fmt.Errorf("parakeet-cpp: %q is a diarization model but libparakeet.so has "+
+				"no diarization C-API (needs parakeet.cpp ABI >= 7)", opts.ModelFile)
+		}
+		p.isDiarModel = true
+	}
+
 	ctx := CppLoad(opts.ModelFile)
 	if ctx == 0 {
 		// No ctx to ask for last_error (the C-API's last-error buffer
@@ -166,6 +184,19 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error {
 		return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for %q", opts.ModelFile)
 	}
 	p.ctxPtr = ctx
+	if p.isDiarModel {
+		xlog.Info("parakeet-cpp: loaded a speaker diarization model (Diarize only)")
+		return nil
+	}
+
+	// Optional speaker diarization for transcripts (diarize.go).
+	if dm := optString(opts, "diar_model"); dm != "" {
+		if err := p.loadDiarModel(resolveModelPath(opts.ModelPath, dm)); err != nil {
+			CppFree(p.ctxPtr)
+			p.ctxPtr = 0
+			return err
+		}
+	}
 
 	// Dynamic batching knobs (model YAML options:, key:value form). Batching is
 	// OFF by default (batch_max_size:1): each request runs on its own. On GPU,
@@ -287,8 +318,9 @@ func (p *ParakeetCpp) runBatch(reqs []*batchRequest) {
 // OpenAI API, whose default is segment-level); token ids always populate
 // Segment.Tokens.
 //
-// translate/diarize/prompt/temperature/threads are not applicable to parakeet
-// and are ignored; language is honored on the batched + streaming paths (see
+// With a diar_model attached, diarize=true tags each segment with its speaker
+// (diarize.go). translate/prompt/temperature/threads are not applicable to
+// parakeet and are ignored; language is honored on the batched + streaming paths (see
 // opts.GetLanguage() below); streaming is handled by AudioTranscriptionStream
 // (L2).
 func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.TranscriptRequest) (pb.TranscriptResult, error) {
@@ -298,6 +330,9 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip
 	if opts.Dst == "" {
 		return pb.TranscriptResult{}, errors.New("parakeet-cpp: TranscriptRequest.dst (audio path) is required")
 	}
+	if p.isDiarModel {
+		return pb.TranscriptResult{}, errDiarModelNoASR
+	}
 
 	// Fallback when the batched C-API is unavailable: transcribe from a file
 	// path (original behavior, no batching). The C library's audio loader only
@@ -331,26 +366,63 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip
 	if err != nil {
 		return pb.TranscriptResult{}, err
 	}
+	doc, err := p.transcribeDocCtx(ctx, pcm, opts.GetLanguage())
+	if err != nil {
+		return pb.TranscriptResult{}, err
+	}
+
+	// With a diar_model attached, tag each segment with its speaker. The
+	// request's diarize flag can turn it off (the OpenAI endpoint sends true
+	// unless the client passes diarize=false).
+	var speakers []int
+	if p.diarCtx != 0 && opts.GetDiarize() && len(doc.Words) > 0 {
+		dd, err := p.runDiarization(pcm)
+		if err != nil {
+			return pb.TranscriptResult{}, err
+		}
+		speakers = assignSpeakers(doc.Words, dd.Segments)
+	}
+	return transcriptResultWithSpeakers(doc, opts, p.segmentGapFrames, speakers), nil
+}
+
+// errDiarModelNoASR is returned by the transcription RPCs on a diarization
+// model: it answers Diarize, it cannot transcribe.
+var errDiarModelNoASR = status.Error(codes.FailedPrecondition,
+	"parakeet-cpp: this is a speaker diarization model; use the diarization endpoint, "+
+		"or attach it to an ASR model with the diar_model option")
+
+// transcribeDocCtx runs one clip through the batcher and returns its decoded
+// transcript document. Honours ctx cancellation on both channel operations.
+func (p *ParakeetCpp) transcribeDocCtx(ctx context.Context, pcm []float32, lang string) (transcriptJSON, error) {
+	if p.bat == nil {
+		return transcriptJSON{}, errors.New("parakeet-cpp: libparakeet.so has no batched C-API")
+	}
 	rep := make(chan batchReply, 1)
 	select {
-	case p.bat.submit <- &batchRequest{pcm: pcm, decoder: 0, language: opts.GetLanguage(), reply: rep}:
+	case p.bat.submit <- &batchRequest{pcm: pcm, decoder: 0, language: lang, reply: rep}:
 	case <-ctx.Done():
-		return pb.TranscriptResult{}, status.Error(codes.Canceled, "transcription cancelled")
+		return transcriptJSON{}, status.Error(codes.Canceled, "transcription cancelled")
 	}
 	var res batchReply
 	select {
 	case res = <-rep:
 	case <-ctx.Done():
-		return pb.TranscriptResult{}, status.Error(codes.Canceled, "transcription cancelled")
+		return transcriptJSON{}, status.Error(codes.Canceled, "transcription cancelled")
 	}
 	if res.err != nil {
-		return pb.TranscriptResult{}, res.err
+		return transcriptJSON{}, res.err
 	}
 	var doc transcriptJSON
 	if err := json.Unmarshal([]byte(res.json), &doc); err != nil {
-		return pb.TranscriptResult{}, fmt.Errorf("parakeet-cpp: decode transcript json: %w", err)
+		return transcriptJSON{}, fmt.Errorf("parakeet-cpp: decode transcript json: %w", err)
 	}
-	return transcriptResultFromDoc(doc, opts, p.segmentGapFrames), nil
+	return doc, nil
+}
+
+// transcribeDoc is transcribeDocCtx for callers without a request context
+// (the Diarize RPC).
+func (p *ParakeetCpp) transcribeDoc(pcm []float32, lang string) (transcriptJSON, error) {
+	return p.transcribeDocCtx(context.Background(), pcm, lang)
 }
 
 // segmentSeparators is NeMo's default segment_seperators (sentence-ending
@@ -365,6 +437,13 @@ var segmentSeparators = []rune{'.', '?', '!'}
 // the caller requested word granularity; token ids populate each segment's
 // Tokens by time-window membership. Shared by the batched and direct paths.
 func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gapFrames int) pb.TranscriptResult {
+	return transcriptResultWithSpeakers(doc, opts, gapFrames, nil)
+}
+
+// transcriptResultWithSpeakers is transcriptResultFromDoc plus optional
+// per-word speakers (indexed like doc.Words, -1 = none): segments additionally
+// split wherever the speaker changes and carry the speaker's label.
+func transcriptResultWithSpeakers(doc transcriptJSON, opts *pb.TranscriptRequest, gapFrames int, speakers []int) pb.TranscriptResult {
 	text, eou := stripEouMarker(strings.TrimSpace(doc.Text))
 
 	// Frame-unit gap threshold -> seconds (NeMo segment_gap_threshold). 0 = off.
@@ -388,6 +467,11 @@ func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gap
 		}
 	}
 
+	var groupSpeakers []int
+	if len(speakers) == len(doc.Words) && speakers != nil {
+		groups, groupSpeakers = splitAtSpeakerChanges(groups, speakers)
+	}
+
 	wantWords := wordsRequested(opts.TimestampGranularities)
 	segments := make([]*pb.TranscriptSegment, 0, len(groups))
 	for id, group := range groups {
@@ -402,6 +486,9 @@ func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gap
 			Text:   strings.TrimSpace(strings.Join(parts, " ")),
 			Tokens: tokensInWindow(doc.Tokens, group[0].Start, group[len(group)-1].End),
 		}
+		if groupSpeakers != nil {
+			seg.Speaker = speakerLabel(groupSpeakers[id])
+		}
 		if wantWords {
 			ws := make([]*pb.TranscriptWord, len(group))
 			for i, gw := range group {
@@ -695,6 +782,9 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra
 		return status.Error(codes.Canceled, "transcription cancelled")
 	}
 
+	if p.isDiarModel {
+		return errDiarModelNoASR
+	}
 	stream, err := p.streamBegin(opts.GetLanguage())
 	if err != nil {
 		return err
@@ -834,6 +924,12 @@ func (p *ParakeetCpp) Free() error {
 		CppFree(p.ctxPtr)
 		p.ctxPtr = 0
 	}
+	p.diarMu.Lock()
+	defer p.diarMu.Unlock()
+	if p.diarCtx != 0 {
+		CppFree(p.diarCtx)
+		p.diarCtx = 0
+	}
 	return nil
 }
 
diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go
index a6f6af1f04a0..fcd1b8f4f9f4 100644
--- a/backend/go/parakeet-cpp/goparakeetcpp_test.go
+++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go
@@ -59,6 +59,9 @@ func ensureLibLoaded() {
 			purego.RegisterLibFunc(&CppStreamFeedJSON, lib, "parakeet_capi_stream_feed_json")
 			purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json")
 		}
+		if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_pcm"); err == nil && sym != 0 {
+			purego.RegisterLibFunc(&CppDiarizePcm, lib, "parakeet_capi_diarize_pcm")
+		}
 		purego.RegisterLibFunc(&CppFreeString, lib, "parakeet_capi_free_string")
 		purego.RegisterLibFunc(&CppLastError, lib, "parakeet_capi_last_error")
 	})
diff --git a/backend/go/parakeet-cpp/live.go b/backend/go/parakeet-cpp/live.go
index 3d68a2914be1..bfd58e809f34 100644
--- a/backend/go/parakeet-cpp/live.go
+++ b/backend/go/parakeet-cpp/live.go
@@ -43,6 +43,9 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest
 	if p.ctxPtr == 0 {
 		return grpcerrors.ModelNotLoaded("parakeet-cpp")
 	}
+	if p.isDiarModel {
+		return errDiarModelNoASR
+	}
 
 	first, ok := <-in
 	if !ok {
diff --git a/backend/go/parakeet-cpp/main.go b/backend/go/parakeet-cpp/main.go
index 9c6466b131e5..d04f78016aa3 100644
--- a/backend/go/parakeet-cpp/main.go
+++ b/backend/go/parakeet-cpp/main.go
@@ -90,6 +90,13 @@ func main() {
 		purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json")
 	}
 
+	// Speaker diarization (ABI v7): a diarization GGUF, or one attached to an
+	// ASR model with the diar_model option. Same probe pattern; absent in older
+	// libparakeet.so, where diarization is reported as unavailable.
+	if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_pcm"); err == nil && sym != 0 {
+		purego.RegisterLibFunc(&CppDiarizePcm, lib, "parakeet_capi_diarize_pcm")
+	}
+
 	fmt.Fprintf(os.Stderr, "[parakeet-cpp] ABI=%d\n", CppAbiVersion())
 
 	flag.Parse()
diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go
index 886ddd22a185..de54b65e3614 100644
--- a/core/config/backend_capabilities.go
+++ b/core/config/backend_capabilities.go
@@ -477,11 +477,14 @@ var BackendCapabilities = map[string]BackendCapability{
 		DefaultUsecases:  []string{UsecaseTranscript},
 		Description:      "NVIDIA NeMo speech recognition",
 	},
+	// parakeet-cpp: an ASR GGUF transcribes (and tags speakers when a
+	// diarization GGUF is attached with the diar_model option); a diarization
+	// GGUF (Nemotron-3-Diarization) only answers Diarize.
 	"parakeet-cpp": {
-		GRPCMethods:      []GRPCMethod{MethodAudioTranscription},
-		PossibleUsecases: []string{UsecaseTranscript},
+		GRPCMethods:      []GRPCMethod{MethodAudioTranscription, MethodDiarize},
+		PossibleUsecases: []string{UsecaseTranscript, UsecaseDiarization},
 		DefaultUsecases:  []string{UsecaseTranscript},
-		Description:      "NVIDIA NeMo Parakeet ASR (parakeet.cpp)",
+		Description:      "NVIDIA NeMo Parakeet ASR and Nemotron-3-Diarization speaker diarization (parakeet.cpp)",
 	},
 	// nemo-speech-cpp is one gRPC server in front of four NeMo-Speech.cpp model
 	// families, picked at load time from the GGUF general.architecture key, so
diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md
index 83a51343e561..cb8f866ddd4e 100644
--- a/docs/content/features/audio-diarization.md
+++ b/docs/content/features/audio-diarization.md
@@ -9,11 +9,12 @@ url = "/features/audio-diarization/"
 
 Speaker diarization answers the question **"who spoke when?"** - given an audio clip with multiple speakers, it returns time-stamped segments labelled with a stable speaker ID (`SPEAKER_00`, `SPEAKER_01`, …).
 
-LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Four backends are supported today:
+LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Five backends are supported today:
 
 - **[sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx)** - pyannote-3.0 segmentation + a speaker-embedding extractor (3D-Speaker, NeMo, WeSpeaker) + fast clustering. Pure diarization - no transcription cost. Recommended when you only need speaker turns.
 - **[vibevoice.cpp](https://github.com/microsoft/VibeVoice)** - produces speaker-labelled segments as a by-product of its long-form ASR pass, so you can optionally get a transcript per segment for free.
 - **[NeMo-Speech.cpp](https://github.com/NVIDIA/NeMo-Speech.cpp)** - NVIDIA Sortformer, served standalone by the [NeMo-Speech.cpp backend]({{%relref "features/nemo-speech-cpp" %}}). It is end to end, so the speaker capacity is fixed by the checkpoint and the count hints are ignored. The same backend can instead put speaker tags on a transcript, by attaching a Sortformer model to an ASR one.
+- **[parakeet.cpp](https://github.com/mudler/parakeet.cpp)** - NVIDIA Nemotron-3-Diarization (Sortformer, up to 8 speakers), matched against NeMo. A diarization model runs on its own, or attaches to a Parakeet ASR model to put speaker labels on transcripts and text on diarization segments.
 - **[audio.cpp](https://github.com/0xShug0/audio.cpp)** - the `sortformer_diar` family, served by the multi-modality [audio.cpp backend]({{%relref "features/audio-cpp" %}}).
 
 Because diarization is exposed as a regular OpenAI-compatible endpoint, any HTTP client works. There is no Python dependency on pyannote or NeMo on the consumer side.
@@ -105,7 +106,7 @@ curl http://localhost:8080/v1/audio/diarization \
   -F num_speakers=3
 ```
 
-The sections below show how to configure the two supported backends by hand when you want full control over the segmentation and embedding models.
+The sections below show how to configure some of the supported backends by hand when you want full control over the segmentation and embedding models.
 
 ## Backend setup - sherpa-onnx (pure diarization)
 
@@ -157,6 +158,43 @@ curl http://localhost:8080/v1/audio/diarization \
   -F response_format=verbose_json
 ```
 
+## Backend setup - parakeet.cpp (Nemotron-3-Diarization)
+
+The [parakeet-cpp backend]({{%relref "features/audio-to-text#using-the-parakeet-cpp-backend" %}}) runs [nvidia/Nemotron-3-Diarization](https://huggingface.co/nvidia/Nemotron-3-Diarization). Convert the checkpoint to GGUF with parakeet.cpp's converter (Q8_0 is 109 MB and gives the same segments as F32):
+
+```bash
+python scripts/convert_parakeet_to_gguf.py \
+    --model nvidia/Nemotron-3-Diarization --dtype q8_0 \
+    --output nemotron-3-diarization-q8_0.gguf
+```
+
+A diarization-only model:
+
+```yaml
+name: nemotron-diarization
+backend: parakeet-cpp
+parameters:
+  model: nemotron-3-diarization-q8_0.gguf
+known_usecases:
+  - FLAG_DIARIZATION
+```
+
+Attach the same GGUF to a Parakeet ASR model with `diar_model` to get both: transcripts gain a speaker per segment, and `include_text=true` on `/v1/audio/diarization` fills each segment with the words its speaker said.
+
+```yaml
+name: parakeet-speakers
+backend: parakeet-cpp
+parameters:
+  model: tdt-0.6b-v3-f16.gguf
+options:
+  - diar_model:nemotron-3-diarization-q8_0.gguf
+known_usecases:
+  - FLAG_TRANSCRIPT
+  - FLAG_DIARIZATION
+```
+
+Sortformer is end to end: the speaker count, clustering and segment-duration fields of the request have no equivalent and are ignored (the backend logs which ones). It processes audio the way NeMo's `diarize()` does for this checkpoint, in 21 s chunks with a speaker cache, so long recordings keep consistent speaker IDs: a 12 minute, 3 speaker recording diarizes in about 7 s on a desktop CPU.
+
 ## Notes
 
 - **Speaker identity across files**: speaker IDs (`SPEAKER_00`, `SPEAKER_01`, …) are local to each request. To track the same person across multiple recordings, combine `/v1/audio/diarization` with `/v1/voice/embed` (speaker embedding) and maintain your own embedding store.
diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md
index 5a5e833cf3a3..5b3937344107 100644
--- a/docs/content/features/audio-to-text.md
+++ b/docs/content/features/audio-to-text.md
@@ -205,6 +205,21 @@ options:
 - segment_gap_threshold:12   # split on silence > 12 encoder frames (default 0 = off, punctuation-only)
 ```
 
+### Speaker labels
+
+Attach a speaker diarization model with the `diar_model` option and each transcript segment carries a `speaker` (`"0"`, `"1"`, ... in order of first appearance). Segments split wherever the speaker changes, on top of the punctuation split. The model is NVIDIA Nemotron-3-Diarization, converted with parakeet.cpp's converter; see [Speaker Diarization]({{< relref "audio-diarization.md" >}}) for the conversion and for the `/v1/audio/diarization` endpoint the same model serves.
+
+```yaml
+name: parakeet-speakers
+backend: parakeet-cpp
+parameters:
+  model: tdt-0.6b-v3-f16.gguf
+options:
+- diar_model:nemotron-3-diarization-q8_0.gguf   # relative to the models directory
+```
+
+The `diarize` form field turns it off per request (`-F diarize=false`). Streaming transcription does not carry speaker labels.
+
 ### Dynamic batching
 
 The backend can coalesce concurrent transcription requests into a single batched engine call, which improves throughput on GPU when many requests arrive at once. Batching is **off by default** (`batch_max_size:1`, one request at a time); raise it to opt in. Two `options:` knobs control it:
diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md
index 40c6ae4f34e5..69e1cfdc35b1 100644
--- a/docs/content/reference/compatibility-table.md
+++ b/docs/content/reference/compatibility-table.md
@@ -43,7 +43,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({
 | [faster-whisper](https://github.com/SYSTRAN/faster-whisper) | Fast Whisper with CTranslate2 | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal, Jetson L4T |
 | [WhisperX](https://github.com/m-bain/whisperX) | Word-level timestamps and speaker diarization | CPU, CUDA 12/13, Metal, Jetson L4T |
 | [moonshine](https://github.com/moonshine-ai/moonshine) | Ultra-fast transcription for low-end devices (ONNX) | CPU, CUDA 12/13, Metal |
-| [parakeet.cpp](https://github.com/mudler/parakeet.cpp) | C++/GGML port of NVIDIA NeMo Parakeet (tdt/ctc/rnnt/hybrid), with cache-aware streaming | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T |
+| [parakeet.cpp](https://github.com/mudler/parakeet.cpp) | C++/GGML port of NVIDIA NeMo Parakeet (tdt/ctc/rnnt/hybrid), with cache-aware streaming and Nemotron-3-Diarization speaker diarization | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T |
 | [CrispASR](https://github.com/CrispStrobe/CrispASR) | Unified speech engine (whisper.cpp fork) supporting Parakeet, Canary, and many ASR architectures, plus TTS | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T |
 | [voxtral](https://github.com/antirez/voxtral.c) | Voxtral Realtime 4B speech-to-text in pure C | CPU, Metal |
 | [Qwen3-ASR](https://github.com/QwenLM/Qwen3-ASR) | Qwen3 automatic speech recognition | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal, Jetson L4T |

From 761505237d8bb42e59deb59537a215fa057c6b12 Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Mon, 28 Sep 2026 06:57:51 +0000
Subject: [PATCH 2/5] feat(importers): detect the parakeet.cpp diarization GGUF

The Nemotron-3-Diarization GGUFs are now published in
mudler/parakeet-cpp-gguf as nemotron-3-diarization-<quant>.gguf. The
parakeet-cpp importer did not recognise that name, so a direct import
fell through to another importer.

A direct URL to the file now imports with the diarization usecase. A
repo import still picks ASR weights when the repo also ships the
diarization model, and falls back to the diarization weights only
when there are no others. The docs import the published file instead
of converting the checkpoint.

Assisted-by: Claude:claude-opus-5-5 [Claude Code]
---
 core/gallery/importers/parakeet-cpp.go      | 31 +++++++++++++--
 core/gallery/importers/parakeet-cpp_test.go | 42 +++++++++++++++++++++
 docs/content/features/audio-diarization.md  |  8 ++--
 docs/content/features/audio-to-text.md      |  2 +-
 4 files changed, 74 insertions(+), 9 deletions(-)

diff --git a/core/gallery/importers/parakeet-cpp.go b/core/gallery/importers/parakeet-cpp.go
index aa732aa3e7fc..5b758167b457 100644
--- a/core/gallery/importers/parakeet-cpp.go
+++ b/core/gallery/importers/parakeet-cpp.go
@@ -108,6 +108,11 @@ func (i *ParakeetCppImporter) Import(details Details) (gallery.ModelConfig, erro
 
 	uri := downloader.URI(details.URI)
 	directGGUF := isParakeetGGUF(filepath.Base(details.URI))
+	// A speaker diarization GGUF is served by the same backend but answers
+	// /v1/audio/diarization, not transcription.
+	if directGGUF && isParakeetDiarGGUF(filepath.Base(details.URI)) {
+		modelConfig.KnownUsecaseStrings = []string{"diarization"}
+	}
 	switch {
 	case uri.LooksLikeURL() && directGGUF:
 		// Direct file URL (e.g. .../resolve/main/tdt_ctc-110m-f16.gguf). The
@@ -128,12 +133,23 @@ func (i *ParakeetCppImporter) Import(details Details) (gallery.ModelConfig, erro
 		// HF repo: collect every parakeet GGUF, pick the preferred quant, and
 		// nest under parakeet-cpp/models/<name>/ so a multi-quant repo doesn't
 		// collide on disk.
-		var ggufFiles []hfapi.ModelFile
+		// Prefer ASR weights: a repo that also ships the diarization model
+		// (mudler/parakeet-cpp-gguf) imports as a transcription model, and the
+		// diarization GGUF is imported by its direct URL. A repo with only
+		// diarization weights imports as a diarization model.
+		var ggufFiles, diarFiles []hfapi.ModelFile
 		for _, f := range details.HuggingFace.Files {
-			if isParakeetGGUF(filepath.Base(f.Path)) {
+			switch base := filepath.Base(f.Path); {
+			case isParakeetDiarGGUF(base):
+				diarFiles = append(diarFiles, f)
+			case isParakeetGGUF(base):
 				ggufFiles = append(ggufFiles, f)
 			}
 		}
+		if len(ggufFiles) == 0 && len(diarFiles) > 0 {
+			ggufFiles = diarFiles
+			modelConfig.KnownUsecaseStrings = []string{"diarization"}
+		}
 		if chosen, ok := pickPreferredGGMLFile(ggufFiles, quants); ok {
 			target := filepath.Join("parakeet-cpp", "models", name, filepath.Base(chosen.Path))
 			cfg.Files = append(cfg.Files, gallery.File{
@@ -176,5 +192,14 @@ func isParakeetGGUF(name string) bool {
 			return true
 		}
 	}
-	return false
+	return isParakeetDiarGGUF(name)
+}
+
+// isParakeetDiarGGUF reports whether name is the parakeet.cpp speaker
+// diarization GGUF (nemotron-3-diarization-<quant>.gguf). Matched by its
+// published name only, so diarization weights for other backends are not
+// claimed.
+func isParakeetDiarGGUF(name string) bool {
+	lower := strings.ToLower(name)
+	return strings.HasSuffix(lower, ".gguf") && strings.Contains(lower, "nemotron-3-diarization")
 }
diff --git a/core/gallery/importers/parakeet-cpp_test.go b/core/gallery/importers/parakeet-cpp_test.go
index 4aa87c4110e6..8636c929de6d 100644
--- a/core/gallery/importers/parakeet-cpp_test.go
+++ b/core/gallery/importers/parakeet-cpp_test.go
@@ -50,6 +50,11 @@ var _ = Describe("ParakeetCppImporter", func() {
 			Expect(imp.Match(d)).To(BeTrue())
 		})
 
+		It("matches a direct URL to the diarization GGUF", func() {
+			d := parakeetDetails("https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf", `{}`)
+			Expect(imp.Match(d)).To(BeTrue())
+		})
+
 		It("does NOT claim a generic llama-style GGUF", func() {
 			d := parakeetDetails("huggingface://someorg/some-llm-gguf", `{}`,
 				hfapi.ModelFile{Path: "llama-3-8b-instruct-q4_k_m.gguf"},
@@ -66,6 +71,43 @@ var _ = Describe("ParakeetCppImporter", func() {
 	})
 
 	Context("import (Import)", func() {
+		It("imports the diarization GGUF as a diarization model", func() {
+			d := parakeetDetails("https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf",
+				`{"name":"nemotron-diarization"}`)
+			cfg, err := imp.Import(d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(cfg.ConfigFile).To(ContainSubstring("backend: parakeet-cpp"))
+			Expect(cfg.ConfigFile).To(ContainSubstring("diarization"))
+			Expect(cfg.ConfigFile).ToNot(ContainSubstring("transcript"))
+			Expect(cfg.Files).To(HaveLen(1))
+			Expect(cfg.Files[0].Filename).To(HaveSuffix("nemotron-3-diarization-q8_0.gguf"))
+		})
+
+		It("keeps picking ASR weights from a repo that also ships the diarization model", func() {
+			d := parakeetDetails("huggingface://mudler/parakeet-cpp-gguf", `{"name":"parakeet-110m"}`,
+				hfapi.ModelFile{Path: "nemotron-3-diarization-f16.gguf", URL: "https://hf/diar-f16", SHA256: "ddd"},
+				hfapi.ModelFile{Path: "tdt_ctc-110m-f16.gguf", URL: "https://hf/f16", SHA256: "aaa"},
+				hfapi.ModelFile{Path: "nemotron-3-diarization-q8_0.gguf", URL: "https://hf/diar-q8", SHA256: "eee"},
+			)
+			cfg, err := imp.Import(d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(cfg.Files).To(HaveLen(1))
+			Expect(cfg.Files[0].URI).To(Equal("https://hf/f16"))
+			Expect(cfg.ConfigFile).To(ContainSubstring("transcript"))
+		})
+
+		It("imports a diarization-only repo as a diarization model", func() {
+			d := parakeetDetails("huggingface://someone/diar-gguf", `{"name":"diar"}`,
+				hfapi.ModelFile{Path: "nemotron-3-diarization-f16.gguf", URL: "https://hf/diar-f16", SHA256: "ddd"},
+				hfapi.ModelFile{Path: "nemotron-3-diarization-q8_0.gguf", URL: "https://hf/diar-q8", SHA256: "eee"},
+			)
+			cfg, err := imp.Import(d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(cfg.Files).To(HaveLen(1))
+			Expect(cfg.Files[0].URI).To(Equal("https://hf/diar-q8"), "default quant ladder picks q8_0 before f16")
+			Expect(cfg.ConfigFile).To(ContainSubstring("diarization"))
+		})
+
 		It("picks the default quant (q4_k) from a multi-quant HF repo", func() {
 			d := parakeetDetails("huggingface://mudler/parakeet-cpp-gguf", `{"name":"parakeet-110m"}`,
 				hfapi.ModelFile{Path: "tdt_ctc-110m-f16.gguf", URL: "https://hf/f16", SHA256: "aaa"},
diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md
index cb8f866ddd4e..3bfde1aef164 100644
--- a/docs/content/features/audio-diarization.md
+++ b/docs/content/features/audio-diarization.md
@@ -160,15 +160,13 @@ curl http://localhost:8080/v1/audio/diarization \
 
 ## Backend setup - parakeet.cpp (Nemotron-3-Diarization)
 
-The [parakeet-cpp backend]({{%relref "features/audio-to-text#using-the-parakeet-cpp-backend" %}}) runs [nvidia/Nemotron-3-Diarization](https://huggingface.co/nvidia/Nemotron-3-Diarization). Convert the checkpoint to GGUF with parakeet.cpp's converter (Q8_0 is 109 MB and gives the same segments as F32):
+The [parakeet-cpp backend]({{%relref "features/audio-to-text#using-the-parakeet-cpp-backend" %}}) runs [nvidia/Nemotron-3-Diarization](https://huggingface.co/nvidia/Nemotron-3-Diarization). GGUF weights are published in [`mudler/parakeet-cpp-gguf`](https://huggingface.co/mudler/parakeet-cpp-gguf): F16 (191 MB) and Q8_0 (104 MB) both give the same segments as NeMo. Import one directly:
 
 ```bash
-python scripts/convert_parakeet_to_gguf.py \
-    --model nvidia/Nemotron-3-Diarization --dtype q8_0 \
-    --output nemotron-3-diarization-q8_0.gguf
+local-ai models import https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/nemotron-3-diarization-q8_0.gguf
 ```
 
-A diarization-only model:
+Or configure it by hand, as a diarization-only model:
 
 ```yaml
 name: nemotron-diarization
diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md
index 5b3937344107..f927cc121c34 100644
--- a/docs/content/features/audio-to-text.md
+++ b/docs/content/features/audio-to-text.md
@@ -207,7 +207,7 @@ options:
 
 ### Speaker labels
 
-Attach a speaker diarization model with the `diar_model` option and each transcript segment carries a `speaker` (`"0"`, `"1"`, ... in order of first appearance). Segments split wherever the speaker changes, on top of the punctuation split. The model is NVIDIA Nemotron-3-Diarization, converted with parakeet.cpp's converter; see [Speaker Diarization]({{< relref "audio-diarization.md" >}}) for the conversion and for the `/v1/audio/diarization` endpoint the same model serves.
+Attach a speaker diarization model with the `diar_model` option and each transcript segment carries a `speaker` (`"0"`, `"1"`, ... in order of first appearance). Segments split wherever the speaker changes, on top of the punctuation split. The model is NVIDIA Nemotron-3-Diarization, published as `nemotron-3-diarization-f16.gguf` and `nemotron-3-diarization-q8_0.gguf` in [`mudler/parakeet-cpp-gguf`](https://huggingface.co/mudler/parakeet-cpp-gguf); see [Speaker Diarization]({{< relref "audio-diarization.md" >}}) for the conversion and for the `/v1/audio/diarization` endpoint the same model serves.
 
 ```yaml
 name: parakeet-speakers

From 2f0049f979496c3428c08c9b1d5bf4315403dccf Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Mon, 28 Sep 2026 09:40:09 +0000
Subject: [PATCH 3/5] feat(transcription): carry speaker labels on words and
 streamed segments

A diarizing backend could label transcript segments, but two paths
dropped the label: TranscriptWord had no speaker field, so live
transcription words and word-level timestamps could not carry one, and
the stream=true transcript.text.done event left the speaker out of
its segments.

TranscriptWord gains an optional speaker (proto field 4, additive).
It flows through the live event and result mapping, the JSON word
output of the endpoint and the CLI, and transcript.text.done now
includes a segment's speaker when there is one. Empty labels are
omitted, so responses without diarization are unchanged.

Assisted-by: Claude:claude-opus-5-5 [Claude Code]
---
 backend/backend.proto                         |  1 +
 core/backend/transcript.go                    |  7 +++---
 core/backend/transcript_live.go               |  7 +++---
 core/backend/transcript_live_internal_test.go | 14 ++++++++++++
 core/cli/transcript.go                        | 14 +++++++-----
 core/http/endpoints/openai/transcription.go   | 22 ++++++++++++-------
 core/schema/transcription.go                  | 14 +++++++-----
 7 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/backend/backend.proto b/backend/backend.proto
index 54255528e5ad..9ec5f862fd33 100644
--- a/backend/backend.proto
+++ b/backend/backend.proto
@@ -647,6 +647,7 @@ message TranscriptWord {
   int64 start = 1;
   int64 end = 2;
   string text = 3;
+  string speaker = 4;   // backend speaker label when diarizing; empty otherwise
 }
 
 message TranscriptSegment {
diff --git a/core/backend/transcript.go b/core/backend/transcript.go
index 0ddee3adddff..27b13bc55316 100644
--- a/core/backend/transcript.go
+++ b/core/backend/transcript.go
@@ -213,9 +213,10 @@ func transcriptResultFromProto(r *proto.TranscriptResult) *schema.TranscriptionR
 		var words []schema.TranscriptionWord
 		for _, w := range s.Words {
 			var word = schema.TranscriptionWord{
-				Start: time.Duration(w.Start),
-				End:   time.Duration(w.End),
-				Text:  w.Text,
+				Start:   time.Duration(w.Start),
+				End:     time.Duration(w.End),
+				Text:    w.Text,
+				Speaker: w.Speaker,
 			}
 			words = append(words, word)
 			tr.Words = append(tr.Words, word)
diff --git a/core/backend/transcript_live.go b/core/backend/transcript_live.go
index 0ad6d72e1456..d0dc59d871bf 100644
--- a/core/backend/transcript_live.go
+++ b/core/backend/transcript_live.go
@@ -298,9 +298,10 @@ func liveEventFromProto(r *proto.TranscriptLiveResponse) LiveTranscriptionEvent
 	}
 	for _, w := range r.GetWords() {
 		ev.Words = append(ev.Words, schema.TranscriptionWord{
-			Start: time.Duration(w.Start),
-			End:   time.Duration(w.End),
-			Text:  w.Text,
+			Start:   time.Duration(w.Start),
+			End:     time.Duration(w.End),
+			Text:    w.Text,
+			Speaker: w.Speaker,
 		})
 	}
 	if r.GetFinalResult() != nil {
diff --git a/core/backend/transcript_live_internal_test.go b/core/backend/transcript_live_internal_test.go
index 6f6bed6a4b5c..975ac844bcc0 100644
--- a/core/backend/transcript_live_internal_test.go
+++ b/core/backend/transcript_live_internal_test.go
@@ -54,6 +54,20 @@ var _ = Describe("liveEventFromProto", func() {
 		Expect(ev.Final).To(BeNil())
 	})
 
+	It("carries word speakers and final segment speakers from a diarizing backend", func() {
+		ev := liveEventFromProto(&proto.TranscriptLiveResponse{
+			Words: []*proto.TranscriptWord{{Text: "hi", Speaker: "1"}},
+		})
+		Expect(ev.Words[0].Speaker).To(Equal("1"))
+		ev = liveEventFromProto(&proto.TranscriptLiveResponse{
+			FinalResult: &proto.TranscriptResult{
+				Text:     "hi there",
+				Segments: []*proto.TranscriptSegment{{Text: "hi", Speaker: "0"}, {Text: "there", Speaker: "1"}},
+			},
+		})
+		Expect(ev.Final.Segments[1].Speaker).To(Equal("1"))
+	})
+
 	It("maps the eob backchannel flag separately from eou", func() {
 		ev := liveEventFromProto(&proto.TranscriptLiveResponse{Delta: "uh-huh", Eob: true})
 		Expect(ev.Eob).To(BeTrue())
diff --git a/core/cli/transcript.go b/core/cli/transcript.go
index 06764f4dd277..e187453a6fa9 100644
--- a/core/cli/transcript.go
+++ b/core/cli/transcript.go
@@ -93,18 +93,20 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error {
 		}
 		for _, word := range(tr.Words) {
 			trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{
-				Start: word.Start.Seconds(),
-				End:   word.End.Seconds(),
-				Text:  word.Text,
+				Start:   word.Start.Seconds(),
+				End:     word.End.Seconds(),
+				Text:    word.Text,
+				Speaker: word.Speaker,
 			})
 		}
 		for _, seg := range(tr.Segments) {
 			segWords := []schema.TranscriptionWordSeconds{}
 			for _, word := range(seg.Words) {
 				segWords = append(segWords, schema.TranscriptionWordSeconds{
-					Start: word.Start.Seconds(),
-					End:   word.End.Seconds(),
-					Text:  word.Text,
+					Start:   word.Start.Seconds(),
+					End:     word.End.Seconds(),
+					Text:    word.Text,
+					Speaker: word.Speaker,
 				})
 			}
 			trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{
diff --git a/core/http/endpoints/openai/transcription.go b/core/http/endpoints/openai/transcription.go
index 920094f84e08..a7e01ba950de 100644
--- a/core/http/endpoints/openai/transcription.go
+++ b/core/http/endpoints/openai/transcription.go
@@ -191,18 +191,20 @@ func TranscriptEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
 			}
 			for _, word := range tr.Words {
 				trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{
-					Start: word.Start.Seconds(),
-					End:   word.End.Seconds(),
-					Text:  word.Text,
+					Start:   word.Start.Seconds(),
+					End:     word.End.Seconds(),
+					Text:    word.Text,
+					Speaker: word.Speaker,
 				})
 			}
 			for _, seg := range tr.Segments {
 				segWords := []schema.TranscriptionWordSeconds{}
 				for _, word := range seg.Words {
 					segWords = append(segWords, schema.TranscriptionWordSeconds{
-						Start: word.Start.Seconds(),
-						End:   word.End.Seconds(),
-						Text:  word.Text,
+						Start:   word.Start.Seconds(),
+						End:     word.End.Seconds(),
+						Text:    word.Text,
+						Speaker: word.Speaker,
 					})
 				}
 				trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{
@@ -309,12 +311,16 @@ func streamTranscription(c echo.Context, req backend.TranscriptionRequest, ml *m
 	if len(finalResult.Segments) > 0 {
 		segs := make([]map[string]any, 0, len(finalResult.Segments))
 		for _, seg := range finalResult.Segments {
-			segs = append(segs, map[string]any{
+			entry := map[string]any{
 				"id":    seg.Id,
 				"start": seg.Start.Seconds(),
 				"end":   seg.End.Seconds(),
 				"text":  seg.Text,
-			})
+			}
+			if seg.Speaker != "" {
+				entry["speaker"] = seg.Speaker
+			}
+			segs = append(segs, entry)
 		}
 		doneEvent["segments"] = segs
 	}
diff --git a/core/schema/transcription.go b/core/schema/transcription.go
index 8414fd0ba805..aab914f704e9 100644
--- a/core/schema/transcription.go
+++ b/core/schema/transcription.go
@@ -13,9 +13,10 @@ type TranscriptionSegment struct {
 }
 
 type TranscriptionWord struct {
-	Start time.Duration `json:"start"`
-	End   time.Duration `json:"end"`
-	Text  string        `json:"text"`
+	Start   time.Duration `json:"start"`
+	End     time.Duration `json:"end"`
+	Text    string        `json:"text"`
+	Speaker string        `json:"speaker,omitempty"`
 }
 
 type TranscriptionResult struct {
@@ -42,9 +43,10 @@ type TranscriptionSegmentSeconds struct {
 }
 
 type TranscriptionWordSeconds struct {
-	Start float64 `json:"start"`
-	End   float64 `json:"end"`
-	Text  string  `json:"text"`
+	Start   float64 `json:"start"`
+	End     float64 `json:"end"`
+	Text    string  `json:"text"`
+	Speaker string  `json:"speaker,omitempty"`
 }
 
 type TranscriptionResultSeconds struct {

From c9c7e5e4d2cfb8a9a73b8ca9e1d8c4a6f0fee917 Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Mon, 28 Sep 2026 09:40:09 +0000
Subject: [PATCH 4/5] feat(parakeet-cpp): speaker labels on streaming and live
 transcription

With a diar_model attached, only unary transcription carried speakers.

- stream=true: the final segments carry the speaker who said most of
  each utterance, from the same diarization as a unary request.
- Live transcription: a low-latency diarization stream runs next to
  the ASR stream and gets each audio payload first. Every finalized
  word carries a speaker; an open speaker segment is assumed to
  continue through words diarization has not reached. The final
  result lists one segment per speaker turn, relabelled once the whole
  session is diarized. Live config param diarize=false opts out.
- diar_latency picks the model card's mode (low 1.04 s, default;
  very_low 0.64 s; ultra_low 0.32 s; model 21.12 s).
- Word-level timestamps on unary requests carry their speaker too.

The streaming diarization entry points are probed with Dlsym, so live
labels need parakeet.cpp with the latency modes
(mudler/parakeet.cpp#73); an older libparakeet.so still transcribes
live, without speakers.

Assisted-by: Claude:claude-opus-5-5 [Claude Code]
---
 backend/go/parakeet-cpp/diarize.go            | 172 ++++++++++++++++++
 backend/go/parakeet-cpp/diarize_test.go       | 105 +++++++++++
 backend/go/parakeet-cpp/goparakeetcpp.go      |  42 ++++-
 backend/go/parakeet-cpp/goparakeetcpp_test.go |   7 +
 backend/go/parakeet-cpp/live.go               |  56 +++++-
 backend/go/parakeet-cpp/main.go               |  10 +
 docs/content/features/audio-to-text.md        |  24 ++-
 7 files changed, 407 insertions(+), 9 deletions(-)

diff --git a/backend/go/parakeet-cpp/diarize.go b/backend/go/parakeet-cpp/diarize.go
index 9511846cd353..a1eb4decde09 100644
--- a/backend/go/parakeet-cpp/diarize.go
+++ b/backend/go/parakeet-cpp/diarize.go
@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"unsafe"
 
 	gguf "github.com/gpustack/gguf-parser-go"
 	"github.com/mudler/LocalAI/pkg/grpc/grpcerrors"
@@ -41,6 +42,177 @@ import (
 // Present only in libparakeet.so with ABI >= 7; nil disables diarization.
 var CppDiarizePcm func(ctx uintptr, samples []float32, nSamples int32, sampleRate int32) uintptr
 
+// Streaming diarization (parakeet_capi_diarize_stream_*), used by live
+// transcription. begin_latency takes a PARAKEET_DIAR_LATENCY_* mode; feed and
+// active return a malloc'd parakeet_diar_segment array through out/nOut (free
+// with CppFreeDiarSegments) and 0 on success. Present only in newer
+// libparakeet.so; nil disables speaker labels on live transcripts.
+var (
+	CppDiarizeStreamBeginLatency func(ctx uintptr, latency int32) uintptr
+	CppDiarizeStreamFeed         func(s uintptr, pcm []float32, nSamples int32, isLast int32, out unsafe.Pointer, nOut unsafe.Pointer) int32
+	CppDiarizeStreamActive       func(s uintptr, out unsafe.Pointer, nOut unsafe.Pointer) int32
+	CppDiarizeStreamFree         func(s uintptr)
+	CppFreeDiarSegments          func(segs uintptr)
+)
+
+// cDiarSegment mirrors parakeet_diar_segment {int speaker; float start, end}.
+type cDiarSegment struct {
+	Speaker int32
+	Start   float32
+	End     float32
+}
+
+// Latency modes of parakeet_capi_diarize_stream_begin_latency
+// (PARAKEET_DIAR_LATENCY_*), by diar_latency option value.
+var diarLatencyModes = map[string]int32{
+	"model":     0, // the checkpoint's configuration (21.12 s for Nemotron-3)
+	"low":       1, // 1.04 s
+	"very_low":  2, // 0.64 s
+	"ultra_low": 3, // 0.32 s
+}
+
+// defaultLiveDiarLatency is used for live transcription when diar_latency is
+// unset: the model card's recommended low-latency configuration.
+const defaultLiveDiarLatency = "low"
+
+// liveDiarizationAvailable reports whether the library can diarize live.
+func liveDiarizationAvailable() bool {
+	return CppDiarizeStreamBeginLatency != nil && CppDiarizeStreamFeed != nil &&
+		CppDiarizeStreamActive != nil && CppDiarizeStreamFree != nil && CppFreeDiarSegments != nil
+}
+
+// takeDiarSegments copies a C parakeet_diar_segment array and frees it.
+func takeDiarSegments(ptr uintptr, n int32) []diarSegmentDoc {
+	if ptr == 0 || n <= 0 {
+		return nil
+	}
+	// C-owned malloc'd array, not Go-GC memory; copied out before freeing.
+	cs := unsafe.Slice((*cDiarSegment)(unsafe.Pointer(ptr)), int(n)) //nolint:govet // see goStringFromCPtr
+	out := make([]diarSegmentDoc, len(cs))
+	for i, c := range cs {
+		out[i] = diarSegmentDoc{Speaker: int(c.Speaker), Start: float64(c.Start), End: float64(c.End)}
+	}
+	CppFreeDiarSegments(ptr)
+	return out
+}
+
+// liveDiarizer runs a streaming diarization session next to a live ASR
+// stream. It keeps every closed segment (a live session is bounded by its
+// caller, and segments are small) so the final transcript can be relabelled
+// with complete information.
+type liveDiarizer struct {
+	p      *ParakeetCpp
+	stream uintptr
+	closed []diarSegmentDoc
+	active []diarSegmentDoc
+}
+
+func (p *ParakeetCpp) newLiveDiarizer(latency int32) (*liveDiarizer, error) {
+	p.diarMu.Lock()
+	defer p.diarMu.Unlock()
+	if p.diarCtx == 0 {
+		return nil, grpcerrors.ModelNotLoaded("parakeet-cpp")
+	}
+	s := CppDiarizeStreamBeginLatency(p.diarCtx, latency)
+	if s == 0 {
+		return nil, fmt.Errorf("parakeet-cpp: diarize stream begin failed: %s", CppLastError(p.diarCtx))
+	}
+	return &liveDiarizer{p: p, stream: s}, nil
+}
+
+// feed pushes PCM (and, with last, flushes the stream), recording the segments
+// that closed and those still open.
+func (d *liveDiarizer) feed(pcm []float32, last bool) error {
+	d.p.diarMu.Lock()
+	defer d.p.diarMu.Unlock()
+	if d.p.diarCtx == 0 {
+		return grpcerrors.ModelNotLoaded("parakeet-cpp")
+	}
+	var ptr uintptr
+	var n int32
+	isLast := int32(0)
+	if last {
+		isLast = 1
+	}
+	if CppDiarizeStreamFeed(d.stream, pcm, int32(len(pcm)), isLast, unsafe.Pointer(&ptr), unsafe.Pointer(&n)) != 0 {
+		return fmt.Errorf("parakeet-cpp: diarize stream feed failed: %s", CppLastError(d.p.diarCtx))
+	}
+	d.closed = append(d.closed, takeDiarSegments(ptr, n)...)
+	ptr, n = 0, 0
+	if CppDiarizeStreamActive(d.stream, unsafe.Pointer(&ptr), unsafe.Pointer(&n)) != 0 {
+		return fmt.Errorf("parakeet-cpp: diarize stream active failed: %s", CppLastError(d.p.diarCtx))
+	}
+	d.active = takeDiarSegments(ptr, n)
+	return nil
+}
+
+// speakersFor labels words with what is known so far. Diarization trails the
+// audio by its latency, so an open segment is assumed to go on through the
+// words it has not reached yet; a word is only mislabelled right at a turn
+// that diarization has not seen.
+func (d *liveDiarizer) speakersFor(words []transcriptWord) []int {
+	segs := append([]diarSegmentDoc(nil), d.closed...)
+	var lastEnd float64
+	for _, w := range words {
+		lastEnd = max(lastEnd, w.End)
+	}
+	for _, a := range d.active {
+		a.End = max(a.End, lastEnd)
+		segs = append(segs, a)
+	}
+	return assignSpeakers(words, segs)
+}
+
+func (d *liveDiarizer) free() {
+	if d == nil || d.stream == 0 {
+		return
+	}
+	d.p.diarMu.Lock()
+	defer d.p.diarMu.Unlock()
+	CppDiarizeStreamFree(d.stream)
+	d.stream = 0
+}
+
+// speakerTurns groups labelled words into one segment per speaker turn.
+func speakerTurns(words []transcriptWord, speakers []int) []*pb.TranscriptSegment {
+	var out []*pb.TranscriptSegment
+	start := 0
+	for i := 1; i <= len(words); i++ {
+		if i < len(words) && speakers[i] == speakers[start] {
+			continue
+		}
+		parts := make([]string, 0, i-start)
+		for _, w := range words[start:i] {
+			parts = append(parts, w.W)
+		}
+		out = append(out, &pb.TranscriptSegment{
+			Id:      int32(len(out)),
+			Start:   secondsToNanos(words[start].Start),
+			End:     secondsToNanos(words[i-1].End),
+			Text:    strings.TrimSpace(strings.Join(parts, " ")),
+			Speaker: speakerLabel(speakers[start]),
+		})
+		start = i
+	}
+	return out
+}
+
+// majoritySpeaker is the speaker covering most of the words' duration, or -1.
+func majoritySpeaker(words []transcriptWord, speakers []int) int {
+	dur := map[int]float64{}
+	best, bestDur := -1, 0.0
+	for i, w := range words {
+		if speakers[i] < 0 {
+			continue
+		}
+		dur[speakers[i]] += max(w.End-w.Start, 1e-3)
+		if dur[speakers[i]] > bestDur {
+			best, bestDur = speakers[i], dur[speakers[i]]
+		}
+	}
+	return best
+}
+
 // diarModelArch is the parakeet.arch GGUF value of a diarization model.
 const diarModelArch = "diarization"
 
diff --git a/backend/go/parakeet-cpp/diarize_test.go b/backend/go/parakeet-cpp/diarize_test.go
index 10581fe1acd4..ac0d2729e285 100644
--- a/backend/go/parakeet-cpp/diarize_test.go
+++ b/backend/go/parakeet-cpp/diarize_test.go
@@ -89,6 +89,11 @@ var _ = Describe("speaker diarization helpers", func() {
 			Expect(res.Segments[1].Speaker).To(Equal("1"))
 			Expect(res.Segments[2].Text).To(Equal("hello"))
 			Expect(res.Segments[2].Speaker).To(Equal("1"))
+
+			// With word timestamps requested, words carry their speaker too.
+			res = transcriptResultWithSpeakers(doc, &pb.TranscriptRequest{TimestampGranularities: []string{"word"}}, 0, []int{0, 1, 1})
+			Expect(res.Segments[0].Words[0].Speaker).To(Equal("0"))
+			Expect(res.Segments[2].Words[0].Speaker).To(Equal("1"))
 			for i, seg := range res.Segments {
 				Expect(seg.Id).To(Equal(int32(i)))
 			}
@@ -120,6 +125,40 @@ var _ = Describe("speaker diarization helpers", func() {
 		})
 	})
 
+	Context("live speaker labels", func() {
+		It("assumes an open segment continues through words diarization has not reached", func() {
+			d := &liveDiarizer{
+				closed: []diarSegmentDoc{ds(0, 0.0, 2.0)},
+				active: []diarSegmentDoc{ds(1, 2.5, 3.0)}, // diarized up to 3.0 s
+			}
+			words := []transcriptWord{tw("a", 0.5, 0.9), tw("b", 2.6, 2.9), tw("c", 3.4, 3.9)}
+			Expect(d.speakersFor(words)).To(Equal([]int{0, 1, 1}))
+		})
+
+		It("groups labelled words into speaker turns", func() {
+			words := []transcriptWord{tw("hi", 0, 0.3), tw("there.", 0.3, 0.6), tw("hello", 1.0, 1.4)}
+			segs := speakerTurns(words, []int{0, 0, 1})
+			Expect(segs).To(HaveLen(2))
+			Expect(segs[0].Text).To(Equal("hi there."))
+			Expect(segs[0].Speaker).To(Equal("0"))
+			Expect(segs[1].Text).To(Equal("hello"))
+			Expect(segs[1].Speaker).To(Equal("1"))
+			Expect(segs[1].Id).To(Equal(int32(1)))
+		})
+
+		It("picks the speaker covering most of an utterance", func() {
+			words := []transcriptWord{tw("a", 0, 0.2), tw("b", 0.2, 1.5), tw("c", 1.5, 1.6)}
+			Expect(majoritySpeaker(words, []int{0, 1, 0})).To(Equal(1))
+			Expect(majoritySpeaker(words, []int{-1, -1, -1})).To(Equal(-1))
+		})
+
+		It("maps diar_latency option values to the C-API modes", func() {
+			Expect(diarLatencyModes).To(HaveKeyWithValue("model", int32(0)))
+			Expect(diarLatencyModes).To(HaveKeyWithValue(defaultLiveDiarLatency, int32(1)))
+			Expect(diarLatencyModes).To(HaveKeyWithValue("ultra_low", int32(3)))
+		})
+	})
+
 	Context("parakeetArch", func() {
 		It("reads parakeet.arch from the GGUF metadata", func() {
 			path := filepath.Join(GinkgoT().TempDir(), "m.gguf")
@@ -255,3 +294,69 @@ var _ = Describe("ParakeetCpp speaker diarization", func() {
 		Expect(status.Code(err)).To(Equal(codes.Unimplemented))
 	})
 })
+
+var _ = Describe("ParakeetCpp live speaker labels", func() {
+	It("labels live words and returns speaker turns in the final result", func() {
+		diarModel := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_MODEL")
+		streamModel := os.Getenv("PARAKEET_BACKEND_TEST_STREAM_MODEL")
+		wavPath := os.Getenv("PARAKEET_BACKEND_TEST_DIAR_WAV")
+		if diarModel == "" || streamModel == "" || wavPath == "" {
+			Skip("set PARAKEET_BACKEND_TEST_DIAR_MODEL, PARAKEET_BACKEND_TEST_STREAM_MODEL " +
+				"(a cache-aware streaming ASR model) and PARAKEET_BACKEND_TEST_DIAR_WAV")
+		}
+		ensureLibLoaded()
+		if !liveDiarizationAvailable() {
+			Skip("libparakeet.so has no streaming diarization C-API")
+		}
+		p := &ParakeetCpp{}
+		Expect(p.Load(&pb.ModelOptions{
+			ModelFile: streamModel,
+			Options:   []string{"diar_model:" + diarModel},
+		})).To(Succeed())
+		defer func() { _ = p.Free() }()
+
+		pcm, _, err := decodeWavMono16k(wavPath)
+		Expect(err).ToNot(HaveOccurred())
+		in := make(chan *pb.TranscriptLiveRequest)
+		out := make(chan *pb.TranscriptLiveResponse, 1024)
+		errCh := make(chan error, 1)
+		go func() { errCh <- p.AudioTranscriptionLive(in, out) }()
+		in <- &pb.TranscriptLiveRequest{Payload: &pb.TranscriptLiveRequest_Config{Config: &pb.TranscriptLiveConfig{}}}
+		for lo := 0; lo < len(pcm); lo += 1600 {
+			hi := min(lo+1600, len(pcm))
+			in <- &pb.TranscriptLiveRequest{Payload: &pb.TranscriptLiveRequest_Audio{
+				Audio: &pb.TranscriptLiveAudio{Pcm: pcm[lo:hi]}}}
+		}
+		close(in)
+		Expect(<-errCh).To(Succeed())
+
+		var live []string
+		var final *pb.TranscriptResult
+		for r := range out {
+			for _, w := range r.GetWords() {
+				live = append(live, w.GetSpeaker())
+			}
+			if r.GetFinalResult() != nil {
+				final = r.GetFinalResult()
+			}
+		}
+		Expect(live).ToNot(BeEmpty())
+		labelled := 0
+		for _, l := range live {
+			if l != "" {
+				labelled++
+			}
+		}
+		// Words finalized before diarization's first chunk (1.04 s) have no
+		// speaker yet; nearly all others do.
+		Expect(labelled).To(BeNumerically(">=", len(live)*9/10), "live labels: %v", live)
+		Expect(turns(live)).To(ContainElements("0", "1"))
+
+		Expect(final).ToNot(BeNil())
+		labels := make([]string, len(final.Segments))
+		for i, s := range final.Segments {
+			labels[i] = s.Speaker
+		}
+		Expect(turns(labels)).To(Equal([]string{"0", "1", "0", "1"}), "final turns: %v", labels)
+	})
+})
diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go
index 95621e1c1be2..de5269a8bd20 100644
--- a/backend/go/parakeet-cpp/goparakeetcpp.go
+++ b/backend/go/parakeet-cpp/goparakeetcpp.go
@@ -156,6 +156,9 @@ type ParakeetCpp struct {
 	isDiarModel bool
 	diarCtx     uintptr
 	diarMu      sync.Mutex
+	// liveDiarLatency is the PARAKEET_DIAR_LATENCY_* mode live transcription
+	// diarizes with (diar_latency option, default low = 1.04 s).
+	liveDiarLatency int32
 }
 
 // Load is the LocalAI gRPC entry point for LoadModel: it calls
@@ -190,6 +193,15 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error {
 	}
 
 	// Optional speaker diarization for transcripts (diarize.go).
+	lat := optString(opts, "diar_latency")
+	if lat == "" {
+		lat = defaultLiveDiarLatency
+	}
+	mode, ok := diarLatencyModes[lat]
+	if !ok {
+		return fmt.Errorf("parakeet-cpp: unknown diar_latency %q (model, low, very_low, ultra_low)", lat)
+	}
+	p.liveDiarLatency = mode
 	if dm := optString(opts, "diar_model"); dm != "" {
 		if err := p.loadDiarModel(resolveModelPath(opts.ModelPath, dm)); err != nil {
 			CppFree(p.ctxPtr)
@@ -492,7 +504,8 @@ func transcriptResultWithSpeakers(doc transcriptJSON, opts *pb.TranscriptRequest
 		if wantWords {
 			ws := make([]*pb.TranscriptWord, len(group))
 			for i, gw := range group {
-				ws[i] = &pb.TranscriptWord{Start: secondsToNanos(gw.Start), End: secondsToNanos(gw.End), Text: gw.W}
+				ws[i] = &pb.TranscriptWord{Start: secondsToNanos(gw.Start), End: secondsToNanos(gw.End), Text: gw.W,
+					Speaker: seg.Speaker}
 			}
 			seg.Words = ws
 		}
@@ -590,7 +603,8 @@ func tokensInWindow(tokens []transcriptToken, start, end float64) []int32 {
 // text-only library (no words) it falls back to segmenting the delta text, so
 // the same assembler serves both paths.
 type streamSegmenter struct {
-	segs    []*pb.TranscriptSegment
+	segs     []*pb.TranscriptSegment
+	segWords [][]transcriptWord // words of each segment (nil for text-only ones)
 	cur     []transcriptWord // words for the open segment (ABI v4 JSON path)
 	curText []string         // delta text for the open segment (text-only path)
 	nextID  int32
@@ -621,12 +635,14 @@ func (s *streamSegmenter) flush() {
 			End:   secondsToNanos(s.cur[len(s.cur)-1].End),
 			Text:  strings.TrimSpace(strings.Join(parts, " ")),
 		})
+		s.segWords = append(s.segWords, s.cur)
 		s.nextID++
 	case len(s.curText) > 0:
 		// No words this segment: emit a text-only segment (no timestamps),
 		// skipping a purely-whitespace one as the legacy text path did.
 		if t := strings.TrimSpace(strings.Join(s.curText, "")); t != "" {
 			s.segs = append(s.segs, &pb.TranscriptSegment{Id: s.nextID, Text: t})
+			s.segWords = append(s.segWords, nil)
 			s.nextID++
 		}
 	}
@@ -843,6 +859,28 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra
 	// The single-segment fallback stays trimmed.
 	fullText := full.String()
 	segments := seg.segments()
+
+	// With a diar_model attached, label each utterance with the speaker who
+	// said most of it. The whole file is available, so this runs the same
+	// diarization as the unary path rather than a low-latency stream.
+	if p.diarCtx != 0 && opts.GetDiarize() && len(seg.segWords) == len(segments) {
+		var all []transcriptWord
+		for _, ws := range seg.segWords {
+			all = append(all, ws...)
+		}
+		if len(all) > 0 {
+			dd, err := p.runDiarization(data)
+			if err != nil {
+				return err
+			}
+			speakers := assignSpeakers(all, dd.Segments)
+			k := 0
+			for i, ws := range seg.segWords {
+				segments[i].Speaker = speakerLabel(majoritySpeaker(ws, speakers[k:k+len(ws)]))
+				k += len(ws)
+			}
+		}
+	}
 	if trimmed := strings.TrimSpace(fullText); len(segments) == 0 && trimmed != "" {
 		segments = append(segments, &pb.TranscriptSegment{Id: 0, Text: trimmed})
 	}
diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go
index fcd1b8f4f9f4..2a96091696b1 100644
--- a/backend/go/parakeet-cpp/goparakeetcpp_test.go
+++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go
@@ -62,6 +62,13 @@ func ensureLibLoaded() {
 		if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_pcm"); err == nil && sym != 0 {
 			purego.RegisterLibFunc(&CppDiarizePcm, lib, "parakeet_capi_diarize_pcm")
 		}
+		if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_stream_begin_latency"); err == nil && sym != 0 {
+			purego.RegisterLibFunc(&CppDiarizeStreamBeginLatency, lib, "parakeet_capi_diarize_stream_begin_latency")
+			purego.RegisterLibFunc(&CppDiarizeStreamFeed, lib, "parakeet_capi_diarize_stream_feed")
+			purego.RegisterLibFunc(&CppDiarizeStreamActive, lib, "parakeet_capi_diarize_stream_active")
+			purego.RegisterLibFunc(&CppDiarizeStreamFree, lib, "parakeet_capi_diarize_stream_free")
+			purego.RegisterLibFunc(&CppFreeDiarSegments, lib, "parakeet_capi_free_diar_segments")
+		}
 		purego.RegisterLibFunc(&CppFreeString, lib, "parakeet_capi_free_string")
 		purego.RegisterLibFunc(&CppLastError, lib, "parakeet_capi_last_error")
 	})
diff --git a/backend/go/parakeet-cpp/live.go b/backend/go/parakeet-cpp/live.go
index bfd58e809f34..0bb4cc204e61 100644
--- a/backend/go/parakeet-cpp/live.go
+++ b/backend/go/parakeet-cpp/live.go
@@ -71,6 +71,26 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest
 	// current when the RPC unwinds.
 	defer func() { p.streamFree(stream) }()
 
+	// Speaker labels (diarize.go): a low-latency diarization stream runs next
+	// to the ASR stream when a diar_model is attached, unless the session
+	// opts out with params diarize=false.
+	var diar *liveDiarizer
+	var allWords []transcriptWord
+	startDiar := func(c *pb.TranscriptLiveConfig) error {
+		diar.free()
+		diar, allWords = nil, nil
+		if p.diarCtx == 0 || !liveDiarizationAvailable() || c.GetParams()["diarize"] == "false" {
+			return nil
+		}
+		var err error
+		diar, err = p.newLiveDiarizer(p.liveDiarLatency)
+		return err
+	}
+	if err := startDiar(cfg); err != nil {
+		return err
+	}
+	defer func() { diar.free() }()
+
 	out <- &pb.TranscriptLiveResponse{Ready: true}
 
 	var (
@@ -97,11 +117,18 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest
 			full.WriteString(r.Delta)
 		}
 		if r.Delta != "" || r.Eou || r.Eob || len(r.Words) > 0 {
+			words := liveWordsToProto(r.Words)
+			if diar != nil && len(r.Words) > 0 {
+				allWords = append(allWords, r.Words...)
+				for i, spk := range diar.speakersFor(r.Words) {
+					words[i].Speaker = speakerLabel(spk)
+				}
+			}
 			out <- &pb.TranscriptLiveResponse{
 				Delta: r.Delta,
 				Eou:   r.Eou,
 				Eob:   r.Eob,
-				Words: liveWordsToProto(r.Words),
+				Words: words,
 			}
 		}
 		return nil
@@ -125,11 +152,20 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest
 			}
 			full.Reset()
 			fedSecs = 0
+			if err := startDiar(payload.Config); err != nil {
+				return err
+			}
 		case *pb.TranscriptLiveRequest_Audio:
 			pcm := payload.Audio.GetPcm()
 			audioSec := float64(len(pcm)) / liveSampleRate
 			fedSecs += audioSec
 			start := time.Now()
+			// Diarize first so the words this feed finalizes can be labelled.
+			if diar != nil {
+				if err := diar.feed(pcm, false); err != nil {
+					return err
+				}
+			}
 			// nil ctx: a live session is bounded by this request channel, not a
 			// context — cancellation is the caller closing the stream.
 			if err := p.feedSlices(nil, stream, pcm, emit); err != nil {
@@ -153,15 +189,23 @@ func (p *ParakeetCpp) AudioTranscriptionLive(in <-chan *pb.TranscriptLiveRequest
 	}
 
 	// Send side closed: flush the streaming tail and emit the final transcript.
-	// The live FinalResult carries only Text — the authoritative full-turn
-	// transcript the realtime core commits. Per-utterance segments, duration,
-	// and the terminal <EOU> flag are not produced on the live path.
+	// The live FinalResult carries Text — the authoritative full-turn
+	// transcript the realtime core commits — plus, when diarizing, one segment
+	// per speaker turn. Duration and the terminal <EOU> flag are not produced
+	// on the live path.
 	if err := p.flushTail(stream, emit); err != nil {
 		return err
 	}
-	out <- &pb.TranscriptLiveResponse{
-		FinalResult: &pb.TranscriptResult{Text: strings.TrimSpace(full.String())},
+	final := &pb.TranscriptResult{Text: strings.TrimSpace(full.String())}
+	// With speakers, the final result also carries one segment per speaker
+	// turn, relabelled now that diarization has seen the whole session.
+	if diar != nil && len(allWords) > 0 {
+		if err := diar.feed(nil, true); err != nil {
+			return err
+		}
+		final.Segments = speakerTurns(allWords, assignSpeakers(allWords, diar.closed))
 	}
+	out <- &pb.TranscriptLiveResponse{FinalResult: final}
 	return nil
 }
 
diff --git a/backend/go/parakeet-cpp/main.go b/backend/go/parakeet-cpp/main.go
index d04f78016aa3..fb2c38fd3300 100644
--- a/backend/go/parakeet-cpp/main.go
+++ b/backend/go/parakeet-cpp/main.go
@@ -97,6 +97,16 @@ func main() {
 		purego.RegisterLibFunc(&CppDiarizePcm, lib, "parakeet_capi_diarize_pcm")
 	}
 
+	// Streaming diarization with latency modes, for speaker labels on live
+	// transcripts. All-or-nothing: live diarization needs every entry point.
+	if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_stream_begin_latency"); err == nil && sym != 0 {
+		purego.RegisterLibFunc(&CppDiarizeStreamBeginLatency, lib, "parakeet_capi_diarize_stream_begin_latency")
+		purego.RegisterLibFunc(&CppDiarizeStreamFeed, lib, "parakeet_capi_diarize_stream_feed")
+		purego.RegisterLibFunc(&CppDiarizeStreamActive, lib, "parakeet_capi_diarize_stream_active")
+		purego.RegisterLibFunc(&CppDiarizeStreamFree, lib, "parakeet_capi_diarize_stream_free")
+		purego.RegisterLibFunc(&CppFreeDiarSegments, lib, "parakeet_capi_free_diar_segments")
+	}
+
 	fmt.Fprintf(os.Stderr, "[parakeet-cpp] ABI=%d\n", CppAbiVersion())
 
 	flag.Parse()
diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md
index f927cc121c34..aa67edc82824 100644
--- a/docs/content/features/audio-to-text.md
+++ b/docs/content/features/audio-to-text.md
@@ -218,7 +218,29 @@ options:
 - diar_model:nemotron-3-diarization-q8_0.gguf   # relative to the models directory
 ```
 
-The `diarize` form field turns it off per request (`-F diarize=false`). Streaming transcription does not carry speaker labels.
+The `diarize` form field turns it off per request (`-F diarize=false`). With word timestamps (`timestamp_granularities[]=word`) each word carries its speaker too.
+
+Streaming works as well:
+
+- `stream=true`: the closing `transcript.text.done` event lists the segments with their `speaker`. Speakers are assigned once the whole file is transcribed, with the same diarization as a non-streaming request.
+- Live transcription (a cache-aware streaming model such as `realtime_eou_120m-v1`, driven through the live transcription API): a low-latency diarization stream runs alongside the recognizer, each finalized word carries a `speaker`, and the final result lists one segment per speaker turn. Pass the live config parameter `diarize=false` to turn it off for a session.
+
+Live labels trail the audio by the diarization latency, set with `diar_latency` (the Nemotron-3-Diarization model card's modes):
+
+| `diar_latency` | Latency | |
+|---|---|---|
+| `low` (default) | 1.04 s | the model card's recommended configuration |
+| `very_low` | 0.64 s | |
+| `ultra_low` | 0.32 s | |
+| `model` | 21.12 s | the checkpoint's own configuration, what non-live requests use |
+
+A word said before diarization has caught up takes the speaker who is talking at that moment, so right after a speaker change the first word or two can carry the previous speaker; the final result is relabelled once the whole session has been diarized.
+
+```yaml
+options:
+- diar_model:nemotron-3-diarization-q8_0.gguf
+- diar_latency:very_low
+```
 
 ### Dynamic batching
 

From 706aa80966d1dec4146c5f6100a04faebdc12e6f Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Tue, 29 Sep 2026 16:17:42 +0000
Subject: [PATCH 5/5] chore(parakeet-cpp): pin parakeet.cpp to 6dea76a for the
 latency modes

The previous pin (238057c) had speaker diarization but not the
streaming latency modes that live speaker labels need
(mudler/parakeet.cpp#73). 6dea76a is current master: it includes #73,
plus sound-event detection (additive C-API, ABI 8) and a GCC 16 build
fix. It adds the third_party/ced.cpp submodule, which the recursive
submodule fetch picks up.

Assisted-by: Claude:claude-opus-5-5 [Claude Code]
---
 backend/go/parakeet-cpp/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile
index f14e507715f3..4d462d11b46b 100644
--- a/backend/go/parakeet-cpp/Makefile
+++ b/backend/go/parakeet-cpp/Makefile
@@ -1,6 +1,6 @@
 # parakeet-cpp backend Makefile.
 #
-# Upstream pin lives below as PARAKEET_VERSION?=238057cb707fdb6c4b185c853da095328930b539
+# Upstream pin lives below as PARAKEET_VERSION?=6dea76a81ee0bba83c6dad320dccc2fbeef111e2
 # (.github/bump_deps.sh) can find and update it - matches the
 # whisper.cpp / ds4 / vibevoice-cpp convention.
 #
@@ -15,7 +15,7 @@
 # That's what the L0 smoke test uses. The default target below does the
 # proper clone-at-pin + cmake build so CI doesn't need a side-checkout.
 
-PARAKEET_VERSION?=238057cb707fdb6c4b185c853da095328930b539
+PARAKEET_VERSION?=6dea76a81ee0bba83c6dad320dccc2fbeef111e2
 PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp
 
 GOCMD?=go