diff --git a/scripts/test-siri-tts-stream-regression.sh b/scripts/test-siri-tts-stream-regression.sh index abf6e06e8..650f85487 100755 --- a/scripts/test-siri-tts-stream-regression.sh +++ b/scripts/test-siri-tts-stream-regression.sh @@ -15,7 +15,7 @@ clang \ -framework AVFoundation \ -framework AudioToolbox \ -framework CoreAudio \ - "$repo_root/src-tauri/native/tests/siri_tts_stream_regression.m" \ + "$repo_root/src-tauri/crates/berd-voice/native/tests/siri_tts_stream_regression.m" \ -o "$test_binary" "$test_binary" diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 539f0d573..a8a7b3e12 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -7,20 +7,16 @@ name = "Berd" version = "0.6.2" dependencies = [ "anyhow", - "audioadapter-buffers", "base64 0.22.1", "berd-voice", "block2", "builderbot-auth", "bytes", - "bzip2 0.6.1", - "cc", "chrono", "coreaudio-rs", "dirs", "doctor", "dunce", - "earshot", "etcetera 0.11.0", "fern", "flate2", @@ -45,13 +41,10 @@ dependencies = [ "portable-pty", "reqwest 0.13.4", "rodio", - "rubato", - "rustls", "semver", "serde", "serde_json", "sha2", - "sherpa-onnx", "sqlx", "swift-rs", "sysinfo", @@ -74,7 +67,6 @@ dependencies = [ "tempfile", "time", "tokio", - "tokio-tungstenite", "toml 1.1.4+spec-1.1.0", "url", "uuid", @@ -601,14 +593,35 @@ dependencies = [ name = "berd-voice" version = "0.1.0" dependencies = [ + "audioadapter-buffers", + "base64 0.22.1", + "bzip2 0.6.1", + "cc", + "claxon", + "coreaudio-rs", + "earshot", + "fs2", + "futures-util", + "hound", + "libc", "ort", "ort-sys", "rand 0.10.2", + "reqwest 0.13.4", + "rubato", + "rustls", "sentencepiece-model", "serde", "serde_json", + "sha2", "sherpa-onnx", + "swift-rs", + "tar", + "tempfile", "tokenizers", + "tokio", + "tokio-tungstenite", + "uuid", ] [[package]] @@ -999,6 +1012,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "claxon" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bfbf56724aa9eca8afa4fcfadeb479e722935bb2a0900c2d37e0cc477af0688" + [[package]] name = "clipboard-win" version = "5.4.1" @@ -2663,6 +2682,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "html5ever" version = "0.38.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 602b59f4c..69f2915fb 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -24,7 +24,6 @@ exclude = ["plugins/app-test-driver"] [build-dependencies] tauri-build = { version = "2", features = [] } -cc = "1" [target.'cfg(target_os = "macos")'.build-dependencies] swift-rs = { version = "1.0.7", features = ["build"] } @@ -34,7 +33,6 @@ anyhow = "1" base64 = "0.22" builderbot-auth = { path = "../crates/builderbot-auth", features = ["blocking-client"] } bytes = "1" -bzip2 = "0.6" berd-voice = { path = "crates/berd-voice", features = ["static"] } chrono = { version = "0.4", features = ["serde"] } dirs = "6.0.0" @@ -58,24 +56,7 @@ plist = "1" portable-pty = "0.9.0" reqwest = { version = "0.13.3", default-features = false, features = ["json", "multipart", "rustls", "stream"] } rodio = { version = "0.22", default-features = false } -audioadapter-buffers = "3.0" -earshot = "1.0" -rubato = "3.0" tar = "0.4" -# Held below 1.13: sherpa-onnx-sys downloads prebuilt native libs into a -# version-stamped dir under the target dir, and rust-cache strips the files out -# of it when saving. The restored empty skeleton defeats the build script's -# `lib_dir.is_dir()` early return, so it never re-downloads and linking fails -# with "could not find native static library `sherpa-onnx-c-api`". CI's Linux -# cache is poisoned for v1.13.5; this pin resolves to a path the cache has -# never gutted. Tilde, not caret — `^1.12` floats back to 1.13.5. -# -# NOTE: sherpa-onnx-sys is the crate whose version actually names that path, -# and it declares `^1.12.40`, so nothing in this manifest constrains it — it is -# held at 1.12.40 by Cargo.lock alone. A bare `cargo update` floats it back to -# 1.13.5 and reintroduces the CI failure; re-pin it with -# `cargo update -p sherpa-onnx-sys --precise 1.12.40`. -sherpa-onnx = "~1.12.40" semver = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -100,8 +81,6 @@ tauri-plugin-updater = "2" tauri-plugin-window-state = "2" time = { version = "0.3", features = ["formatting"] } tokio = { version = "1.50.0", features = ["full"] } -tokio-tungstenite = { version = "0.27", features = ["rustls-tls-webpki-roots"] } -rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12"] } url = "2" uuid = { version = "1", features = ["v4", "serde"] } zip = { version = "2", default-features = false, features = ["deflate"] } diff --git a/src-tauri/build.rs b/src-tauri/build.rs index 5f78c7978..fbaa9e8c9 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -9,15 +9,6 @@ fn main() { #[cfg(target_os = "macos")] { - println!("cargo:rerun-if-changed=native/siri_tts_bridge.h"); - println!("cargo:rerun-if-changed=native/siri_tts_bridge.m"); - cc::Build::new() - .file("native/siri_tts_bridge.m") - .flag("-fobjc-arc") - .compile("berd_siri_tts_bridge"); - for framework in ["Foundation", "AVFoundation", "AudioToolbox", "CoreAudio"] { - println!("cargo:rustc-link-lib=framework={framework}"); - } if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { swift_rs::SwiftLinker::new("14.0") .with_package("BerdAirPodsBridge", "swift/BerdAirPodsBridge") diff --git a/src-tauri/crates/berd-voice/Cargo.toml b/src-tauri/crates/berd-voice/Cargo.toml index bdddf62ad..0a0f41131 100644 --- a/src-tauri/crates/berd-voice/Cargo.toml +++ b/src-tauri/crates/berd-voice/Cargo.toml @@ -2,21 +2,65 @@ name = "berd-voice" version = "0.1.0" edition = "2021" -license = "Apache-2.0" +# Rust sources are Apache-2.0; the embedded LibriSpeech benchmark fixture is +# CC-BY-4.0 and carries its attribution notice in every benchmark report. +license = "Apache-2.0 AND CC-BY-4.0" repository = "https://github.com/block/berd" description = "Device-local voice primitives for Berd" +[[bin]] +name = "berd-voice" +path = "src/main.rs" + [features] default = ["static"] static = ["sherpa-onnx/static"] shared = ["sherpa-onnx/shared"] [dependencies] +audioadapter-buffers = "3.0" +base64 = "0.22" +bzip2 = "0.6" +claxon = "0.4.3" +earshot = "1.0" +fs2 = "0.4" +futures-util = "0.3" +hound = "3.5" +libc = "0.2" ort = { version = "=2.0.0-rc.12", default-features = false, features = ["api-24", "ndarray", "std"] } ort-sys = { version = "=2.0.0-rc.12", features = ["disable-linking"] } rand = "0.10" +reqwest = { version = "0.13.3", default-features = false, features = ["json", "rustls", "stream"] } +rubato = "3.0" +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12"] } sentencepiece-model = "0.1" serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" +tar = "0.4" +tempfile = "3" +# Held below 1.13: sherpa-onnx-sys downloads prebuilt native libs into a +# version-stamped dir under the target dir, and rust-cache strips the files out +# of it when saving. The restored empty skeleton defeats the build script's +# `lib_dir.is_dir()` early return, so it never re-downloads and linking fails +# with "could not find native static library `sherpa-onnx-c-api`". CI's Linux +# cache is poisoned for v1.13.5; this pin resolves to a path the cache has +# never gutted. Tilde, not caret — `^1.12` floats back to 1.13.5. +# +# NOTE: sherpa-onnx-sys is the crate whose version actually names that path, +# and it declares `^1.12.40`, so nothing in this manifest constrains it — it is +# held at 1.12.40 by Cargo.lock alone. A bare `cargo update` floats it back to +# 1.13.5 and reintroduces the CI failure; re-pin it with +# `cargo update -p sherpa-onnx-sys --precise 1.12.40`. sherpa-onnx = { version = "~1.12.40", default-features = false } tokenizers = { version = "0.22", default-features = false, features = ["fancy-regex"] } +tokio = { version = "1", features = ["fs", "io-util", "rt-multi-thread", "macros", "net", "sync", "time"] } +tokio-tungstenite = { version = "0.27", features = ["rustls-tls-webpki-roots"] } +uuid = { version = "1", features = ["v4"] } + +[target.'cfg(target_os = "macos")'.dependencies] +coreaudio-rs = "0.14.2" + +[target.'cfg(target_os = "macos")'.build-dependencies] +cc = "1" +swift-rs = { version = "1.0.7", features = ["build"] } diff --git a/src-tauri/crates/berd-voice/PROTOCOL.md b/src-tauri/crates/berd-voice/PROTOCOL.md new file mode 100644 index 000000000..9e6107fa7 --- /dev/null +++ b/src-tauri/crates/berd-voice/PROTOCOL.md @@ -0,0 +1,330 @@ +# berd-voice session protocol + +`berd-voice session` is a development, full-authority voice session. The child +owns speech recognition, finalized-input order, confirmation, speak admission, +synthesis, source-frame delivery, playback lifecycle, and barge-in. The parent +owns capture and playback devices: it writes normalized microphone PCM on stdin +and consumes synthesized PCM from a dedicated inherited pipe. The child writes +flushed JSONL events to stdout. Diagnostics go only to stderr. + +## Startup + +The child selects closed TTS and STT backends at startup: + +```text +berd-voice session --pcm-output-fd FD [--tts-backend siri] --voice NAME --language BCP47 [--rate 0.5..2.0] +berd-voice session --pcm-output-fd FD --tts-backend openai [--rate 0.75..2.0] +berd-voice session --pcm-output-fd FD --tts-backend pocket --model-dir ABS --voice ID [--rate 0.75..2.0] + +berd-voice session [--stt-backend macos] +berd-voice session --stt-backend parakeet --stt-model-dir ABS +berd-voice session --stt-backend openai +``` + +Siri TTS and macOS STT are the defaults. Siri selection is exact and requires +an installed sirittsd voice; omitting its voice or language fails startup with +setup guidance. There is no fallback to OpenAI or another cloud engine. Pocket +requires an explicit self-contained bundle +containing its ONNX/tokenizer assets and `voices/.wav`; it never searches a +Berd cache. macOS STT uses the current locale and requires its model to be +installed before startup; an unavailable model fails startup with installation +guidance. Parakeet requires an explicit self-contained bundle. +OpenAI credentials and optional endpoint/model configuration come only from the +child environment, never arguments or wire messages. TTS and STT validation and +initialization finish before `ready`. STT must report readiness within 60 +seconds; otherwise the child emits a sanitized fatal event and performs bounded +runtime cleanup before exiting. + +Siri startup preflight validates the exact case-sensitive installed name, +normalized BCP-47 language, and a responsive sirittsd availability query. It +does not guarantee that a later synthesis request cannot fail; those failures +remain terminal speech events. + +`FD` must be an inherited writable descriptor at least 3. It is required and +has no device-owning or stdout-multiplexed fallback. + +The first request must be `hello`. `input_during_tts` is the host's resolved +initial policy; a host-specific `auto` mode must be resolved before the request: + +```json +{"type":"hello","id":1,"input_during_tts":"allow_barge_in"} +``` + +The response retains `protocol:2` as a fixed wire-integrity marker, not a +negotiated mode: + +```json +{"type":"ready","id":1,"protocol":2,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} +``` + +The `session.tts` object is the authoritative, sanitized TTS configuration. +OpenAI snapshots contain `model`, `voice`, and `rate`; Siri contains `voice`, +`language`, and `rate`; Pocket contains its public `model` identifier, `voice`, +and `rate`. Credentials, endpoints, and bundle paths never appear on stdout. +Detailed backend errors are diagnostics on stderr only; protocol rejection and +fatal messages are sanitized at the stdout boundary. +`session.input_during_tts` is the authoritative effective assistant-input +policy and has its own revision. + +## Stdin framing + +Every stdin message is an eight-byte header followed by exactly `length` bytes: + +```text +0x42 0x56 0x02 kind length:u32-little-endian +``` + +`0x02` is a fixed framing marker. Kind `1` is one UTF-8 JSON request, bounded to +1 MiB. Kind `2` is exactly 3840 bytes: one 20 ms frame of 960 little-endian, +finite Float32 mono samples at 48 kHz. Wrong magic, marker, kind, length, JSON, +PCM shape, or non-finite PCM is fatal. Frames are processed in order and bounded +before payload allocation. Stdout remains unframed, flushed JSONL. + +JSON controls and acknowledgements use an unbounded control path inside the +child. Microphone PCM uses a bounded nonblocking path; its first overflow is +fatal. A control remains ordered after every earlier accepted microphone frame, +but later PCM cannot block an urgent audio acknowledgement or cancellation. + +## Dedicated PCM output pipe + +The child is the sole writer of self-framed records on `--pcm-output-fd`. Each +record has an eight-byte header followed by exactly `length` bytes: + +```text +0x42 0x41 0x02 kind length:u32-little-endian +``` + +Kinds and little-endian payloads are: + +```text +1 Begin: speech_id:u64 sample_rate:u32 playback_rate:f32 +2 Chunk: speech_id:u64 sequence:u64 samples:[f32] +3 End: speech_id:u64 last_sequence:u64 total_frames:u64 +4 Cancel: speech_id:u64 +``` + +PCM is finite, unit-scale, mono Float32 at the Begin sample rate. The current +closed sample rates are 24 kHz and 48 kHz; playback rate is finite `0.5..2.0`. +Chunk sequence starts at 1 and is contiguous. A Chunk is nonempty and contains +at most 4096 source frames. Pipe writes, Begin acceptance, each Chunk acceptance, +credit release, and cancellation acknowledgement are each bounded to two +seconds. A partial record that cannot finish within the pipe deadline poisons +the transport; Cancel never overtakes bytes from an unfinished record. + +The parent acknowledges on framed stdin: + +```text +{"type":"audio_begin_accepted","speech_id":u64} +{"type":"audio_begin_failed","speech_id":u64,"played_frames":u64,"message":string} +{"type":"audio_chunk_accepted","speech_id":u64,"sequence":u64} +{"type":"audio_played","speech_id":u64,"played_frames":u64} +{"type":"audio_suspended","speech_id":u64,"played_frames":u64} +{"type":"audio_resumed","speech_id":u64,"played_frames":u64} +{"type":"audio_drained","speech_id":u64,"sequence":u64,"played_frames":u64} +{"type":"audio_failed","speech_id":u64,"played_frames":u64,"message":string} +{"type":"audio_cancelled","speech_id":u64,"played_frames":u64} +``` + +For provisional barge-in, the child writes one of these flushed JSONL commands on stdout: + +```text +{"type":"audio_suspend","speech_id":u64} +{"type":"audio_resume","speech_id":u64} +``` + +Suspend and Resume are nonterminal controls for the same speech and the same host player. `audio_suspended` is an audible-quiescence barrier: the host has paused the player, included its bounded presentation latency, settled callbacks, retained already queued audio in place, and reports the cumulative unique source frames actually played. No callback or audible progress from the suspended generation may cross that barrier. `audio_resumed` confirms that the same retained player is ready to continue, with an unchanged cumulative `played_frames`, before the child releases more PCM. Both acknowledgements are bounded to two seconds. While fully suspended, the child blocks synthesis delivery without applying the normal two-second played-credit deadline; memory remains bounded by the existing backend and transport queues, and cancellation, shutdown, host failure, or EOF wakes the wait. + +Stdout and the PCM pipe are independently observed. A host must therefore correlate a Suspend that arrives before Begin, acknowledge it at zero only after proving the route quiescent, and keep the later Begin paused until Resume. The child emits Suspend only for the active speech, emits no later Chunk or End while waiting for its acknowledgement, and does not emit Resume until both speaking and recognition-pending state have cleared without a final. If no-result settlement races the Suspend acknowledgement, Resume follows that exact barrier. A real final, targeted cancellation, host mute/reset of a provisional hold, pause, or shutdown suppresses Resume, waits any already-emitted Suspend or Resume acknowledgement, then serializes Cancel. End or Drained racing ahead of Suspend remains a provisional logical hold rather than publishing completion; no-result Resume releases exactly one completion, while terminal cancellation clears the held host state and publishes exactly one interruption. Stale, mismatched, regressed, post-barrier, or unsolicited suspension acknowledgements are fatal. + +Begin must be accepted before the first Chunk. Only one Chunk may await +acceptance. Accepted-but-not-fully-played credit is measured in cumulative +source frames and is duration-derived to retain at least 400 ms of runway at +the Begin sample and playback rates. The child coalesces backend callbacks into +4096-frame records plus one final tail. The host nevertheless validates source +duration rather than assuming full records and also caps the queue at 64 records +(at most 1 MiB of Float32 PCM). Every individual record remains bounded to 4096 +source frames. +The child flushes `output_ready_result(accepted)` before writing Begin, but +stdout and the PCM pipe are independently observed. The host may therefore +buffer one valid Begin for the single reserved speech for at most two seconds; +it must not acknowledge Begin until its session actor has applied `accepted`. +A second Begin, a speech mismatch, a stale result, or rendezvous expiry is +fatal. +`audio_played.played_frames` is cumulative, monotonic source-frame truth and may +legitimately lag newer accepted sequences. It cannot exceed accepted frames. +End follows the last accepted Chunk. Drained must name End's last sequence and +confirm every source frame played before completion is published. A device host +must include its bounded post-render route latency before stopping the engine +and sending Drained; node buffer consumption alone is insufficient. That grace +must remain within the child's two-second End-to-Drained deadline. Cancellation +or failure bypasses the grace and quiesces immediately. + +Cancel is an ordered pipe record. `audio_cancelled` is a quiescence barrier and +its settled played count is the authoritative partial-delivery snapshot before +the child publishes interruption. A cancellation timeout or host failure +publishes `speech_failed`, not `speech_interrupted`. When cancellation overtakes +an already-written Begin, Chunk, or End, the exact in-flight acceptance, +progress, Drained, or Failed acknowledgement may settle before the pipe-ordered +Cancelled acknowledgement. Cancelled is then the quiescence barrier: failure +wins the speech outcome, otherwise cancellation wins. Callbacks after that +barrier are protocol-fatal. Outside this narrow race, Drained, Failed, and +Cancelled are terminal. Only an exact duplicate in-phase played count is +idempotent. Unknown or future speech IDs, sequence gaps or regressions, +impossible counts, and stale terminal +acknowledgements are fatal. +If a backend or pipe write fails and the child cannot obtain a quiescent cancel +barrier, it emits that speech's failure and terminates the session data plane; +it never admits a replacement while old host audio may still be active. + +## Parent requests + +```text +{"type":"hello","id":u64,"input_during_tts":"allow_barge_in"|"suppress_input"} +{"type":"set_paused","active":bool} +{"type":"set_input_muted","id":u64,"active":bool} +{"type":"set_tts_settings","id":u64,"expected_revision":u64,"settings":TtsSettings} +{"type":"set_input_during_tts","id":u64,"expected_revision":u64,"policy":"allow_barge_in"|"suppress_input"} +{"type":"reset_input","id":u64} +{"type":"prepare_speak","id":u64,"acknowledgement":u64|null,"text":string} +{"type":"output_ready","id":u64,"speech_id":u64} +audio acknowledgements listed above +{"type":"query_state","id":u64,"after":u64} +{"type":"cancel","id":u64} +{"type":"shutdown"} +``` + +Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. +The parent cannot author speaking state or finalized input; those are derived +only from PCM by the child runtime. + +`set_tts_settings` accepts the same tagged public object projected by `ready`, +without `revision`. It changes settings only for the already-active backend: + +```text +{"backend":"openai","model":string,"voice":string,"rate":0.75..2.0} +{"backend":"siri","voice":string,"language":string,"rate":0.5..2.0} +{"backend":"pocket","model":string,"voice":string,"rate":0.75..2.0} +``` + +The child constructs and validates a replacement without blocking input or +playback processing, then atomically commits it only if `expected_revision` +still matches. It responds with: + +```text +{"type":"tts_settings_result","id":u64,"outcome":"applied","snapshot":TtsConfigurationSnapshot} +{"type":"tts_settings_result","id":u64,"outcome":"rejected","snapshot":TtsConfigurationSnapshot,"message":string} +``` + +The snapshot is authoritative in both outcomes. Invalid, stale, cross-backend, +concurrent, timed-out, or shutdown-interrupted updates are nonfatal and leave +the prior configuration active. The applied response is the client-visible +linearization point. A speech reservation holds a configuration lease: speech +admitted before the response retains its old backend/settings, while later +admission receives the new revision. Pocket's public model identifier cannot be +changed without selecting and validating another bundle at process startup. + +`set_input_during_tts` changes the policy for later admissions. Its expected +revision must be positive. The correlated result is always authoritative: + +```text +{"type":"input_during_tts_result","id":u64,"outcome":"applied"|"rejected","snapshot":{"revision":u64,"policy":"allow_barge_in"|"suppress_input"}} +``` + +A stale revision is rejected nonfatally with no mutation. Each speech leases +the current policy when it is admitted; a held prepare leases only when it is +eventually admitted. Updating the policy never changes an already-admitted +utterance. + +`set_input_muted` controls the host-mute reason. Assistant suppression is a +separate guard-owned reason, so clearing either reason cannot clear the other. +The effective input-mute epoch advances only when the composed state changes. +`set_input_muted` and `reset_input` return exact correlated acknowledgements: + +```text +{"type":"input_mute_applied","id":u64,"active":bool} +{"type":"input_reset_applied","id":u64} +``` + +## Authoritative input + +The child emits: + +```text +{"type":"input_speaking","active":bool} +{"type":"recognition_pending","active":bool} +{"type":"user_final","token":u64,"text":string} +``` + +For every final, the child allocates a strictly increasing token, stores it in +`SessionCore`, acknowledges the runtime storage receipt, emits `user_final`, and +only then interrupts reserved or playing assistant output. Final text is at +most 64 KiB. + +## Confirmation and admission + +`acknowledgement` is a request-local causal cutoff. `null` uses the stored +confirmed cursor. `0` is the exact zero cutoff. Any existing token is the exact +cutoff, even when older than the stored cursor. Naming an existing token advances +the stored cursor monotonically but never moves it backward. A missing or future +token falls back to the stored cursor. Finals after the request-local cutoff +produce `pending`. + +Prepare evaluation order is fixed: reject empty text; while input is speaking or +recognition is pending, hold one prepare indefinitely without applying its +acknowledgement; then apply the cutoff and return pending finals; then reject +paused; then reject an in-progress speech; otherwise reserve. A second prepare +while one is held returns `in_progress` without mutation. + +Reservation emits: + +```text +{"type":"admitted","id":u64,"speech_id":u64,"confirmed_token":u64} +``` + +It does not begin synthesis. The parent replies with the originating prepare +ID and speech ID when it is ready to accept output: + +```text +{"type":"output_ready","id":u64,"speech_id":u64} +{"type":"output_ready_result","id":u64,"speech_id":u64,"outcome":"accepted"|"stale"} +``` + +Before emitting `accepted`, the child installs the admitted speech's leased +assistant-activity guard. `suppress_input` stops PCM admission so user input +cannot interrupt the speech; `allow_barge_in` continues PCM admission with the +assistant-sensitive VAD threshold. `accepted` transfers output authority. The +child then writes Begin and waits for `audio_begin_accepted` before delivering +PCM. Readiness is bounded to two seconds; expiry emits `speech_failed` with zero output. With `allow_barge_in`, speaking or recognition pending provisionally suspends started output. If both clear without a final, the same speech and player resume. A final, targeted cancellation, host mute/reset while provisionally suspended, pause, or shutdown discards the hold and interrupts exactly once. +Accepted output owns exactly one assistant-activity guard. The child removes +that guard before emitting completion, interruption, or failure, including +cancellation and shutdown terminals. + +## State, cancellation, and output events + +```text +{"type":"pending","id":u64,"utterances":[{"token":u64,"text":string}]} +{"type":"not_admitted","id":u64,"reason":"paused"|"in_progress"|"cancelled"|"empty_text"} +{"type":"state","id":u64,"confirmed_token":u64,"utterances_after":[{"token":u64,"text":string}]} +{"type":"cancel_result","id":u64,"outcome":"cancelled"|"stale","speech_id":u64|null} +{"type":"speech_started","id":u64,"speech_id":u64} +{"type":"speech_completed","id":u64,"speech_id":u64} +{"type":"speech_interrupted","id":u64,"speech_id":u64,"spoken_through_utf8":u64} +{"type":"speech_failed","id":u64,"speech_id":u64,"message":string} +{"type":"fatal","message":string} +``` + +`query_state.after` is an exclusive token cutoff; `0` requests all. `cancel.id` +targets the originating `prepare_speak.id`. `cancel_result` is emitted first. A +live held target then emits `not_admitted(cancelled)`; a live admitted target +then emits `speech_interrupted`. `spoken_through_utf8` is Berd Voice's conservative UTF-8 byte boundary through the last fully played word; hosts may use it to distinguish the estimated spoken prefix from the unspoken suffix without recreating delivery policy. Repeated or unknown cancellation is stale. +Every speech event carries the originating prepare ID. `speech_started` appears +only after the first PCM Chunk is accepted by the host, and exactly one terminal +message follows every admission. + +On `shutdown`, all complete earlier frames are processed in order. The parent +keeps stdin open while the child cancels and drains output. The child then +finishes the input runtime while continuing to drain events and storage +receipts, flushes, and exits. EOF, malformed framing, fatal +input failure, or process death cancels both authorities without transparent +restart. A fatal error is flushed exactly once and followed by no protocol +output. diff --git a/src-tauri/crates/berd-voice/README.md b/src-tauri/crates/berd-voice/README.md index 32e167120..3e14b5b3c 100644 --- a/src-tauri/crates/berd-voice/README.md +++ b/src-tauri/crates/berd-voice/README.md @@ -1,6 +1,301 @@ # berd-voice -Berd-owned Pocket TTS inference. +Berd-owned voice primitives, TTS, and speech recognition. -This crate owns the April ONNX runtime and text chunking used by Berd's native -voice commands. +This crate owns the neutral PCM output contract and backend-neutral TTS stream +used by Berd, plus the April ONNX runtime and text chunking used by Berd's native +voice commands. It also owns the concrete Parakeet model loader and complete +16 kHz utterance recognizer, plus the OpenAI Realtime transcription websocket +client and macOS SpeechTranscriber engine used by Berd's existing native STT +workers. The concrete voice-input runtime accepts bounded 20 ms, 48 kHz mono +Float32 frames and owns Berd's adaptive VAD, resampling, utterance boundaries, +logical mute/reset epochs, recognition-pending state, stale-result rejection, +and bounded engine shutdown. Hosts retain capture devices, optional physical +mute effects, engine configuration resolution, transcript storage and delivery, +and UI projection. Logical host mute and assistant input suppression compose +inside the shared runtime. OpenAI emits 24 kHz mono Float32 PCM. On macOS, the shared +Siri bridge emits normalized 48 kHz mono Float32 PCM without opening an audio +device; the existing Berd Siri player and the CLI use the same decoder. + +`berd-voice session` exposes the development voice-session protocol documented +in [PROTOCOL.md](PROTOCOL.md). Siri TTS and macOS speech recognition are the +defaults: + +```text +berd-voice session --voice Aaron --language en-US --rate 1.0 +berd-voice session --tts-backend openai --rate 1.0 +berd-voice session --tts-backend pocket --model-dir /path/to/native-voice-v2 --voice george --rate 1.0 +berd-voice session --stt-backend parakeet --stt-model-dir /path/to/parakeet +berd-voice session --stt-backend openai +``` + +The default Siri backend still requires an exact installed voice name and +language. Missing or unavailable Siri voice configuration and an unavailable +current-locale macOS speech model fail startup with setup guidance. The session +never silently falls back to OpenAI or another cloud engine. +Siri preflight validates the exact case-sensitive installed name, normalized +BCP-47 language, and a responsive sirittsd availability query; later synthesis +can still fail and is reported through the normal terminal speech lifecycle. + +## Synthesize to WAV + +`synthesize` renders through the same TTS backends without opening an audio device. It writes mono signed 16-bit little-endian PCM WAV to a new file: + +```sh +berd-voice synthesize --tts-backend siri --voice Aaron --language en-US \ + --rate 1.0 --text "Hello" --output hello.wav +berd-voice synthesize --tts-backend pocket \ + --model-dir /absolute/path/to/native-voice-v2 --voice mary --rate 1.0 \ + --text "Hello" --output hello.wav +berd-voice synthesize --tts-backend openai --model gpt-4o-mini-tts \ + --voice marin --rate 1.0 --allow-paid-openai \ + --text "Hello" --output hello.wav +``` + +The command rejects an existing output before constructing the backend, writes to a same-directory temporary file, synchronizes the completed WAV, and publishes it without clobbering a target that appears concurrently. Failure leaves no partial target. Input text is nonempty and at most 16 KiB; output is bounded to ten minutes of finite source PCM in `[-1, 1]`. Backend cancellation, empty or invalid PCM, and a non-1x PCM playback specification fail rather than publishing a misleading file. + +Pocket rendering supports only rate `1.0`: its other rates are a host playback time-stretch policy and are not encoded into synthesized PCM. Siri applies its rate during native synthesis. OpenAI requires an explicit model, voice, and `--allow-paid-openai`; one invocation makes at most one request and reads the credential only from `OPENAI_API_KEY` after output preflight. + +Success emits one schema-version-one JSON line with the public backend identity, requested rate, and WAV encoding, sample rate, channels, bit depth, source frames, duration, and byte count. It never serializes the prompt, credential, endpoint, Pocket bundle path, or temporary path. Operation failure emits one sanitized error line and exits 1; usage failure emits no JSON and exits 2. Stdout is reserved for this machine-readable terminal record, not audio data. + +The public `berd_voice::siri` management API is also the single native boundary +used by Berd for Siri catalog discovery, represented languages, exact installed +voice validation, and download. A voice identity is its case-sensitive catalog +name plus a normalized BCP-47 language tag; private Apple identifiers are never +persisted or exposed. Download is a blocking terminal success/error operation +with a validated availability-polling bound and no invented byte progress. +Native validation and subscription have separate bounded waits before that +polling deadline. Berd continues to own persisted selection, fallback policy, +settings/UI events, and management preview playback. There is intentionally no +host settings or fallback policy in the standalone management commands. + +## Voice and model management + +The standalone commands are thin projections of the same shared management +APIs used by Berd: + +```sh +berd-voice voices list +berd-voice voices list --language en-US +berd-voice voices download --voice Aaron --language en-US +berd-voice voices download --voice Aaron --language en-US \ + --availability-wait-seconds 300 +berd-voice models macos status +berd-voice models macos install +berd-voice models pocket status --store-root /absolute/portable-store +berd-voice models pocket install --store-root /absolute/portable-store +berd-voice models pocket voices +berd-voice models parakeet status --store-root /absolute/portable-store +berd-voice models parakeet install --store-root /absolute/portable-store +``` + +The Siri language filter is an exact normalized BCP-47 language, not a prefix. +Download identifies a voice by its exact case-sensitive catalog name plus that +language. Its optional `1..1800` second bound applies only to the final native +availability poll; validation and subscription retain their separate bounded +waits. Siri download reports no invented byte progress. + +Management stdout is machine-first JSONL. Every line contains +`schemaVersion: 1`, an `operation`, and an `event`. Read-only commands and Siri +download emit exactly one terminal `result`; macOS model installation may emit +native `progress` fractions followed by exactly one terminal `result` or +`error`. Operation failures emit a sanitized structured `error` on stdout and +diagnostic detail on stderr. Usage errors emit no JSON and exit 2; operation or +unsupported-mutation failures exit 1; successful read-only status on an +unsupported platform still exits 0 with `supported: false`. These blocking +commands have no cancellation protocol, and interruption by a process signal +does not promise a terminal JSON line. None opens an audio device, starts a +voice session, applies host settings, chooses fallbacks, or emits Tauri events. + +Pocket and Parakeet require an explicit absolute coordination root; there is no +default and no coupling to Berd's app cache. The CLI derives the closed portable +layout `/native-voice-v2`, with Parakeet nested at `stt`. Status does +not create a missing store. Install is per engine, is idempotent when that +engine is already Ready, and preserves the other Ready engine through the +shared transaction. Progress reports only the shared phase, downloaded bytes, +and total download bytes. A successful result identifies `alreadyReady` or +`installed`, verified bytes, and whether cleanup remains; any retained recovery +path is diagnostic stderr data, never JSON. + +The public `pocket_assets` and `parakeet_assets` modules define the immutable +asset catalogs used by Berd and inspect an explicit portable bundle root as +`Missing`, `Invalid`, or `Ready { verified_bytes }`. Verification opens each +file, checks that it is a regular file with the pinned size, and streams its +SHA-256 through a bounded buffer. Pocket exposes its model files and twelve +voice descriptors; Parakeet has its own model identity and pins the runtime +model, tokens, and exact attribution/license file. These modules do not choose +a default voice, cache root, removal policy, or UI representation. Their +concrete installers accept an explicit closed Pocket/Parakeet root layout, +download only pinned HTTPS assets with bounded streaming size and checksum +verification, extract only exact Parakeet manifest entries, and publish a fully +verified combined tree. A short cross-process transaction lock coordinates +publication with model-loading readers and host-owned removal; interrupted +publication is recovered from one unambiguous verified backup, while ambiguous +or failed rollback state is returned with recovery paths. Downloads and archive +preparation remain outside that lock. Progress describes concrete phases plus +monotonic downloaded bytes; it does not invent extraction progress. Berd keeps +root selection, Tauri queue/revisions/events, settings and fallback policy, +live-stop policy, and removal UI. + +Once the new combined tree passes final verification it is authoritative. If +deleting the retired backup then fails, installation still returns success with +`cleanup_pending`; Berd logs the retained path and the next locked preflight +retries cleanup. This avoids reporting a failed install after the model has +already been applied. + +`voices.list` returns `backend`, normalized `languageFilter`, +`availableLanguages`, and exact voice records. `voices.download` returns the +canonical voice, `installed: true`, and `availabilityWaitSeconds`; a missing +exact catalog identity fails with `voice_not_found` before any native download +request. macOS status and install results contain `supported`, `locale`, +`localeSupported`, `modelStatus`, and `ready`. Install progress records contain +the native finite fraction clamped to `0...1`; nonfinite callbacks are omitted. +`models.pocket.voices` is a separate immutable catalog because Pocket installs +one pinned all-voices bundle rather than managing OS voices one at a time. It +returns only public model/license IDs and `{id,name}` records—no paths, hashes, +or source URLs. Pocket and Parakeet status return `missing|invalid|ready`, +nullable verified bytes, and the pinned total download bytes. Their install +progress uses schema-version-one JSONL envelopes with lowercase phases and +monotonic byte counts. + +The session host owns both physical devices. Stdin is one framed stream +containing JSON controls and exact 20 ms, 48 kHz mono Float32 microphone frames; +controls are priority-routed while PCM forwarding remains strictly bounded. A +required inherited `--pcm-output-fd` carries independently framed synthesized +PCM records so device backpressure cannot block stdout lifecycle events or +stdin cancellation. The child retains source-frame delivery, drain, partial +delivery, and terminal authority through correlated acceptance, played, +drained, failed, and cancelled acknowledgements. The shared runtime owns Berd's +adaptive VAD, recognition-pending state, final-token storage, admission, and +barge-in. Omitting `--stt-backend` selects macOS speech recognition. + +The session's `ready` event projects a sanitized, revisioned TTS snapshot. +Same-backend voice, language/model, and normalized rate updates validate off the +session loop, commit atomically, and apply to the next admitted utterance. +Already-admitted speech retains its configuration lease. Failed or stale +updates keep the prior configuration, and private credentials, endpoints, and +bundle paths never enter the snapshot. + +The session also projects a separately revisioned `input_during_tts` snapshot. +`allow_barge_in` keeps PCM flowing through assistant-sensitive VAD, while +`suppress_input` drops PCM at the shared runtime for the admitted utterance. +Live policy updates apply to the next admission; host mute remains an +independent reason, so neither state can accidentally clear the other. + +Pocket's model path is the exact portable bundle directory, not a Berd cache +root. The CLI resolves an exact voice ID through the shared +`voices/.wav` bundle layout and validates both model and voice before +`ready`; callers may point it at a Berd-downloaded bundle explicitly, but no +application-specific cache path is assumed. + +## TTS benchmarks + +`benchmark tts` exercises the same backend PCM source without opening an audio +device. It emits one JSON report on stdout and diagnostics on stderr: + +```text +berd-voice benchmark tts --tts-backend siri --voice Aaron --language en-US \ + --prompt-manifest english-short-v1 --mode fresh-backend +berd-voice benchmark tts --tts-backend pocket \ + --model-dir /path/to/native-voice-v2 --voice mary \ + --prompt-manifest english-short-v1 --mode warm +``` + +The built-in `english-short-v1` manifest has one separate warm-up prompt and +five distinct, similarly sized measured prompts. `fresh-backend` constructs a +backend for each measured prompt. `warm` constructs one backend, synthesizes the +separate unmeasured prompt, then reuses that backend for the five measured +prompts. Neither mode promises a fresh process, provider daemon, native +framework, model-file cache, or operating-system cache. Warm OpenAI mode makes +one additional billable warm-up request. + +An explicit `--text TEXT --runs COUNT` remains available for intentional +exact-prompt cache experiments. Reports label that scenario +`exact_prompt_repeat`; the manifest path is labeled +`distinct_prompt_manifest`. This distinction matters for Siri: exact repeats +have been observed to return decoded PCM within a few milliseconds, likely +benefiting from hot system or daemon state. That does not measure novel +synthesis or audible onset; the private sirittsd implementation does not let us +attribute the effect to a particular internal cache. + +Each run reports initialization time when applicable, time to first nonempty +PCM, total synthesis time, mono PCM frame count and sample rate, finite and +nonfinite frame counts, peak amplitude, global RMS, PCM audio duration, +real-time factor (`synthesis duration / PCM audio duration`), and a structured +outcome or error stage. Completed output containing nonfinite PCM or no +sustained signal is an error. `playback_rate` is metadata only: benchmarks +measure generated PCM duration and never playback or output-device drain. + +Signal onset uses 20 ms RMS windows with a 10 ms hop and requires three +consecutive windows at or above `max(1e-6, peak_window_rms * 0.01)`. Reports +include the threshold, the source-timeline offset of the first qualifying +window, and the callback time that supplied that source frame. They also +simulate immediate zero-device-latency PCM playout, stalling the source timeline +when a callback arrives too late, as +`estimated_earliest_realtime_signal_ms`. This is a device-free PCM scheduling +estimate, not actual or audible onset: it excludes player buffering, operating +system scheduling, output devices, transducers, volume, and hearing. +Every run identifies its prompt ID, UTF-8 byte count, and SHA-256 without +printing the prompt itself. Manifest reports include its stable ID, language, +and pinned content hash. Prompts are distinct within a manifest invocation, but +`prior_cache_state` remains explicitly uncontrolled because provider and system +caches can survive earlier processes. `planned_workload` includes the warm-up +when present; individual results show what actually ran. OpenAI reports whether +its endpoint came from the built-in default or the `OPENAI_BASE_URL` +environment, but never includes the URL. + +OpenAI benchmarking is disabled unless the command includes +`--allow-paid-openai`. The CLI preflights the full workload, including the warm +mode's extra request, and rejects more than 20 requests or 65,536 total prompt +bytes before constructing the backend. A missing `OPENAI_API_KEY` still fails as +a structured initialization error without making a request. + +## STT benchmarks + +`benchmark stt` feeds a small, immutable LibriSpeech `test-clean` fixture pack +through the same `VoiceInputRuntime` used by Berd and the voice session. It does +not open an input device: + +```text +berd-voice benchmark stt --stt-backend macos --runs 1 --mode cold +berd-voice benchmark stt --stt-backend parakeet \ + --stt-model-dir /path/to/parakeet --runs 3 --mode warm +``` + +The checked-in pack contains three unmodified 16 kHz mono FLAC utterances from +OpenSLR SLR12. Its manifest records the official archive URL and MD5, CC BY 4.0 +license, exact transcripts, decoded stream metadata, and per-file SHA-256. +Benchmark startup verifies those hashes and metadata, decodes the audio, and +uses deterministic linear interpolation to convert it to the runtime's 48 kHz +mono Float32 contract. The report records that conversion and embeds the full +fixture attribution notice, so standalone binaries and packaged applications +retain the notice. Rust sources remain Apache 2.0; the embedded corpus files are +CC BY 4.0, as reflected by the crate's aggregate package-license metadata. + +Input is paced in real time as exact 960-sample frames every 20 ms. Each clip +has one second of leading silence and 6.5 seconds of trailing silence. The long +tail deliberately keeps continuous recognizers supplied with capture-like PCM +through VAD settlement and the runtime's five-second live no-result bound; it +is included in the reported workload. A final transcript is validated and +stored in its per-utterance result before its storage receipt is acknowledged, +and the next clip does not begin until authoritative speaking and +recognition-pending state are both idle. + +Cold mode creates a fresh `VoiceInputRuntime` for each measured run. It does not +start a fresh process, so operating-system, provider, and model-file caches may +remain warm. Warm mode creates one runtime, records one unmeasured fixture-pack +warm-up, then reuses that resident runtime for the measured runs. + +Reports contain fixture provenance, sanitized engine/environment metadata, +planned recognition commits and streamed duration, initialization and turn +timings, hypotheses, and aggregate word error rate. WER normalization retains +ASCII letters, digits, and apostrophes, converts them to uppercase, maps other +punctuation to whitespace, and reports substitutions, deletions, and insertions +alongside the aggregate rate. + +OpenAI STT benchmarking requires `--allow-paid-openai` and reads its key only +from `OPENAI_API_KEY`. Before resolving credentials or connecting, the CLI +rejects a warmup-inclusive workload above 20 recognition commits or 120 seconds +of streamed PCM. Endpoint and model overrides use the same environment variables +as the session; reports record only which source supplied them and never include +the key or endpoint value. diff --git a/src-tauri/crates/berd-voice/build.rs b/src-tauri/crates/berd-voice/build.rs new file mode 100644 index 000000000..16244a377 --- /dev/null +++ b/src-tauri/crates/berd-voice/build.rs @@ -0,0 +1,20 @@ +fn main() { + #[cfg(target_os = "macos")] + { + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + println!("cargo:rerun-if-changed=native/siri_tts_bridge.h"); + println!("cargo:rerun-if-changed=native/siri_tts_bridge.m"); + cc::Build::new() + .file("native/siri_tts_bridge.m") + .flag("-fobjc-arc") + .compile("berd_siri_tts_bridge"); + swift_rs::SwiftLinker::new("14.0") + .with_package("BerdMacSpeechBridge", "swift/BerdMacSpeechBridge") + .link(); + for framework in ["Foundation", "AVFoundation", "AudioToolbox", "CoreAudio"] { + println!("cargo:rustc-link-lib=framework={framework}"); + } + println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift"); + } + } +} diff --git a/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/1089-134686-0002.flac b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/1089-134686-0002.flac new file mode 100644 index 000000000..0ee470052 Binary files /dev/null and b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/1089-134686-0002.flac differ diff --git a/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/1221-135766-0002.flac b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/1221-135766-0002.flac new file mode 100644 index 000000000..3ab10154a Binary files /dev/null and b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/1221-135766-0002.flac differ diff --git a/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/1284-1180-0003.flac b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/1284-1180-0003.flac new file mode 100644 index 000000000..59c9b6be9 Binary files /dev/null and b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/1284-1180-0003.flac differ diff --git a/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/NOTICE.md b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/NOTICE.md new file mode 100644 index 000000000..b2d4d44de --- /dev/null +++ b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/NOTICE.md @@ -0,0 +1,33 @@ +# LibriSpeech `test-clean` benchmark fixture + +The upstream archive includes this license notice: + +> LibriSpeech (c) 2014 by Vassil Panayotov +> +> LibriSpeech ASR corpus is licensed under a Creative Commons Attribution 4.0 +> International License. +> +> See . + +These three unmodified FLAC files and their reference transcripts were +extracted from the LibriSpeech ASR corpus, OpenSLR resource SLR12: + +- Source: https://www.openslr.org/12/ +- Archive: https://www.openslr.org/resources/12/test-clean.tar.gz +- Published archive MD5: `32fa31d27d2e1cad72775fee3f4849a9` +- License: [Creative Commons Attribution 4.0 International](https://creativecommons.org/licenses/by/4.0/) + +The source archive's published checksum was verified before extracting the +three paths named in `manifest.json`. The transcript text was copied exactly +from each corresponding `*.trans.txt` file in that archive. The audio has not +been modified. `manifest.json` records the SHA-256, byte length, and decoded +stream metadata for each extracted file. + +Please cite the corpus as: + +> Vassil Panayotov, Guoguo Chen, Daniel Povey, and Sanjeev Khudanpur. +> “LibriSpeech: An ASR Corpus Based on Public Domain Audio Books.” +> ICASSP 2015. + +The corpus was prepared from LibriVox public-domain audiobook recordings. The +Rust source code in this crate remains licensed under Apache License 2.0. diff --git a/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/manifest.json b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/manifest.json new file mode 100644 index 000000000..29b9a8169 --- /dev/null +++ b/src-tauri/crates/berd-voice/fixtures/stt/librispeech-test-clean-mini/manifest.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "corpus": "LibriSpeech ASR corpus", + "resource_id": "SLR12", + "subset": "test-clean", + "language": "en", + "license": "CC BY 4.0", + "license_url": "https://creativecommons.org/licenses/by/4.0/", + "source_url": "https://www.openslr.org/12/", + "archive_url": "https://www.openslr.org/resources/12/test-clean.tar.gz", + "archive_md5": "32fa31d27d2e1cad72775fee3f4849a9", + "utterances": [ + { + "id": "1089-134686-0002", + "file": "1089-134686-0002.flac", + "sha256": "66277a3fa3df407261dc2a3ce685a7ceef19999ab0c10531bee5257cb64cb59d", + "bytes": 116299, + "sample_rate_hz": 16000, + "channels": 1, + "bits_per_sample": 16, + "samples": 106000, + "transcript": "AFTER EARLY NIGHTFALL THE YELLOW LAMPS WOULD LIGHT UP HERE AND THERE THE SQUALID QUARTER OF THE BROTHELS" + }, + { + "id": "1221-135766-0002", + "file": "1221-135766-0002.flac", + "sha256": "1f02bae046f9567010f0163e8f204612fd1035a751383586a7aef6cb0fcc6e60", + "bytes": 104776, + "sample_rate_hz": 16000, + "channels": 1, + "bits_per_sample": 16, + "samples": 77200, + "transcript": "YET THESE THOUGHTS AFFECTED HESTER PRYNNE LESS WITH HOPE THAN APPREHENSION" + }, + { + "id": "1284-1180-0003", + "file": "1284-1180-0003.flac", + "sha256": "e9850c58c0b066ec8eeceda7ded87ba42cb6823f69b935bfe5d3619c46fe166a", + "bytes": 96521, + "sample_rate_hz": 16000, + "channels": 1, + "bits_per_sample": 16, + "samples": 77360, + "transcript": "FOR A LONG TIME HE HAD WISHED TO EXPLORE THE BEAUTIFUL LAND OF OZ IN WHICH THEY LIVED" + } + ] +} diff --git a/src-tauri/crates/berd-voice/fixtures/tts/english-short-v1.json b/src-tauri/crates/berd-voice/fixtures/tts/english-short-v1.json new file mode 100644 index 000000000..a9fa47dcf --- /dev/null +++ b/src-tauri/crates/berd-voice/fixtures/tts/english-short-v1.json @@ -0,0 +1,30 @@ +{ + "id": "english-short-v1", + "language": "en-US", + "warmup": { + "id": "warmup", + "text": "Fresh violet clouds drift slowly above the harbor." + }, + "prompts": [ + { + "id": "copper-fox", + "text": "A bright copper fox races past the quiet pond." + }, + { + "id": "silver-bird", + "text": "The small silver bird glides over the calm lake." + }, + { + "id": "autumn-wind", + "text": "A gentle autumn wind moves through the tall grass." + }, + { + "id": "blue-heron", + "text": "The patient blue heron waits beside the clear stream." + }, + { + "id": "rain-window", + "text": "A curious young cat watches rain cross the window." + } + ] +} diff --git a/src-tauri/native/siri_tts_bridge.h b/src-tauri/crates/berd-voice/native/siri_tts_bridge.h similarity index 71% rename from src-tauri/native/siri_tts_bridge.h rename to src-tauri/crates/berd-voice/native/siri_tts_bridge.h index 5223f7abc..153c51815 100644 --- a/src-tauri/native/siri_tts_bridge.h +++ b/src-tauri/crates/berd-voice/native/siri_tts_bridge.h @@ -8,17 +8,18 @@ extern "C" { #endif -/// Returns a malloc-owned JSON array of Siri voices for the requested language -/// prefix. Each item contains name, language, sizeBytes, and installed. Returns -/// NULL and sets error_out on failure. -char *berd_siri_tts_catalog_json(const char *language_prefix, char **error_out); +/// Returns a malloc-owned JSON array of Siri voices for the requested exact +/// normalized language, or every language when empty. Each item contains name, +/// language, sizeBytes, and installed. Returns NULL and sets error_out on failure. +char *berd_siri_tts_catalog_json(const char *language, char **error_out); /// Returns the locale tags represented in the complete Siri voice catalog. /// This does not perform per-voice daemon validation. char *berd_siri_tts_languages_json(char **error_out); /// Downloads and validates one exact Siri voice. This call blocks until the -/// voice is usable or the timeout elapses. +/// voice is usable or the availability-polling timeout elapses. Validation and +/// subscription are separately bounded and occur before that polling deadline. bool berd_siri_tts_download_voice( const char *language, const char *voice_name, @@ -28,6 +29,33 @@ bool berd_siri_tts_download_voice( typedef bool (*BerdSiriTTSShouldStop)(void *context); typedef void (*BerdSiriTTSPlaybackStarted)(void *context); +typedef bool (*BerdSiriTTSPcmFrames)( + const float *samples, + uint32_t frame_count, + void *context +); + +/// Validates that an exact Siri voice is downloaded and usable through +/// sirittsd. This does not synthesize or play audio. +bool berd_siri_tts_validate_voice( + const char *language, + const char *voice_name, + char **error_out +); + +/// Synthesizes one utterance and emits normalized 48 kHz mono Float32 PCM. +/// This call blocks until synthesis and converter flushing complete. It never +/// opens an audio device. Returning false from `pcm_frames` cancels synthesis. +bool berd_siri_tts_synthesize_pcm( + const char *text, + const char *language, + const char *voice_name, + float rate, + BerdSiriTTSShouldStop should_stop, + BerdSiriTTSPcmFrames pcm_frames, + void *context, + char **error_out +); /// Plays the small per-voice sample bundled with macOS. This works before the /// full Siri voice has been downloaded. @@ -87,7 +115,6 @@ bool berd_pocket_audio_player_enqueue( uint32_t frame_count, char **error_out ); -bool berd_pocket_audio_player_set_rate(void *player, float rate, char **error_out); uint64_t berd_pocket_audio_player_completed_source_frames(void *player); uint64_t berd_pocket_audio_player_pending_buffers(void *player); bool berd_pocket_audio_player_failed(void *player); diff --git a/src-tauri/native/siri_tts_bridge.m b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m similarity index 84% rename from src-tauri/native/siri_tts_bridge.m rename to src-tauri/crates/berd-voice/native/siri_tts_bridge.m index 43b7dfbe5..ec80eb370 100644 --- a/src-tauri/native/siri_tts_bridge.m +++ b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m @@ -296,14 +296,26 @@ @interface BerdSiriDeliverySegment : NSObject @implementation BerdSiriDeliverySegment @end -@interface BerdSiriSpeechPlayer : NSObject -@property(nonatomic, strong) dispatch_queue_t queue; -@property(nonatomic, strong) AVAudioEngine *engine; -@property(nonatomic, strong) AVAudioPlayerNode *player; +/// Stateful sirittsd packet decoder. Both the app-owned streaming player and +/// the standalone berd-voice session use this one normalization path. +@interface BerdSiriAudioDecoder : NSObject @property(nonatomic, strong) AVAudioConverter *opusConverter; @property(nonatomic, strong) AVAudioFormat *opusSourceFormat; @property(nonatomic, strong) AVAudioConverter *pcmConverter; @property(nonatomic, strong) AVAudioFormat *pcmSourceFormat; +- (AVAudioPCMBuffer *)decodeData:(NSData *)data + format:(AudioStreamBasicDescription)description + packetCount:(UInt32)packetCount + packetDescriptions:(NSData *)packetDescriptions + error:(NSError **)error; +- (NSArray *)finishConversion:(NSError **)error; +@end + +@interface BerdSiriSpeechPlayer : NSObject +@property(nonatomic, strong) dispatch_queue_t queue; +@property(nonatomic, strong) AVAudioEngine *engine; +@property(nonatomic, strong) AVAudioPlayerNode *player; +@property(nonatomic, strong) BerdSiriAudioDecoder *decoder; @property(nonatomic, strong) BerdSiriSynthesisSession *session; @property(nonatomic, strong) NSMutableArray *pendingTexts; @property(nonatomic, strong) NSMutableArray *deliverySegments; @@ -327,32 +339,7 @@ - (void)cancel; - (NSString *)deliveryJSON; @end -@implementation BerdSiriSpeechPlayer -- (instancetype)init { - self = [super init]; - if (self) { - _queue = dispatch_queue_create("com.block.berd.sirittsd", DISPATCH_QUEUE_SERIAL); - dispatch_queue_set_specific(_queue, BerdSiriSpeechQueueKey, - BerdSiriSpeechQueueKey, NULL); - _completionSemaphore = dispatch_semaphore_create(0); - _pendingTexts = [NSMutableArray array]; - _deliverySegments = [NSMutableArray array]; - } - return self; -} -- (void)finish:(NSError *)error { - if (self.finished) return; - self.finished = YES; - self.error = error; - self.progressGeneration += 1; - dispatch_semaphore_signal(self.completionSemaphore); -} -- (void)finishIfReady { - if (self.inputFinished && !self.session && self.pendingTexts.count == 0 && - self.pendingBuffers == 0) { - [self finish:self.error]; - } -} +@implementation BerdSiriAudioDecoder - (AVAudioPCMBuffer *)decodeData:(NSData *)data format:(AudioStreamBasicDescription)description packetCount:(UInt32)packetCount @@ -505,6 +492,35 @@ - (AVAudioPCMBuffer *)decodeData:(NSData *)data self.opusSourceFormat = nil; return buffers; } +@end + +@implementation BerdSiriSpeechPlayer +- (instancetype)init { + self = [super init]; + if (self) { + _queue = dispatch_queue_create("com.block.berd.sirittsd", DISPATCH_QUEUE_SERIAL); + dispatch_queue_set_specific(_queue, BerdSiriSpeechQueueKey, + BerdSiriSpeechQueueKey, NULL); + _completionSemaphore = dispatch_semaphore_create(0); + _pendingTexts = [NSMutableArray array]; + _deliverySegments = [NSMutableArray array]; + _decoder = [BerdSiriAudioDecoder new]; + } + return self; +} +- (void)finish:(NSError *)error { + if (self.finished) return; + self.finished = YES; + self.error = error; + self.progressGeneration += 1; + dispatch_semaphore_signal(self.completionSemaphore); +} +- (void)finishIfReady { + if (self.inputFinished && !self.session && self.pendingTexts.count == 0 && + self.pendingBuffers == 0) { + [self finish:self.error]; + } +} - (BOOL)ensurePlayer:(NSError **)error { if (self.player) return YES; self.engine = [AVAudioEngine new]; @@ -557,7 +573,7 @@ - (void)enqueueData:(NSData *)data format:(AudioStreamBasicDescription)format if (self.finished || !data.length) return; self.progressGeneration += 1; NSError *error = nil; - AVAudioPCMBuffer *buffer = [self decodeData:data format:format packetCount:packetCount + AVAudioPCMBuffer *buffer = [self.decoder decodeData:data format:format packetCount:packetCount packetDescriptions:packetDescriptions error:&error]; if (error || !buffer) { if (error) [self finish:error]; @@ -595,7 +611,7 @@ - (void)startNextSynthesis { if (!error) { NSError *conversionError = nil; NSArray *buffers = - [weakSelf finishConversion:&conversionError]; + [weakSelf.decoder finishConversion:&conversionError]; if (!buffers) { [weakSelf finish:conversionError]; return; @@ -694,7 +710,6 @@ @interface BerdPocketAudioPlayer : NSObject @property(nonatomic, strong) AVAudioFormat *format; @property(nonatomic, assign) uint64_t pendingBuffers; @property(nonatomic, assign) uint64_t completedSourceFrames; -@property(nonatomic, assign) BOOL playbackFailed; @property(nonatomic, assign) BOOL stopped; - (instancetype)initWithSampleRate:(double)sampleRate rate:(float)rate @@ -703,7 +718,6 @@ - (instancetype)initWithSampleRate:(double)sampleRate - (BOOL)enqueueSamples:(const float *)samples frameCount:(AVAudioFrameCount)frameCount error:(NSError **)error; -- (BOOL)setPlaybackRate:(float)rate error:(NSError **)error; - (uint64_t)completedSourceFramesSnapshot; - (void)stop; @end @@ -716,11 +730,34 @@ - (instancetype)initWithSampleRate:(double)sampleRate self = [super init]; if (!self) return nil; if (!(sampleRate > 0) || !isfinite(rate) || rate < 0.75f || rate > 2.0f) { - if (error) *error = BerdError(30, @"Pocket playback speed or sample rate is invalid."); + if (error) *error = BerdError(30, @"PCM playback speed or sample rate is invalid."); return nil; } _engine = [AVAudioEngine new]; + // This output has an explicit stop lifecycle. Keep the engine alive while + // a streaming backend is producing its first PCM buffer; otherwise macOS + // may auto-shut down a freshly recreated engine during that synthesis gap. + _engine.autoShutdownEnabled = NO; + + // Select the physical route before connecting the graph. AVAudioEngine + // negotiates the main mixer's channel layout while nodes are connected; + // changing from the default stereo device to a multi-channel device after + // that point can stop the engine as soon as playback begins. + if (outputDeviceID != kAudioObjectUnknown) { + AudioUnit outputUnit = _engine.outputNode.audioUnit; + OSStatus status = AudioUnitSetProperty(outputUnit, + kAudioOutputUnitProperty_CurrentDevice, + kAudioUnitScope_Global, + 0, + &outputDeviceID, + sizeof(outputDeviceID)); + if (status != noErr) { + if (error) *error = BerdError(32, @"Could not select the configured audio output."); + return nil; + } + } + _player = [AVAudioPlayerNode new]; _timePitch = [AVAudioUnitTimePitch new]; _timePitch.rate = rate; @@ -731,7 +768,7 @@ - (instancetype)initWithSampleRate:(double)sampleRate channels:1 interleaved:NO]; if (!_format) { - if (error) *error = BerdError(31, @"Could not create the Pocket PCM format."); + if (error) *error = BerdError(31, @"Could not create the PCM playback format."); return nil; } @@ -740,20 +777,6 @@ - (instancetype)initWithSampleRate:(double)sampleRate [_engine connect:_player to:_timePitch format:_format]; [_engine connect:_timePitch to:_engine.mainMixerNode format:_format]; - if (outputDeviceID != kAudioObjectUnknown) { - AudioUnit outputUnit = _engine.outputNode.audioUnit; - OSStatus status = AudioUnitSetProperty(outputUnit, - kAudioOutputUnitProperty_CurrentDevice, - kAudioUnitScope_Global, - 0, - &outputDeviceID, - sizeof(outputDeviceID)); - if (status != noErr) { - if (error) *error = BerdError(32, @"Could not select the configured audio output."); - return nil; - } - } - [_engine prepare]; if (![_engine startAndReturnError:error]) return nil; return self; @@ -764,7 +787,7 @@ - (BOOL)enqueueSamples:(const float *)samples error:(NSError **)error { @synchronized (self) { if (self.stopped) { - if (error) *error = BerdError(NSUserCancelledError, @"Pocket playback stopped."); + if (error) *error = BerdError(NSUserCancelledError, @"PCM playback stopped."); return NO; } } @@ -772,7 +795,7 @@ - (BOOL)enqueueSamples:(const float *)samples AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:self.format frameCapacity:frameCount]; if (!buffer || !buffer.floatChannelData[0]) { - if (error) *error = BerdError(34, @"Could not allocate a Pocket playback buffer."); + if (error) *error = BerdError(34, @"Could not allocate a PCM playback buffer."); return NO; } buffer.frameLength = frameCount; @@ -781,15 +804,20 @@ - (BOOL)enqueueSamples:(const float *)samples __weak typeof(self) weakSelf = self; [self.player scheduleBuffer:buffer completionCallbackType:AVAudioPlayerNodeCompletionDataPlayedBack - completionHandler:^(__unused AVAudioPlayerNodeCompletionCallbackType type) { + completionHandler:^(AVAudioPlayerNodeCompletionCallbackType type) { BerdPocketAudioPlayer *strongSelf = weakSelf; if (!strongSelf) return; @synchronized (strongSelf) { - BOOL playing = strongSelf.engine.isRunning && strongSelf.player.isPlaying; - if (!strongSelf.stopped && playing) { + // `AVAudioPlayerNode.stop` may synchronously run this + // callback while holding AVFAudio's realtime-messenger + // lock. Calling `engine.isRunning` or `player.isPlaying` + // here attempts to reacquire that lock and deadlocks + // cancellation. `stop` sets this flag before stopping + // either node, so it is also the authoritative delivery + // distinction for disposed versus played buffers. + if (!strongSelf.stopped && + type == AVAudioPlayerNodeCompletionDataPlayedBack) { strongSelf.completedSourceFrames += frameCount; - } else if (!strongSelf.stopped) { - strongSelf.playbackFailed = YES; } strongSelf.pendingBuffers = strongSelf.pendingBuffers > 0 ? strongSelf.pendingBuffers - 1 @@ -800,22 +828,6 @@ - (BOOL)enqueueSamples:(const float *)samples return YES; } -- (BOOL)setPlaybackRate:(float)rate error:(NSError **)error { - if (!isfinite(rate) || rate < 0.75f || rate > 2.0f) { - if (error) *error = BerdError(38, @"Pocket playback speed is invalid."); - return NO; - } - @synchronized (self) { - if (self.stopped) { - if (error) *error = BerdError(NSUserCancelledError, @"Pocket playback stopped."); - return NO; - } - self.timePitch.rate = rate; - self.timePitch.bypass = fabsf(rate - 1.0f) <= 0.0001f; - } - return YES; -} - - (uint64_t)completedSourceFramesSnapshot { @synchronized (self) { return self.completedSourceFrames; } } @@ -917,7 +929,7 @@ static void BerdDownloadedVoices( NSString *candidateLanguage = candidate[@"language"]; NSString *normalizedCandidate = [[candidateLanguage stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; - if ([candidateName caseInsensitiveCompare:voiceName] == NSOrderedSame && + if ([candidateName isEqualToString:voiceName] && [normalizedCandidate isEqualToString:normalizedLanguage]) { return candidate; } @@ -926,7 +938,7 @@ static void BerdDownloadedVoices( } static NSArray *> *BerdDiscoverVoices( - NSString *languagePrefix, + NSString *languageFilter, NSError **error ) { if (!BerdLoadFramework( @@ -952,8 +964,8 @@ static void BerdDownloadedVoices( return nil; } NSArray *resources = ((SendObject)objc_msgSend)(manager, voicesSelector, nil); - NSString *normalizedPrefix = - [[languagePrefix stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; + NSString *normalizedFilter = + [[languageFilter stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; NSMutableDictionary *> *byKey = [NSMutableDictionary dictionary]; for (id resource in resources ?: @[]) { @@ -962,11 +974,10 @@ static void BerdDownloadedVoices( NSString *language = [resource valueForKey:@"language"] ?: @""; NSString *normalizedLanguage = [[language stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; - if (normalizedPrefix.length && ![normalizedLanguage isEqualToString:normalizedPrefix]) continue; + if (normalizedFilter.length && ![normalizedLanguage isEqualToString:normalizedFilter]) continue; NSString *name = [resource valueForKey:@"name"] ?: @""; if (!name.length || !language.length) continue; - NSString *key = [NSString stringWithFormat:@"%@|%@", name.lowercaseString, - normalizedLanguage]; + NSString *key = [NSString stringWithFormat:@"%@|%@", name, normalizedLanguage]; byKey[key] = @{ @"name" : name, @"language" : language, @@ -1056,15 +1067,15 @@ static BOOL BerdTriggerDownload(NSString *language, NSError **error) { return YES; } -char *berd_siri_tts_catalog_json(const char *languagePrefix, char **errorOut) { +char *berd_siri_tts_catalog_json(const char *languageValue, char **errorOut) { @autoreleasepool { if (errorOut) *errorOut = NULL; - NSString *prefix = languagePrefix - ? [NSString stringWithUTF8String:languagePrefix] + NSString *language = languageValue + ? [NSString stringWithUTF8String:languageValue] : @""; NSError *error = nil; NSArray *> *candidates = - BerdDiscoverVoices(prefix, &error); + BerdDiscoverVoices(language, &error); if (!candidates) { BerdSetError(errorOut, error); return NULL; @@ -1075,15 +1086,19 @@ static BOOL BerdTriggerDownload(NSString *language, NSError **error) { for (NSDictionary *candidate in candidates) { NSString *name = candidate[@"name"]; NSString *language = candidate[@"language"]; - NSString *key = [NSString stringWithFormat:@"%@|%@", name.lowercaseString, - language.lowercaseString]; + NSString *normalizedLanguage = + [[language stringByReplacingOccurrencesOfString:@"_" withString:@"-"] lowercaseString]; + NSString *key = [NSString stringWithFormat:@"%@|%@", name, normalizedLanguage]; dispatch_group_enter(group); BerdDownloadedVoices(language, name, ^(NSArray *voices, NSError *failure) { BOOL exact = NO; if (!failure) { for (NSDictionary *voice in voices) { - if ([voice[@"name"] caseInsensitiveCompare:name] == NSOrderedSame && - [voice[@"language"] caseInsensitiveCompare:language] == NSOrderedSame) { + NSString *downloadedLanguage = + [[voice[@"language"] stringByReplacingOccurrencesOfString:@"_" + withString:@"-"] lowercaseString]; + if ([voice[@"name"] isEqualToString:name] && + [downloadedLanguage isEqualToString:normalizedLanguage]) { exact = YES; break; } @@ -1097,14 +1112,18 @@ static BOOL BerdTriggerDownload(NSString *language, NSError **error) { group, dispatch_time(DISPATCH_TIME_NOW, (int64_t)(4 * NSEC_PER_SEC)) ); + NSDictionary *installedSnapshot = nil; + @synchronized (installed) { installedSnapshot = [installed copy]; } NSMutableArray *result = [NSMutableArray arrayWithCapacity:candidates.count]; for (NSDictionary *candidate in candidates) { + NSString *normalizedLanguage = + [[candidate[@"language"] stringByReplacingOccurrencesOfString:@"_" + withString:@"-"] lowercaseString]; NSString *key = [NSString stringWithFormat:@"%@|%@", - [candidate[@"name"] lowercaseString], - [candidate[@"language"] lowercaseString]]; + candidate[@"name"], normalizedLanguage]; NSMutableDictionary *voice = [candidate mutableCopy]; - voice[@"installed"] = installed[key] ?: @NO; + voice[@"installed"] = installedSnapshot[key] ?: @NO; [result addObject:voice]; } NSData *json = [NSJSONSerialization dataWithJSONObject:result options:0 error:&error]; @@ -1275,6 +1294,180 @@ bool berd_siri_tts_play_sample( } } +bool berd_siri_tts_validate_voice( + const char *languageValue, + const char *voiceNameValue, + char **errorOut +) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + if (!languageValue || !voiceNameValue) { + BerdSetError(errorOut, BerdError(20, @"A Siri voice name and language are required.")); + return false; + } + NSString *language = [NSString stringWithUTF8String:languageValue]; + NSString *voiceName = [NSString stringWithUTF8String:voiceNameValue]; + NSError *error = nil; + if (BerdDownloadedVoiceSync(language, voiceName, &error)) return true; + BerdSetError(errorOut, error ?: BerdError(21, + [NSString stringWithFormat:@"Siri voice %@ (%@) is not installed.", + voiceName, language])); + return false; + } +} + +static BOOL BerdEmitSiriPCMBuffer( + AVAudioPCMBuffer *buffer, + BerdSiriTTSPcmFrames pcmFrames, + void *context, + NSError **error +) { + if (!buffer.frameLength) return YES; + if (![buffer.format isEqual:BerdSiriPlaybackFormat()] || !buffer.floatChannelData) { + if (error) *error = BerdError(27, @"Siri audio was not normalized to mono Float32 PCM."); + return NO; + } + if (pcmFrames && !pcmFrames(buffer.floatChannelData[0], buffer.frameLength, context)) { + if (error) *error = BerdError(NSUserCancelledError, @"Siri synthesis cancelled."); + return NO; + } + return YES; +} + +/// Owns the borrowed Rust callback boundary while a synchronous synthesis call +/// is active. Callers serialize emit/close with the decoder lock, so a queued +/// XPC callback can observe closure but can never retain a usable raw context +/// after the blocking FFI call returns. +@interface BerdSiriPCMCallbackGate : NSObject +@property(nonatomic, assign) BerdSiriTTSPcmFrames pcmFrames; +@property(nonatomic, assign) void *context; +@property(nonatomic, assign) BOOL closed; +- (instancetype)initWithFrames:(BerdSiriTTSPcmFrames)pcmFrames context:(void *)context; +- (BOOL)emitBuffer:(AVAudioPCMBuffer *)buffer error:(NSError **)error; +- (void)close; +@end + +@implementation BerdSiriPCMCallbackGate +- (instancetype)initWithFrames:(BerdSiriTTSPcmFrames)pcmFrames context:(void *)context { + self = [super init]; + if (self) { + _pcmFrames = pcmFrames; + _context = context; + } + return self; +} +- (BOOL)emitBuffer:(AVAudioPCMBuffer *)buffer error:(NSError **)error { + if (self.closed) return YES; + return BerdEmitSiriPCMBuffer(buffer, self.pcmFrames, self.context, error); +} +- (void)close { + self.closed = YES; + self.pcmFrames = NULL; + self.context = NULL; +} +@end + +static bool BerdCountTestPCMFrames( + const float *samples, + uint32_t frameCount, + void *context +) { + (void)samples; + (void)frameCount; + (*(uint32_t *)context) += 1; + return true; +} + +bool berd_siri_tts_test_closed_pcm_gate_ignores_late_callback(void) { + @autoreleasepool { + uint32_t callbacks = 0; + BerdSiriPCMCallbackGate *gate = [[BerdSiriPCMCallbackGate alloc] + initWithFrames:BerdCountTestPCMFrames context:&callbacks]; + [gate close]; + NSError *error = nil; + BOOL accepted = [gate emitBuffer:nil error:&error]; + return accepted && !error && callbacks == 0 && gate.pcmFrames == NULL && + gate.context == NULL; + } +} + +bool berd_siri_tts_synthesize_pcm( + const char *textValue, + const char *languageValue, + const char *voiceNameValue, + float rate, + BerdSiriTTSShouldStop shouldStop, + BerdSiriTTSPcmFrames pcmFrames, + void *context, + char **errorOut +) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + if (!textValue || !languageValue || !voiceNameValue || !pcmFrames) { + BerdSetError(errorOut, BerdError(20, + @"Text, voice name, language, and a PCM receiver are required.")); + return false; + } + NSString *text = [NSString stringWithUTF8String:textValue]; + NSString *language = [NSString stringWithUTF8String:languageValue]; + NSString *voiceName = [NSString stringWithUTF8String:voiceNameValue]; + BerdSiriAudioDecoder *decoder = [BerdSiriAudioDecoder new]; + BerdSiriPCMCallbackGate *pcmGate = [[BerdSiriPCMCallbackGate alloc] + initWithFrames:pcmFrames context:context]; + dispatch_semaphore_t completion = dispatch_semaphore_create(0); + __block NSError *terminalError = nil; + __block BerdSiriSynthesisSession *session = nil; + session = [[BerdSiriSynthesisSession alloc] + initWithAudioHandler:^(NSData *data, AudioStreamBasicDescription format, + UInt32 packetCount, NSData *descriptions) { + @synchronized (decoder) { + if (pcmGate.closed || terminalError) return; + NSError *decodeError = nil; + AVAudioPCMBuffer *buffer = [decoder decodeData:data format:format + packetCount:packetCount packetDescriptions:descriptions + error:&decodeError]; + if (decodeError || + (buffer && ![pcmGate emitBuffer:buffer error:&decodeError])) { + terminalError = decodeError; + } + } + }]; + [session synthesizeText:text language:language voiceName:voiceName rate:rate + completion:^(NSError *error) { + @synchronized (decoder) { + if (!terminalError && error) terminalError = error; + if (!terminalError) { + NSError *flushError = nil; + NSArray *buffers = [decoder finishConversion:&flushError]; + if (!buffers) terminalError = flushError; + for (AVAudioPCMBuffer *buffer in buffers ?: @[]) { + if (![pcmGate emitBuffer:buffer error:&flushError]) { + terminalError = flushError; + break; + } + } + } + // The reply can race audio callbacks already queued by XPC. Close + // their path to the borrowed Rust context before waking the + // blocking caller; later callbacks may decode no further PCM. + [pcmGate close]; + } + dispatch_semaphore_signal(completion); + }]; + while (dispatch_semaphore_wait( + completion, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(10 * NSEC_PER_MSEC))) != 0) { + if (shouldStop && shouldStop(context)) [session cancel]; + } + session = nil; + if (terminalError && terminalError.code != NSUserCancelledError) { + BerdSetError(errorOut, terminalError); + return false; + } + return true; + } +} + void *berd_siri_tts_stream_create( const char *languageValue, const char *voiceNameValue, @@ -1426,7 +1619,7 @@ bool berd_siri_tts_speak( outputDeviceID:outputDeviceID error:&error]; if (!player) { - BerdSetError(errorOut, error ?: BerdError(35, @"Could not start Pocket playback.")); + BerdSetError(errorOut, error ?: BerdError(35, @"Could not start PCM playback.")); return NULL; } return (__bridge_retained void *)player; @@ -1442,32 +1635,17 @@ bool berd_pocket_audio_player_enqueue( @autoreleasepool { if (errorOut) *errorOut = NULL; if (!playerValue) { - BerdSetError(errorOut, BerdError(36, @"Pocket playback is unavailable.")); + BerdSetError(errorOut, BerdError(36, @"PCM playback is unavailable.")); return false; } NSError *error = nil; BOOL enqueued = [(__bridge BerdPocketAudioPlayer *)playerValue enqueueSamples:samples frameCount:frameCount error:&error]; - if (!enqueued) BerdSetError(errorOut, error ?: BerdError(37, @"Could not queue Pocket audio.")); + if (!enqueued) BerdSetError(errorOut, error ?: BerdError(37, @"Could not queue PCM audio.")); return enqueued; } } -bool berd_pocket_audio_player_set_rate(void *playerValue, float rate, char **errorOut) { - @autoreleasepool { - if (errorOut) *errorOut = NULL; - if (!playerValue) { - BerdSetError(errorOut, BerdError(36, @"Pocket playback is unavailable.")); - return false; - } - NSError *error = nil; - BOOL updated = [(__bridge BerdPocketAudioPlayer *)playerValue - setPlaybackRate:rate error:&error]; - if (!updated) BerdSetError(errorOut, error ?: BerdError(38, @"Could not update Pocket playback speed.")); - return updated; - } -} - uint64_t berd_pocket_audio_player_completed_source_frames(void *playerValue) { if (!playerValue) return 0; return [(__bridge BerdPocketAudioPlayer *)playerValue completedSourceFramesSnapshot]; @@ -1483,7 +1661,7 @@ bool berd_pocket_audio_player_failed(void *playerValue) { if (!playerValue) return true; BerdPocketAudioPlayer *player = (__bridge BerdPocketAudioPlayer *)playerValue; @synchronized (player) { - return player.playbackFailed || (!player.stopped && !player.engine.isRunning); + return !player.stopped && !player.engine.isRunning; } } diff --git a/src-tauri/native/tests/siri_tts_stream_regression.m b/src-tauri/crates/berd-voice/native/tests/siri_tts_stream_regression.m similarity index 98% rename from src-tauri/native/tests/siri_tts_stream_regression.m rename to src-tauri/crates/berd-voice/native/tests/siri_tts_stream_regression.m index ab2944f20..b86534017 100644 --- a/src-tauri/native/tests/siri_tts_stream_regression.m +++ b/src-tauri/crates/berd-voice/native/tests/siri_tts_stream_regression.m @@ -87,7 +87,7 @@ @implementation BerdCapturedSiriPacket static BOOL BerdDecodePackets( NSArray *packets, - BerdSiriSpeechPlayer *decoder, + BerdSiriAudioDecoder *decoder, NSMutableData *samples, NSError **error ) { @@ -126,7 +126,7 @@ static BOOL BerdTestPCMNormalization(NSError **error) { }; const int16_t sourceSamples[] = { 0, INT16_MAX, INT16_MIN, 16384, -16384 }; NSData *data = [NSData dataWithBytes:sourceSamples length:sizeof(sourceSamples)]; - BerdSiriSpeechPlayer *decoder = [BerdSiriSpeechPlayer new]; + BerdSiriAudioDecoder *decoder = [BerdSiriAudioDecoder new]; AVAudioPCMBuffer *buffer = [decoder decodeData:data format:format packetCount:5 @@ -239,7 +239,7 @@ int main(void) { return 2; } - BerdSiriSpeechPlayer *reusedDecoder = [BerdSiriSpeechPlayer new]; + BerdSiriAudioDecoder *reusedDecoder = [BerdSiriAudioDecoder new]; if (!BerdDecodePackets(firstPackets, reusedDecoder, nil, &error)) { fprintf(stderr, "decode first sentence: %s\n", error.localizedDescription.UTF8String); return 2; @@ -265,7 +265,7 @@ int main(void) { length:buffer.frameLength * sizeof(float)]; } - BerdSiriSpeechPlayer *freshDecoder = [BerdSiriSpeechPlayer new]; + BerdSiriAudioDecoder *freshDecoder = [BerdSiriAudioDecoder new]; NSMutableData *freshSamples = [NSMutableData data]; if (!BerdDecodePackets(secondPackets, freshDecoder, freshSamples, &error)) { fprintf(stderr, "decode second sentence with fresh decoder: %s\n", diff --git a/src-tauri/crates/berd-voice/src/asset_verification.rs b/src-tauri/crates/berd-voice/src/asset_verification.rs new file mode 100644 index 000000000..669c61cc3 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/asset_verification.rs @@ -0,0 +1,262 @@ +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::fs::File; +use std::io::{ErrorKind, Read}; +use std::path::{Component, Path}; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct PinnedAsset { + pub(crate) relative_path: &'static str, + pub(crate) size_bytes: u64, + pub(crate) sha256: &'static str, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AssetInspection { + Missing, + Invalid, + Ready { verified_bytes: u64 }, +} + +pub(crate) fn inspect_assets( + root: &Path, + assets: &[PinnedAsset], +) -> Result { + validate_manifest(assets)?; + let expected_bytes = assets.iter().try_fold(0_u64, |total, asset| { + total + .checked_add(asset.size_bytes) + .ok_or_else(|| "pinned asset byte total overflow".to_string()) + })?; + let mut found = false; + let mut verified_bytes = 0_u64; + for asset in assets { + let path = root.join(asset.relative_path); + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + return Ok(AssetInspection::Invalid) + } + Ok(_) => {} + Err(error) if error.kind() == ErrorKind::NotFound => continue, + Err(_) => return Ok(AssetInspection::Invalid), + } + let mut file = match File::open(&path) { + Ok(file) => { + found = true; + file + } + Err(error) if error.kind() == ErrorKind::NotFound => continue, + Err(_) => return Ok(AssetInspection::Invalid), + }; + let Ok(metadata) = file.metadata() else { + return Ok(AssetInspection::Invalid); + }; + if !metadata.is_file() || metadata.len() != asset.size_bytes { + return Ok(AssetInspection::Invalid); + } + + let mut hasher = Sha256::new(); + let mut read_bytes = 0_u64; + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = match file.read(&mut buffer) { + Ok(read) => read, + Err(_) => return Ok(AssetInspection::Invalid), + }; + if read == 0 { + break; + } + read_bytes = match read_bytes.checked_add(read as u64) { + Some(total) if total <= asset.size_bytes => total, + _ => return Ok(AssetInspection::Invalid), + }; + hasher.update(&buffer[..read]); + } + if read_bytes != asset.size_bytes || format!("{:x}", hasher.finalize()) != asset.sha256 { + return Ok(AssetInspection::Invalid); + } + verified_bytes = verified_bytes + .checked_add(read_bytes) + .ok_or_else(|| "pinned asset byte total overflow".to_string())?; + } + + if !found { + Ok(AssetInspection::Missing) + } else if verified_bytes == expected_bytes { + Ok(AssetInspection::Ready { verified_bytes }) + } else { + Ok(AssetInspection::Invalid) + } +} + +fn validate_manifest(assets: &[PinnedAsset]) -> Result<(), String> { + if assets.is_empty() { + return Err("pinned asset manifest cannot be empty".into()); + } + let mut paths = HashSet::new(); + for asset in assets { + let path = Path::new(asset.relative_path); + if asset.relative_path.is_empty() + || path.is_absolute() + || path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(format!( + "pinned asset path must be a safe relative path: {}", + asset.relative_path + )); + } + if !paths.insert(asset.relative_path) { + return Err(format!( + "pinned asset path must be unique: {}", + asset.relative_path + )); + } + if asset.size_bytes == 0 { + return Err(format!( + "pinned asset size must be nonzero: {}", + asset.relative_path + )); + } + if asset.sha256.len() != 64 + || !asset + .sha256 + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(format!( + "pinned asset SHA-256 must be 64 lowercase hexadecimal characters: {}", + asset.relative_path + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{inspect_assets, AssetInspection, PinnedAsset}; + use sha2::{Digest, Sha256}; + use std::fs; + use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicU64, Ordering}; + + struct TestDirectory(PathBuf); + + impl TestDirectory { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(1); + let path = std::env::temp_dir().join(format!( + "berd-voice-assets-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).expect("create temporary directory"); + Self(path) + } + + fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for TestDirectory { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + fn asset(path: &'static str, contents: &[u8]) -> PinnedAsset { + PinnedAsset { + relative_path: path, + size_bytes: contents.len() as u64, + sha256: Box::leak(format!("{:x}", Sha256::digest(contents)).into_boxed_str()), + } + } + + #[test] + fn inspection_distinguishes_missing_valid_and_same_size_corruption() { + let directory = TestDirectory::new(); + let manifest = [asset("model.bin", b"model")]; + assert_eq!( + inspect_assets(directory.path(), &manifest).expect("inspect missing"), + AssetInspection::Missing + ); + + fs::write(directory.path().join("model.bin"), b"model").expect("write model"); + assert_eq!( + inspect_assets(directory.path(), &manifest).expect("inspect valid"), + AssetInspection::Ready { verified_bytes: 5 } + ); + + fs::write(directory.path().join("model.bin"), b"other").expect("corrupt model"); + assert_eq!( + inspect_assets(directory.path(), &manifest).expect("inspect corrupt"), + AssetInspection::Invalid + ); + } + + #[test] + fn partial_manifest_is_invalid_not_missing() { + let directory = TestDirectory::new(); + let manifest = [asset("one", b"one"), asset("two", b"two")]; + fs::write(directory.path().join("one"), b"one").expect("write first asset"); + assert_eq!( + inspect_assets(directory.path(), &manifest).expect("inspect partial"), + AssetInspection::Invalid + ); + } + + #[cfg(unix)] + #[test] + fn pinned_file_symlinks_are_invalid_even_when_the_target_matches() { + use std::os::unix::fs::symlink; + + let directory = TestDirectory::new(); + let manifest = [asset("model.bin", b"model")]; + fs::write(directory.path().join("target.bin"), b"model").expect("write target"); + symlink("target.bin", directory.path().join("model.bin")).expect("create symlink"); + assert_eq!( + inspect_assets(directory.path(), &manifest).expect("inspect symlink"), + AssetInspection::Invalid + ); + } + + #[test] + fn unsafe_duplicate_zero_and_malformed_hash_manifest_entries_are_rejected() { + let directory = TestDirectory::new(); + let valid_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + for manifest in [ + vec![PinnedAsset { + relative_path: "../outside", + size_bytes: 1, + sha256: valid_hash, + }], + vec![ + PinnedAsset { + relative_path: "same", + size_bytes: 1, + sha256: valid_hash, + }, + PinnedAsset { + relative_path: "same", + size_bytes: 1, + sha256: valid_hash, + }, + ], + vec![PinnedAsset { + relative_path: "zero", + size_bytes: 0, + sha256: valid_hash, + }], + vec![PinnedAsset { + relative_path: "hash", + size_bytes: 1, + sha256: "ABC", + }], + ] { + assert!(inspect_assets(directory.path(), &manifest).is_err()); + } + } +} diff --git a/src-tauri/crates/berd-voice/src/audio_output.rs b/src-tauri/crates/berd-voice/src/audio_output.rs new file mode 100644 index 000000000..cd5cbde08 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/audio_output.rs @@ -0,0 +1,139 @@ +/// A destination for streamed mono, unit-scale `f32` PCM audio. +/// +/// The host constructs the output with its chosen input sample rate and device. +/// In unit-scale PCM, `-1.0` and `1.0` represent negative and positive full +/// scale respectively. +pub trait PcmAudioOutput { + /// Queues source frames for playback. + /// + /// A successful return means every borrowed frame was synchronously + /// accepted or copied; the implementation does not retain `samples`. + fn write(&self, samples: &[f32]) -> Result<(), String>; + + /// Permanently cancels queued and active playback for this output instance. + /// + /// Further writes are not accepted. Status and progress may still be + /// observed, and repeated cancellation must be harmless. + fn cancel(&self); + + /// Cancels playback and returns the final confirmed source-frame count. + /// + /// The default preserves local-output semantics by freezing progress before + /// cancellation can discard native bookkeeping. Outputs with a remote + /// quiescence acknowledgement may override this to cancel first and return + /// the settled count reported by the remote host. + fn cancel_and_snapshot(&self) -> Result { + let played_frames = self.played_frames(); + self.cancel(); + Ok(played_frames) + } + + /// Returns whether all queued source frames have drained. + fn is_drained(&self) -> bool; + + /// Checks for a playback failure that may occur after a successful write. + fn check_health(&self) -> Result<(), String>; + + /// Returns the cumulative, monotonic count of source frames confirmed + /// played since construction, measured at the configured input sample rate. + fn played_frames(&self) -> u64; +} + +/// Waits for queued PCM to drain, cancelling when session authority is revoked. +pub fn wait_until_drained( + output: &dyn PcmAudioOutput, + active: &std::sync::atomic::AtomicBool, + poll_interval: std::time::Duration, +) -> Result { + use std::sync::atomic::Ordering; + + while !output.is_drained() { + if !active.load(Ordering::SeqCst) { + output.cancel(); + return Ok(false); + } + output.check_health()?; + std::thread::sleep(poll_interval); + } + if !active.load(Ordering::SeqCst) { + output.cancel(); + return Ok(false); + } + output.check_health()?; + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::{wait_until_drained, PcmAudioOutput}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::Duration; + + struct FakeOutput { + polls: AtomicUsize, + cancelled: AtomicBool, + fail: bool, + } + + impl PcmAudioOutput for FakeOutput { + fn write(&self, _samples: &[f32]) -> Result<(), String> { + Ok(()) + } + fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + } + fn is_drained(&self) -> bool { + self.polls.fetch_add(1, Ordering::SeqCst) >= 1 + } + fn check_health(&self) -> Result<(), String> { + if self.fail { + Err("failed".into()) + } else { + Ok(()) + } + } + fn played_frames(&self) -> u64 { + 0 + } + } + + #[test] + fn fake_output_proves_drain_and_cancellation_contract() { + let output = FakeOutput { + polls: AtomicUsize::new(0), + cancelled: AtomicBool::new(false), + fail: false, + }; + assert!(wait_until_drained(&output, &AtomicBool::new(true), Duration::ZERO).unwrap()); + let cancelled = FakeOutput { + polls: AtomicUsize::new(0), + cancelled: AtomicBool::new(false), + fail: false, + }; + assert!(!wait_until_drained(&cancelled, &AtomicBool::new(false), Duration::ZERO).unwrap()); + assert!(cancelled.cancelled.load(Ordering::SeqCst)); + + let already_drained = FakeOutput { + polls: AtomicUsize::new(1), + cancelled: AtomicBool::new(false), + fail: false, + }; + assert!( + !wait_until_drained(&already_drained, &AtomicBool::new(false), Duration::ZERO).unwrap() + ); + assert!(already_drained.cancelled.load(Ordering::SeqCst)); + } + + #[test] + fn asynchronous_output_failure_propagates() { + let output = FakeOutput { + polls: AtomicUsize::new(0), + cancelled: AtomicBool::new(false), + fail: true, + }; + assert_eq!( + wait_until_drained(&output, &AtomicBool::new(true), Duration::ZERO).unwrap_err(), + "failed" + ); + } +} diff --git a/src-tauri/crates/berd-voice/src/benchmark.rs b/src-tauri/crates/berd-voice/src/benchmark.rs new file mode 100644 index 000000000..943010fce --- /dev/null +++ b/src-tauri/crates/berd-voice/src/benchmark.rs @@ -0,0 +1,1173 @@ +use std::sync::atomic::AtomicBool; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::{TtsBackend, TtsOutcome, TtsSynthesisEvent}; + +mod stt; + +#[cfg(test)] +pub(crate) use stt::first_bundled_fixture_frames_for_test; +pub use stt::{ + benchmark_stt, load_bundled_stt_fixture_pack, SttBenchmarkEnvironment, SttBenchmarkMode, + SttBenchmarkReport, SttBenchmarkTarget, SttBenchmarkWorkload, SttFixturePack, +}; + +const TTS_PROMPT_MANIFEST_ENGLISH_SHORT_V1: &str = + include_str!("../fixtures/tts/english-short-v1.json"); +const MAX_TTS_PROMPT_BYTES: usize = 16 * 1024; +const SIGNAL_WINDOW_MS: u32 = 20; +const SIGNAL_HOP_MS: u32 = 10; +const SIGNAL_RELATIVE_THRESHOLD_DB: f64 = -40.0; +const SIGNAL_RELATIVE_THRESHOLD_RATIO: f64 = 0.01; +const SIGNAL_RMS_FLOOR: f64 = 1.0e-6; +const SIGNAL_CONSECUTIVE_WINDOWS: usize = 3; +const SIGNAL_PLAYOUT_ASSUMPTION: &str = + "immediate_playout_zero_device_latency_with_underrun_stalls"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TtsBenchmarkMode { + FreshBackend, + Warm, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TtsBenchmarkScenario { + ExactPromptRepeat, + DistinctPromptManifest, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct TtsBenchmarkPrompt { + pub id: String, + pub text: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TtsBenchmarkPromptManifest { + pub id: String, + pub language: String, + pub sha256: String, + pub warmup: TtsBenchmarkPrompt, + pub prompts: Vec, +} + +#[derive(Deserialize)] +struct RawTtsBenchmarkPromptManifest { + id: String, + language: String, + warmup: TtsBenchmarkPrompt, + prompts: Vec, +} + +#[derive(Debug, Serialize)] +pub struct TtsBenchmarkReport { + pub schema_version: u32, + pub target: TtsBenchmarkTarget, + pub mode: TtsBenchmarkMode, + pub scenario: TtsBenchmarkScenario, + pub prior_cache_state: &'static str, + pub signal_onset_method: TtsSignalOnsetMethod, + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_manifest: Option, + pub requested_runs: usize, + pub planned_workload: TtsBenchmarkWorkload, + #[serde(skip_serializing_if = "Option::is_none")] + pub warmup: Option, + pub runs: Vec, +} + +#[derive(Debug, Serialize)] +pub struct TtsSignalOnsetMethod { + pub algorithm: &'static str, + pub window_ms: u32, + pub hop_ms: u32, + pub relative_threshold_db: f64, + pub rms_floor: f64, + pub consecutive_windows: usize, + pub playout_assumption: &'static str, +} + +impl Default for TtsSignalOnsetMethod { + fn default() -> Self { + Self { + algorithm: "relative_rms_v1", + window_ms: SIGNAL_WINDOW_MS, + hop_ms: SIGNAL_HOP_MS, + relative_threshold_db: SIGNAL_RELATIVE_THRESHOLD_DB, + rms_floor: SIGNAL_RMS_FLOOR, + consecutive_windows: SIGNAL_CONSECUTIVE_WINDOWS, + playout_assumption: SIGNAL_PLAYOUT_ASSUMPTION, + } + } +} + +#[derive(Debug, Serialize)] +pub struct TtsBenchmarkPromptManifestReport { + pub id: String, + pub language: String, + pub sha256: String, +} + +#[derive(Debug, Serialize)] +pub struct TtsBenchmarkWorkload { + pub synthesis_requests: usize, + pub total_text_bytes: usize, +} + +#[derive(Debug, Serialize)] +pub struct TtsBenchmarkTarget { + pub backend: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub voice: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rate: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub endpoint_source: Option, +} + +impl TtsBenchmarkReport { + pub fn succeeded(&self) -> bool { + self.warmup + .iter() + .chain(self.runs.iter()) + .all(|run| run.error.is_none() && run.outcome == Some(TtsOutcomeLabel::Completed)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TtsOutcomeLabel { + Completed, + Cancelled, +} + +#[derive(Debug, Serialize)] +pub struct TtsBenchmarkRun { + pub run: usize, + pub measured: bool, + pub prompt_id: String, + pub text_bytes: usize, + pub text_sha256: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub initialization_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub time_to_first_pcm_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub synthesis_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sample_rate_hz: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub playback_rate: Option, + pub pcm_frames: u64, + pub finite_pcm_frames: u64, + pub nonfinite_pcm_frames: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub peak_abs_amplitude: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub global_rms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub peak_window_rms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub onset_threshold_rms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub leading_sustained_signal_offset_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_sustained_signal_callback_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub estimated_earliest_realtime_signal_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub audio_duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub realtime_factor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub outcome: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_stage: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl TtsBenchmarkRun { + fn initialization_error( + run: usize, + measured: bool, + prompt: &TtsBenchmarkPrompt, + elapsed: Duration, + error: String, + ) -> Self { + Self { + run, + measured, + prompt_id: prompt.id.clone(), + text_bytes: prompt.text.len(), + text_sha256: text_sha256(&prompt.text), + initialization_ms: Some(milliseconds(elapsed)), + time_to_first_pcm_ms: None, + synthesis_ms: None, + sample_rate_hz: None, + playback_rate: None, + pcm_frames: 0, + finite_pcm_frames: 0, + nonfinite_pcm_frames: 0, + peak_abs_amplitude: None, + global_rms: None, + peak_window_rms: None, + onset_threshold_rms: None, + leading_sustained_signal_offset_ms: None, + first_sustained_signal_callback_ms: None, + estimated_earliest_realtime_signal_ms: None, + audio_duration_ms: None, + realtime_factor: None, + outcome: None, + error_stage: Some("initialization"), + error: Some(error), + } + } +} + +pub fn load_bundled_tts_prompt_manifest(id: &str) -> Result { + if id != "english-short-v1" { + return Err(format!("unsupported TTS prompt manifest: {id}")); + } + let raw: RawTtsBenchmarkPromptManifest = + serde_json::from_str(TTS_PROMPT_MANIFEST_ENGLISH_SHORT_V1) + .map_err(|error| format!("invalid bundled TTS prompt manifest: {error}"))?; + if raw.id != id || raw.language.trim().is_empty() { + return Err("bundled TTS prompt manifest identity is invalid".into()); + } + if !(5..=10).contains(&raw.prompts.len()) { + return Err("bundled TTS prompt manifest must contain 5 to 10 measured prompts".into()); + } + let all = std::iter::once(&raw.warmup).chain(raw.prompts.iter()); + let mut ids = std::collections::HashSet::new(); + let mut texts = std::collections::HashSet::new(); + for prompt in all { + if prompt.id.trim().is_empty() + || prompt.text.trim().is_empty() + || prompt.text.len() > MAX_TTS_PROMPT_BYTES + { + return Err("bundled TTS prompt is empty or oversized".into()); + } + if !ids.insert(prompt.id.as_str()) || !texts.insert(prompt.text.as_str()) { + return Err("bundled TTS prompt IDs and texts must be distinct".into()); + } + } + Ok(TtsBenchmarkPromptManifest { + id: raw.id, + language: raw.language, + sha256: text_sha256(TTS_PROMPT_MANIFEST_ENGLISH_SHORT_V1), + warmup: raw.warmup, + prompts: raw.prompts, + }) +} + +/// Benchmarks exact-prompt cache reuse without constructing an audio output. +pub fn benchmark_tts( + target: TtsBenchmarkTarget, + text: &str, + requested_runs: usize, + mode: TtsBenchmarkMode, + create_backend: impl FnMut() -> Result, String>, +) -> TtsBenchmarkReport { + let prompt = TtsBenchmarkPrompt { + id: "repeated".into(), + text: text.into(), + }; + let warmup = (mode == TtsBenchmarkMode::Warm).then_some(&prompt); + let prompts = std::iter::repeat_n(prompt.clone(), requested_runs).collect::>(); + benchmark_tts_prompts( + target, + TtsBenchmarkScenario::ExactPromptRepeat, + None, + warmup, + &prompts, + mode, + create_backend, + ) +} + +/// Benchmarks prompts that are distinct within this invocation from a fixed +/// manifest, without constructing an audio output. Provider and system cache +/// state from earlier invocations remains uncontrolled. +pub fn benchmark_tts_manifest( + target: TtsBenchmarkTarget, + manifest: &TtsBenchmarkPromptManifest, + mode: TtsBenchmarkMode, + create_backend: impl FnMut() -> Result, String>, +) -> TtsBenchmarkReport { + let warmup = (mode == TtsBenchmarkMode::Warm).then_some(&manifest.warmup); + benchmark_tts_prompts( + target, + TtsBenchmarkScenario::DistinctPromptManifest, + Some(TtsBenchmarkPromptManifestReport { + id: manifest.id.clone(), + language: manifest.language.clone(), + sha256: manifest.sha256.clone(), + }), + warmup, + &manifest.prompts, + mode, + create_backend, + ) +} + +fn benchmark_tts_prompts( + target: TtsBenchmarkTarget, + scenario: TtsBenchmarkScenario, + prompt_manifest: Option, + warmup_prompt: Option<&TtsBenchmarkPrompt>, + prompts: &[TtsBenchmarkPrompt], + mode: TtsBenchmarkMode, + mut create_backend: impl FnMut() -> Result, String>, +) -> TtsBenchmarkReport { + let synthesis_requests = prompts.len() + usize::from(warmup_prompt.is_some()); + let total_text_bytes = prompts.iter().fold(0_usize, |total, prompt| { + total.saturating_add(prompt.text.len()) + }) + warmup_prompt.map_or(0, |prompt| prompt.text.len()); + let mut report = TtsBenchmarkReport { + schema_version: 3, + target, + mode, + scenario, + prior_cache_state: "uncontrolled_system_and_provider_state", + signal_onset_method: TtsSignalOnsetMethod::default(), + prompt_manifest, + requested_runs: prompts.len(), + planned_workload: TtsBenchmarkWorkload { + synthesis_requests, + total_text_bytes, + }, + warmup: None, + runs: Vec::with_capacity(prompts.len()), + }; + + match mode { + TtsBenchmarkMode::FreshBackend => { + for (index, prompt) in prompts.iter().enumerate() { + let started = Instant::now(); + match create_backend() { + Ok(backend) => report.runs.push(run_synthesis( + index + 1, + true, + Some(started.elapsed()), + backend.as_ref(), + prompt, + )), + Err(error) => report.runs.push(TtsBenchmarkRun::initialization_error( + index + 1, + true, + prompt, + started.elapsed(), + error, + )), + } + } + } + TtsBenchmarkMode::Warm => { + let prompt = warmup_prompt.expect("warm benchmark always provides a warm-up prompt"); + let started = Instant::now(); + match create_backend() { + Ok(backend) => { + report.warmup = Some(run_synthesis( + 0, + false, + Some(started.elapsed()), + backend.as_ref(), + prompt, + )); + if report.warmup.as_ref().is_some_and(|run| { + run.error.is_none() && run.outcome == Some(TtsOutcomeLabel::Completed) + }) { + for (index, prompt) in prompts.iter().enumerate() { + report.runs.push(run_synthesis( + index + 1, + true, + None, + backend.as_ref(), + prompt, + )); + } + } + } + Err(error) => { + report.warmup = Some(TtsBenchmarkRun::initialization_error( + 0, + false, + prompt, + started.elapsed(), + error, + )); + } + } + } + } + report +} + +#[derive(Clone, Copy, Debug)] +struct PcmChunkTiming { + source_start_frame: u64, + frame_count: u64, + callback_elapsed: Duration, +} + +#[derive(Clone, Copy, Debug)] +struct SignalWindow { + source_start_frame: u64, + rms: f64, +} + +#[derive(Debug, Default)] +struct SignalAnalysis { + finite_pcm_frames: u64, + nonfinite_pcm_frames: u64, + peak_abs_amplitude: Option, + global_rms: Option, + peak_window_rms: Option, + onset_threshold_rms: Option, + leading_sustained_signal_offset_ms: Option, + first_sustained_signal_callback_ms: Option, + estimated_earliest_realtime_signal_ms: Option, +} + +struct PcmSignalAnalyzer { + sample_rate: u32, + window_frames: usize, + hop_frames: usize, + frames_seen: u64, + finite_frames: u64, + nonfinite_frames: u64, + finite_square_sum: f64, + peak_abs: f64, + rolling_squares: std::collections::VecDeque, + rolling_square_sum: f64, + windows: Vec, + chunks: Vec, +} + +impl PcmSignalAnalyzer { + fn new(sample_rate: u32) -> Self { + let window_frames = frames_for_milliseconds(sample_rate, SIGNAL_WINDOW_MS); + let hop_frames = frames_for_milliseconds(sample_rate, SIGNAL_HOP_MS); + Self { + sample_rate, + window_frames, + hop_frames, + frames_seen: 0, + finite_frames: 0, + nonfinite_frames: 0, + finite_square_sum: 0.0, + peak_abs: 0.0, + rolling_squares: std::collections::VecDeque::with_capacity(window_frames), + rolling_square_sum: 0.0, + windows: Vec::new(), + chunks: Vec::new(), + } + } + + fn observe(&mut self, frames: &[f32], callback_elapsed: Duration) { + if frames.is_empty() { + return; + } + self.chunks.push(PcmChunkTiming { + source_start_frame: self.frames_seen, + frame_count: frames.len() as u64, + callback_elapsed, + }); + for &sample in frames { + let square = if sample.is_finite() { + let sample = f64::from(sample); + let square = sample * sample; + self.finite_frames += 1; + self.finite_square_sum += square; + self.peak_abs = self.peak_abs.max(sample.abs()); + square + } else { + self.nonfinite_frames += 1; + 0.0 + }; + self.rolling_squares.push_back(square); + self.rolling_square_sum += square; + if self.rolling_squares.len() > self.window_frames { + self.rolling_square_sum -= self.rolling_squares.pop_front().unwrap_or_default(); + } + self.frames_seen += 1; + if self.window_frames > 0 + && self.hop_frames > 0 + && self.rolling_squares.len() == self.window_frames + && (self.frames_seen - self.window_frames as u64) + .is_multiple_of(self.hop_frames as u64) + { + self.windows.push(SignalWindow { + source_start_frame: self.frames_seen - self.window_frames as u64, + rms: (self.rolling_square_sum / self.window_frames as f64).sqrt(), + }); + } + } + } + + fn finish(&self) -> SignalAnalysis { + let peak_window_rms = self + .windows + .iter() + .map(|window| window.rms) + .reduce(f64::max); + let onset_threshold_rms = peak_window_rms + .map(|peak| SIGNAL_RMS_FLOOR.max(peak * SIGNAL_RELATIVE_THRESHOLD_RATIO)); + let onset_frame = onset_threshold_rms.and_then(|threshold| { + let mut qualifying = 0_usize; + for (index, window) in self.windows.iter().enumerate() { + if window.rms >= threshold { + qualifying += 1; + if qualifying == SIGNAL_CONSECUTIVE_WINDOWS { + return Some( + self.windows[index + 1 - SIGNAL_CONSECUTIVE_WINDOWS].source_start_frame, + ); + } + } else { + qualifying = 0; + } + } + None + }); + let onset_chunk = onset_frame.and_then(|frame| { + self.chunks.iter().position(|chunk| { + frame >= chunk.source_start_frame + && frame < chunk.source_start_frame + chunk.frame_count + }) + }); + let estimated_earliest_realtime_signal_ms = onset_chunk.map(|onset_chunk| { + let mut playout_ms = milliseconds(self.chunks[0].callback_elapsed); + for index in 1..=onset_chunk { + let previous = self.chunks[index - 1]; + let previous_end_ms = + playout_ms + frames_to_milliseconds(previous.frame_count, self.sample_rate); + playout_ms = previous_end_ms.max(milliseconds(self.chunks[index].callback_elapsed)); + } + playout_ms + + frames_to_milliseconds( + onset_frame.unwrap_or_default() - self.chunks[onset_chunk].source_start_frame, + self.sample_rate, + ) + }); + SignalAnalysis { + finite_pcm_frames: self.finite_frames, + nonfinite_pcm_frames: self.nonfinite_frames, + peak_abs_amplitude: (self.finite_frames > 0).then_some(self.peak_abs), + global_rms: (self.finite_frames > 0) + .then(|| (self.finite_square_sum / self.finite_frames as f64).sqrt()), + peak_window_rms, + onset_threshold_rms, + leading_sustained_signal_offset_ms: onset_frame + .map(|frame| frames_to_milliseconds(frame, self.sample_rate)), + first_sustained_signal_callback_ms: onset_chunk + .map(|index| milliseconds(self.chunks[index].callback_elapsed)), + estimated_earliest_realtime_signal_ms, + } + } +} + +fn frames_for_milliseconds(sample_rate: u32, duration_ms: u32) -> usize { + ((u64::from(sample_rate) * u64::from(duration_ms)) / 1_000) as usize +} + +fn frames_to_milliseconds(frames: u64, sample_rate: u32) -> f64 { + if sample_rate == 0 { + return 0.0; + } + frames as f64 * 1_000.0 / f64::from(sample_rate) +} + +fn run_synthesis( + run: usize, + measured: bool, + initialization: Option, + backend: &dyn TtsBackend, + prompt: &TtsBenchmarkPrompt, +) -> TtsBenchmarkRun { + let spec = backend.pcm_spec(); + let active = AtomicBool::new(true); + let started = Instant::now(); + let mut first_pcm = None; + let mut pcm_frames = 0_u64; + let mut signal = PcmSignalAnalyzer::new(spec.sample_rate); + let result = backend.synthesize_with_poll(&prompt.text, &active, &mut |event| { + if let TtsSynthesisEvent::Frames(frames) = event { + let callback_elapsed = started.elapsed(); + if !frames.is_empty() && first_pcm.is_none() { + first_pcm = Some(callback_elapsed); + } + pcm_frames = pcm_frames.saturating_add(frames.len() as u64); + signal.observe(frames, callback_elapsed); + } + Ok(()) + }); + let synthesis = started.elapsed(); + let signal = signal.finish(); + let audio_duration = (spec.sample_rate > 0) + .then(|| Duration::from_secs_f64(pcm_frames as f64 / f64::from(spec.sample_rate))); + let realtime_factor = audio_duration + .filter(|duration| !duration.is_zero()) + .map(|duration| synthesis.as_secs_f64() / duration.as_secs_f64()); + let (outcome, error_stage, error) = match result { + Ok(TtsOutcome::Completed) if spec.sample_rate == 0 => ( + None, + Some("synthesis"), + Some("backend reported a zero PCM sample rate".into()), + ), + Ok(TtsOutcome::Completed) if pcm_frames == 0 => ( + None, + Some("synthesis"), + Some("synthesis completed without PCM".into()), + ), + Ok(TtsOutcome::Completed) if signal.nonfinite_pcm_frames > 0 => ( + None, + Some("synthesis"), + Some("synthesis produced non-finite PCM".into()), + ), + Ok(TtsOutcome::Completed) if signal.leading_sustained_signal_offset_ms.is_none() => ( + None, + Some("synthesis"), + Some("synthesis completed without sustained PCM signal".into()), + ), + Ok(TtsOutcome::Completed) => (Some(TtsOutcomeLabel::Completed), None, None), + Ok(TtsOutcome::Cancelled) => (Some(TtsOutcomeLabel::Cancelled), None, None), + Err(error) => (None, Some("synthesis"), Some(error)), + }; + TtsBenchmarkRun { + run, + measured, + prompt_id: prompt.id.clone(), + text_bytes: prompt.text.len(), + text_sha256: text_sha256(&prompt.text), + initialization_ms: initialization.map(milliseconds), + time_to_first_pcm_ms: first_pcm.map(milliseconds), + synthesis_ms: Some(milliseconds(synthesis)), + sample_rate_hz: Some(spec.sample_rate), + playback_rate: Some(spec.playback_rate), + pcm_frames, + finite_pcm_frames: signal.finite_pcm_frames, + nonfinite_pcm_frames: signal.nonfinite_pcm_frames, + peak_abs_amplitude: signal.peak_abs_amplitude, + global_rms: signal.global_rms, + peak_window_rms: signal.peak_window_rms, + onset_threshold_rms: signal.onset_threshold_rms, + leading_sustained_signal_offset_ms: signal.leading_sustained_signal_offset_ms, + first_sustained_signal_callback_ms: signal.first_sustained_signal_callback_ms, + estimated_earliest_realtime_signal_ms: signal.estimated_earliest_realtime_signal_ms, + audio_duration_ms: audio_duration.map(milliseconds), + realtime_factor, + outcome, + error_stage, + error, + } +} + +fn text_sha256(text: &str) -> String { + format!("{:x}", Sha256::digest(text.as_bytes())) +} + +fn milliseconds(duration: Duration) -> f64 { + duration.as_secs_f64() * 1_000.0 +} + +#[cfg(test)] +mod tests { + use super::{ + benchmark_tts, benchmark_tts_manifest, load_bundled_tts_prompt_manifest, PcmSignalAnalyzer, + TtsBenchmarkMode, TtsBenchmarkScenario, TtsBenchmarkTarget, TtsOutcomeLabel, + }; + use crate::{TtsBackend, TtsOutcome, TtsPcmSpec}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::Arc; + use std::time::Duration; + + struct FakeTts; + + struct EmptyTts; + + struct ZeroRateTts; + + struct PartialErrorTts; + + struct CancelledTts; + + struct PollingTts; + + struct SilentTts; + + struct NonfiniteTts; + + fn target() -> TtsBenchmarkTarget { + TtsBenchmarkTarget { + backend: "fake".into(), + model: None, + voice: Some("test".into()), + language: None, + rate: Some(1.0), + endpoint_source: None, + } + } + + impl TtsBackend for FakeTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 1_000, + playback_rate: 1.5, + } + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + on_frames(&[])?; + on_frames(&[0.25; 100])?; + Ok(TtsOutcome::Completed) + } + } + + impl TtsBackend for EmptyTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + } + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + _on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + Ok(TtsOutcome::Completed) + } + } + + impl TtsBackend for ZeroRateTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 0, + playback_rate: 1.0, + } + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + on_frames(&[0.0])?; + Ok(TtsOutcome::Completed) + } + } + + impl TtsBackend for PartialErrorTts { + fn pcm_spec(&self) -> TtsPcmSpec { + FakeTts.pcm_spec() + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + on_frames(&[0.0; 20])?; + Err("provider disconnected".into()) + } + } + + impl TtsBackend for CancelledTts { + fn pcm_spec(&self) -> TtsPcmSpec { + FakeTts.pcm_spec() + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + _on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + Ok(TtsOutcome::Cancelled) + } + } + + impl TtsBackend for PollingTts { + fn pcm_spec(&self) -> TtsPcmSpec { + FakeTts.pcm_spec() + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + _on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + unreachable!("benchmark uses synthesize_with_poll") + } + + fn synthesize_with_poll( + &self, + _text: &str, + _active: &AtomicBool, + on_event: &mut dyn FnMut(crate::TtsSynthesisEvent<'_>) -> Result<(), String>, + ) -> Result { + on_event(crate::TtsSynthesisEvent::Poll)?; + on_event(crate::TtsSynthesisEvent::Frames(&[0.25; 40]))?; + Ok(TtsOutcome::Completed) + } + } + + impl TtsBackend for SilentTts { + fn pcm_spec(&self) -> TtsPcmSpec { + FakeTts.pcm_spec() + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + on_frames(&[0.0; 100])?; + Ok(TtsOutcome::Completed) + } + } + + impl TtsBackend for NonfiniteTts { + fn pcm_spec(&self) -> TtsPcmSpec { + FakeTts.pcm_spec() + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + let mut frames = [0.25; 100]; + frames[30] = f32::NAN; + frames[70] = f32::INFINITY; + on_frames(&frames)?; + Ok(TtsOutcome::Completed) + } + } + + #[test] + fn fresh_backend_mode_constructs_each_run_and_reports_pcm_metrics() { + let constructions = AtomicUsize::new(0); + let report = benchmark_tts(target(), "hello", 2, TtsBenchmarkMode::FreshBackend, || { + constructions.fetch_add(1, Ordering::SeqCst); + Ok(Arc::new(FakeTts)) + }); + + assert_eq!(constructions.load(Ordering::SeqCst), 2); + assert!(report.warmup.is_none()); + assert_eq!(report.scenario, TtsBenchmarkScenario::ExactPromptRepeat); + assert_eq!(report.runs.len(), 2); + for run in report.runs { + assert!(run.measured); + assert!(run.initialization_ms.is_some()); + assert!(run.time_to_first_pcm_ms.is_some()); + assert_eq!(run.pcm_frames, 100); + assert_eq!(run.audio_duration_ms, Some(100.0)); + assert_eq!(run.outcome, Some(TtsOutcomeLabel::Completed)); + } + } + + #[test] + fn warm_mode_records_warmup_then_reuses_one_backend() { + let constructions = AtomicUsize::new(0); + let report = benchmark_tts(target(), "hello", 2, TtsBenchmarkMode::Warm, || { + constructions.fetch_add(1, Ordering::SeqCst); + Ok(Arc::new(FakeTts)) + }); + + assert_eq!(constructions.load(Ordering::SeqCst), 1); + assert!(!report.warmup.as_ref().unwrap().measured); + assert!(report.warmup.as_ref().unwrap().initialization_ms.is_some()); + assert_eq!(report.runs.len(), 2); + assert!(report + .runs + .iter() + .all(|run| run.measured && run.initialization_ms.is_none())); + assert!(report.succeeded()); + } + + #[test] + fn initialization_errors_remain_structured() { + let report = benchmark_tts(target(), "hello", 2, TtsBenchmarkMode::FreshBackend, || { + Err("missing model".into()) + }); + + assert_eq!(report.runs.len(), 2); + assert_eq!(report.runs[0].error_stage, Some("initialization")); + assert_eq!(report.runs[0].error.as_deref(), Some("missing model")); + assert!(!report.succeeded()); + } + + #[test] + fn completed_synthesis_without_pcm_is_an_error() { + let report = benchmark_tts(target(), "hello", 1, TtsBenchmarkMode::FreshBackend, || { + Ok(Arc::new(EmptyTts)) + }); + + assert_eq!(report.runs[0].error_stage, Some("synthesis")); + assert_eq!( + report.runs[0].error.as_deref(), + Some("synthesis completed without PCM") + ); + assert!(!report.succeeded()); + } + + #[test] + fn poll_is_not_pcm_and_partial_errors_keep_measurements() { + let polled = benchmark_tts(target(), "hello", 1, TtsBenchmarkMode::FreshBackend, || { + Ok(Arc::new(PollingTts)) + }); + assert_eq!(polled.runs[0].pcm_frames, 40); + assert_eq!(polled.runs[0].audio_duration_ms, Some(40.0)); + assert!(polled.runs[0].realtime_factor.is_some()); + assert_eq!(polled.runs[0].outcome, Some(TtsOutcomeLabel::Completed)); + + let failed = benchmark_tts(target(), "hello", 1, TtsBenchmarkMode::FreshBackend, || { + Ok(Arc::new(PartialErrorTts)) + }); + assert_eq!(failed.runs[0].pcm_frames, 20); + assert_eq!(failed.runs[0].audio_duration_ms, Some(20.0)); + assert!(failed.runs[0].time_to_first_pcm_ms.is_some()); + assert!(failed.runs[0].synthesis_ms.is_some()); + assert_eq!(failed.runs[0].error_stage, Some("synthesis")); + assert_eq!( + failed.runs[0].error.as_deref(), + Some("provider disconnected") + ); + } + + #[test] + fn cancellation_and_invalid_sample_rate_are_terminal_results() { + let cancelled = benchmark_tts(target(), "hello", 1, TtsBenchmarkMode::FreshBackend, || { + Ok(Arc::new(CancelledTts)) + }); + assert_eq!(cancelled.runs[0].outcome, Some(TtsOutcomeLabel::Cancelled)); + assert!(!cancelled.succeeded()); + + let invalid = benchmark_tts(target(), "hello", 1, TtsBenchmarkMode::FreshBackend, || { + Ok(Arc::new(ZeroRateTts)) + }); + assert_eq!(invalid.runs[0].error_stage, Some("synthesis")); + assert_eq!( + invalid.runs[0].error.as_deref(), + Some("backend reported a zero PCM sample rate") + ); + assert!(invalid.runs[0].audio_duration_ms.is_none()); + assert!(invalid.runs[0].realtime_factor.is_none()); + } + + #[test] + fn completed_silence_and_nonfinite_pcm_are_terminal_errors() { + let silent = benchmark_tts(target(), "hello", 1, TtsBenchmarkMode::FreshBackend, || { + Ok(Arc::new(SilentTts)) + }); + assert_eq!(silent.runs[0].peak_abs_amplitude, Some(0.0)); + assert_eq!(silent.runs[0].global_rms, Some(0.0)); + assert_eq!(silent.runs[0].leading_sustained_signal_offset_ms, None); + assert_eq!( + silent.runs[0].error.as_deref(), + Some("synthesis completed without sustained PCM signal") + ); + + let nonfinite = benchmark_tts(target(), "hello", 1, TtsBenchmarkMode::FreshBackend, || { + Ok(Arc::new(NonfiniteTts)) + }); + assert_eq!(nonfinite.runs[0].nonfinite_pcm_frames, 2); + assert_eq!( + nonfinite.runs[0].error.as_deref(), + Some("synthesis produced non-finite PCM") + ); + assert!(!nonfinite.succeeded()); + } + + fn signal_with_leading( + sample_rate: u32, + leading_ms: usize, + signal_ms: usize, + amplitude: f32, + ) -> Vec { + let mut samples = vec![0.0; sample_rate as usize * leading_ms / 1_000]; + samples.extend(vec![amplitude; sample_rate as usize * signal_ms / 1_000]); + samples + } + + #[test] + fn sustained_signal_detection_is_scaled_and_sample_rate_independent() { + for sample_rate in [24_000, 48_000] { + let samples = signal_with_leading(sample_rate, 50, 100, 0.2); + let mut analyzer = PcmSignalAnalyzer::new(sample_rate); + analyzer.observe(&samples, Duration::from_millis(7)); + let analysis = analyzer.finish(); + assert_eq!(analysis.nonfinite_pcm_frames, 0); + assert!((analysis.peak_abs_amplitude.unwrap() - 0.2).abs() < 1.0e-6); + assert!((analysis.peak_window_rms.unwrap() - 0.2).abs() < 1.0e-6); + assert!((analysis.onset_threshold_rms.unwrap() - 0.002).abs() < 1.0e-6); + // A 20 ms window can backdate the sustained transition by at most + // one hop for this aligned fixture. + assert_eq!(analysis.leading_sustained_signal_offset_ms, Some(40.0)); + assert_eq!(analysis.first_sustained_signal_callback_ms, Some(7.0)); + assert_eq!(analysis.estimated_earliest_realtime_signal_ms, Some(47.0)); + } + + let quiet = signal_with_leading(48_000, 50, 100, 0.0001); + let mut analyzer = PcmSignalAnalyzer::new(48_000); + analyzer.observe(&quiet, Duration::from_millis(3)); + let analysis = analyzer.finish(); + assert_eq!(analysis.onset_threshold_rms, Some(1.0e-6)); + assert_eq!(analysis.leading_sustained_signal_offset_ms, Some(40.0)); + } + + #[test] + fn detector_crosses_callbacks_ignores_low_noise_and_rejects_a_click() { + let mut samples = vec![0.0005; 60]; + samples.extend([0.1; 100]); + let mut contiguous = PcmSignalAnalyzer::new(1_000); + contiguous.observe(&samples, Duration::from_millis(5)); + let contiguous = contiguous.finish(); + + let mut split = PcmSignalAnalyzer::new(1_000); + let mut start = 0; + for (end, arrival) in [(7, 1), (23, 2), (61, 3), (104, 4), (160, 5)] { + split.observe(&samples[start..end], Duration::from_millis(arrival)); + start = end; + } + let split = split.finish(); + assert_eq!( + split.leading_sustained_signal_offset_ms, + contiguous.leading_sustained_signal_offset_ms + ); + assert_eq!(split.peak_abs_amplitude, contiguous.peak_abs_amplitude); + assert_eq!(split.global_rms, contiguous.global_rms); + assert_eq!(split.peak_window_rms, contiguous.peak_window_rms); + assert_eq!(split.leading_sustained_signal_offset_ms, Some(50.0)); + + let mut click = PcmSignalAnalyzer::new(1_000); + let mut samples = vec![0.0; 100]; + samples[50] = 1.0; + click.observe(&samples, Duration::ZERO); + let click = click.finish(); + assert!(click.peak_window_rms.unwrap() > 0.0); + assert_eq!(click.leading_sustained_signal_offset_ms, None); + } + + #[test] + fn estimated_realtime_signal_accounts_for_late_chunks_and_underruns() { + let mut analyzer = PcmSignalAnalyzer::new(1_000); + analyzer.observe(&[0.0; 40], Duration::from_millis(10)); + let mut remainder = vec![0.0; 20]; + remainder.extend([0.25; 100]); + analyzer.observe(&remainder, Duration::from_millis(200)); + let analysis = analyzer.finish(); + assert_eq!(analysis.leading_sustained_signal_offset_ms, Some(50.0)); + assert_eq!(analysis.first_sustained_signal_callback_ms, Some(200.0)); + assert_eq!(analysis.estimated_earliest_realtime_signal_ms, Some(210.0)); + + let all = signal_with_leading(1_000, 50, 100, 0.25); + let mut batched = PcmSignalAnalyzer::new(1_000); + batched.observe(&all, Duration::from_millis(10)); + let batched = batched.finish(); + assert_eq!(batched.leading_sustained_signal_offset_ms, Some(40.0)); + assert_eq!(batched.estimated_earliest_realtime_signal_ms, Some(50.0)); + } + + #[test] + fn report_is_stable_structured_json() { + let report = benchmark_tts(target(), "hello", 1, TtsBenchmarkMode::FreshBackend, || { + Ok(Arc::new(FakeTts)) + }); + let value = serde_json::to_value(report).unwrap(); + + assert_eq!(value["schema_version"], 3); + assert_eq!(value["target"]["backend"], "fake"); + assert_eq!(value["target"]["voice"], "test"); + assert_eq!(value["mode"], "fresh_backend"); + assert_eq!(value["scenario"], "exact_prompt_repeat"); + assert_eq!( + value["prior_cache_state"], + "uncontrolled_system_and_provider_state" + ); + assert_eq!( + value["runs"][0]["text_sha256"], + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + ); + assert_eq!(value["runs"][0]["prompt_id"], "repeated"); + assert_eq!(value["requested_runs"], 1); + assert_eq!(value["planned_workload"]["synthesis_requests"], 1); + assert_eq!(value["planned_workload"]["total_text_bytes"], 5); + assert_eq!(value["runs"][0]["pcm_frames"], 100); + assert_eq!(value["runs"][0]["finite_pcm_frames"], 100); + assert_eq!(value["signal_onset_method"]["algorithm"], "relative_rms_v1"); + assert_eq!( + value["signal_onset_method"]["playout_assumption"], + "immediate_playout_zero_device_latency_with_underrun_stalls" + ); + assert_eq!(value["runs"][0]["nonfinite_pcm_frames"], 0); + assert_eq!(value["runs"][0]["leading_sustained_signal_offset_ms"], 0.0); + assert!(value["runs"][0]["estimated_earliest_realtime_signal_ms"].is_number()); + assert_eq!(value["runs"][0]["outcome"], "completed"); + + let changed = benchmark_tts(target(), "jello", 1, TtsBenchmarkMode::FreshBackend, || { + Ok(Arc::new(FakeTts)) + }); + assert_ne!( + value["runs"][0]["text_sha256"], + serde_json::to_value(changed).unwrap()["runs"][0]["text_sha256"] + ); + } + + #[test] + fn bundled_manifest_is_fixed_distinct_and_uses_separate_warmup() { + let manifest = load_bundled_tts_prompt_manifest("english-short-v1").unwrap(); + assert_eq!( + manifest.sha256, + "ab41a51ef214f0a632f517b1c3dca288505a9edafe70f7d58b2c4b4782594e0d" + ); + assert_eq!(manifest.prompts.len(), 5); + assert!(manifest + .prompts + .iter() + .all(|prompt| prompt.text != manifest.warmup.text)); + + let report = benchmark_tts_manifest(target(), &manifest, TtsBenchmarkMode::Warm, || { + Ok(Arc::new(FakeTts)) + }); + assert_eq!( + report.scenario, + TtsBenchmarkScenario::DistinctPromptManifest + ); + assert_eq!(report.warmup.as_ref().unwrap().prompt_id, "warmup"); + assert_eq!(report.runs.len(), 5); + assert!(report + .runs + .iter() + .all(|run| run.text_sha256 != report.warmup.as_ref().unwrap().text_sha256)); + assert_eq!(report.planned_workload.synthesis_requests, 6); + assert_eq!( + report.prompt_manifest.as_ref().unwrap().id, + "english-short-v1" + ); + } +} diff --git a/src-tauri/crates/berd-voice/src/benchmark/stt.rs b/src-tauri/crates/berd-voice/src/benchmark/stt.rs new file mode 100644 index 000000000..a25569403 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/benchmark/stt.rs @@ -0,0 +1,1148 @@ +use std::io::Cursor; +use std::time::{Duration, Instant}; + +use claxon::FlacReader; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::sync::mpsc; + +use crate::input::{ + VoiceInputEvent, VoiceInputFrame, VoiceInputRuntime, INPUT_FRAME_SAMPLES, INPUT_SAMPLE_RATE, +}; + +const MANIFEST_JSON: &str = + include_str!("../../fixtures/stt/librispeech-test-clean-mini/manifest.json"); +const ATTRIBUTION_NOTICE: &str = + include_str!("../../fixtures/stt/librispeech-test-clean-mini/NOTICE.md"); +const LEADING_SILENCE_FRAMES: usize = 50; +// Keep supplying capture-like silence through the runtime's five-second live +// no-result bounds. Continuous engines such as SpeechTranscriber settle from +// PCM time, not from the benchmark merely waiting without sending frames. +const TRAILING_SILENCE_FRAMES: usize = 325; +const FRAME_DURATION: Duration = Duration::from_millis(20); +const INITIALIZATION_TIMEOUT: Duration = Duration::from_secs(60); +const RESULT_TIMEOUT: Duration = Duration::from_secs(7); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SttBenchmarkMode { + Cold, + Warm, +} + +#[derive(Clone, Debug, Serialize)] +pub struct SttBenchmarkTarget { + pub backend: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub locale: Option, + pub vad_threshold: f32, + #[serde(skip_serializing_if = "Option::is_none")] + pub endpoint_source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub model_source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub credential_source: Option, +} + +#[derive(Clone, Debug, Serialize)] +pub struct SttBenchmarkEnvironment { + pub os: String, + pub architecture: String, + pub berd_voice_version: String, +} + +impl Default for SttBenchmarkEnvironment { + fn default() -> Self { + Self { + os: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + berd_voice_version: env!("CARGO_PKG_VERSION").to_string(), + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SttFixtureManifest { + pub schema_version: u32, + pub corpus: String, + pub resource_id: String, + pub subset: String, + pub language: String, + pub license: String, + pub license_url: String, + pub source_url: String, + pub archive_url: String, + pub archive_md5: String, + pub utterances: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SttFixtureMetadata { + pub id: String, + pub file: String, + pub sha256: String, + pub bytes: usize, + pub sample_rate_hz: u32, + pub channels: u32, + pub bits_per_sample: u32, + pub samples: u64, + pub transcript: String, +} + +pub struct SttFixturePack { + manifest: SttFixtureManifest, + manifest_sha256: String, + utterances: Vec, +} + +struct PreparedUtterance { + metadata: SttFixtureMetadata, + samples_48k: Vec, +} + +#[derive(Debug, Serialize)] +pub struct SttBenchmarkReport { + pub schema_version: u32, + pub target: SttBenchmarkTarget, + pub environment: SttBenchmarkEnvironment, + pub mode: SttBenchmarkMode, + pub runtime_scope: &'static str, + pub fixture_conversion: &'static str, + pub input_pacing: &'static str, + pub word_error_normalization: &'static str, + pub requested_runs: usize, + pub fixture: SttFixtureSummary, + pub planned_workload: SttBenchmarkWorkload, + #[serde(skip_serializing_if = "Option::is_none")] + pub warmup: Option, + pub runs: Vec, + pub aggregate: SttWordError, +} + +#[derive(Debug, Serialize)] +pub struct SttFixtureSummary { + pub corpus: String, + pub resource_id: String, + pub subset: String, + pub language: String, + pub license: String, + pub license_url: String, + pub source_url: String, + pub archive_url: String, + pub archive_md5: String, + pub manifest_sha256: String, + pub attribution_notice: &'static str, + pub utterances: Vec, +} + +#[derive(Debug, Serialize)] +pub struct SttBenchmarkWorkload { + pub runtime_initializations: usize, + pub recognition_commits: usize, + pub source_audio_seconds: f64, + pub streamed_audio_seconds: f64, + pub includes_warmup: bool, +} + +#[derive(Debug, Serialize)] +pub struct SttBenchmarkRun { + pub run: usize, + pub measured: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub initialization_ms: Option, + pub total_ms: f64, + pub utterances: Vec, + pub aggregate: SttWordError, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_stage: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, Serialize)] +pub struct SttUtteranceResult { + pub id: String, + pub reference: String, + pub hypothesis: String, + pub source_duration_ms: f64, + pub streamed_duration_ms: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub speaking_started_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub speaking_duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub recognition_pending_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub final_latency_ms: Option, + pub total_ms: f64, + pub word_error: SttWordError, + pub outcome: SttUtteranceOutcome, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SttUtteranceOutcome { + Completed, + NoResult, + Failed, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize)] +pub struct SttWordError { + pub reference_words: usize, + pub substitutions: usize, + pub deletions: usize, + pub insertions: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub word_error_rate: Option, +} + +impl SttWordError { + fn add(&mut self, other: &Self) { + self.reference_words += other.reference_words; + self.substitutions += other.substitutions; + self.deletions += other.deletions; + self.insertions += other.insertions; + self.word_error_rate = rate(self); + } +} + +impl SttBenchmarkReport { + pub fn succeeded(&self) -> bool { + self.warmup.iter().chain(self.runs.iter()).all(|run| { + run.error.is_none() + && run + .utterances + .iter() + .all(|utterance| utterance.outcome == SttUtteranceOutcome::Completed) + }) && self.runs.len() == self.requested_runs + } +} + +pub fn load_bundled_stt_fixture_pack() -> Result { + let manifest: SttFixtureManifest = serde_json::from_str(MANIFEST_JSON) + .map_err(|error| format!("decode STT fixture manifest: {error}"))?; + if manifest.schema_version != 1 { + return Err(format!( + "unsupported STT fixture manifest schema {}", + manifest.schema_version + )); + } + let mut utterances = Vec::with_capacity(manifest.utterances.len()); + for metadata in &manifest.utterances { + let bytes = fixture_bytes(&metadata.file) + .ok_or_else(|| format!("fixture manifest names unknown file {}", metadata.file))?; + verify_fixture_bytes(metadata, bytes)?; + let samples_16k = decode_flac(metadata, bytes)?; + utterances.push(PreparedUtterance { + metadata: metadata.clone(), + samples_48k: upsample_16k_to_48k(&samples_16k), + }); + } + if utterances.is_empty() { + return Err("STT fixture manifest has no utterances".into()); + } + Ok(SttFixturePack { + manifest, + manifest_sha256: format!("{:x}", Sha256::digest(MANIFEST_JSON.as_bytes())), + utterances, + }) +} + +pub async fn benchmark_stt( + target: SttBenchmarkTarget, + environment: SttBenchmarkEnvironment, + pack: &SttFixturePack, + requested_runs: usize, + mode: SttBenchmarkMode, + mut create_runtime: impl FnMut() -> Result< + (VoiceInputRuntime, mpsc::Receiver), + String, + >, +) -> SttBenchmarkReport { + let mut report = SttBenchmarkReport { + schema_version: 1, + target, + environment, + mode, + runtime_scope: match mode { + SttBenchmarkMode::Cold => "fresh_voice_input_runtime_per_measured_run", + SttBenchmarkMode::Warm => "one_resident_voice_input_runtime_with_unmeasured_warmup", + }, + fixture_conversion: "FLAC PCM decoded at 16kHz and linearly interpolated to 48kHz", + input_pacing: "real_time_48khz_mono_f32_960_samples_every_20ms", + word_error_normalization: + "ASCII alphanumeric and apostrophe words, uppercase, punctuation as whitespace", + requested_runs, + fixture: pack.summary(), + planned_workload: pack.workload(requested_runs, mode), + warmup: None, + runs: Vec::with_capacity(requested_runs), + aggregate: SttWordError::default(), + }; + + match mode { + SttBenchmarkMode::Cold => { + for run in 1..=requested_runs { + let result = run_with_new_runtime(run, true, pack, &mut create_runtime).await; + let can_continue = result.error.is_none(); + report.aggregate.add(&result.aggregate); + report.runs.push(result); + if !can_continue { + break; + } + } + } + SttBenchmarkMode::Warm => { + let initialization_started = Instant::now(); + match create_runtime() { + Err(error) => { + report.warmup = Some(SttBenchmarkRun::initialization_error( + 0, + false, + initialization_started.elapsed(), + error, + )); + } + Ok((runtime, mut events)) => match wait_until_ready(&mut events).await { + Err(error) => { + runtime.cancel(); + report.warmup = Some(SttBenchmarkRun::initialization_error( + 0, + false, + initialization_started.elapsed(), + error, + )); + if let Err(error) = finish_runtime(runtime, &mut events).await { + record_shutdown_failure( + report.warmup.as_mut().expect("warmup was just recorded"), + error, + ); + } + } + Ok(()) => { + let initialization = initialization_started.elapsed(); + let warmup = + run_pack(0, false, Some(initialization), pack, &runtime, &mut events) + .await; + let warmup_ok = warmup.error.is_none() + && warmup.utterances.iter().all(|utterance| { + utterance.outcome == SttUtteranceOutcome::Completed + }); + report.warmup = Some(warmup); + if warmup_ok { + for run in 1..=requested_runs { + let result = + run_pack(run, true, None, pack, &runtime, &mut events).await; + let can_continue = result.error.is_none(); + report.aggregate.add(&result.aggregate); + report.runs.push(result); + if !can_continue { + break; + } + } + } + if let Err(error) = finish_runtime(runtime, &mut events).await { + let target = report + .runs + .last_mut() + .or(report.warmup.as_mut()) + .expect("warm mode records a warmup before shutdown"); + record_shutdown_failure(target, error); + } + } + }, + } + } + } + report +} + +async fn run_with_new_runtime( + run: usize, + measured: bool, + pack: &SttFixturePack, + create_runtime: &mut impl FnMut() -> Result< + (VoiceInputRuntime, mpsc::Receiver), + String, + >, +) -> SttBenchmarkRun { + let initialization_started = Instant::now(); + let (runtime, mut events) = match create_runtime() { + Ok(value) => value, + Err(error) => { + return SttBenchmarkRun::initialization_error( + run, + measured, + initialization_started.elapsed(), + error, + ) + } + }; + if let Err(error) = wait_until_ready(&mut events).await { + runtime.cancel(); + let mut result = SttBenchmarkRun::initialization_error( + run, + measured, + initialization_started.elapsed(), + error, + ); + if let Err(error) = finish_runtime(runtime, &mut events).await { + record_shutdown_failure(&mut result, error); + } + return result; + } + let initialization = initialization_started.elapsed(); + let mut result = run_pack( + run, + measured, + Some(initialization), + pack, + &runtime, + &mut events, + ) + .await; + if let Err(error) = finish_runtime(runtime, &mut events).await { + record_shutdown_failure(&mut result, error); + } + result +} + +async fn wait_until_ready(events: &mut mpsc::Receiver) -> Result<(), String> { + let deadline = tokio::time::Instant::now() + INITIALIZATION_TIMEOUT; + match tokio::time::timeout_at(deadline, events.recv()).await { + Err(_) => Err("voice input initialization exceeded 60 seconds".into()), + Ok(None) => Err("voice input stopped before readiness".into()), + Ok(Some(VoiceInputEvent::Ready)) => Ok(()), + Ok(Some(VoiceInputEvent::Failed(error))) => Err(error), + Ok(Some(_)) => Err("voice input emitted data before readiness".into()), + } +} + +async fn run_pack( + run: usize, + measured: bool, + initialization: Option, + pack: &SttFixturePack, + runtime: &VoiceInputRuntime, + events: &mut mpsc::Receiver, +) -> SttBenchmarkRun { + let started = Instant::now(); + let mut utterances = Vec::with_capacity(pack.utterances.len()); + let mut error = None; + for fixture in &pack.utterances { + let result = run_utterance(fixture, runtime, events).await; + let terminal = result.outcome == SttUtteranceOutcome::Failed; + utterances.push(result); + if terminal { + error = utterances.last().and_then(|result| result.error.clone()); + runtime.cancel(); + break; + } + } + let aggregate = aggregate_utterances(&utterances); + SttBenchmarkRun { + run, + measured, + initialization_ms: initialization.map(milliseconds), + total_ms: milliseconds(started.elapsed()), + utterances, + aggregate, + error_stage: error.as_ref().map(|_| "recognition"), + error, + } +} + +async fn run_utterance( + fixture: &PreparedUtterance, + runtime: &VoiceInputRuntime, + events: &mut mpsc::Receiver, +) -> SttUtteranceResult { + let started = Instant::now(); + let frames = framed_utterance(&fixture.samples_48k); + let source_end = started + + FRAME_DURATION * LEADING_SILENCE_FRAMES as u32 + + Duration::from_secs_f64(fixture.metadata.samples as f64 / 16_000.0); + let result_deadline = started + FRAME_DURATION * frames.len() as u32 + RESULT_TIMEOUT; + let mut next_frame = 0; + let mut tracker = UtteranceTracker::default(); + + while next_frame < frames.len() || !tracker.settled() { + if deadline_expired(Instant::now(), result_deadline) { + tracker.error = Some("recognition did not settle within 7 seconds after input".into()); + break; + } + let frame_deadline = started + FRAME_DURATION * (next_frame as u32 + 1); + if next_frame < frames.len() { + tokio::select! { + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(frame_deadline)) => { + if let Err(error) = runtime.try_push_frame( + VoiceInputFrame::try_from_samples(&frames[next_frame]) + .expect("benchmark frames have the runtime's exact shape"), + ) { + tracker.error = Some(error); + break; + } + next_frame += 1; + } + event = events.recv() => { + if observe_event(event, &mut tracker, started).is_err() { + break; + } + } + } + } else { + let remaining = result_deadline.saturating_duration_since(Instant::now()); + match tokio::time::timeout(remaining, events.recv()).await { + Err(_) => { + tracker.error = + Some("recognition did not settle within 7 seconds after input".into()); + break; + } + Ok(event) => { + if observe_event(event, &mut tracker, started).is_err() { + break; + } + } + } + } + } + + let hypothesis = tracker.final_text.unwrap_or_default(); + let word_error = word_error(&fixture.metadata.transcript, &hypothesis); + let no_result = tracker.error.is_none() && hypothesis.is_empty(); + SttUtteranceResult { + id: fixture.metadata.id.clone(), + reference: fixture.metadata.transcript.clone(), + hypothesis, + source_duration_ms: fixture.metadata.samples as f64 / 16.0, + streamed_duration_ms: milliseconds(FRAME_DURATION * frames.len() as u32), + speaking_started_ms: tracker + .speaking_started + .map(|time| milliseconds(time - started)), + speaking_duration_ms: duration_between(tracker.speaking_started, tracker.speaking_ended), + recognition_pending_ms: duration_between(tracker.pending_started, tracker.pending_ended), + final_latency_ms: tracker + .final_received + .map(|time| milliseconds(time.saturating_duration_since(source_end))), + total_ms: milliseconds(started.elapsed()), + word_error, + outcome: if tracker.error.is_some() { + SttUtteranceOutcome::Failed + } else if no_result { + SttUtteranceOutcome::NoResult + } else { + SttUtteranceOutcome::Completed + }, + error: tracker + .error + .or_else(|| no_result.then(|| "recognizer returned no transcript".into())), + } +} + +fn observe_event( + event: Option, + tracker: &mut UtteranceTracker, + run_started: Instant, +) -> Result<(), ()> { + let now = Instant::now(); + match event { + None => tracker.error = Some("voice input stopped before recognition settled".into()), + Some(VoiceInputEvent::Ready) => { + tracker.error = Some("voice input emitted duplicate readiness".into()) + } + Some(VoiceInputEvent::SpeakingChanged(active)) => { + tracker.speaking = active; + if active { + tracker.speaking_started.get_or_insert(now); + } else if tracker.speaking_started.is_some() { + tracker.speaking_ended = Some(now); + } + } + Some(VoiceInputEvent::RecognitionPendingChanged(active)) => { + tracker.pending = active; + if active { + tracker.pending_seen = true; + tracker.pending_started.get_or_insert(now); + } else if tracker.pending_started.is_some() { + tracker.pending_ended = Some(now); + } + } + Some(VoiceInputEvent::FinalTranscript { + text, + storage_receipt, + }) => { + if tracker.final_text.is_some() { + tracker.error = + Some("recognizer emitted more than one final for one fixture".into()); + } else { + tracker.final_text = Some(text); + tracker.final_received = Some(now.max(run_started)); + storage_receipt.stored(); + } + } + Some(VoiceInputEvent::Failed(error)) => tracker.error = Some(error), + } + if tracker.error.is_some() { + Err(()) + } else { + Ok(()) + } +} + +async fn finish_runtime( + runtime: VoiceInputRuntime, + events: &mut mpsc::Receiver, +) -> Result<(), String> { + let finish = runtime.finish(); + tokio::pin!(finish); + loop { + tokio::select! { + result = &mut finish => return result.map_err(|error| error.to_string()), + event = events.recv() => match event { + Some(event) => observe_shutdown_event(event)?, + None => return finish.await.map_err(|error| error.to_string()), + } + } + } +} + +fn observe_shutdown_event(event: VoiceInputEvent) -> Result<(), String> { + match event { + VoiceInputEvent::FinalTranscript { .. } => Err( + "voice input emitted a final after benchmark utterances settled; transcript was not stored" + .into(), + ), + VoiceInputEvent::Failed(error) => Err(error), + _ => Ok(()), + } +} + +#[derive(Default)] +struct UtteranceTracker { + speaking: bool, + pending: bool, + pending_seen: bool, + speaking_started: Option, + speaking_ended: Option, + pending_started: Option, + pending_ended: Option, + final_received: Option, + final_text: Option, + error: Option, +} + +impl UtteranceTracker { + fn settled(&self) -> bool { + self.error.is_some() + || (!self.speaking && !self.pending && (self.final_text.is_some() || self.pending_seen)) + } +} + +impl SttFixturePack { + fn summary(&self) -> SttFixtureSummary { + SttFixtureSummary { + corpus: self.manifest.corpus.clone(), + resource_id: self.manifest.resource_id.clone(), + subset: self.manifest.subset.clone(), + language: self.manifest.language.clone(), + license: self.manifest.license.clone(), + license_url: self.manifest.license_url.clone(), + source_url: self.manifest.source_url.clone(), + archive_url: self.manifest.archive_url.clone(), + archive_md5: self.manifest.archive_md5.clone(), + manifest_sha256: self.manifest_sha256.clone(), + attribution_notice: ATTRIBUTION_NOTICE, + utterances: self.manifest.utterances.clone(), + } + } + + pub fn workload(&self, requested_runs: usize, mode: SttBenchmarkMode) -> SttBenchmarkWorkload { + let attempts = requested_runs + usize::from(mode == SttBenchmarkMode::Warm); + let source_audio_seconds = self + .utterances + .iter() + .map(|fixture| fixture.metadata.samples as f64 / 16_000.0) + .sum::(); + let streamed_audio_seconds = self + .utterances + .iter() + .map(|fixture| { + framed_utterance(&fixture.samples_48k).len() as f64 * FRAME_DURATION.as_secs_f64() + }) + .sum::(); + SttBenchmarkWorkload { + runtime_initializations: match mode { + SttBenchmarkMode::Cold => requested_runs, + SttBenchmarkMode::Warm => 1, + }, + recognition_commits: self.utterances.len().saturating_mul(attempts), + source_audio_seconds: source_audio_seconds * attempts as f64, + streamed_audio_seconds: streamed_audio_seconds * attempts as f64, + includes_warmup: mode == SttBenchmarkMode::Warm, + } + } +} + +impl SttBenchmarkRun { + fn initialization_error( + run: usize, + measured: bool, + initialization: Duration, + error: String, + ) -> Self { + Self { + run, + measured, + initialization_ms: Some(milliseconds(initialization)), + total_ms: milliseconds(initialization), + utterances: Vec::new(), + aggregate: SttWordError::default(), + error_stage: Some("initialization"), + error: Some(error), + } + } +} + +fn record_shutdown_failure(run: &mut SttBenchmarkRun, error: String) { + if let Some(existing) = run.error.take() { + run.error = Some(format!("{existing}; shutdown: {error}")); + } else { + run.error_stage = Some("shutdown"); + run.error = Some(error); + } +} + +fn fixture_bytes(file: &str) -> Option<&'static [u8]> { + match file { + "1089-134686-0002.flac" => Some(include_bytes!( + "../../fixtures/stt/librispeech-test-clean-mini/1089-134686-0002.flac" + )), + "1221-135766-0002.flac" => Some(include_bytes!( + "../../fixtures/stt/librispeech-test-clean-mini/1221-135766-0002.flac" + )), + "1284-1180-0003.flac" => Some(include_bytes!( + "../../fixtures/stt/librispeech-test-clean-mini/1284-1180-0003.flac" + )), + _ => None, + } +} + +fn verify_fixture_bytes(metadata: &SttFixtureMetadata, bytes: &[u8]) -> Result<(), String> { + if bytes.len() != metadata.bytes { + return Err(format!( + "fixture {} has {} bytes; manifest expects {}", + metadata.id, + bytes.len(), + metadata.bytes + )); + } + let sha256 = format!("{:x}", Sha256::digest(bytes)); + if sha256 != metadata.sha256 { + return Err(format!( + "fixture {} SHA-256 mismatch: {sha256}", + metadata.id + )); + } + Ok(()) +} + +fn decode_flac(metadata: &SttFixtureMetadata, bytes: &[u8]) -> Result, String> { + let mut reader = FlacReader::new(Cursor::new(bytes)) + .map_err(|error| format!("decode fixture {}: {error}", metadata.id))?; + let info = reader.streaminfo(); + if info.sample_rate != metadata.sample_rate_hz + || info.channels != metadata.channels + || info.bits_per_sample != metadata.bits_per_sample + || info.samples != Some(metadata.samples) + { + return Err(format!( + "fixture {} decoded stream metadata does not match manifest", + metadata.id + )); + } + if metadata.channels != 1 || metadata.sample_rate_hz != 16_000 { + return Err(format!("fixture {} must be 16 kHz mono FLAC", metadata.id)); + } + let scale = (1_u64 << (metadata.bits_per_sample - 1)) as f32; + let samples = reader + .samples() + .map(|sample| { + sample + .map(|sample| (sample as f32 / scale).clamp(-1.0, 1.0)) + .map_err(|error| format!("decode fixture {}: {error}", metadata.id)) + }) + .collect::, _>>()?; + if samples.len() as u64 != metadata.samples { + return Err(format!( + "fixture {} decoded {} samples; manifest expects {}", + metadata.id, + samples.len(), + metadata.samples + )); + } + Ok(samples) +} + +fn upsample_16k_to_48k(samples: &[f32]) -> Vec { + debug_assert_eq!(INPUT_SAMPLE_RATE, 48_000); + let mut output = Vec::with_capacity(samples.len() * 3); + for (index, current) in samples.iter().copied().enumerate() { + let next = samples.get(index + 1).copied().unwrap_or(current); + output.push(current); + output.push(current + (next - current) / 3.0); + output.push(current + (next - current) * (2.0 / 3.0)); + } + output +} + +fn framed_utterance(samples: &[f32]) -> Vec<[f32; INPUT_FRAME_SAMPLES]> { + let audio_frames = samples.len().div_ceil(INPUT_FRAME_SAMPLES); + let mut frames = vec![ + [0.0; INPUT_FRAME_SAMPLES]; + LEADING_SILENCE_FRAMES + audio_frames + TRAILING_SILENCE_FRAMES + ]; + for (target, sample) in frames[LEADING_SILENCE_FRAMES..] + .iter_mut() + .flatten() + .zip(samples) + { + *target = *sample; + } + frames +} + +#[cfg(test)] +pub(crate) fn first_bundled_fixture_frames_for_test() -> Vec<[f32; INPUT_FRAME_SAMPLES]> { + let pack = load_bundled_stt_fixture_pack().expect("checked bundled STT fixture"); + framed_utterance(&pack.utterances[0].samples_48k) +} + +fn word_error(reference: &str, hypothesis: &str) -> SttWordError { + let reference = normalized_words(reference); + let hypothesis = normalized_words(hypothesis); + let mut distance = vec![vec![0_usize; hypothesis.len() + 1]; reference.len() + 1]; + for (index, row) in distance.iter_mut().enumerate() { + row[0] = index; + } + for (index, cell) in distance[0].iter_mut().enumerate() { + *cell = index; + } + for i in 1..=reference.len() { + for j in 1..=hypothesis.len() { + distance[i][j] = if reference[i - 1] == hypothesis[j - 1] { + distance[i - 1][j - 1] + } else { + (distance[i - 1][j - 1] + 1) + .min(distance[i - 1][j] + 1) + .min(distance[i][j - 1] + 1) + }; + } + } + let (mut i, mut j) = (reference.len(), hypothesis.len()); + let (mut substitutions, mut deletions, mut insertions) = (0, 0, 0); + while i > 0 || j > 0 { + if i > 0 && j > 0 && reference[i - 1] == hypothesis[j - 1] { + i -= 1; + j -= 1; + } else if i > 0 && j > 0 && distance[i][j] == distance[i - 1][j - 1] + 1 { + substitutions += 1; + i -= 1; + j -= 1; + } else if i > 0 && distance[i][j] == distance[i - 1][j] + 1 { + deletions += 1; + i -= 1; + } else { + insertions += 1; + j -= 1; + } + } + let mut result = SttWordError { + reference_words: reference.len(), + substitutions, + deletions, + insertions, + word_error_rate: None, + }; + result.word_error_rate = rate(&result); + result +} + +fn normalized_words(text: &str) -> Vec { + let normalized = text + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || character == '\'' { + character.to_ascii_uppercase() + } else { + ' ' + } + }) + .collect::(); + normalized + .split_whitespace() + .map(ToString::to_string) + .collect() +} + +fn aggregate_utterances(utterances: &[SttUtteranceResult]) -> SttWordError { + let mut aggregate = SttWordError::default(); + for utterance in utterances { + aggregate.add(&utterance.word_error); + } + aggregate +} + +fn rate(error: &SttWordError) -> Option { + (error.reference_words > 0).then(|| { + (error.substitutions + error.deletions + error.insertions) as f64 + / error.reference_words as f64 + }) +} + +fn duration_between(start: Option, end: Option) -> Option { + start.zip(end).map(|(start, end)| milliseconds(end - start)) +} + +fn milliseconds(duration: Duration) -> f64 { + duration.as_secs_f64() * 1_000.0 +} + +fn deadline_expired(now: Instant, deadline: Instant) -> bool { + now >= deadline +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bundled_manifest_hashes_and_decoded_metadata_are_exact() { + let pack = load_bundled_stt_fixture_pack().unwrap(); + assert_eq!(pack.manifest.resource_id, "SLR12"); + assert_eq!( + pack.manifest.archive_md5, + "32fa31d27d2e1cad72775fee3f4849a9" + ); + assert_eq!(pack.utterances.len(), 3); + assert_eq!(pack.utterances[0].samples_48k.len(), 318_000); + assert_eq!(pack.utterances[1].samples_48k.len(), 231_600); + assert_eq!(pack.utterances[2].samples_48k.len(), 232_080); + assert!(pack + .utterances + .iter() + .flat_map(|fixture| &fixture.samples_48k) + .all(|sample| sample.is_finite() && (-1.0..=1.0).contains(sample))); + } + + #[test] + fn fixture_framing_matches_the_production_contract_and_timing() { + let pack = load_bundled_stt_fixture_pack().unwrap(); + let frames = framed_utterance(&pack.utterances[0].samples_48k); + assert_eq!(frames.len(), 50 + 332 + 325); + assert!(frames[..50].iter().flatten().all(|sample| *sample == 0.0)); + assert!(frames[50 + 332..] + .iter() + .flatten() + .all(|sample| *sample == 0.0)); + assert_eq!(INPUT_SAMPLE_RATE, 48_000); + assert_eq!(INPUT_FRAME_SAMPLES, 960); + } + + #[test] + fn workload_includes_warmup_and_exact_stream_padding() { + let pack = load_bundled_stt_fixture_pack().unwrap(); + let cold = pack.workload(2, SttBenchmarkMode::Cold); + assert_eq!(cold.runtime_initializations, 2); + assert_eq!(cold.recognition_commits, 6); + assert!((cold.source_audio_seconds - 32.57).abs() < 0.000_001); + assert!((cold.streamed_audio_seconds - 77.64).abs() < 0.000_001); + + let warm = pack.workload(2, SttBenchmarkMode::Warm); + assert_eq!(warm.runtime_initializations, 1); + assert_eq!(warm.recognition_commits, 9); + assert!((warm.streamed_audio_seconds - 116.46).abs() < 0.000_001); + assert!(warm.includes_warmup); + } + + #[test] + fn word_error_normalizes_case_and_punctuation_and_reports_sdi() { + assert_eq!( + word_error("Hello, brave new world!", "hello brave old worlds"), + SttWordError { + reference_words: 4, + substitutions: 2, + deletions: 0, + insertions: 0, + word_error_rate: Some(0.5), + } + ); + assert_eq!( + word_error("ONE TWO THREE", "ZERO ONE THREE FOUR"), + SttWordError { + reference_words: 3, + substitutions: 2, + deletions: 0, + insertions: 1, + word_error_rate: Some(1.0), + } + ); + assert_eq!(word_error("ONE TWO THREE", "ONE THREE").deletions, 1); + assert_eq!(word_error("", "noise").word_error_rate, None); + } + + #[test] + fn recognition_deadline_is_inclusive_and_bounded() { + let now = Instant::now(); + let deadline = now + RESULT_TIMEOUT; + assert!(!deadline_expired( + deadline - Duration::from_nanos(1), + deadline + )); + assert!(deadline_expired(deadline, deadline)); + assert!(deadline_expired( + deadline + Duration::from_nanos(1), + deadline + )); + } + + #[test] + fn shutdown_failures_are_terminal_and_preserve_an_earlier_failure() { + let mut clean = SttBenchmarkRun { + run: 1, + measured: true, + initialization_ms: None, + total_ms: 0.0, + utterances: Vec::new(), + aggregate: SttWordError::default(), + error_stage: None, + error: None, + }; + record_shutdown_failure(&mut clean, "worker stuck".into()); + assert_eq!(clean.error_stage, Some("shutdown")); + assert_eq!(clean.error.as_deref(), Some("worker stuck")); + + let mut initialization = + SttBenchmarkRun::initialization_error(0, false, Duration::ZERO, "not ready".into()); + record_shutdown_failure(&mut initialization, "worker stuck".into()); + assert_eq!(initialization.error_stage, Some("initialization")); + assert_eq!( + initialization.error.as_deref(), + Some("not ready; shutdown: worker stuck") + ); + } + + #[test] + fn receipt_is_acknowledged_only_after_the_first_final_is_stored() { + let started = Instant::now(); + let mut tracker = UtteranceTracker::default(); + let (first_receipt, first_ack) = crate::input::FinalTranscriptStorageReceipt::test_pair(); + observe_event( + Some(VoiceInputEvent::FinalTranscript { + text: "stored words".into(), + storage_receipt: first_receipt, + }), + &mut tracker, + started, + ) + .unwrap(); + assert_eq!(tracker.final_text.as_deref(), Some("stored words")); + assert_eq!(first_ack.recv_timeout(Duration::from_millis(10)), Ok(())); + + let (duplicate_receipt, duplicate_ack) = + crate::input::FinalTranscriptStorageReceipt::test_pair(); + assert!(observe_event( + Some(VoiceInputEvent::FinalTranscript { + text: "duplicate words".into(), + storage_receipt: duplicate_receipt, + }), + &mut tracker, + started, + ) + .is_err()); + assert_eq!(tracker.final_text.as_deref(), Some("stored words")); + assert_eq!( + duplicate_ack.try_recv(), + Err(std::sync::mpsc::TryRecvError::Disconnected) + ); + } + + #[test] + fn late_final_is_a_shutdown_failure_and_is_not_acknowledged() { + let (receipt, acknowledgement) = crate::input::FinalTranscriptStorageReceipt::test_pair(); + let error = observe_shutdown_event(VoiceInputEvent::FinalTranscript { + text: "late words".into(), + storage_receipt: receipt, + }) + .unwrap_err(); + assert!(error.contains("after benchmark utterances settled")); + assert_eq!( + acknowledgement.try_recv(), + Err(std::sync::mpsc::TryRecvError::Disconnected) + ); + } + + #[test] + fn tracker_requires_authoritative_idle_after_final_or_no_result() { + let now = Instant::now(); + let mut tracker = UtteranceTracker { + speaking: true, + pending: true, + pending_seen: true, + speaking_started: Some(now), + pending_started: Some(now), + final_text: Some("words".into()), + ..Default::default() + }; + assert!(!tracker.settled()); + tracker.speaking = false; + tracker.pending = false; + assert!(tracker.settled()); + + let no_result = UtteranceTracker { + pending_seen: true, + ..Default::default() + }; + assert!(no_result.settled()); + } + + #[test] + fn report_json_exposes_provenance_without_fixture_audio_or_secrets() { + let pack = load_bundled_stt_fixture_pack().unwrap(); + let report = SttBenchmarkReport { + schema_version: 1, + target: SttBenchmarkTarget { + backend: "openai".into(), + model: Some("test-model".into()), + locale: None, + vad_threshold: 0.5, + endpoint_source: Some("default".into()), + model_source: Some("environment".into()), + credential_source: Some("OPENAI_API_KEY environment variable".into()), + }, + environment: SttBenchmarkEnvironment::default(), + mode: SttBenchmarkMode::Cold, + runtime_scope: "fresh_voice_input_runtime_per_measured_run", + fixture_conversion: "FLAC PCM decoded at 16kHz and linearly interpolated to 48kHz", + input_pacing: "real_time_48khz_mono_f32_960_samples_every_20ms", + word_error_normalization: + "ASCII alphanumeric and apostrophe words, uppercase, punctuation as whitespace", + requested_runs: 1, + fixture: pack.summary(), + planned_workload: pack.workload(1, SttBenchmarkMode::Cold), + warmup: None, + runs: Vec::new(), + aggregate: SttWordError::default(), + }; + let json = serde_json::to_string(&report).unwrap(); + assert!(json.contains("32fa31d27d2e1cad72775fee3f4849a9")); + assert!(json.contains("OPENAI_API_KEY environment variable")); + assert!(json.contains("Creative Commons Attribution 4.0 International")); + assert!(json.contains("LibriSpeech (c) 2014 by Vassil Panayotov")); + assert!(!json.contains("sk-")); + } +} diff --git a/src-tauri/crates/berd-voice/src/configured_tts.rs b/src-tauri/crates/berd-voice/src/configured_tts.rs new file mode 100644 index 000000000..f62c9e33d --- /dev/null +++ b/src-tauri/crates/berd-voice/src/configured_tts.rs @@ -0,0 +1,723 @@ +use std::fmt; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use serde::{Deserialize, Serialize}; + +use crate::openai::OpenAiSpeechConfig; +#[cfg(target_os = "macos")] +use crate::SiriTts; + +use crate::{OpenAiTts, PocketTtsBackend, TtsBackend}; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(tag = "backend", rename_all = "snake_case", deny_unknown_fields)] +pub enum TtsSettings { + #[serde(rename = "openai")] + OpenAi { + model: String, + voice: String, + rate: f32, + }, + Siri { + voice: String, + language: String, + rate: f32, + }, + Pocket { + model: String, + voice: String, + rate: f32, + }, +} + +impl TtsSettings { + pub fn voice(&self) -> &str { + match self { + Self::OpenAi { voice, .. } | Self::Siri { voice, .. } | Self::Pocket { voice, .. } => { + voice + } + } + } + + pub fn rate(&self) -> f32 { + match self { + Self::OpenAi { rate, .. } | Self::Siri { rate, .. } | Self::Pocket { rate, .. } => { + *rate + } + } + } + + fn backend_name(&self) -> &'static str { + match self { + Self::OpenAi { .. } => "openai", + Self::Siri { .. } => "siri", + Self::Pocket { .. } => "pocket", + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TtsConfigurationSnapshot { + pub revision: u64, + #[serde(flatten)] + pub settings: TtsSettings, +} + +#[derive(Clone)] +pub enum TtsConfiguration { + OpenAi { + endpoint: String, + api_key: String, + model: String, + voice: String, + rate: f32, + }, + Siri { + voice: String, + language: String, + rate: f32, + }, + Pocket { + model_dir: PathBuf, + model: String, + voice: String, + rate: f32, + }, +} + +impl fmt::Debug for TtsConfiguration { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("TtsConfiguration") + .field("settings", &self.settings()) + .finish_non_exhaustive() + } +} + +impl TtsConfiguration { + pub fn openai( + endpoint: String, + api_key: String, + model: String, + voice: String, + rate: f32, + ) -> Self { + Self::OpenAi { + endpoint, + api_key, + model, + voice, + rate, + } + } + + pub fn siri(voice: String, language: String, rate: f32) -> Self { + Self::Siri { + voice, + language, + rate, + } + } + + pub fn pocket(model_dir: PathBuf, model: String, voice: String, rate: f32) -> Self { + Self::Pocket { + model_dir, + model, + voice, + rate, + } + } + + pub fn settings(&self) -> TtsSettings { + match self { + Self::OpenAi { + model, voice, rate, .. + } => TtsSettings::OpenAi { + model: model.clone(), + voice: voice.clone(), + rate: *rate, + }, + Self::Siri { + voice, + language, + rate, + } => TtsSettings::Siri { + voice: voice.clone(), + language: language.clone(), + rate: *rate, + }, + Self::Pocket { + model, voice, rate, .. + } => TtsSettings::Pocket { + model: model.clone(), + voice: voice.clone(), + rate: *rate, + }, + } + } + + fn replacement( + &self, + settings: TtsSettings, + ) -> Result { + match (self, settings) { + ( + Self::OpenAi { + endpoint, api_key, .. + }, + TtsSettings::OpenAi { model, voice, rate }, + ) => Ok(Self::openai( + endpoint.clone(), + api_key.clone(), + model, + voice, + rate, + )), + ( + Self::Siri { .. }, + TtsSettings::Siri { + voice, + language, + rate, + }, + ) => Ok(Self::siri(voice, language, rate)), + ( + Self::Pocket { + model_dir, + model: current_model, + .. + }, + TtsSettings::Pocket { model, voice, rate }, + ) if &model == current_model => Ok(Self::pocket(model_dir.clone(), model, voice, rate)), + (Self::Pocket { .. }, TtsSettings::Pocket { .. }) => Err(( + TtsConfigurationRejectionKind::InvalidSettings, + "Pocket model cannot be changed without selecting a new bundle".into(), + )), + (current, requested) => Err(( + TtsConfigurationRejectionKind::BackendMismatch, + format!( + "cannot apply {} settings while {} TTS is active", + requested.backend_name(), + current.settings().backend_name() + ), + )), + } + } + + fn build(&self) -> Result, String> { + validate_settings(&self.settings())?; + match self { + Self::OpenAi { + endpoint, + api_key, + model, + voice, + rate, + } => OpenAiTts::new(OpenAiSpeechConfig { + endpoint: endpoint.clone(), + api_key: api_key.clone(), + model: model.clone(), + voice: voice.clone(), + speed: *rate, + }) + .map(|backend| Arc::new(backend) as Arc), + #[cfg(target_os = "macos")] + Self::Siri { + voice, + language, + rate, + } => SiriTts::new(language, voice, *rate) + .map(|backend| Arc::new(backend) as Arc), + #[cfg(not(target_os = "macos"))] + Self::Siri { .. } => Err("Siri TTS is only available on macOS".into()), + Self::Pocket { + model_dir, + voice, + rate, + .. + } => PocketTtsBackend::new(model_dir, voice, *rate) + .map(|backend| Arc::new(backend) as Arc), + } + } +} + +fn validate_settings(settings: &TtsSettings) -> Result<(), String> { + let nonempty = |name: &str, value: &str| { + (!value.trim().is_empty()) + .then_some(()) + .ok_or_else(|| format!("{name} must be nonempty")) + }; + match settings { + TtsSettings::OpenAi { model, voice, rate } => { + nonempty("OpenAI model", model)?; + nonempty("OpenAI voice", voice)?; + if !rate.is_finite() || !(0.75..=2.0).contains(rate) { + return Err("OpenAI rate must be between 0.75 and 2.0".into()); + } + } + TtsSettings::Siri { + voice, + language, + rate, + } => { + nonempty("Siri voice", voice)?; + nonempty("Siri language", language)?; + if !rate.is_finite() || !(0.5..=2.0).contains(rate) { + return Err("Siri rate must be between 0.5 and 2.0".into()); + } + } + TtsSettings::Pocket { model, voice, rate } => { + nonempty("Pocket model", model)?; + nonempty("Pocket voice", voice)?; + if !rate.is_finite() || !(0.75..=2.0).contains(rate) { + return Err("Pocket rate must be between 0.75 and 2.0".into()); + } + } + } + Ok(()) +} + +struct ConfiguredTtsState { + configuration: TtsConfiguration, + backend: Arc, + snapshot: TtsConfigurationSnapshot, +} + +pub struct ConfiguredTtsSlot { + inner: Mutex, +} + +impl fmt::Debug for ConfiguredTtsSlot { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ConfiguredTtsSlot") + .field("snapshot", &self.snapshot()) + .finish_non_exhaustive() + } +} + +#[derive(Clone)] +pub struct TtsConfigurationLease { + backend: Arc, + snapshot: TtsConfigurationSnapshot, +} + +impl TtsConfigurationLease { + pub fn backend(&self) -> &Arc { + &self.backend + } + pub fn snapshot(&self) -> &TtsConfigurationSnapshot { + &self.snapshot + } +} + +pub struct TtsConfigurationReplacement { + base_revision: u64, + configuration: TtsConfiguration, + backend: Arc, + settings: TtsSettings, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct TtsConfigurationRejection { + pub kind: TtsConfigurationRejectionKind, + pub message: String, + pub snapshot: TtsConfigurationSnapshot, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TtsConfigurationRejectionKind { + StaleRevision, + BackendMismatch, + InvalidSettings, + Initialization, + Internal, +} + +impl ConfiguredTtsSlot { + pub fn new(configuration: TtsConfiguration) -> Result { + let settings = configuration.settings(); + let backend = configuration.build()?; + Ok(Self::from_parts(configuration, backend, settings)) + } + + fn from_parts( + configuration: TtsConfiguration, + backend: Arc, + settings: TtsSettings, + ) -> Self { + Self { + inner: Mutex::new(ConfiguredTtsState { + configuration, + backend, + snapshot: TtsConfigurationSnapshot { + revision: 1, + settings, + }, + }), + } + } + + pub fn snapshot(&self) -> Result { + self.inner + .lock() + .map(|state| state.snapshot.clone()) + .map_err(|_| "configured TTS lock was poisoned".into()) + } + + pub fn lease(&self) -> Result { + let state = self + .inner + .lock() + .map_err(|_| "configured TTS lock was poisoned".to_string())?; + Ok(TtsConfigurationLease { + backend: Arc::clone(&state.backend), + snapshot: state.snapshot.clone(), + }) + } + + /// Builds and validates a replacement without holding the slot lock. + pub fn prepare_replacement( + &self, + expected_revision: u64, + settings: TtsSettings, + ) -> Result { + let (configuration, snapshot) = { + let state = self.inner.lock().map_err(|_| TtsConfigurationRejection { + kind: TtsConfigurationRejectionKind::Internal, + message: "configured TTS lock was poisoned".into(), + snapshot: TtsConfigurationSnapshot { + revision: 0, + settings: settings.clone(), + }, + })?; + (state.configuration.clone(), state.snapshot.clone()) + }; + if snapshot.revision != expected_revision { + return Err(TtsConfigurationRejection { + kind: TtsConfigurationRejectionKind::StaleRevision, + message: format!( + "stale TTS configuration revision: expected {expected_revision}, current {}", + snapshot.revision + ), + snapshot, + }); + } + let configuration = + configuration + .replacement(settings.clone()) + .map_err(|(kind, message)| TtsConfigurationRejection { + kind, + message, + snapshot: snapshot.clone(), + })?; + validate_settings(&settings).map_err(|message| TtsConfigurationRejection { + kind: TtsConfigurationRejectionKind::InvalidSettings, + message, + snapshot: snapshot.clone(), + })?; + let backend = configuration + .build() + .map_err(|message| TtsConfigurationRejection { + kind: TtsConfigurationRejectionKind::Initialization, + message, + snapshot: snapshot.clone(), + })?; + Ok(TtsConfigurationReplacement { + base_revision: expected_revision, + configuration, + backend, + settings, + }) + } + + pub fn commit_replacement( + &self, + replacement: TtsConfigurationReplacement, + ) -> Result { + let mut state = self.inner.lock().map_err(|_| TtsConfigurationRejection { + kind: TtsConfigurationRejectionKind::Internal, + message: "configured TTS lock was poisoned".into(), + snapshot: TtsConfigurationSnapshot { + revision: 0, + settings: replacement.settings.clone(), + }, + })?; + if state.snapshot.revision != replacement.base_revision { + return Err(TtsConfigurationRejection { + kind: TtsConfigurationRejectionKind::StaleRevision, + message: format!( + "stale TTS configuration revision: expected {}, current {}", + replacement.base_revision, state.snapshot.revision + ), + snapshot: state.snapshot.clone(), + }); + } + let revision = + state + .snapshot + .revision + .checked_add(1) + .ok_or_else(|| TtsConfigurationRejection { + kind: TtsConfigurationRejectionKind::Internal, + message: "TTS configuration revision overflow".into(), + snapshot: state.snapshot.clone(), + })?; + state.configuration = replacement.configuration; + state.backend = replacement.backend; + state.snapshot = TtsConfigurationSnapshot { + revision, + settings: replacement.settings, + }; + Ok(state.snapshot.clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{TtsOutcome, TtsPcmSpec}; + use std::sync::atomic::AtomicBool; + + struct FakeTts; + impl TtsBackend for FakeTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + } + } + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + _on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + Ok(TtsOutcome::Completed) + } + } + + impl ConfiguredTtsSlot { + fn new_for_test(settings: TtsSettings) -> Self { + let TtsSettings::OpenAi { model, voice, rate } = &settings else { + panic!("test helper expects OpenAI") + }; + Self::from_parts( + TtsConfiguration::openai( + "endpoint".into(), + "key".into(), + model.clone(), + voice.clone(), + *rate, + ), + Arc::new(FakeTts), + settings, + ) + } + + fn replace_for_test( + &self, + expected_revision: u64, + settings: TtsSettings, + ) -> Result { + let snapshot = self.snapshot().unwrap(); + if snapshot.revision != expected_revision { + return Err(TtsConfigurationRejection { + kind: TtsConfigurationRejectionKind::StaleRevision, + message: "stale".into(), + snapshot, + }); + } + let configuration = self + .inner + .lock() + .unwrap() + .configuration + .replacement(settings.clone()) + .unwrap(); + self.commit_replacement(TtsConfigurationReplacement { + base_revision: expected_revision, + configuration, + backend: Arc::new(FakeTts), + settings, + }) + } + } + + #[test] + fn snapshot_is_sanitized_and_revisioned() { + let slot = ConfiguredTtsSlot::new(TtsConfiguration::openai( + "https://private.invalid/v1/audio/speech".into(), + "secret-key".into(), + "gpt-4o-mini-tts".into(), + "marin".into(), + 1.0, + )) + .unwrap(); + let json = serde_json::to_string(&slot.snapshot().unwrap()).unwrap(); + assert!(json.contains("gpt-4o-mini-tts")); + assert!(json.contains("marin")); + assert!(!json.contains("secret-key")); + assert!(!json.contains("private.invalid")); + assert_eq!(slot.snapshot().unwrap().revision, 1); + } + + #[test] + fn active_lease_keeps_old_configuration_and_next_lease_gets_update() { + let slot = ConfiguredTtsSlot::new_for_test(TtsSettings::OpenAi { + model: "model".into(), + voice: "old".into(), + rate: 1.0, + }); + let old = slot.lease().unwrap(); + slot.replace_for_test( + 1, + TtsSettings::OpenAi { + model: "model".into(), + voice: "new".into(), + rate: 2.0, + }, + ) + .unwrap(); + assert_eq!(old.snapshot().revision, 1); + assert_eq!(old.snapshot().settings.voice(), "old"); + let next = slot.lease().unwrap(); + assert_eq!(next.snapshot().revision, 2); + assert_eq!(next.snapshot().settings.voice(), "new"); + assert_eq!(next.snapshot().settings.rate(), 2.0); + } + + #[test] + fn stale_replacement_preserves_authoritative_configuration() { + let slot = ConfiguredTtsSlot::new_for_test(TtsSettings::OpenAi { + model: "model".into(), + voice: "old".into(), + rate: 1.0, + }); + slot.replace_for_test( + 1, + TtsSettings::OpenAi { + model: "model".into(), + voice: "new".into(), + rate: 2.0, + }, + ) + .unwrap(); + let rejection = slot + .replace_for_test( + 1, + TtsSettings::OpenAi { + model: "model".into(), + voice: "stale".into(), + rate: 1.5, + }, + ) + .unwrap_err(); + assert_eq!(rejection.snapshot.revision, 2); + assert_eq!(slot.snapshot().unwrap().settings.voice(), "new"); + } + + #[test] + fn invalid_or_cross_backend_settings_leave_the_slot_unchanged() { + let slot = ConfiguredTtsSlot::new_for_test(TtsSettings::OpenAi { + model: "model".into(), + voice: "voice".into(), + rate: 1.0, + }); + for settings in [ + TtsSettings::OpenAi { + model: "model".into(), + voice: "voice".into(), + rate: 2.1, + }, + TtsSettings::Siri { + voice: "Aaron".into(), + language: "en-US".into(), + rate: 1.0, + }, + ] { + assert!(slot.prepare_replacement(1, settings).is_err()); + assert_eq!(slot.snapshot().unwrap().revision, 1); + } + } + + #[test] + fn concurrent_preparations_commit_only_from_the_current_revision() { + let slot = ConfiguredTtsSlot::new(TtsConfiguration::openai( + "https://example.invalid/audio/speech".into(), + "secret".into(), + "model".into(), + "initial".into(), + 1.0, + )) + .unwrap(); + let first = slot + .prepare_replacement( + 1, + TtsSettings::OpenAi { + model: "model".into(), + voice: "first".into(), + rate: 1.5, + }, + ) + .unwrap(); + let second = slot + .prepare_replacement( + 1, + TtsSettings::OpenAi { + model: "model".into(), + voice: "second".into(), + rate: 2.0, + }, + ) + .unwrap(); + + assert_eq!(slot.commit_replacement(first).unwrap().revision, 2); + let rejection = slot.commit_replacement(second).unwrap_err(); + assert_eq!(rejection.snapshot.revision, 2); + assert_eq!(rejection.snapshot.settings.voice(), "first"); + } + + #[test] + fn every_backend_snapshot_contains_only_public_configuration() { + for (settings, backend) in [ + ( + TtsSettings::OpenAi { + model: "model".into(), + voice: "marin".into(), + rate: 2.0, + }, + "openai", + ), + ( + TtsSettings::Siri { + voice: "Aaron".into(), + language: "en-US".into(), + rate: 1.25, + }, + "siri", + ), + ( + TtsSettings::Pocket { + model: crate::pocket_assets::MODEL_ID.into(), + voice: "mary".into(), + rate: 1.5, + }, + "pocket", + ), + ] { + let json = serde_json::to_value(TtsConfigurationSnapshot { + revision: 3, + settings, + }) + .unwrap(); + assert_eq!(json["backend"], backend); + assert_eq!(json["revision"], 3); + assert!(json.get("endpoint").is_none()); + assert!(json.get("api_key").is_none()); + assert!(json.get("model_dir").is_none()); + } + } +} diff --git a/src-tauri/crates/berd-voice/src/input.rs b/src-tauri/crates/berd-voice/src/input.rs new file mode 100644 index 000000000..deef651ec --- /dev/null +++ b/src-tauri/crates/berd-voice/src/input.rs @@ -0,0 +1,3553 @@ +//! Host-independent voice-input runtime. +//! +//! Hosts provide exact normalized PCM frames and own capture devices and final +//! transcript delivery. This module owns recognition, VAD, mute/reset epochs, +//! assistant-sensitive interruption thresholds, and bounded engine shutdown. + +use std::{ + collections::{BTreeMap, HashMap, VecDeque}, + path::PathBuf, + sync::{ + atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, + mpsc::{self, Receiver, SyncSender, TrySendError}, + Arc, Condvar, Mutex, + }, + thread, + time::{Duration, Instant}, +}; + +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc as tokio_mpsc; + +use crate::{ + openai_realtime::{ + OpenAiRealtimeTranscriptionClient, OpenAiRealtimeTranscriptionConfig, + OpenAiRealtimeTranscriptionError as TranscriptionError, OpenAiRealtimeTranscriptionEvent, + }, + ParakeetRecognizer, +}; + +pub const INPUT_SAMPLE_RATE: usize = 48_000; +pub const INPUT_FRAME_SAMPLES: usize = 960; +const INPUT_FRAME_DURATION: Duration = Duration::from_millis(20); +const INPUT_QUEUE_FRAMES: usize = 50; +const PARAKEET_RESULT_QUEUE_DEPTH: usize = 1; + +const EVENT_QUEUE_DEPTH: usize = 64; +const MAX_SPEECH_SAMPLES: usize = 16_000 * 30; +const VAD_FRAME_SAMPLES: usize = 256; +const SILENCE_FLUSH_FRAMES: usize = 75; +const OPENAI_NETWORK_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +const OPENAI_LIVE_RESULT_TIMEOUT: Duration = Duration::from_secs(5); +const OPENAI_FINAL_WRITE_TIMEOUT: Duration = Duration::from_secs(1); +const OPENAI_PRE_ROLL_FRAMES: usize = 15; +const FINAL_STORAGE_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(target_os = "macos")] +const MAC_LIVE_NO_RESULT_TIMEOUT: Duration = Duration::from_secs(5); +const FINISH_TIMEOUT: Duration = + Duration::from_secs(crate::MAC_SPEECH_RECOGNITION_FINISH_TIMEOUT_SECONDS + 5 + 1); + +/// One 20 ms frame of 48 kHz mono, finite, unit-scale Float32 PCM. +pub struct VoiceInputFrame([f32; INPUT_FRAME_SAMPLES]); + +impl VoiceInputFrame { + pub fn try_from_samples(samples: &[f32]) -> Result { + if samples.len() != INPUT_FRAME_SAMPLES { + return Err(format!( + "voice input frame has {} samples; expected {INPUT_FRAME_SAMPLES}", + samples.len() + )); + } + let mut frame = [0.0; INPUT_FRAME_SAMPLES]; + for (target, sample) in frame.iter_mut().zip(samples) { + if !sample.is_finite() { + return Err("voice input frame contains a non-finite sample".to_string()); + } + *target = sample.clamp(-1.0, 1.0); + } + Ok(Self(frame)) + } + + fn samples(&self) -> &[f32; INPUT_FRAME_SAMPLES] { + &self.0 + } +} + +pub enum VoiceInputEngineConfig { + Parakeet { + model_dir: PathBuf, + }, + #[cfg(target_os = "macos")] + MacSpeech, + OpenAi { + endpoint: String, + api_key: String, + model: String, + }, +} + +pub struct VoiceInputConfig { + pub engine: VoiceInputEngineConfig, + pub speech_vad_threshold: f32, + pub controls: VoiceInputControls, +} + +pub struct FinalTranscriptStorageReceipt(Option>); + +impl FinalTranscriptStorageReceipt { + /// Confirms that the host accepted this transcript into its authoritative + /// recovery storage. + pub fn stored(mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } + + #[cfg(test)] + pub(crate) fn test_pair() -> (Self, mpsc::Receiver<()>) { + // A single buffered acknowledgement lets unit tests inspect receipt + // ordering without introducing a helper thread. Production receipts + // remain rendezvous channels created by `send_final`. + let (sender, receiver) = mpsc::sync_channel(1); + (Self(Some(sender)), receiver) + } +} + +pub enum VoiceInputEvent { + Ready, + SpeakingChanged(bool), + RecognitionPendingChanged(bool), + FinalTranscript { + text: String, + storage_receipt: FinalTranscriptStorageReceipt, + }, + Failed(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum VoiceInputFinishError { + WorkerPanicked, + Quarantined { timeout: Duration }, +} + +impl VoiceInputFinishError { + pub fn is_quarantined(&self) -> bool { + matches!(self, Self::Quarantined { .. }) + } +} + +impl std::fmt::Display for VoiceInputFinishError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::WorkerPanicked => formatter.write_str("voice input runtime worker panicked"), + Self::Quarantined { timeout } => write!( + formatter, + "voice input runtime did not stop within {timeout:?}; the worker was quarantined" + ), + } + } +} + +impl std::error::Error for VoiceInputFinishError {} + +#[derive(Clone)] +pub struct VoiceInputControls { + shared: Arc, +} + +struct ControlState { + transition: Mutex, + muted: AtomicBool, + mute_epoch: AtomicU64, + assistant_speaking: AtomicBool, + assistant_vad_threshold: AtomicU32, +} + +#[derive(Default)] +struct ControlTransitionState { + host_muted: bool, + assistant_activity: AssistantActivityState, +} + +#[derive(Default)] +struct AssistantActivityState { + generation: u64, + lifetimes: BTreeMap, +} + +struct AssistantActivity { + vad_threshold: u32, + input_policy: InputDuringTtsPolicy, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum InputDuringTtsPolicy { + AllowBargeIn, + SuppressInput, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +pub struct InputDuringTtsSnapshot { + pub revision: u64, + pub policy: InputDuringTtsPolicy, +} + +pub struct InputDuringTtsSlot { + snapshot: Mutex, +} + +impl InputDuringTtsSlot { + pub fn new(policy: InputDuringTtsPolicy) -> Self { + Self { + snapshot: Mutex::new(InputDuringTtsSnapshot { + revision: 1, + policy, + }), + } + } + + pub fn snapshot(&self) -> Result { + self.snapshot + .lock() + .map(|snapshot| *snapshot) + .map_err(|_| "input-during-TTS policy lock was poisoned".into()) + } + + pub fn update( + &self, + expected_revision: u64, + policy: InputDuringTtsPolicy, + ) -> Result { + let mut snapshot = self + .snapshot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if snapshot.revision != expected_revision { + return Err(*snapshot); + } + let Some(revision) = snapshot.revision.checked_add(1) else { + return Err(*snapshot); + }; + *snapshot = InputDuringTtsSnapshot { revision, policy }; + Ok(*snapshot) + } +} + +impl Default for VoiceInputControls { + fn default() -> Self { + Self { + shared: Arc::new(ControlState { + transition: Mutex::new(ControlTransitionState::default()), + muted: AtomicBool::new(false), + mute_epoch: AtomicU64::new(0), + assistant_speaking: AtomicBool::new(false), + assistant_vad_threshold: AtomicU32::new(0.5_f32.to_bits()), + }), + } + } +} + +impl VoiceInputControls { + pub fn is_muted(&self) -> bool { + self.shared.muted.load(Ordering::Acquire) + } + + /// Changes the host-mute reason. Each effective composed edge advances the + /// epoch so queued audio and provider results from the prior state cannot + /// cross it. + pub fn set_host_muted(&self, muted: bool) { + let mut transition = self + .shared + .transition + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + transition.host_muted = muted; + self.publish_effective_mute(&transition); + } + + pub fn is_host_muted(&self) -> bool { + self.shared + .transition + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .host_muted + } + + /// Discards buffered audio and stale provider results without changing mute. + pub fn reset(&self) { + let _transition = self + .shared + .transition + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.shared.mute_epoch.fetch_add(1, Ordering::AcqRel); + } + + pub fn begin_assistant_activity( + &self, + vad_threshold: f32, + input_policy: InputDuringTtsPolicy, + ) -> Result { + if !vad_threshold.is_finite() || !(0.0..=1.0).contains(&vad_threshold) { + return Err("assistant VAD threshold must be finite and between 0 and 1".to_string()); + } + let mut transition = self + .shared + .transition + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let generation = transition + .assistant_activity + .generation + .checked_add(1) + .ok_or_else(|| "assistant activity generation is exhausted".to_string())?; + transition.assistant_activity.generation = generation; + transition.assistant_activity.lifetimes.insert( + generation, + AssistantActivity { + vad_threshold: vad_threshold.to_bits(), + input_policy, + }, + ); + self.publish_assistant_activity(&transition.assistant_activity); + self.publish_effective_mute(&transition); + Ok(AssistantActivityGuard { + controls: self.clone(), + generation, + }) + } + + fn mute_epoch(&self) -> u64 { + self.shared.mute_epoch.load(Ordering::Acquire) + } + + fn vad_threshold(&self, speech_threshold: f32) -> f32 { + if self.shared.assistant_speaking.load(Ordering::Acquire) { + f32::from_bits(self.shared.assistant_vad_threshold.load(Ordering::Acquire)) + } else { + speech_threshold + } + } + + fn publish_effective_mute(&self, transition: &ControlTransitionState) { + let assistant_suppressed = transition + .assistant_activity + .lifetimes + .values() + .any(|activity| activity.input_policy == InputDuringTtsPolicy::SuppressInput); + let muted = transition.host_muted || assistant_suppressed; + if self.shared.muted.swap(muted, Ordering::AcqRel) != muted { + self.shared.mute_epoch.fetch_add(1, Ordering::AcqRel); + } + } + + fn publish_assistant_activity(&self, activity: &AssistantActivityState) { + if let Some((_, current)) = activity.lifetimes.last_key_value() { + self.shared + .assistant_vad_threshold + .store(current.vad_threshold, Ordering::Release); + self.shared + .assistant_speaking + .store(true, Ordering::Release); + } else { + self.shared + .assistant_speaking + .store(false, Ordering::Release); + } + } +} + +#[must_use = "assistant activity ends when the guard is dropped"] +pub struct AssistantActivityGuard { + controls: VoiceInputControls, + generation: u64, +} + +impl Drop for AssistantActivityGuard { + fn drop(&mut self) { + let mut transition = self + .controls + .shared + .transition + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + transition + .assistant_activity + .lifetimes + .remove(&self.generation); + self.controls + .publish_assistant_activity(&transition.assistant_activity); + self.controls.publish_effective_mute(&transition); + } +} + +struct QueuedFrame { + frame: VoiceInputFrame, + mute_epoch: u64, +} + +pub struct VoiceInputRuntime { + frame_tx: SyncSender, + controls: VoiceInputControls, + shutdown: Arc, + discard_on_shutdown: Arc, + shutdown_mute_epoch: Arc, + worker: Option>, +} + +impl VoiceInputRuntime { + pub fn start( + config: VoiceInputConfig, + ) -> Result<(Self, tokio_mpsc::Receiver), String> { + if !config.speech_vad_threshold.is_finite() + || !(0.0..=1.0).contains(&config.speech_vad_threshold) + { + return Err("speech VAD threshold must be finite and between 0 and 1".to_string()); + } + let (frame_tx, frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let (event_tx, event_rx) = tokio_mpsc::channel(EVENT_QUEUE_DEPTH); + let controls = config.controls; + let shutdown = Arc::new(AtomicBool::new(false)); + let discard_on_shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_mute_epoch = Arc::new(AtomicU64::new(0)); + let worker_controls = controls.clone(); + let worker_shutdown = Arc::clone(&shutdown); + let worker_discard = Arc::clone(&discard_on_shutdown); + let worker_shutdown_epoch = Arc::clone(&shutdown_mute_epoch); + let speech_threshold = config.speech_vad_threshold; + let (name, work): (&str, Box) = match config.engine { + VoiceInputEngineConfig::Parakeet { model_dir } => ( + "berd-parakeet-stt", + Box::new(move || { + parakeet_worker( + model_dir, + frame_rx, + event_tx, + worker_shutdown, + worker_discard, + worker_shutdown_epoch, + worker_controls, + speech_threshold, + ) + }), + ), + #[cfg(target_os = "macos")] + VoiceInputEngineConfig::MacSpeech => ( + "berd-macos-stt", + Box::new(move || { + mac_speech_worker( + frame_rx, + event_tx, + worker_shutdown, + worker_discard, + worker_shutdown_epoch, + worker_controls, + speech_threshold, + ) + }), + ), + VoiceInputEngineConfig::OpenAi { + endpoint, + api_key, + model, + } => ( + "berd-openai-stt", + Box::new(move || { + openai_worker( + OpenAiRealtimeTranscriptionConfig::new(endpoint, api_key, model), + frame_rx, + event_tx, + worker_shutdown, + worker_discard, + worker_shutdown_epoch, + worker_controls, + speech_threshold, + ) + }), + ), + }; + let worker = thread::Builder::new() + .name(name.to_string()) + .spawn(work) + .map_err(|error| format!("start voice input runtime: {error}"))?; + Ok(( + Self { + frame_tx, + controls, + shutdown, + discard_on_shutdown, + shutdown_mute_epoch, + worker: Some(worker), + }, + event_rx, + )) + } + + pub fn controls(&self) -> VoiceInputControls { + self.controls.clone() + } + + pub fn try_push_frame(&self, frame: VoiceInputFrame) -> Result<(), String> { + if self.shutdown.load(Ordering::Acquire) { + return Err("Voice input recognition is no longer running.".to_string()); + } + if self.controls.is_muted() { + return Ok(()); + } + let mute_epoch = self.controls.mute_epoch(); + if self.controls.is_muted() || mute_epoch != self.controls.mute_epoch() { + return Ok(()); + } + match self.frame_tx.try_send(QueuedFrame { frame, mute_epoch }) { + Ok(()) => Ok(()), + Err(TrySendError::Full(_)) => Err( + "Voice input overrun: recognition could not keep up with 20 ms frames.".to_string(), + ), + Err(TrySendError::Disconnected(_)) => { + Err("Voice input recognition is no longer running.".to_string()) + } + } + } + + pub fn cancel(&self) { + self.signal_shutdown(true); + } + + pub async fn finish(mut self) -> Result<(), VoiceInputFinishError> { + let worker = self.begin_shutdown(); + if let Some(worker) = worker { + finish_worker(worker, FINISH_TIMEOUT).await?; + } + Ok(()) + } + + fn signal_shutdown(&self, discard: bool) { + let _transition = self + .controls + .shared + .transition + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.shutdown.load(Ordering::Acquire) { + return; + } + self.shutdown_mute_epoch + .store(self.controls.mute_epoch(), Ordering::Release); + if discard || self.controls.is_muted() { + self.discard_on_shutdown.store(true, Ordering::Release); + } + self.shutdown.store(true, Ordering::Release); + } + + fn begin_shutdown(&mut self) -> Option> { + self.signal_shutdown(false); + self.worker.take() + } +} + +impl Drop for VoiceInputRuntime { + fn drop(&mut self) { + if let Some(worker) = self.begin_shutdown() { + reap_dropped_worker(worker); + } + } +} + +async fn finish_worker( + worker: thread::JoinHandle<()>, + timeout: Duration, +) -> Result<(), VoiceInputFinishError> { + let deadline = tokio::time::Instant::now() + timeout; + while !worker.is_finished() { + if tokio::time::Instant::now() >= deadline { + // Rust cannot safely terminate an arbitrary thread blocked inside + // native code. Detaching this handle avoids adding a second + // permanently blocked reaper thread; the caller must quarantine + // the runtime and forbid replacement work in this process. + drop(worker); + return Err(VoiceInputFinishError::Quarantined { timeout }); + } + tokio::time::sleep(Duration::from_millis(10).min(timeout)).await; + } + worker + .join() + .map_err(|_| VoiceInputFinishError::WorkerPanicked) +} + +fn reap_dropped_worker(worker: thread::JoinHandle<()>) { + // Drop must not block on a native recognizer. Explicit shutdown uses the + // bounded quarantine path above when the caller needs a completion result. + let _ = thread::Builder::new() + .name("berd-voice-input-reaper".to_string()) + .spawn(move || { + let _ = worker.join(); + }); +} + +struct PendingRecognitions { + count: usize, +} + +impl PendingRecognitions { + fn new() -> Self { + Self { count: 0 } + } + + fn begin(&mut self, events: &tokio_mpsc::Sender) -> Result<(), ()> { + self.count += 1; + if self.count == 1 { + events + .blocking_send(VoiceInputEvent::RecognitionPendingChanged(true)) + .map_err(|_| ())?; + } + Ok(()) + } + + fn resolve(&mut self, events: &tokio_mpsc::Sender) -> Result<(), ()> { + if self.count == 0 { + return Ok(()); + } + self.count -= 1; + if self.count == 0 { + events + .blocking_send(VoiceInputEvent::RecognitionPendingChanged(false)) + .map_err(|_| ())?; + } + Ok(()) + } + + fn reset(&mut self, events: &tokio_mpsc::Sender) -> Result<(), ()> { + if self.count > 0 { + self.count = 0; + events + .blocking_send(VoiceInputEvent::RecognitionPendingChanged(false)) + .map_err(|_| ())?; + } + Ok(()) + } +} + +fn effective_mute_epoch( + controls: &VoiceInputControls, + shutdown: &AtomicBool, + shutdown_mute_epoch: &AtomicU64, +) -> (bool, u64) { + let live_epoch = controls.mute_epoch(); + if shutdown.load(Ordering::Acquire) { + // Freeze the epoch at shutdown initiation. A later hardware-mute + // callback must not retroactively discard an already accepted final + // utterance from an unmuted shutdown. + (true, shutdown_mute_epoch.load(Ordering::Acquire)) + } else { + (false, live_epoch) + } +} + +fn resample(resampler: &mut rubato::Fft, samples: &[f32]) -> Vec { + use audioadapter_buffers::direct::InterleavedSlice; + use rubato::Resampler; + + let Ok(input) = InterleavedSlice::new(samples, 1, samples.len()) else { + return Vec::new(); + }; + let output_capacity = resampler.output_frames_max(); + let mut output = vec![0.0; output_capacity]; + let Ok(mut output_buffer) = InterleavedSlice::new_mut(&mut output, 1, output_capacity) else { + return Vec::new(); + }; + let Ok((_, produced)) = resampler.process_into_buffer(&input, &mut output_buffer, None) else { + return Vec::new(); + }; + output.truncate(produced); + output +} + +fn clamp_vad_frame(samples: &[f32]) -> Vec { + samples + .iter() + .map(|sample| sample.clamp(-1.0, 1.0)) + .collect() +} + +fn send_final( + events: &tokio_mpsc::Sender, + text: String, + storage_deadline: Option, +) -> Result<(), ()> { + let (receipt, receiver) = if storage_deadline.is_some() { + let (sender, receiver) = mpsc::sync_channel(0); + (FinalTranscriptStorageReceipt(Some(sender)), Some(receiver)) + } else { + (FinalTranscriptStorageReceipt(None), None) + }; + events + .blocking_send(VoiceInputEvent::FinalTranscript { + text, + storage_receipt: receipt, + }) + .map_err(|_| ())?; + if let (Some(deadline), Some(receiver)) = (storage_deadline, receiver) { + let remaining = deadline.saturating_duration_since(Instant::now()); + if !remaining.is_zero() { + let _ = receiver.recv_timeout(remaining); + } + } + Ok(()) +} + +fn clear_speech_state( + speech: &mut Vec, + input_48k: &mut Vec, + leftover_16k: &mut Vec, + silence_frames: &mut usize, + in_speech: &mut bool, + events: &tokio_mpsc::Sender, +) { + speech.clear(); + input_48k.clear(); + leftover_16k.clear(); + *silence_frames = 0; + if std::mem::take(in_speech) { + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); + } +} + +struct ParakeetUtterance { + sequence: u64, + speech: Vec, + mute_epoch: u64, +} + +struct ParakeetRecognition { + sequence: u64, + text: String, + mute_epoch: u64, +} + +struct PendingParakeetRecognition { + sequence: u64, + mute_epoch: u64, +} + +struct ParakeetRecognitionLedger { + next_sequence: u64, + pending: VecDeque, +} + +impl ParakeetRecognitionLedger { + fn new() -> Self { + Self { + next_sequence: 0, + pending: VecDeque::new(), + } + } + + fn next_sequence(&self) -> Result { + self.next_sequence + .checked_add(1) + .ok_or_else(|| "Parakeet recognition sequence is exhausted.".to_string()) + } + + fn record(&mut self, sequence: u64, mute_epoch: u64) { + self.next_sequence = sequence; + self.pending.push_back(PendingParakeetRecognition { + sequence, + mute_epoch, + }); + } + + fn take(&mut self, sequence: u64) -> Result { + let Some(expected) = self.pending.pop_front() else { + return Err("Parakeet recognition produced an unexpected result.".to_string()); + }; + if expected.sequence != sequence { + return Err("Parakeet recognition results arrived out of order.".to_string()); + } + Ok(expected) + } + + fn remove(&mut self, sequence: u64) -> Result { + let Some(index) = self + .pending + .iter() + .position(|pending| pending.sequence == sequence) + else { + return Err("Parakeet recognition replacement was not pending.".to_string()); + }; + self.pending + .remove(index) + .ok_or_else(|| "Parakeet recognition replacement disappeared.".to_string()) + } +} + +struct ParakeetMailbox { + state: Mutex, + ready: Condvar, +} + +struct ParakeetMailboxState { + open: bool, + waiting: Option, +} + +impl ParakeetMailbox { + fn new() -> Self { + Self { + state: Mutex::new(ParakeetMailboxState { + open: true, + waiting: None, + }), + ready: Condvar::new(), + } + } + + fn try_submit( + &self, + utterance: ParakeetUtterance, + ) -> Result, String> { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !state.open { + return Err("Parakeet recognition is no longer running.".to_string()); + } + let displaced = match state.waiting.as_ref() { + Some(waiting) if waiting.mute_epoch == utterance.mute_epoch => { + return Err( + "Parakeet recognition overrun: completed utterances arrived faster than they could be decoded." + .to_string(), + ); + } + Some(_) => state.waiting.replace(utterance), + None => { + state.waiting = Some(utterance); + None + } + }; + self.ready.notify_one(); + Ok(displaced) + } + + fn receive(&self) -> Option { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + loop { + if let Some(utterance) = state.waiting.take() { + return Some(utterance); + } + if !state.open { + return None; + } + state = self + .ready + .wait(state) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + + fn close(&self) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.open = false; + self.ready.notify_one(); + } +} + +struct ParakeetDecoder { + utterances: Option>, + results: Receiver, + worker: Option>, +} + +impl ParakeetDecoder { + fn start( + controls: VoiceInputControls, + shutdown: Arc, + discard_on_shutdown: Arc, + shutdown_mute_epoch: Arc, + mut recognize: impl FnMut(&[f32]) -> String + Send + 'static, + ) -> Result { + let utterances = Arc::new(ParakeetMailbox::new()); + let worker_utterances = Arc::clone(&utterances); + // Only the capture worker submits utterances and drains results. While + // it is occupied, at most the in-flight decode plus the bounded + // utterance queue can produce results, so this channel is structurally + // bounded without making the decoder wait on capture. + let (result_tx, result_rx) = mpsc::sync_channel(PARAKEET_RESULT_QUEUE_DEPTH); + let worker = thread::Builder::new() + .name("berd-parakeet-decode".to_string()) + .spawn(move || { + while let Some(utterance) = worker_utterances.receive() { + let (shutting_down, current_epoch) = + effective_mute_epoch(&controls, &shutdown, &shutdown_mute_epoch); + let discard = shutting_down && discard_on_shutdown.load(Ordering::Acquire); + let text = if !discard && current_epoch == utterance.mute_epoch { + recognize(&utterance.speech) + } else { + String::new() + }; + if result_tx + .send(ParakeetRecognition { + sequence: utterance.sequence, + text, + mute_epoch: utterance.mute_epoch, + }) + .is_err() + { + break; + } + } + }) + .map_err(|error| format!("start Parakeet recognition worker: {error}"))?; + Ok(Self { + utterances: Some(utterances), + results: result_rx, + worker: Some(worker), + }) + } + + fn try_submit( + &self, + utterance: ParakeetUtterance, + ) -> Result, String> { + let Some(sender) = self.utterances.as_ref() else { + return Err("Parakeet recognition is no longer running.".to_string()); + }; + sender.try_submit(utterance) + } + + fn close(&mut self) { + if let Some(sender) = self.utterances.take() { + sender.close(); + } + } + + fn is_finished(&self) -> bool { + self.worker + .as_ref() + .is_none_or(thread::JoinHandle::is_finished) + } + + fn discard_ready_results(&self) { + while self.results.try_recv().is_ok() {} + } + + fn finish(&mut self) -> thread::Result<()> { + self.close(); + if let Some(worker) = self.worker.take() { + worker.join() + } else { + Ok(()) + } + } +} + +fn submit_parakeet_utterance( + decoder: &ParakeetDecoder, + speech: Vec, + mute_epoch: u64, + ledger: &mut ParakeetRecognitionLedger, + pending: &mut PendingRecognitions, + events: &tokio_mpsc::Sender, +) -> Result<(), String> { + if speech.is_empty() { + return Ok(()); + } + let sequence = ledger.next_sequence()?; + let displaced = decoder.try_submit(ParakeetUtterance { + sequence, + speech, + mute_epoch, + })?; + if let Some(displaced) = displaced { + ledger.remove(displaced.sequence)?; + } + ledger.record(sequence, mute_epoch); + pending + .begin(events) + .map_err(|_| "voice input event receiver closed".to_string()) +} + +#[allow(clippy::too_many_arguments)] +fn drain_parakeet_results( + decoder: &ParakeetDecoder, + events: &tokio_mpsc::Sender, + ledger: &mut ParakeetRecognitionLedger, + pending: &mut PendingRecognitions, + storage_deadline: Option, + controls: &VoiceInputControls, + shutdown: &AtomicBool, + discard_on_shutdown: &AtomicBool, + shutdown_mute_epoch: &AtomicU64, +) -> Result<(), String> { + loop { + let result = match decoder.results.try_recv() { + Ok(result) => result, + Err(mpsc::TryRecvError::Empty) => return Ok(()), + Err(mpsc::TryRecvError::Disconnected) if decoder.utterances.is_none() => return Ok(()), + Err(mpsc::TryRecvError::Disconnected) => { + return Err("Parakeet recognition stopped unexpectedly.".to_string()); + } + }; + let expected = ledger.take(result.sequence)?; + if expected.mute_epoch != result.mute_epoch { + return Err("Parakeet recognition result epoch did not match its request.".to_string()); + } + let (shutting_down, current_epoch) = + effective_mute_epoch(controls, shutdown, shutdown_mute_epoch); + if (shutting_down && discard_on_shutdown.load(Ordering::Acquire)) + || result.mute_epoch != current_epoch + { + continue; + } + complete_parakeet_recognition(result.text, events, pending, storage_deadline); + } +} + +#[allow(clippy::too_many_arguments)] +fn parakeet_worker( + model_dir: PathBuf, + frames: Receiver, + events: tokio_mpsc::Sender, + shutdown: Arc, + discard_on_shutdown: Arc, + shutdown_mute_epoch: Arc, + controls: VoiceInputControls, + speech_vad_threshold: f32, +) { + use rubato::{Fft, FixedSync}; + + let resampler = match Fft::::new(INPUT_SAMPLE_RATE, 16_000, 1024, 2, 1, FixedSync::Input) { + Ok(resampler) => resampler, + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(format!( + "Could not initialize native audio resampling: {error}" + ))); + return; + } + }; + let recognizer = match ParakeetRecognizer::load(&model_dir) { + Ok(recognizer) => recognizer, + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + }; + parakeet_coordinator( + resampler, + frames, + events, + shutdown, + discard_on_shutdown, + shutdown_mute_epoch, + controls, + speech_vad_threshold, + move |speech| recognizer.recognize_utterance(speech), + ); +} + +#[allow(clippy::too_many_arguments)] +fn parakeet_coordinator( + mut resampler: rubato::Fft, + frames: Receiver, + events: tokio_mpsc::Sender, + shutdown: Arc, + discard_on_shutdown: Arc, + shutdown_mute_epoch: Arc, + controls: VoiceInputControls, + speech_vad_threshold: f32, + recognize: impl FnMut(&[f32]) -> String + Send + 'static, +) { + use rubato::Resampler; + + let mut decoder = match ParakeetDecoder::start( + controls.clone(), + Arc::clone(&shutdown), + Arc::clone(&discard_on_shutdown), + Arc::clone(&shutdown_mute_epoch), + recognize, + ) { + Ok(decoder) => decoder, + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + }; + let chunk_in = resampler.input_frames_next(); + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + let mut input_48k = Vec::new(); + let mut leftover_16k = Vec::new(); + let mut speech = Vec::new(); + let mut silence_frames = 0; + let mut in_speech = false; + let mut observed_epoch = controls.mute_epoch(); + let mut pending = PendingRecognitions::new(); + let mut ledger = ParakeetRecognitionLedger::new(); + let mut terminal_failure = false; + if events.blocking_send(VoiceInputEvent::Ready).is_err() { + let _ = decoder.finish(); + return; + } + + 'capture: loop { + if let Err(error) = drain_parakeet_results( + &decoder, + &events, + &mut ledger, + &mut pending, + None, + &controls, + &shutdown, + &discard_on_shutdown, + &shutdown_mute_epoch, + ) { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + terminal_failure = true; + break; + } + let frame = match frames.recv_timeout(Duration::from_millis(50)) { + Ok(frame) => Some(frame), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + let (shutting_down, current_epoch) = + effective_mute_epoch(&controls, &shutdown, &shutdown_mute_epoch); + if current_epoch != observed_epoch { + observed_epoch = current_epoch; + clear_speech_state( + &mut speech, + &mut input_48k, + &mut leftover_16k, + &mut silence_frames, + &mut in_speech, + &events, + ); + let _ = pending.reset(&events); + } + if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || frame.is_none()) { + break; + } + if !shutting_down && controls.is_muted() { + continue; + } + let Some(frame) = frame else { continue }; + if frame.mute_epoch != observed_epoch { + continue; + } + input_48k.extend_from_slice(frame.frame.samples()); + while input_48k.len() >= chunk_in { + let chunk: Vec = input_48k.drain(..chunk_in).collect(); + leftover_16k.extend_from_slice(&resample(&mut resampler, &chunk)); + while leftover_16k.len() >= VAD_FRAME_SAMPLES { + let frame: Vec = leftover_16k.drain(..VAD_FRAME_SAMPLES).collect(); + let clamped = clamp_vad_frame(&frame); + let threshold = controls.vad_threshold(speech_vad_threshold); + if vad.predict_f32(&clamped) > threshold { + silence_frames = 0; + speech.extend_from_slice(&frame); + if !in_speech { + in_speech = true; + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(true)); + } + } else if in_speech { + silence_frames += 1; + speech.extend_from_slice(&frame); + if silence_frames >= SILENCE_FLUSH_FRAMES { + silence_frames = 0; + in_speech = false; + let utterance = std::mem::take(&mut speech); + let queued = submit_parakeet_utterance( + &decoder, + utterance, + observed_epoch, + &mut ledger, + &mut pending, + &events, + ); + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); + if let Err(error) = queued { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + terminal_failure = true; + break 'capture; + } + } + } + if speech.len() >= MAX_SPEECH_SAMPLES { + let utterance = std::mem::take(&mut speech); + silence_frames = 0; + let was_speaking = std::mem::take(&mut in_speech); + let queued = submit_parakeet_utterance( + &decoder, + utterance, + observed_epoch, + &mut ledger, + &mut pending, + &events, + ); + if was_speaking { + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); + } + if let Err(error) = queued { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + terminal_failure = true; + break 'capture; + } + } + } + } + } + + let discard = discard_on_shutdown.load(Ordering::Acquire); + if !terminal_failure && !discard && !speech.is_empty() { + if let Err(error) = submit_parakeet_utterance( + &decoder, + speech, + observed_epoch, + &mut ledger, + &mut pending, + &events, + ) { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + terminal_failure = true; + } + } + decoder.close(); + let storage_deadline = Instant::now() + FINAL_STORAGE_TIMEOUT; + while !decoder.is_finished() { + if !terminal_failure && !discard { + if let Err(error) = drain_parakeet_results( + &decoder, + &events, + &mut ledger, + &mut pending, + Some(storage_deadline), + &controls, + &shutdown, + &discard_on_shutdown, + &shutdown_mute_epoch, + ) { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + terminal_failure = true; + } + } else { + decoder.discard_ready_results(); + } + thread::sleep(Duration::from_millis(5)); + } + if !terminal_failure && !discard { + if let Err(error) = drain_parakeet_results( + &decoder, + &events, + &mut ledger, + &mut pending, + Some(storage_deadline), + &controls, + &shutdown, + &discard_on_shutdown, + &shutdown_mute_epoch, + ) { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + } + } + let decoder_result = decoder.finish(); + let _ = pending.reset(&events); + if let Err(payload) = decoder_result { + if !terminal_failure { + let _ = events.blocking_send(VoiceInputEvent::Failed( + "Parakeet recognition stopped unexpectedly.".to_string(), + )); + } + std::panic::resume_unwind(payload); + } +} + +fn complete_parakeet_recognition( + text: String, + events: &tokio_mpsc::Sender, + pending: &mut PendingRecognitions, + storage_deadline: Option, +) { + if !text.is_empty() { + let _ = send_final(events, text, storage_deadline); + } + let _ = pending.resolve(events); +} + +#[cfg(target_os = "macos")] +fn new_mac_recognizer() -> Result< + ( + crate::mac_speech::MacSpeechRecognizer, + tokio_mpsc::UnboundedReceiver, + ), + String, +> { + crate::mac_speech::MacSpeechRecognizer::new() +} + +#[cfg(target_os = "macos")] +fn forward_mac_events( + recognition_events: &mut tokio_mpsc::UnboundedReceiver< + crate::mac_speech::MacSpeechRecognitionEvent, + >, + events: &tokio_mpsc::Sender, + pending: &mut PendingRecognitions, + settle_deadline: &mut Option, + storage_deadline: Option, +) -> Result<(), ()> { + while let Ok(event) = recognition_events.try_recv() { + match event { + crate::mac_speech::MacSpeechRecognitionEvent::Final(text) => { + let text = text.trim().to_string(); + if !text.is_empty() { + send_final(events, text, storage_deadline)?; + } + pending.resolve(events)?; + if pending.count == 0 { + *settle_deadline = None; + } + } + crate::mac_speech::MacSpeechRecognitionEvent::Finished => { + if storage_deadline.is_none() { + pending.reset(events)?; + *settle_deadline = None; + let _ = events.blocking_send(VoiceInputEvent::Failed( + "macOS speech recognition stopped unexpectedly.".to_string(), + )); + return Err(()); + } + // SpeechTranscriber exposes no per-utterance no-result event. + // Native finish is the bounded no-result resolution point. + pending.reset(events)?; + *settle_deadline = None; + } + crate::mac_speech::MacSpeechRecognitionEvent::Failed(message) => { + if storage_deadline.is_none() { + let _ = events.blocking_send(VoiceInputEvent::Failed(message)); + } + pending.reset(events)?; + return Err(()); + } + } + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn begin_mac_turn( + pending: &mut PendingRecognitions, + events: &tokio_mpsc::Sender, + settle_deadline: &mut Option, +) -> Result<(), ()> { + pending.begin(events)?; + *settle_deadline = None; + Ok(()) +} + +#[cfg(target_os = "macos")] +fn end_mac_turn( + pending: &PendingRecognitions, + settle_deadline: &mut Option, + now: Instant, +) { + if pending.count > 0 { + *settle_deadline = Some(now + MAC_LIVE_NO_RESULT_TIMEOUT); + } +} + +#[cfg(target_os = "macos")] +fn mac_settle_expired( + pending: &PendingRecognitions, + settle_deadline: Option, + now: Instant, +) -> bool { + pending.count > 0 && settle_deadline.is_some_and(|deadline| now >= deadline) +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn mac_speech_worker( + frames: Receiver, + events: tokio_mpsc::Sender, + shutdown: Arc, + discard_on_shutdown: Arc, + shutdown_mute_epoch: Arc, + controls: VoiceInputControls, + speech_vad_threshold: f32, +) { + use rubato::{Fft, FixedSync, Resampler}; + + let (mut recognizer, mut recognition_events) = match new_mac_recognizer() { + Ok(session) => session, + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + }; + let mut resampler = + match Fft::::new(INPUT_SAMPLE_RATE, 16_000, 1024, 2, 1, FixedSync::Input) { + Ok(resampler) => resampler, + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(format!( + "Could not initialize native audio resampling: {error}" + ))); + return; + } + }; + let chunk_in = resampler.input_frames_next(); + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + let mut input_48k = Vec::new(); + let mut leftover_16k = Vec::new(); + let mut silence_frames = 0; + let mut in_speech = false; + let mut observed_epoch = controls.mute_epoch(); + let mut pending = PendingRecognitions::new(); + let mut settle_deadline = None; + let mut received_audio = false; + if events.blocking_send(VoiceInputEvent::Ready).is_err() { + return; + } + + loop { + if forward_mac_events( + &mut recognition_events, + &events, + &mut pending, + &mut settle_deadline, + None, + ) + .is_err() + { + return; + } + if mac_settle_expired(&pending, settle_deadline, Instant::now()) { + // SpeechTranscriber has no per-utterance no-result callback. End + // this recognizer generation at the bounded idle deadline so its + // suppressed late callbacks cannot resolve a later generation. + recognizer.cancel(); + let _ = pending.reset(&events); + settle_deadline = None; + input_48k.clear(); + leftover_16k.clear(); + silence_frames = 0; + vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + match new_mac_recognizer() { + Ok((next, next_events)) => { + recognizer = next; + recognition_events = next_events; + received_audio = false; + } + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + } + } + let frame = match frames.recv_timeout(Duration::from_millis(50)) { + Ok(frame) => Some(frame), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + let (shutting_down, current_epoch) = + effective_mute_epoch(&controls, &shutdown, &shutdown_mute_epoch); + if current_epoch != observed_epoch { + observed_epoch = current_epoch; + input_48k.clear(); + leftover_16k.clear(); + silence_frames = 0; + if std::mem::take(&mut in_speech) { + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); + } + let _ = pending.reset(&events); + settle_deadline = None; + recognizer.cancel(); + if shutting_down { + break; + } + match new_mac_recognizer() { + Ok((next, next_events)) => { + recognizer = next; + recognition_events = next_events; + received_audio = false; + } + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + } + } + if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || frame.is_none()) { + break; + } + if !shutting_down && controls.is_muted() { + continue; + } + let Some(frame) = frame else { continue }; + if frame.mute_epoch != observed_epoch { + continue; + } + if let Err(error) = recognizer.push_48khz_mono_f32(frame.frame.samples()) { + let _ = pending.reset(&events); + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + received_audio = true; + + input_48k.extend_from_slice(frame.frame.samples()); + while input_48k.len() >= chunk_in { + let chunk: Vec = input_48k.drain(..chunk_in).collect(); + leftover_16k.extend_from_slice(&resample(&mut resampler, &chunk)); + while leftover_16k.len() >= VAD_FRAME_SAMPLES { + let frame: Vec = leftover_16k.drain(..VAD_FRAME_SAMPLES).collect(); + let clamped = clamp_vad_frame(&frame); + if vad.predict_f32(&clamped) > controls.vad_threshold(speech_vad_threshold) { + silence_frames = 0; + if !in_speech { + in_speech = true; + if begin_mac_turn(&mut pending, &events, &mut settle_deadline).is_err() { + return; + } + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(true)); + } + } else if in_speech { + silence_frames += 1; + if silence_frames >= SILENCE_FLUSH_FRAMES { + silence_frames = 0; + in_speech = false; + end_mac_turn(&pending, &mut settle_deadline, Instant::now()); + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); + } + } + } + } + if forward_mac_events( + &mut recognition_events, + &events, + &mut pending, + &mut settle_deadline, + None, + ) + .is_err() + { + return; + } + } + + if in_speech { + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); + } + if discard_on_shutdown.load(Ordering::Acquire) || !received_audio { + recognizer.cancel(); + let _ = pending.reset(&events); + return; + } + if let Err(error) = recognizer.finish() { + let _ = pending.reset(&events); + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + let deadline = Instant::now() + FINAL_STORAGE_TIMEOUT; + let _ = forward_mac_events( + &mut recognition_events, + &events, + &mut pending, + &mut settle_deadline, + Some(deadline), + ); + let _ = pending.reset(&events); +} + +#[derive(Debug, PartialEq, Eq)] +struct OpenAiCommittedTurn { + item_id: String, + mute_epoch: u64, + settle_deadline: Instant, +} + +#[derive(Debug, PartialEq, Eq)] +struct OpenAiPendingCommit { + mute_epoch: u64, + settle_deadline: Instant, +} + +fn block_on_openai_operation( + runtime: &tokio::runtime::Runtime, + shutdown: &AtomicBool, + future: F, + action: &str, +) -> Result, String> +where + F: std::future::Future>, + E: std::fmt::Display, +{ + runtime.block_on(async { + let wait_for_shutdown = async { + while !shutdown.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }; + tokio::select! { + result = tokio::time::timeout(OPENAI_NETWORK_OPERATION_TIMEOUT, future) => { + match result { + Ok(Ok(value)) => Ok(Some(value)), + Ok(Err(error)) => Err(format!("{action}: {error}")), + Err(_) => Err(format!("{action}: operation timed out")), + } + } + () = wait_for_shutdown => Ok(None), + } + }) +} + +fn block_on_openai_timeout( + runtime: &tokio::runtime::Runtime, + timeout: Duration, + future: F, +) -> Result +where + F: std::future::Future, +{ + runtime.block_on(async { tokio::time::timeout(timeout, future).await }) +} + +fn record_openai_event( + event: OpenAiRealtimeTranscriptionEvent, + current_epoch: u64, + pending_commits: &mut VecDeque, + committed: &mut VecDeque, + completed: &mut HashMap, +) -> Option { + match event { + OpenAiRealtimeTranscriptionEvent::Committed { item_id } => { + let pending = pending_commits.pop_front()?; + let turn = OpenAiCommittedTurn { + item_id, + mute_epoch: pending.mute_epoch, + settle_deadline: pending.settle_deadline, + }; + if turn.mute_epoch == current_epoch { + committed.push_back(OpenAiCommittedTurn { + item_id: turn.item_id.clone(), + mute_epoch: turn.mute_epoch, + settle_deadline: turn.settle_deadline, + }); + } + Some(turn) + } + OpenAiRealtimeTranscriptionEvent::Completed { + item_id, + transcript, + } => { + if committed + .iter() + .any(|turn| turn.item_id == item_id && turn.mute_epoch == current_epoch) + { + completed.insert(item_id, transcript); + } + None + } + } +} + +fn openai_transcription_failure_is_current( + item_id: &str, + current_epoch: u64, + committed: &VecDeque, +) -> bool { + committed + .iter() + .any(|turn| turn.item_id == item_id && turn.mute_epoch == current_epoch) +} + +fn openai_live_result_expired( + pending_commits: &VecDeque, + committed: &VecDeque, + now: Instant, +) -> bool { + pending_commits + .iter() + .map(|turn| turn.settle_deadline) + .chain(committed.iter().map(|turn| turn.settle_deadline)) + .min() + .is_some_and(|deadline| now >= deadline) +} + +fn track_openai_commit(pending_commits: &mut VecDeque, mute_epoch: u64) { + pending_commits.push_back(OpenAiPendingCommit { + mute_epoch, + settle_deadline: Instant::now() + OPENAI_LIVE_RESULT_TIMEOUT, + }); +} + +fn deliver_openai_turns( + committed: &mut VecDeque, + completed: &mut HashMap, + events: &tokio_mpsc::Sender, + pending: &mut PendingRecognitions, + final_item_id: Option<&str>, + final_storage_deadline: Option, +) { + while committed + .front() + .is_some_and(|turn| completed.contains_key(&turn.item_id)) + { + let turn = committed.pop_front().expect("checked front"); + let text = completed.remove(&turn.item_id).unwrap_or_default(); + let storage_deadline = (Some(turn.item_id.as_str()) == final_item_id) + .then_some(final_storage_deadline) + .flatten(); + if !text.is_empty() { + let _ = send_final(events, text, storage_deadline); + } + let _ = pending.resolve(events); + } +} + +fn push_pre_roll(pre_roll: &mut VecDeque>, pcm: Vec) { + pre_roll.push_back(pcm); + while pre_roll.len() > OPENAI_PRE_ROLL_FRAMES { + pre_roll.pop_front(); + } +} + +#[allow(clippy::too_many_arguments)] +fn openai_worker( + config: OpenAiRealtimeTranscriptionConfig, + frames: Receiver, + events: tokio_mpsc::Sender, + shutdown: Arc, + discard_on_shutdown: Arc, + shutdown_mute_epoch: Arc, + controls: VoiceInputControls, + speech_vad_threshold: f32, +) { + use rubato::{Fft, FixedSync, Resampler}; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(format!( + "Could not initialize OpenAI realtime transcription: {error}" + ))); + return; + } + }; + let connection = block_on_openai_operation( + &runtime, + &shutdown, + OpenAiRealtimeTranscriptionClient::connect(config), + "connect to OpenAI realtime transcription", + ); + let mut client = match connection { + Ok(Some(client)) => client, + Ok(None) => return, + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + }; + match block_on_openai_operation( + &runtime, + &shutdown, + client.configure(), + "configure OpenAI realtime transcription", + ) { + Ok(Some(())) => {} + Ok(None) => return, + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + } + + let mut resampler = match Fft::::new( + INPUT_SAMPLE_RATE, + 24_000, + INPUT_FRAME_SAMPLES, + 2, + 1, + FixedSync::Input, + ) { + Ok(resampler) => resampler, + Err(error) => { + let _ = events.blocking_send(VoiceInputEvent::Failed(format!( + "Could not initialize OpenAI audio resampling: {error}" + ))); + return; + } + }; + let chunk_in = resampler.input_frames_next(); + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + let mut input_48k = Vec::new(); + let mut vad_16k = Vec::new(); + let mut silence_frames = 0; + let mut in_speech = false; + let mut turn_has_audio = false; + let mut turn_samples_16k = 0_usize; + let mut pre_roll = VecDeque::>::new(); + let mut observed_epoch = controls.mute_epoch(); + let mut pending_commits = VecDeque::::new(); + let mut committed = VecDeque::::new(); + let mut completed = HashMap::::new(); + let mut pending = PendingRecognitions::new(); + if events.blocking_send(VoiceInputEvent::Ready).is_err() { + return; + } + + macro_rules! send_operation { + ($operation:expr, $action:literal, $on_shutdown:block) => { + match block_on_openai_operation(&runtime, &shutdown, $operation, $action) { + Ok(Some(())) => {} + Ok(None) => { + let _ = pending.reset(&events); + $on_shutdown + } + Err(error) => { + let _ = pending.reset(&events); + let _ = events.blocking_send(VoiceInputEvent::Failed(error)); + return; + } + } + }; + } + + 'worker: loop { + let (shutting_down, current_epoch) = + effective_mute_epoch(&controls, &shutdown, &shutdown_mute_epoch); + if shutting_down { + break; + } + if current_epoch != observed_epoch { + observed_epoch = current_epoch; + input_48k.clear(); + vad_16k.clear(); + silence_frames = 0; + turn_has_audio = false; + turn_samples_16k = 0; + pre_roll.clear(); + pending_commits.clear(); + committed.clear(); + completed.clear(); + let _ = pending.reset(&events); + if std::mem::take(&mut in_speech) { + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); + } + send_operation!(client.clear(), "clear reset OpenAI transcription audio", { + break 'worker; + }); + } + + while let Ok(event) = + block_on_openai_timeout(&runtime, Duration::from_millis(1), client.next_event()) + { + let (shutting_down, current_epoch) = + effective_mute_epoch(&controls, &shutdown, &shutdown_mute_epoch); + if shutting_down { + break 'worker; + } + if current_epoch != observed_epoch { + continue 'worker; + } + let event = match event { + Ok(event) => event, + Err(TranscriptionError::TranscriptionFailed { item_id, message }) => { + let failure_is_current = openai_transcription_failure_is_current( + &item_id, + observed_epoch, + &committed, + ); + if failure_is_current { + let _ = pending.reset(&events); + let _ = events.blocking_send(VoiceInputEvent::Failed(message)); + return; + } + continue; + } + Err(error) => { + let _ = pending.reset(&events); + let _ = events.blocking_send(VoiceInputEvent::Failed(error.to_string())); + return; + } + }; + record_openai_event( + event, + observed_epoch, + &mut pending_commits, + &mut committed, + &mut completed, + ); + deliver_openai_turns( + &mut committed, + &mut completed, + &events, + &mut pending, + None, + None, + ); + } + if openai_live_result_expired(&pending_commits, &committed, Instant::now()) { + let _ = pending.reset(&events); + let _ = events.blocking_send(VoiceInputEvent::Failed( + "OpenAI transcription did not complete within 5 seconds.".to_string(), + )); + return; + } + + let frame = match frames.recv_timeout(INPUT_FRAME_DURATION) { + Ok(frame) => Some(frame), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + let (shutting_down, current_epoch) = + effective_mute_epoch(&controls, &shutdown, &shutdown_mute_epoch); + if shutting_down { + break; + } + if current_epoch != observed_epoch { + continue 'worker; + } + if controls.is_muted() { + continue; + } + let Some(frame) = frame else { continue }; + if frame.mute_epoch != observed_epoch { + continue; + } + input_48k.extend_from_slice(frame.frame.samples()); + while input_48k.len() >= chunk_in { + let chunk: Vec = input_48k.drain(..chunk_in).collect(); + let pcm_24k = resample(&mut resampler, &chunk); + let pcm_bytes: Vec = pcm_24k + .iter() + .flat_map(|sample| { + ((sample.clamp(-1.0, 1.0) * i16::MAX as f32).round() as i16).to_le_bytes() + }) + .collect(); + vad_16k.extend(chunk.iter().step_by(3).copied()); + let mut speech_started = false; + let mut should_commit = false; + while vad_16k.len() >= VAD_FRAME_SAMPLES { + let frame: Vec = vad_16k.drain(..VAD_FRAME_SAMPLES).collect(); + if vad.predict_f32(&frame) > controls.vad_threshold(speech_vad_threshold) { + silence_frames = 0; + if !in_speech { + in_speech = true; + speech_started = true; + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(true)); + } + } else if in_speech { + silence_frames += 1; + if silence_frames >= SILENCE_FLUSH_FRAMES { + should_commit = true; + silence_frames = 0; + in_speech = false; + } + } + } + if speech_started { + while let Some(bytes) = pre_roll.pop_front() { + send_operation!( + client.append_pcm16le_24khz(&bytes), + "stream OpenAI pre-roll", + { + break 'worker; + } + ); + } + } + if speech_started || in_speech || should_commit { + send_operation!( + client.append_pcm16le_24khz(&pcm_bytes), + "stream audio to OpenAI transcription", + { break 'worker } + ); + turn_has_audio = true; + turn_samples_16k = turn_samples_16k.saturating_add(chunk.len() / 3); + } else { + push_pre_roll(&mut pre_roll, pcm_bytes); + } + if should_commit || turn_samples_16k >= MAX_SPEECH_SAMPLES { + if pending.begin(&events).is_err() { + return; + } + send_operation!(client.commit(), "commit OpenAI transcription turn", { + break 'worker; + }); + track_openai_commit(&mut pending_commits, observed_epoch); + turn_has_audio = false; + turn_samples_16k = 0; + silence_frames = 0; + if std::mem::take(&mut in_speech) || should_commit { + let _ = events.blocking_send(VoiceInputEvent::SpeakingChanged(false)); + } + } + } + } + + if discard_on_shutdown.load(Ordering::Acquire) { + let _ = pending.reset(&events); + return; + } + let mut final_item_id = None::; + if turn_has_audio { + if pending.begin(&events).is_err() { + return; + } + let final_write = + block_on_openai_timeout(&runtime, OPENAI_FINAL_WRITE_TIMEOUT, client.commit()); + if matches!(final_write, Ok(Ok(()))) { + track_openai_commit(&mut pending_commits, observed_epoch); + } else { + let _ = pending.resolve(&events); + } + } + let deadline = Instant::now() + FINAL_STORAGE_TIMEOUT; + while Instant::now() < deadline { + if committed.is_empty() && pending_commits.is_empty() && pending.count == 0 { + break; + } + let Ok(event) = + block_on_openai_timeout(&runtime, Duration::from_millis(50), client.next_event()) + else { + continue; + }; + let event = match event { + Ok(event) => event, + Err(TranscriptionError::TranscriptionFailed { item_id, message }) => { + let failure_is_current = + openai_transcription_failure_is_current(&item_id, observed_epoch, &committed); + if failure_is_current { + let _ = pending.reset(&events); + let _ = events.blocking_send(VoiceInputEvent::Failed(message)); + break; + } + continue; + } + Err(TranscriptionError::Provider(message)) => { + let _ = pending.reset(&events); + let _ = events.blocking_send(VoiceInputEvent::Failed(message)); + break; + } + Err(TranscriptionError::Disconnected) | Err(TranscriptionError::Socket(_)) => continue, + }; + if let Some(turn) = record_openai_event( + event, + observed_epoch, + &mut pending_commits, + &mut committed, + &mut completed, + ) { + if turn.mute_epoch == observed_epoch && pending_commits.is_empty() && turn_has_audio { + final_item_id = Some(turn.item_id); + } + } + deliver_openai_turns( + &mut committed, + &mut completed, + &events, + &mut pending, + final_item_id.as_deref(), + Some(deadline), + ); + } + let _ = pending.reset(&events); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn silence_frame() -> VoiceInputFrame { + VoiceInputFrame::try_from_samples(&[0.0; INPUT_FRAME_SAMPLES]).unwrap() + } + + #[test] + fn exact_frame_contract_rejects_shape_and_nonfinite_and_clamps_unit_scale() { + assert!(VoiceInputFrame::try_from_samples(&[0.0; 959]).is_err()); + let mut samples = [0.0; INPUT_FRAME_SAMPLES]; + samples[0] = f32::NAN; + assert!(VoiceInputFrame::try_from_samples(&samples).is_err()); + samples[0] = 2.0; + samples[1] = -2.0; + let frame = VoiceInputFrame::try_from_samples(&samples).unwrap(); + assert_eq!(frame.samples()[0], 1.0); + assert_eq!(frame.samples()[1], -1.0); + } + + #[test] + fn mute_and_reset_advance_the_authoritative_epoch() { + let controls = VoiceInputControls::default(); + assert_eq!(controls.mute_epoch(), 0); + controls.set_host_muted(true); + assert_eq!(controls.mute_epoch(), 1); + controls.set_host_muted(true); + assert_eq!(controls.mute_epoch(), 1); + controls.set_host_muted(false); + assert_eq!(controls.mute_epoch(), 2); + controls.reset(); + assert_eq!(controls.mute_epoch(), 3); + } + + #[test] + fn host_mute_and_assistant_suppression_cannot_clear_each_other() { + let controls = VoiceInputControls::default(); + controls.set_host_muted(true); + let assistant = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::SuppressInput) + .unwrap(); + controls.set_host_muted(false); + assert!(controls.is_muted()); + assert_eq!(controls.mute_epoch(), 1); + drop(assistant); + assert!(!controls.is_muted()); + assert_eq!(controls.mute_epoch(), 2); + + let assistant = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::SuppressInput) + .unwrap(); + controls.set_host_muted(true); + drop(assistant); + assert!(controls.is_muted()); + assert_eq!(controls.mute_epoch(), 3); + controls.set_host_muted(false); + assert!(!controls.is_muted()); + assert_eq!(controls.mute_epoch(), 4); + } + + #[test] + fn overlapping_assistant_guards_snapshot_policy_and_drop_out_of_order() { + let controls = VoiceInputControls::default(); + let suppress = controls + .begin_assistant_activity(0.8, InputDuringTtsPolicy::SuppressInput) + .unwrap(); + let allow = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + assert!(controls.is_muted()); + assert_eq!(controls.vad_threshold(0.5), 0.65); + drop(suppress); + assert!(!controls.is_muted()); + assert_eq!(controls.vad_threshold(0.5), 0.65); + drop(allow); + assert_eq!(controls.vad_threshold(0.5), 0.5); + } + + #[test] + fn input_during_tts_policy_updates_are_revisioned_and_nonmutating_when_stale() { + let slot = InputDuringTtsSlot::new(InputDuringTtsPolicy::AllowBargeIn); + let leased = slot.snapshot().unwrap(); + let applied = slot.update(1, InputDuringTtsPolicy::SuppressInput).unwrap(); + let stale = slot + .update(1, InputDuringTtsPolicy::AllowBargeIn) + .unwrap_err(); + + assert_eq!(leased.revision, 1); + assert_eq!(leased.policy, InputDuringTtsPolicy::AllowBargeIn); + assert_eq!(applied.revision, 2); + assert_eq!(applied.policy, InputDuringTtsPolicy::SuppressInput); + assert_eq!(stale, applied); + assert_eq!(slot.snapshot().unwrap(), applied); + } + + #[test] + fn assistant_activity_guards_restore_the_newest_overlapping_threshold() { + let controls = VoiceInputControls::default(); + let first = controls + .begin_assistant_activity(0.8, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + assert_eq!(controls.vad_threshold(0.5), 0.8); + let second = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + assert_eq!(controls.vad_threshold(0.5), 0.65); + drop(first); + assert_eq!(controls.vad_threshold(0.5), 0.65); + drop(second); + assert_eq!(controls.vad_threshold(0.5), 0.5); + } + + #[test] + fn concurrent_assistant_starts_publish_the_highest_generation_threshold() { + let controls = VoiceInputControls::default(); + let barrier = Arc::new(std::sync::Barrier::new(8)); + let (result_tx, result_rx) = mpsc::channel(); + let threads = (1..=8) + .map(|index| { + let controls = controls.clone(); + let barrier = Arc::clone(&barrier); + let result_tx = result_tx.clone(); + thread::spawn(move || { + let threshold = index as f32 / 10.0; + barrier.wait(); + let guard = controls + .begin_assistant_activity(threshold, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + result_tx + .send((guard.generation, threshold, guard)) + .unwrap(); + }) + }) + .collect::>(); + drop(result_tx); + let guards = result_rx.into_iter().collect::>(); + for thread in threads { + thread.join().unwrap(); + } + let expected = guards + .iter() + .max_by_key(|(generation, _, _)| *generation) + .map(|(_, threshold, _)| *threshold) + .unwrap(); + + assert_eq!(controls.vad_threshold(0.5), expected); + drop(guards); + assert_eq!(controls.vad_threshold(0.5), 0.5); + } + + #[test] + fn cloned_controls_carry_assistant_activity_across_runtime_replacement() { + let controls = VoiceInputControls::default(); + let replacement_controls = controls.clone(); + let guard = controls + .begin_assistant_activity(0.8, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + + assert_eq!(replacement_controls.vad_threshold(0.5), 0.8); + drop(guard); + assert_eq!(replacement_controls.vad_threshold(0.5), 0.5); + } + + #[test] + fn recognition_pending_is_counted_and_clears_only_after_the_last_resolution() { + let (events, mut receiver) = tokio_mpsc::channel(8); + let mut pending = PendingRecognitions::new(); + pending.begin(&events).unwrap(); + pending.begin(&events).unwrap(); + assert!(matches!( + receiver.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(true)) + )); + pending.resolve(&events).unwrap(); + assert!(receiver.try_recv().is_err()); + pending.resolve(&events).unwrap(); + assert!(matches!( + receiver.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(false)) + )); + pending.resolve(&events).unwrap(); + assert!(receiver.try_recv().is_err()); + } + + #[test] + fn parakeet_pending_precedes_idle_and_ordered_final_resolution_follows() { + let (events, mut receiver) = tokio_mpsc::channel(8); + let mut pending = PendingRecognitions::new(); + pending.begin(&events).unwrap(); + events + .blocking_send(VoiceInputEvent::SpeakingChanged(false)) + .unwrap(); + complete_parakeet_recognition("final words".to_string(), &events, &mut pending, None); + + assert!(matches!( + receiver.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(true)) + )); + assert!(matches!( + receiver.try_recv(), + Ok(VoiceInputEvent::SpeakingChanged(false)) + )); + assert!(matches!( + receiver.try_recv(), + Ok(VoiceInputEvent::FinalTranscript { text, .. }) if text == "final words" + )); + assert!(matches!( + receiver.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(false)) + )); + } + + #[test] + fn parakeet_decoder_orders_one_waiter_and_rejects_a_third_utterance() { + let controls = VoiceInputControls::default(); + let shutdown = Arc::new(AtomicBool::new(false)); + let discard = Arc::new(AtomicBool::new(false)); + let shutdown_epoch = Arc::new(AtomicU64::new(0)); + let (decode_started_tx, decode_started_rx) = mpsc::sync_channel(0); + let (release_decode_tx, release_decode_rx) = mpsc::sync_channel(0); + let mut decode_count = 0; + let mut decoder = ParakeetDecoder::start( + controls, + Arc::clone(&shutdown), + discard, + shutdown_epoch, + move |speech| { + decode_count += 1; + if decode_count == 1 { + decode_started_tx.send(()).unwrap(); + release_decode_rx.recv().unwrap(); + } + format!("recognized {}", speech[0]) + }, + ) + .unwrap(); + assert!(decoder + .try_submit(ParakeetUtterance { + sequence: 1, + speech: vec![0.25; VAD_FRAME_SAMPLES], + mute_epoch: 0, + }) + .unwrap() + .is_none()); + decode_started_rx + .recv_timeout(Duration::from_millis(100)) + .expect("recognizer starts"); + assert!(decoder + .try_submit(ParakeetUtterance { + sequence: 2, + speech: vec![0.5; VAD_FRAME_SAMPLES], + mute_epoch: 0, + }) + .expect("one waiting utterance is bounded and accepted") + .is_none()); + let third = decoder.try_submit(ParakeetUtterance { + sequence: 3, + speech: vec![0.75; VAD_FRAME_SAMPLES], + mute_epoch: 0, + }); + assert_eq!( + third.err().expect("third same-epoch utterance is full"), + "Parakeet recognition overrun: completed utterances arrived faster than they could be decoded." + ); + + release_decode_tx.send(()).unwrap(); + let first = decoder + .results + .recv_timeout(Duration::from_millis(100)) + .expect("first recognition result"); + let second = decoder + .results + .recv_timeout(Duration::from_millis(100)) + .expect("ordered waiting recognition result"); + assert_eq!( + (first.sequence, first.text.as_str()), + (1, "recognized 0.25") + ); + assert_eq!( + (second.sequence, second.text.as_str()), + (2, "recognized 0.5") + ); + decoder.finish().unwrap(); + } + + #[tokio::test] + async fn blocked_parakeet_decode_does_not_overrun_the_production_frame_coordinator() { + use rubato::{Fft, FixedSync}; + + let controls = VoiceInputControls::default(); + let shutdown = Arc::new(AtomicBool::new(false)); + let discard = Arc::new(AtomicBool::new(false)); + let shutdown_epoch = Arc::new(AtomicU64::new(0)); + let (frame_tx, frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let (event_tx, mut event_rx) = tokio_mpsc::channel(EVENT_QUEUE_DEPTH); + let (decode_started_tx, decode_started_rx) = mpsc::sync_channel(0); + let (release_decode_tx, release_decode_rx) = mpsc::sync_channel(0); + let resampler = Fft::::new(INPUT_SAMPLE_RATE, 16_000, 1024, 2, 1, FixedSync::Input) + .expect("resampler"); + let worker_controls = controls.clone(); + let worker_shutdown = Arc::clone(&shutdown); + let worker_discard = Arc::clone(&discard); + let worker_shutdown_epoch = Arc::clone(&shutdown_epoch); + let worker = thread::spawn(move || { + parakeet_coordinator( + resampler, + frame_rx, + event_tx, + worker_shutdown, + worker_discard, + worker_shutdown_epoch, + worker_controls, + 0.5, + move |_| { + decode_started_tx.send(()).unwrap(); + release_decode_rx.recv().unwrap(); + "fixture final".to_string() + }, + ) + }); + let runtime = VoiceInputRuntime { + frame_tx, + controls, + shutdown, + discard_on_shutdown: discard, + shutdown_mute_epoch: shutdown_epoch, + worker: Some(worker), + }; + + let mut decode_started = false; + for samples in crate::benchmark::first_bundled_fixture_frames_for_test() { + runtime + .try_push_frame(VoiceInputFrame::try_from_samples(&samples).unwrap()) + .expect("fixture frame reaches the production coordinator"); + if decode_started_rx.try_recv().is_ok() { + decode_started = true; + break; + } + thread::sleep(Duration::from_millis(2)); + } + assert!(decode_started, "checked fixture reaches a VAD boundary"); + + for _ in 0..INPUT_QUEUE_FRAMES + 5 { + runtime + .try_push_frame(silence_frame()) + .expect("20 ms capture remains drainable during blocked inference"); + thread::sleep(INPUT_FRAME_DURATION); + } + release_decode_tx.send(()).unwrap(); + + let mut observed = Vec::new(); + while !observed.contains(&"pending:false") { + let event = tokio::time::timeout(Duration::from_secs(1), event_rx.recv()) + .await + .expect("coordinator event timeout") + .expect("coordinator event channel"); + match event { + VoiceInputEvent::Ready => observed.push("ready"), + VoiceInputEvent::SpeakingChanged(true) => observed.push("speaking:true"), + VoiceInputEvent::SpeakingChanged(false) => observed.push("speaking:false"), + VoiceInputEvent::RecognitionPendingChanged(true) => observed.push("pending:true"), + VoiceInputEvent::RecognitionPendingChanged(false) => observed.push("pending:false"), + VoiceInputEvent::FinalTranscript { + text, + storage_receipt, + } => { + assert_eq!(text, "fixture final"); + storage_receipt.stored(); + observed.push("final"); + } + VoiceInputEvent::Failed(error) => panic!("unexpected input failure: {error}"), + } + } + assert_eq!( + observed, + [ + "ready", + "speaking:true", + "pending:true", + "speaking:false", + "final", + "pending:false", + ] + ); + runtime.finish().await.unwrap(); + } + + #[test] + fn parakeet_inference_panic_emits_one_terminal_failure() { + let controls = VoiceInputControls::default(); + let shutdown = Arc::new(AtomicBool::new(false)); + let discard = Arc::new(AtomicBool::new(false)); + let shutdown_epoch = Arc::new(AtomicU64::new(0)); + let mut decoder = ParakeetDecoder::start( + controls.clone(), + Arc::clone(&shutdown), + Arc::clone(&discard), + Arc::clone(&shutdown_epoch), + move |_| panic!("synthetic inference panic"), + ) + .unwrap(); + let (events, mut event_rx) = tokio_mpsc::channel(8); + let mut pending = PendingRecognitions::new(); + let mut ledger = ParakeetRecognitionLedger::new(); + submit_parakeet_utterance( + &decoder, + vec![0.25; VAD_FRAME_SAMPLES], + 0, + &mut ledger, + &mut pending, + &events, + ) + .unwrap(); + let deadline = Instant::now() + Duration::from_millis(100); + let failure = loop { + match drain_parakeet_results( + &decoder, + &events, + &mut ledger, + &mut pending, + None, + &controls, + &shutdown, + &discard, + &shutdown_epoch, + ) { + Ok(()) if Instant::now() < deadline => thread::yield_now(), + Ok(()) => panic!("inference worker did not disconnect"), + Err(error) => break error, + } + }; + events + .blocking_send(VoiceInputEvent::Failed(failure)) + .unwrap(); + let decoder_result = decoder.finish(); + pending.reset(&events).unwrap(); + assert!(decoder_result.is_err()); + + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(true)) + )); + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::Failed(message)) + if message == "Parakeet recognition stopped unexpectedly." + )); + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(false)) + )); + assert!(event_rx.try_recv().is_err()); + } + + #[test] + fn reset_replaces_a_stale_waiter_without_resolving_fresh_pending() { + let controls = VoiceInputControls::default(); + let shutdown = Arc::new(AtomicBool::new(false)); + let discard = Arc::new(AtomicBool::new(false)); + let shutdown_epoch = Arc::new(AtomicU64::new(0)); + let (first_started_tx, first_started_rx) = mpsc::sync_channel(0); + let (release_first_tx, release_first_rx) = mpsc::sync_channel(0); + let (second_started_tx, second_started_rx) = mpsc::sync_channel(0); + let (release_second_tx, release_second_rx) = mpsc::sync_channel(0); + let mut decode_count = 0; + let mut decoder = ParakeetDecoder::start( + controls.clone(), + Arc::clone(&shutdown), + Arc::clone(&discard), + Arc::clone(&shutdown_epoch), + move |_| { + decode_count += 1; + if decode_count == 1 { + first_started_tx.send(()).unwrap(); + release_first_rx.recv().unwrap(); + "stale".to_string() + } else { + second_started_tx.send(()).unwrap(); + release_second_rx.recv().unwrap(); + "fresh".to_string() + } + }, + ) + .unwrap(); + let (events, mut event_rx) = tokio_mpsc::channel(8); + let mut pending = PendingRecognitions::new(); + let mut ledger = ParakeetRecognitionLedger::new(); + + submit_parakeet_utterance( + &decoder, + vec![0.25; VAD_FRAME_SAMPLES], + 0, + &mut ledger, + &mut pending, + &events, + ) + .unwrap(); + first_started_rx.recv().unwrap(); + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(true)) + )); + submit_parakeet_utterance( + &decoder, + vec![0.375; VAD_FRAME_SAMPLES], + 0, + &mut ledger, + &mut pending, + &events, + ) + .expect("one old-epoch utterance may wait behind the active decode"); + assert_eq!(pending.count, 2); + assert!(event_rx.try_recv().is_err()); + + controls.reset(); + pending.reset(&events).unwrap(); + submit_parakeet_utterance( + &decoder, + vec![0.5; VAD_FRAME_SAMPLES], + controls.mute_epoch(), + &mut ledger, + &mut pending, + &events, + ) + .expect("fresh utterance replaces the stale waiting utterance"); + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(false)) + )); + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(true)) + )); + + release_first_tx.send(()).unwrap(); + second_started_rx.recv().unwrap(); + drain_parakeet_results( + &decoder, + &events, + &mut ledger, + &mut pending, + None, + &controls, + &shutdown, + &discard, + &shutdown_epoch, + ) + .unwrap(); + assert_eq!(pending.count, 1); + assert!(event_rx.try_recv().is_err()); + + release_second_tx.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_millis(100); + while ledger.pending.len() == 1 && Instant::now() < deadline { + drain_parakeet_results( + &decoder, + &events, + &mut ledger, + &mut pending, + None, + &controls, + &shutdown, + &discard, + &shutdown_epoch, + ) + .unwrap(); + thread::yield_now(); + } + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::FinalTranscript { text, .. }) if text == "fresh" + )); + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(false)) + )); + assert!(event_rx.try_recv().is_err()); + decoder.finish().unwrap(); + } + + #[test] + fn cancel_rejects_a_late_parakeet_result() { + let controls = VoiceInputControls::default(); + let shutdown = Arc::new(AtomicBool::new(false)); + let discard = Arc::new(AtomicBool::new(false)); + let shutdown_epoch = Arc::new(AtomicU64::new(0)); + let (started_tx, started_rx) = mpsc::sync_channel(0); + let (release_tx, release_rx) = mpsc::sync_channel(0); + let mut decoder = ParakeetDecoder::start( + controls.clone(), + Arc::clone(&shutdown), + Arc::clone(&discard), + Arc::clone(&shutdown_epoch), + move |_| { + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + "too late".to_string() + }, + ) + .unwrap(); + let (events, mut event_rx) = tokio_mpsc::channel(8); + let mut pending = PendingRecognitions::new(); + let mut ledger = ParakeetRecognitionLedger::new(); + submit_parakeet_utterance( + &decoder, + vec![0.25; VAD_FRAME_SAMPLES], + 0, + &mut ledger, + &mut pending, + &events, + ) + .unwrap(); + started_rx.recv().unwrap(); + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(true)) + )); + + discard.store(true, Ordering::Release); + shutdown.store(true, Ordering::Release); + pending.reset(&events).unwrap(); + release_tx.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_millis(100); + while ledger.pending.len() == 1 && Instant::now() < deadline { + drain_parakeet_results( + &decoder, + &events, + &mut ledger, + &mut pending, + None, + &controls, + &shutdown, + &discard, + &shutdown_epoch, + ) + .unwrap(); + thread::yield_now(); + } + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(false)) + )); + assert!(event_rx.try_recv().is_err()); + decoder.finish().unwrap(); + } + + #[tokio::test] + async fn blocked_parakeet_inference_preserves_outer_quarantine() { + let controls = VoiceInputControls::default(); + let shutdown = Arc::new(AtomicBool::new(false)); + let (started_tx, started_rx) = mpsc::sync_channel(0); + let (release_tx, release_rx) = mpsc::sync_channel(0); + let mut decoder = ParakeetDecoder::start( + controls, + shutdown, + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicU64::new(0)), + move |_| { + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + String::new() + }, + ) + .unwrap(); + assert!(decoder + .try_submit(ParakeetUtterance { + sequence: 1, + speech: vec![0.25; VAD_FRAME_SAMPLES], + mute_epoch: 0, + }) + .unwrap() + .is_none()); + started_rx.recv().unwrap(); + let (finished_tx, finished_rx) = mpsc::sync_channel(0); + let outer = thread::spawn(move || { + decoder.finish().unwrap(); + finished_tx.send(()).unwrap(); + }); + + assert_eq!( + finish_worker(outer, Duration::from_millis(20)) + .await + .unwrap_err(), + VoiceInputFinishError::Quarantined { + timeout: Duration::from_millis(20) + } + ); + release_tx.send(()).unwrap(); + finished_rx + .recv_timeout(Duration::from_millis(100)) + .expect("quarantined nested worker can still finish after release"); + } + + #[test] + fn shutdown_freezes_the_accepted_mute_epoch() { + let controls = VoiceInputControls::default(); + let shutdown = AtomicBool::new(false); + let shutdown_epoch = AtomicU64::new(controls.mute_epoch()); + shutdown.store(true, Ordering::Release); + controls.set_host_muted(true); + + assert_eq!( + effective_mute_epoch(&controls, &shutdown, &shutdown_epoch), + (true, 0) + ); + } + + #[test] + fn final_storage_receipt_unblocks_the_bounded_sender() { + let (events, mut receiver) = tokio_mpsc::channel(1); + let (done_tx, done_rx) = mpsc::sync_channel(0); + let sender = thread::spawn(move || { + send_final( + &events, + "stored words".to_string(), + Some(Instant::now() + Duration::from_secs(1)), + ) + .unwrap(); + done_tx.send(()).unwrap(); + }); + let VoiceInputEvent::FinalTranscript { + text, + storage_receipt, + } = receiver.blocking_recv().expect("final event") + else { + panic!("expected final transcript") + }; + assert_eq!(text, "stored words"); + assert!(done_rx.try_recv().is_err()); + storage_receipt.stored(); + done_rx + .recv_timeout(Duration::from_millis(100)) + .expect("storage acknowledgement unblocks worker"); + sender.join().unwrap(); + } + + #[test] + fn dropping_an_unstored_receipt_unblocks_without_claiming_storage() { + let (events, mut receiver) = tokio_mpsc::channel(1); + let (done_tx, done_rx) = mpsc::sync_channel(0); + let sender = thread::spawn(move || { + send_final( + &events, + "unstored words".to_string(), + Some(Instant::now() + Duration::from_secs(1)), + ) + .unwrap(); + done_tx.send(()).unwrap(); + }); + let VoiceInputEvent::FinalTranscript { + storage_receipt, .. + } = receiver.blocking_recv().expect("final event") + else { + panic!("expected final transcript") + }; + + drop(storage_receipt); + + done_rx + .recv_timeout(Duration::from_millis(100)) + .expect("dropping an unstored receipt unblocks worker shutdown"); + sender.join().unwrap(); + } + + #[test] + fn multiple_finals_share_one_absolute_storage_deadline() { + let (events, mut receiver) = tokio_mpsc::channel(2); + let (done_tx, done_rx) = mpsc::sync_channel(0); + let deadline = Instant::now() + Duration::from_millis(100); + let sender = thread::spawn(move || { + send_final(&events, "first".to_string(), Some(deadline)).unwrap(); + send_final(&events, "second".to_string(), Some(deadline)).unwrap(); + done_tx.send(()).unwrap(); + }); + let VoiceInputEvent::FinalTranscript { + storage_receipt: first_receipt, + .. + } = receiver.blocking_recv().expect("first final") + else { + panic!("expected first final transcript") + }; + + thread::sleep(Duration::from_millis(120)); + let VoiceInputEvent::FinalTranscript { + storage_receipt: second_receipt, + .. + } = receiver.blocking_recv().expect("second final") + else { + panic!("expected second final transcript") + }; + done_rx + .recv_timeout(Duration::from_millis(50)) + .expect("second final does not start a new storage deadline"); + + drop((first_receipt, second_receipt)); + sender.join().unwrap(); + } + + #[test] + fn queued_frame_keeps_its_old_epoch_across_mute_and_unmute() { + let (frame_tx, frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let controls = VoiceInputControls::default(); + let runtime = VoiceInputRuntime { + frame_tx, + controls: controls.clone(), + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + worker: None, + }; + runtime.try_push_frame(silence_frame()).unwrap(); + + controls.set_host_muted(true); + controls.set_host_muted(false); + + let queued = frame_rx.try_recv().expect("queued pre-mute frame"); + assert_eq!(queued.mute_epoch, 0); + assert_ne!(queued.mute_epoch, controls.mute_epoch()); + } + + #[test] + fn input_during_tts_policy_controls_frame_admission() { + let (frame_tx, frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let controls = VoiceInputControls::default(); + let runtime = VoiceInputRuntime { + frame_tx, + controls: controls.clone(), + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + worker: None, + }; + + let suppress = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::SuppressInput) + .unwrap(); + runtime.try_push_frame(silence_frame()).unwrap(); + assert!(frame_rx.try_recv().is_err()); + drop(suppress); + + let allow = controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::AllowBargeIn) + .unwrap(); + runtime.try_push_frame(silence_frame()).unwrap(); + assert!(frame_rx.try_recv().is_ok()); + drop(allow); + } + + #[test] + fn bounded_frame_queue_reports_overrun_without_blocking() { + let (frame_tx, _frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let controls = VoiceInputControls::default(); + let runtime = VoiceInputRuntime { + frame_tx, + controls, + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + worker: None, + }; + for _ in 0..INPUT_QUEUE_FRAMES { + runtime.try_push_frame(silence_frame()).unwrap(); + } + assert!(runtime.try_push_frame(silence_frame()).is_err()); + } + + #[test] + fn cancelled_runtime_rejects_new_frames() { + let (frame_tx, _frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let runtime = VoiceInputRuntime { + frame_tx, + controls: VoiceInputControls::default(), + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + worker: None, + }; + + runtime.cancel(); + + assert_eq!( + runtime.try_push_frame(silence_frame()).unwrap_err(), + "Voice input recognition is no longer running." + ); + } + + #[test] + fn mute_before_shutdown_discards_the_frozen_epoch() { + let (frame_tx, _frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let controls = VoiceInputControls::default(); + let runtime = VoiceInputRuntime { + frame_tx, + controls: controls.clone(), + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + worker: None, + }; + + controls.set_host_muted(true); + runtime.signal_shutdown(false); + + assert!(runtime.discard_on_shutdown.load(Ordering::Acquire)); + assert_eq!( + effective_mute_epoch(&controls, &runtime.shutdown, &runtime.shutdown_mute_epoch), + (true, 1) + ); + } + + #[test] + fn shutdown_before_mute_accepts_and_freezes_the_prior_epoch() { + let (frame_tx, _frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let controls = VoiceInputControls::default(); + let runtime = VoiceInputRuntime { + frame_tx, + controls: controls.clone(), + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + worker: None, + }; + + runtime.signal_shutdown(false); + controls.set_host_muted(true); + + assert!(!runtime.discard_on_shutdown.load(Ordering::Acquire)); + assert_eq!( + effective_mute_epoch(&controls, &runtime.shutdown, &runtime.shutdown_mute_epoch), + (true, 0) + ); + } + + #[tokio::test] + async fn concurrent_cancel_and_finish_signals_linearize_without_deadlock() { + let (frame_tx, _frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let runtime = VoiceInputRuntime { + frame_tx, + controls: VoiceInputControls::default(), + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + worker: None, + }; + + thread::scope(|scope| { + scope.spawn(|| runtime.cancel()); + scope.spawn(|| runtime.signal_shutdown(false)); + }); + + assert!(runtime.shutdown.load(Ordering::Acquire)); + assert_eq!(runtime.shutdown_mute_epoch.load(Ordering::Acquire), 0); + runtime.finish().await.unwrap(); + } + + #[test] + fn cancel_before_finish_signal_discards_but_finish_before_cancel_stays_accepted() { + let make_runtime = || { + let (frame_tx, _frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + VoiceInputRuntime { + frame_tx, + controls: VoiceInputControls::default(), + shutdown: Arc::new(AtomicBool::new(false)), + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + worker: None, + } + }; + + let cancelled = make_runtime(); + cancelled.cancel(); + cancelled.signal_shutdown(false); + assert!(cancelled.discard_on_shutdown.load(Ordering::Acquire)); + + let finished = make_runtime(); + finished.signal_shutdown(false); + finished.cancel(); + assert!(!finished.discard_on_shutdown.load(Ordering::Acquire)); + } + + #[test] + fn silence_does_not_cross_the_default_earshot_threshold() { + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + assert!(vad.predict_f32(&[0.0; VAD_FRAME_SAMPLES]) <= 0.5); + } + + #[test] + fn resampling_produces_the_expected_nonempty_frame_count() { + use rubato::{Fft, FixedSync, Resampler}; + + let mut resampler = + Fft::::new(48_000, 16_000, 1024, 2, 1, FixedSync::Input).expect("resampler"); + let input_frames = resampler.input_frames_next(); + let expected_output_frames = resampler.output_frames_next(); + let output = resample(&mut resampler, &vec![0.25; input_frames]); + + assert_eq!(output.len(), expected_output_frames); + assert!(!output.is_empty()); + } + + #[test] + fn resampler_overshoot_is_clamped_before_vad() { + assert_eq!( + clamp_vad_frame(&[-1.25, -1.0, 0.25, 1.0, 1.25]), + [-1.0, -1.0, 0.25, 1.0, 1.0] + ); + } + + #[test] + fn openai_transcripts_are_delivered_in_commit_order() { + let deadline = Instant::now() + OPENAI_LIVE_RESULT_TIMEOUT; + let mut pending_commits = VecDeque::from([ + OpenAiPendingCommit { + mute_epoch: 7, + settle_deadline: deadline, + }, + OpenAiPendingCommit { + mute_epoch: 7, + settle_deadline: deadline, + }, + ]); + let mut committed = VecDeque::new(); + let mut completed = HashMap::new(); + for item_id in ["first", "second"] { + record_openai_event( + OpenAiRealtimeTranscriptionEvent::Committed { + item_id: item_id.to_string(), + }, + 7, + &mut pending_commits, + &mut committed, + &mut completed, + ); + } + for (item_id, transcript) in [("second", "two"), ("first", "one")] { + record_openai_event( + OpenAiRealtimeTranscriptionEvent::Completed { + item_id: item_id.to_string(), + transcript: transcript.to_string(), + }, + 7, + &mut pending_commits, + &mut committed, + &mut completed, + ); + } + let (event_tx, mut event_rx) = tokio_mpsc::channel(4); + let mut pending = PendingRecognitions { count: 2 }; + + deliver_openai_turns( + &mut committed, + &mut completed, + &event_tx, + &mut pending, + None, + None, + ); + + let texts = [event_rx.try_recv(), event_rx.try_recv()].map(|event| match event { + Ok(VoiceInputEvent::FinalTranscript { text, .. }) => text, + _ => panic!("expected finalized transcript"), + }); + assert_eq!(texts, ["one", "two"]); + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(false)) + )); + } + + #[test] + fn openai_commits_from_stale_mute_epochs_are_not_deliverable() { + let deadline = Instant::now() + OPENAI_LIVE_RESULT_TIMEOUT; + let mut pending_commits = VecDeque::from([OpenAiPendingCommit { + mute_epoch: 1, + settle_deadline: deadline, + }]); + let mut committed = VecDeque::new(); + let turn = record_openai_event( + OpenAiRealtimeTranscriptionEvent::Committed { + item_id: "stale".to_string(), + }, + 2, + &mut pending_commits, + &mut committed, + &mut HashMap::new(), + ) + .expect("recorded commit"); + + assert_eq!(turn.mute_epoch, 1); + assert_eq!(turn.settle_deadline, deadline); + assert!(committed.is_empty()); + } + + #[test] + fn openai_transcription_failures_only_apply_to_the_current_epoch() { + let committed = VecDeque::from([OpenAiCommittedTurn { + item_id: "current".to_string(), + mute_epoch: 2, + settle_deadline: Instant::now() + OPENAI_LIVE_RESULT_TIMEOUT, + }]); + + assert!(openai_transcription_failure_is_current( + "current", 2, &committed + )); + assert!(!openai_transcription_failure_is_current( + "current", 3, &committed + )); + let discarded_is_current = + openai_transcription_failure_is_current("discarded", 2, &committed); + assert!(!discarded_is_current); + } + + #[test] + fn openai_unmatched_commits_are_not_assigned_to_the_current_epoch() { + let mut committed = VecDeque::new(); + + let turn = record_openai_event( + OpenAiRealtimeTranscriptionEvent::Committed { + item_id: "unmatched".to_string(), + }, + 2, + &mut VecDeque::new(), + &mut committed, + &mut HashMap::new(), + ); + + assert!(turn.is_none()); + assert!(committed.is_empty()); + } + + #[test] + fn openai_commit_transfers_its_original_live_result_deadline() { + let deadline = Instant::now() + Duration::from_secs(1); + let mut pending_commits = VecDeque::from([OpenAiPendingCommit { + mute_epoch: 4, + settle_deadline: deadline, + }]); + let mut committed = VecDeque::new(); + let turn = record_openai_event( + OpenAiRealtimeTranscriptionEvent::Committed { + item_id: "turn".to_string(), + }, + 4, + &mut pending_commits, + &mut committed, + &mut HashMap::new(), + ) + .expect("recorded commit"); + + assert!(pending_commits.is_empty()); + assert_eq!(turn.settle_deadline, deadline); + assert_eq!(committed.front().unwrap().settle_deadline, deadline); + } + + #[test] + fn openai_successful_commit_tracking_starts_a_bounded_deadline() { + let before = Instant::now() + OPENAI_LIVE_RESULT_TIMEOUT; + let mut pending_commits = VecDeque::new(); + + track_openai_commit(&mut pending_commits, 9); + + let after = Instant::now() + OPENAI_LIVE_RESULT_TIMEOUT; + let tracked = pending_commits.front().expect("tracked commit"); + assert_eq!(tracked.mute_epoch, 9); + assert!(tracked.settle_deadline >= before); + assert!(tracked.settle_deadline <= after); + } + + #[test] + fn openai_oldest_unresolved_turn_controls_live_result_expiry() { + let now = Instant::now(); + let pending_commits = VecDeque::from([ + OpenAiPendingCommit { + mute_epoch: 1, + settle_deadline: now + Duration::from_secs(1), + }, + OpenAiPendingCommit { + mute_epoch: 1, + settle_deadline: now + Duration::from_secs(2), + }, + ]); + let committed = VecDeque::new(); + + assert!(!openai_live_result_expired( + &pending_commits, + &committed, + now + )); + assert!(openai_live_result_expired( + &pending_commits, + &committed, + now + Duration::from_secs(1) + )); + } + + #[test] + fn openai_empty_completion_resolves_pending_without_a_final() { + let deadline = Instant::now() + OPENAI_LIVE_RESULT_TIMEOUT; + let mut pending_commits = VecDeque::from([OpenAiPendingCommit { + mute_epoch: 2, + settle_deadline: deadline, + }]); + let mut committed = VecDeque::new(); + let mut completed = HashMap::new(); + record_openai_event( + OpenAiRealtimeTranscriptionEvent::Committed { + item_id: "empty".to_string(), + }, + 2, + &mut pending_commits, + &mut committed, + &mut completed, + ); + record_openai_event( + OpenAiRealtimeTranscriptionEvent::Completed { + item_id: "empty".to_string(), + transcript: String::new(), + }, + 2, + &mut pending_commits, + &mut committed, + &mut completed, + ); + let (event_tx, mut event_rx) = tokio_mpsc::channel(2); + let mut pending = PendingRecognitions { count: 1 }; + + deliver_openai_turns( + &mut committed, + &mut completed, + &event_tx, + &mut pending, + None, + None, + ); + + assert_eq!(pending.count, 0); + assert!(matches!( + event_rx.try_recv(), + Ok(VoiceInputEvent::RecognitionPendingChanged(false)) + )); + assert!(event_rx.try_recv().is_err()); + } + + #[test] + fn openai_timeout_teardown_ignores_late_completion_from_the_old_generation() { + let deadline = Instant::now(); + let mut pending_commits = VecDeque::new(); + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: "old".to_string(), + mute_epoch: 3, + settle_deadline: deadline, + }]); + let mut completed = HashMap::new(); + assert!(openai_live_result_expired( + &pending_commits, + &committed, + deadline + )); + + pending_commits.clear(); + committed.clear(); + completed.clear(); + record_openai_event( + OpenAiRealtimeTranscriptionEvent::Completed { + item_id: "old".to_string(), + transcript: "late".to_string(), + }, + 4, + &mut pending_commits, + &mut committed, + &mut completed, + ); + let (event_tx, mut event_rx) = tokio_mpsc::channel(2); + let mut pending = PendingRecognitions::new(); + deliver_openai_turns( + &mut committed, + &mut completed, + &event_tx, + &mut pending, + None, + None, + ); + + assert!(event_rx.try_recv().is_err()); + assert_eq!(pending.count, 0); + } + + #[test] + fn stalled_openai_operation_observes_shutdown() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("runtime"); + let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_for_thread = Arc::clone(&shutdown); + let signal = thread::spawn(move || { + thread::sleep(Duration::from_millis(30)); + shutdown_for_thread.store(true, Ordering::Release); + }); + + let result = block_on_openai_operation( + &runtime, + shutdown.as_ref(), + std::future::pending::>(), + "stalled operation", + ) + .expect("shutdown is not an error"); + + signal.join().expect("shutdown signal"); + assert_eq!(result, None); + } + + #[test] + fn openai_timeout_constructs_its_timer_inside_the_worker_runtime() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("runtime"); + + let result = + block_on_openai_timeout(&runtime, Duration::from_millis(10), std::future::ready(42)); + + assert_eq!(result, Ok(42)); + } + + #[test] + fn openai_idle_audio_keeps_only_bounded_pre_roll() { + let mut pre_roll = VecDeque::new(); + for index in 0..(OPENAI_PRE_ROLL_FRAMES * 4) { + push_pre_roll(&mut pre_roll, vec![index as u8]); + } + + assert_eq!(pre_roll.len(), OPENAI_PRE_ROLL_FRAMES); + assert_eq!( + pre_roll.front(), + Some(&vec![(OPENAI_PRE_ROLL_FRAMES * 3) as u8]) + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn mac_finals_before_the_deadline_preserve_order_and_clear_pending() { + let (recognition_tx, mut recognition_rx) = tokio_mpsc::unbounded_channel(); + for text in ["first", "second"] { + recognition_tx + .send(crate::mac_speech::MacSpeechRecognitionEvent::Final( + text.to_string(), + )) + .unwrap(); + } + let (event_tx, mut event_rx) = tokio_mpsc::channel(4); + let mut pending = PendingRecognitions { count: 2 }; + let mut settle_deadline = Some(Instant::now() + MAC_LIVE_NO_RESULT_TIMEOUT); + + forward_mac_events( + &mut recognition_rx, + &event_tx, + &mut pending, + &mut settle_deadline, + None, + ) + .unwrap(); + + let texts = [event_rx.try_recv(), event_rx.try_recv()].map(|event| match event { + Ok(VoiceInputEvent::FinalTranscript { text, .. }) => text, + _ => panic!("expected finalized transcript"), + }); + assert_eq!(texts, ["first", "second"]); + assert_eq!(pending.count, 0); + assert!(settle_deadline.is_none()); + } + + #[cfg(target_os = "macos")] + #[test] + fn mac_new_speech_cancels_then_refreshes_the_settle_deadline() { + let (event_tx, _event_rx) = tokio_mpsc::channel(4); + let mut pending = PendingRecognitions::new(); + let mut deadline = None; + let now = Instant::now(); + begin_mac_turn(&mut pending, &event_tx, &mut deadline).unwrap(); + end_mac_turn(&pending, &mut deadline, now); + let first_deadline = deadline.expect("first deadline"); + + begin_mac_turn(&mut pending, &event_tx, &mut deadline).unwrap(); + assert!(deadline.is_none()); + end_mac_turn(&pending, &mut deadline, now + Duration::from_secs(1)); + + assert!(deadline.is_some_and(|deadline| deadline > first_deadline)); + } + + #[cfg(target_os = "macos")] + #[test] + fn mac_timeout_reset_drops_the_old_event_channel_before_new_pending() { + let (event_tx, _event_rx) = tokio_mpsc::channel(8); + let mut pending = PendingRecognitions::new(); + let mut deadline = None; + let now = Instant::now(); + begin_mac_turn(&mut pending, &event_tx, &mut deadline).unwrap(); + end_mac_turn(&pending, &mut deadline, now); + assert!(mac_settle_expired( + &pending, + deadline, + now + MAC_LIVE_NO_RESULT_TIMEOUT + )); + + let (old_tx, old_rx) = tokio_mpsc::unbounded_channel(); + drop(old_rx); + pending.reset(&event_tx).unwrap(); + deadline = None; + begin_mac_turn(&mut pending, &event_tx, &mut deadline).unwrap(); + assert!(old_tx + .send(crate::mac_speech::MacSpeechRecognitionEvent::Final( + "late old final".to_string(), + )) + .is_err()); + assert_eq!(pending.count, 1); + + let (new_tx, mut new_rx) = tokio_mpsc::unbounded_channel(); + new_tx + .send(crate::mac_speech::MacSpeechRecognitionEvent::Final( + "current final".to_string(), + )) + .unwrap(); + forward_mac_events(&mut new_rx, &event_tx, &mut pending, &mut deadline, None).unwrap(); + assert_eq!(pending.count, 0); + } + + #[tokio::test] + async fn finish_is_bounded_and_joins_a_cooperative_worker() { + let (frame_tx, _frame_rx) = mpsc::sync_channel(INPUT_QUEUE_FRAMES); + let controls = VoiceInputControls::default(); + let shutdown = Arc::new(AtomicBool::new(false)); + let worker_shutdown = Arc::clone(&shutdown); + let worker = thread::spawn(move || { + while !worker_shutdown.load(Ordering::Acquire) { + thread::yield_now(); + } + }); + let runtime = VoiceInputRuntime { + frame_tx, + controls, + shutdown, + discard_on_shutdown: Arc::new(AtomicBool::new(false)), + shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), + worker: Some(worker), + }; + runtime.finish().await.unwrap(); + } + + #[tokio::test] + async fn blocked_worker_is_quarantined_at_the_production_finish_seam() { + let (release_tx, release_rx) = mpsc::sync_channel::<()>(0); + let returned = Arc::new(AtomicBool::new(false)); + let worker_returned = Arc::clone(&returned); + let worker = thread::spawn(move || { + let _ = release_rx.recv(); + worker_returned.store(true, Ordering::Release); + }); + + let error = finish_worker(worker, Duration::from_millis(20)) + .await + .expect_err("blocked worker must miss its bounded finish deadline"); + assert_eq!( + error, + VoiceInputFinishError::Quarantined { + timeout: Duration::from_millis(20) + } + ); + assert!(!returned.load(Ordering::Acquire)); + + release_tx + .send(()) + .expect("release quarantined test worker"); + let deadline = Instant::now() + Duration::from_secs(1); + while !returned.load(Ordering::Acquire) && Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert!(returned.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn panicked_worker_is_quiescent_and_not_quarantined() { + let worker = thread::spawn(|| panic!("deliberate worker panic")); + let error = finish_worker(worker, Duration::from_secs(1)) + .await + .expect_err("panic is reported"); + assert_eq!(error, VoiceInputFinishError::WorkerPanicked); + assert!(!error.is_quarantined()); + } +} diff --git a/src-tauri/crates/berd-voice/src/lib.rs b/src-tauri/crates/berd-voice/src/lib.rs index 8b0abc784..2a074cad0 100644 --- a/src-tauri/crates/berd-voice/src/lib.rs +++ b/src-tauri/crates/berd-voice/src/lib.rs @@ -1,8 +1,49 @@ -//! Device-local voice primitives for Berd. +//! Reusable voice primitives for Berd. +/// Maximum time the macOS recognizer waits for native completion after input ends. +pub const MAC_SPEECH_RECOGNITION_FINISH_TIMEOUT_SECONDS: u64 = 5; + +mod asset_verification; +mod audio_output; +pub mod benchmark; +mod configured_tts; +pub mod input; +pub mod local_assets; +#[cfg(target_os = "macos")] +pub mod mac_speech; +#[cfg(target_os = "macos")] +mod macos_audio_output; +pub mod openai; +pub mod openai_realtime; +mod outbound; +mod parakeet; +pub mod parakeet_assets; mod pocket; +pub mod pocket_assets; +pub mod protocol; +pub mod session; +pub mod siri; +mod synthesis; +mod tts; +pub use audio_output::{wait_until_drained, PcmAudioOutput}; +pub use configured_tts::{ + ConfiguredTtsSlot, TtsConfiguration, TtsConfigurationLease, TtsConfigurationRejection, + TtsConfigurationRejectionKind, TtsConfigurationReplacement, TtsConfigurationSnapshot, + TtsSettings, +}; +#[cfg(target_os = "macos")] +pub use macos_audio_output::PocketAudioPlayer; +pub use outbound::{ + estimated_spoken_through_utf8, DeliveryProgress, DeliverySegment, DrainPolicy, + DrainTimeoutOutcome, OutboundFailure, OutboundOutcome, OutboundPlayback, +}; +pub use parakeet::ParakeetRecognizer; pub use pocket::{ - load_text_to_speech, load_voice_style, take_streaming_text_chunks, PocketTts, - StreamingTextChunks, VoiceStyle, SAMPLE_RATE, + load_pocket_voice_style, load_text_to_speech, load_voice_style, take_streaming_text_chunks, + PocketTts, StreamingTextChunks, VoiceStyle, SAMPLE_RATE, }; +#[cfg(target_os = "macos")] +pub use siri::SiriTts; +pub use synthesis::{synthesize_pcm16_wav, WavSynthesis, WavSynthesisError, WavSynthesisErrorKind}; +pub use tts::{OpenAiTts, PocketTtsBackend, TtsBackend, TtsOutcome, TtsPcmSpec, TtsSynthesisEvent}; diff --git a/src-tauri/crates/berd-voice/src/local_assets.rs b/src-tauri/crates/berd-voice/src/local_assets.rs new file mode 100644 index 000000000..71c7bf3cf --- /dev/null +++ b/src-tauri/crates/berd-voice/src/local_assets.rs @@ -0,0 +1,1189 @@ +//! Shared local-model roots, cross-process coordination, and installation helpers. + +use fs2::FileExt; +use futures_util::StreamExt; +use sha2::{Digest, Sha256}; +use std::fmt; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::time::Duration; + +const LOCK_FILE: &str = ".berd-voice-assets.lock"; +const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +const READ_TIMEOUT: Duration = Duration::from_secs(30); +const TOTAL_TIMEOUT: Duration = Duration::from_secs(30 * 60); +const MUTATION_LOCK_TIMEOUT: Duration = Duration::from_secs(15); +const MUTATION_LOCK_RETRY: Duration = Duration::from_millis(20); + +/// The two exact bundle roots in one host-selected local-model store. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LocalAssetRoots { + coordination_root: PathBuf, + pocket_bundle_root: PathBuf, + parakeet_bundle_root: PathBuf, +} + +impl LocalAssetRoots { + /// Define the current closed Pocket/Parakeet store layout. + pub fn new( + coordination_root: impl Into, + pocket_bundle_root: impl Into, + parakeet_bundle_root: impl Into, + ) -> Result { + let roots = Self { + coordination_root: coordination_root.into(), + pocket_bundle_root: pocket_bundle_root.into(), + parakeet_bundle_root: parakeet_bundle_root.into(), + }; + for root in [ + &roots.coordination_root, + &roots.pocket_bundle_root, + &roots.parakeet_bundle_root, + ] { + validate_root(root)?; + } + if roots.parakeet_bundle_root != roots.pocket_bundle_root.join("stt") { + return Err(LocalAssetLockError::InvalidRoot( + "the Parakeet bundle root must be the Pocket bundle root's stt directory" + .to_string(), + )); + } + if roots.pocket_bundle_root.parent() != Some(roots.coordination_root.as_path()) { + return Err(LocalAssetLockError::InvalidRoot( + "the Pocket bundle root must be an immediate child of the coordination root" + .to_string(), + )); + } + Ok(roots) + } + + pub fn coordination_root(&self) -> &Path { + &self.coordination_root + } + + pub fn pocket_bundle_root(&self) -> &Path { + &self.pocket_bundle_root + } + + pub fn parakeet_bundle_root(&self) -> &Path { + &self.parakeet_bundle_root + } +} + +/// Error acquiring the store's advisory process lock. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LocalAssetLockError { + Busy, + InvalidRoot(String), + Io(String), +} + +impl fmt::Display for LocalAssetLockError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Busy => write!( + formatter, + "another local voice asset mutation is in progress" + ), + Self::InvalidRoot(message) | Self::Io(message) => formatter.write_str(message), + } + } +} + +/// Shared reader guard. Model construction holds this across bundle loading. +pub struct LocalAssetReadGuard { + _file: File, +} + +/// Exclusive mutation guard. Install and host-owned removal use the same lock. +pub struct LocalAssetMutationGuard { + _file: File, + roots: LocalAssetRoots, +} + +impl LocalAssetMutationGuard { + /// Recover one unambiguous interrupted publication while this guard owns + /// the store's exclusive process lock. + pub fn recover_interrupted_publication(&self) -> Result<(), LocalInstallError> { + recover_interrupted_publication_with(&self.roots, combined_tree_has_ready_engine) + } + + pub(crate) fn roots(&self) -> &LocalAssetRoots { + &self.roots + } + + pub(crate) fn validate_staging_paths( + &self, + source: &Path, + expected_source: &Path, + destination: &Path, + ) -> Result<(), LocalInstallError> { + validate_root(destination).map_err(LocalInstallError::from)?; + let coordination_root = self.roots.coordination_root(); + let live_root = self.roots.pocket_bundle_root(); + if source != expected_source + || destination == coordination_root + || !destination.starts_with(coordination_root) + || destination.starts_with(live_root) + { + return Err(LocalInstallError::new( + LocalInstallErrorKind::InvalidRoot, + "staging paths do not belong to the locked local asset store", + )); + } + Ok(()) + } +} + +pub fn try_lock_for_read( + roots: &LocalAssetRoots, +) -> Result { + let file = open_lock(roots)?; + match FileExt::try_lock_shared(&file) { + Ok(()) => Ok(LocalAssetReadGuard { _file: file }), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + Err(LocalAssetLockError::Busy) + } + Err(error) => Err(LocalAssetLockError::Io(format!( + "lock local voice assets for reading: {error}" + ))), + } +} + +pub fn try_lock_for_mutation( + roots: &LocalAssetRoots, +) -> Result { + let file = open_lock(roots)?; + match FileExt::try_lock_exclusive(&file) { + Ok(()) => Ok(LocalAssetMutationGuard { + _file: file, + roots: roots.clone(), + }), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + Err(LocalAssetLockError::Busy) + } + Err(error) => Err(LocalAssetLockError::Io(format!( + "lock local voice assets for mutation: {error}" + ))), + } +} + +/// Wait for the short publication transaction lock without discarding a +/// completed download because a reader happened to be constructing a model. +pub async fn lock_for_mutation( + roots: &LocalAssetRoots, +) -> Result { + let deadline = tokio::time::Instant::now() + MUTATION_LOCK_TIMEOUT; + loop { + match try_lock_for_mutation(roots) { + Ok(guard) => return Ok(guard), + Err(LocalAssetLockError::Busy) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(MUTATION_LOCK_RETRY).await; + } + result => return result, + } + } +} + +/// Blocking counterpart for host mutations already running on a blocking +/// worker. Readers remain fail-fast. +pub fn lock_for_mutation_blocking( + roots: &LocalAssetRoots, +) -> Result { + let deadline = std::time::Instant::now() + MUTATION_LOCK_TIMEOUT; + loop { + match try_lock_for_mutation(roots) { + Ok(guard) => return Ok(guard), + Err(LocalAssetLockError::Busy) if std::time::Instant::now() < deadline => { + std::thread::sleep(MUTATION_LOCK_RETRY); + } + result => return result, + } + } +} + +fn recover_interrupted_publication_with( + roots: &LocalAssetRoots, + is_valid: impl Fn(&Path) -> bool, +) -> Result<(), LocalInstallError> { + let entries = fs::read_dir(roots.coordination_root()).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Recovery, + format!("inspect local voice asset recovery state: {error}"), + ) + })?; + let mut backups = Vec::new(); + for entry in entries { + let path = entry + .map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Recovery, + format!("read local voice asset recovery entry: {error}"), + ) + })? + .path(); + if path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(".voice-backup-")) + { + backups.push(path); + } + } + backups.sort(); + if backups.is_empty() { + return Ok(()); + } + let final_root = roots.pocket_bundle_root(); + let final_valid = final_root.exists() && is_valid(final_root); + if final_valid { + for backup in backups { + fs::remove_dir_all(&backup).map_err(|error| { + let mut failure = LocalInstallError::new( + LocalInstallErrorKind::Cleanup, + format!("remove stale local voice asset backup: {error}"), + ); + failure.recovery_paths.push(backup); + failure + })?; + } + return Ok(()); + } + if final_root.exists() || backups.len() != 1 || !is_valid(&backups[0]) { + let mut failure = LocalInstallError::new( + LocalInstallErrorKind::Recovery, + "local voice asset recovery state is ambiguous or invalid", + ); + if final_root.exists() { + failure.recovery_paths.push(final_root.to_path_buf()); + } + failure.recovery_paths.extend(backups); + return Err(failure); + } + fs::rename(&backups[0], final_root).map_err(|error| { + let mut failure = LocalInstallError::new( + LocalInstallErrorKind::Recovery, + format!("restore interrupted local voice asset publication: {error}"), + ); + failure.recovery_paths.push(backups[0].clone()); + failure + }) +} + +fn combined_tree_has_ready_engine(root: &Path) -> bool { + matches!( + crate::pocket_assets::inspect(root), + Ok(crate::pocket_assets::PocketAssetStatus::Ready { .. }) + ) || matches!( + crate::parakeet_assets::inspect(&root.join("stt")), + Ok(crate::parakeet_assets::ParakeetAssetStatus::Ready { .. }) + ) +} + +fn open_lock(roots: &LocalAssetRoots) -> Result { + fs::create_dir_all(&roots.coordination_root).map_err(|error| { + LocalAssetLockError::Io(format!( + "create local voice asset coordination root: {error}" + )) + })?; + OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(roots.coordination_root.join(LOCK_FILE)) + .map_err(|error| LocalAssetLockError::Io(format!("open local asset lock: {error}"))) +} + +fn validate_root(root: &Path) -> Result<(), LocalAssetLockError> { + if !root.is_absolute() { + return Err(LocalAssetLockError::InvalidRoot( + "local voice asset roots must be absolute".to_string(), + )); + } + if root + .components() + .any(|component| matches!(component, Component::ParentDir | Component::CurDir)) + { + return Err(LocalAssetLockError::InvalidRoot( + "local voice asset roots must not contain traversal components".to_string(), + )); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LocalInstallErrorKind { + Busy, + InvalidRoot, + Download, + Integrity, + Extraction, + Io, + Publish, + Rollback, + Recovery, + Cleanup, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LocalInstallPhase { + Downloading, + Extracting, + Verifying, + Publishing, + Complete, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LocalInstallProgress { + pub phase: LocalInstallPhase, + pub downloaded_bytes: u64, + pub total_download_bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LocalInstallError { + pub kind: LocalInstallErrorKind, + pub message: String, + pub recovery_paths: Vec, +} + +impl LocalInstallError { + pub(crate) fn new(kind: LocalInstallErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + recovery_paths: Vec::new(), + } + } +} + +impl fmt::Display for LocalInstallError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.message) + } +} + +impl From for LocalInstallError { + fn from(error: LocalAssetLockError) -> Self { + match error { + LocalAssetLockError::Busy => Self::new(LocalInstallErrorKind::Busy, error.to_string()), + LocalAssetLockError::InvalidRoot(message) => { + Self::new(LocalInstallErrorKind::InvalidRoot, message) + } + LocalAssetLockError::Io(message) => Self::new(LocalInstallErrorKind::Io, message), + } + } +} + +#[derive(Clone, Copy)] +pub(crate) struct DownloadSpec<'a> { + pub source_url: &'a str, + pub relative_path: &'a str, + pub size_bytes: u64, + pub sha256: &'a str, +} + +pub(crate) fn inspect_download_specs( + root: &Path, + specs: &[DownloadSpec<'static>], +) -> Result { + let manifest = specs + .iter() + .map(|spec| crate::asset_verification::PinnedAsset { + relative_path: spec.relative_path, + size_bytes: spec.size_bytes, + sha256: spec.sha256, + }) + .collect::>(); + crate::asset_verification::inspect_assets(root, &manifest) +} + +pub(crate) struct TemporaryDirectory { + path: PathBuf, + keep: bool, +} + +impl TemporaryDirectory { + pub(crate) fn create(parent: &Path, label: &str) -> Result { + fs::create_dir_all(parent).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("create local voice asset root: {error}"), + ) + })?; + let path = parent.join(format!(".{label}-{}", uuid::Uuid::new_v4())); + fs::create_dir(&path).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("create local voice asset temporary directory: {error}"), + ) + })?; + Ok(Self { path, keep: false }) + } + + pub(crate) fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TemporaryDirectory { + fn drop(&mut self) { + if !self.keep { + let _ = fs::remove_dir_all(&self.path); + } + } +} + +pub(crate) struct CombinedPublication { + final_root: PathBuf, + staging: TemporaryDirectory, + backup_root: PathBuf, +} + +impl CombinedPublication { + pub(crate) fn prepare(roots: &LocalAssetRoots) -> Result { + let staging = TemporaryDirectory::create(roots.coordination_root(), "voice-stage")?; + let backup_root = roots + .coordination_root() + .join(format!(".voice-backup-{}", uuid::Uuid::new_v4())); + Ok(Self { + final_root: roots.pocket_bundle_root().to_path_buf(), + staging, + backup_root, + }) + } + + pub(crate) fn root(&self) -> &Path { + self.staging.path() + } + + pub(crate) fn publish( + mut self, + verify: impl Fn(&Path) -> bool, + ) -> Result, LocalInstallError> { + let had_previous = self.final_root.exists(); + if had_previous { + fs::rename(&self.final_root, &self.backup_root).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Publish, + format!("retire prior local voice asset bundle: {error}"), + ) + })?; + } + if let Err(error) = fs::rename(self.staging.path(), &self.final_root) { + if had_previous { + if let Err(rollback_error) = fs::rename(&self.backup_root, &self.final_root) { + let mut failure = LocalInstallError::new( + LocalInstallErrorKind::Rollback, + format!( + "publish local voice asset bundle failed ({error}); restoring the prior bundle also failed: {rollback_error}" + ), + ); + failure.recovery_paths.push(self.backup_root.clone()); + failure + .recovery_paths + .push(self.staging.path().to_path_buf()); + self.staging.keep = true; + return Err(failure); + } + } + return Err(LocalInstallError::new( + LocalInstallErrorKind::Publish, + format!("publish local voice asset bundle: {error}"), + )); + } + self.staging.keep = true; + if !verify(&self.final_root) { + let failed_root = self + .final_root + .with_file_name(format!(".voice-failed-{}", uuid::Uuid::new_v4())); + if let Err(error) = fs::rename(&self.final_root, &failed_root) { + let mut failure = LocalInstallError::new( + LocalInstallErrorKind::Rollback, + format!("preserve invalid published local voice asset bundle: {error}"), + ); + failure.recovery_paths.push(self.final_root.clone()); + if had_previous { + failure.recovery_paths.push(self.backup_root.clone()); + } + return Err(failure); + } + if had_previous { + if let Err(error) = fs::rename(&self.backup_root, &self.final_root) { + let mut failure = LocalInstallError::new( + LocalInstallErrorKind::Rollback, + format!("restore prior local voice asset bundle: {error}"), + ); + failure.recovery_paths.push(self.backup_root.clone()); + failure.recovery_paths.push(failed_root); + return Err(failure); + } + } + if let Err(error) = fs::remove_dir_all(&failed_root) { + let mut failure = LocalInstallError::new( + LocalInstallErrorKind::Cleanup, + format!("remove failed local voice asset publication: {error}"), + ); + failure.recovery_paths.push(failed_root); + return Err(failure); + } + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "published local voice asset bundle failed verification", + )); + } + if had_previous && fs::remove_dir_all(&self.backup_root).is_err() { + // The new verified tree is authoritative. Preserve the cleanup + // evidence without turning an applied install into a failure; + // the next locked recovery preflight retries stale cleanup. + return Ok(Some(self.backup_root)); + } + Ok(None) + } +} + +pub(crate) fn copy_exact_files( + source: &Path, + destination: &Path, + files: impl IntoIterator, +) -> Result<(), LocalInstallError> { + for (relative, size, hash) in files { + let source = source.join(relative); + let metadata = fs::symlink_metadata(&source).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Integrity, + format!("inspect retained local voice asset: {error}"), + ) + })?; + if !metadata.is_file() || metadata.file_type().is_symlink() || metadata.len() != size { + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "retained local voice asset is not a pinned regular file", + )); + } + let destination = destination.join(relative); + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("create staged local voice asset directory: {error}"), + ) + })?; + } + fs::copy(&source, &destination).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("copy retained local voice asset: {error}"), + ) + })?; + let mut file = File::open(&destination).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("open staged local voice asset: {error}"), + ) + })?; + let mut hasher = Sha256::new(); + std::io::copy(&mut file, &mut hasher).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("hash staged local voice asset: {error}"), + ) + })?; + if format!("{:x}", hasher.finalize()) != hash { + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "retained local voice asset checksum changed while staging", + )); + } + } + Ok(()) +} + +pub(crate) fn default_client() -> Result { + reqwest::Client::builder() + .connect_timeout(CONNECT_TIMEOUT) + .read_timeout(READ_TIMEOUT) + .timeout(TOTAL_TIMEOUT) + .build() + .map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Download, + format!("create local voice asset download client: {error}"), + ) + }) +} + +pub(crate) async fn download( + client: &reqwest::Client, + root: &Path, + spec: DownloadSpec<'_>, + mut on_chunk: impl FnMut(u64), +) -> Result<(), LocalInstallError> { + let destination = root.join(spec.relative_path); + if let Some(parent) = destination.parent() { + tokio::fs::create_dir_all(parent).await.map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("create local voice asset directory: {error}"), + ) + })?; + } + let response = client + .get(spec.source_url) + .send() + .await + .and_then(reqwest::Response::error_for_status) + .map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Download, + format!("download local voice asset: {error}"), + ) + })?; + let mut file = tokio::fs::File::create(destination) + .await + .map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("create local voice asset file: {error}"), + ) + })?; + let mut stream = response.bytes_stream(); + let mut size = 0_u64; + let mut hasher = Sha256::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Download, + format!("read local voice asset response: {error}"), + ) + })?; + size = size.checked_add(chunk.len() as u64).ok_or_else(|| { + LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "local voice asset size overflow", + ) + })?; + if size > spec.size_bytes { + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "local voice asset exceeded its pinned size", + )); + } + hasher.update(&chunk); + tokio::io::AsyncWriteExt::write_all(&mut file, &chunk) + .await + .map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("write local voice asset: {error}"), + ) + })?; + on_chunk(chunk.len() as u64); + } + tokio::io::AsyncWriteExt::flush(&mut file) + .await + .map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Io, + format!("flush local voice asset: {error}"), + ) + })?; + if size != spec.size_bytes || format!("{:x}", hasher.finalize()) != spec.sha256 { + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "local voice asset did not match its pinned size and checksum", + )); + } + Ok(()) +} + +pub(crate) fn extract_exact_tar_bz2( + archive_path: &Path, + destination: &Path, + archive_directory: &str, + expected: &[(&str, u64, &str)], +) -> Result<(), LocalInstallError> { + let archive = File::open(archive_path).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Extraction, + format!("open local voice asset archive: {error}"), + ) + })?; + let decoder = bzip2::read::BzDecoder::new(archive); + let mut archive = tar::Archive::new(decoder); + let mut remaining = expected.to_vec(); + for entry in archive.entries().map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Extraction, + format!("read local voice asset archive: {error}"), + ) + })? { + let mut entry = entry.map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Extraction, + format!("read local voice asset archive entry: {error}"), + ) + })?; + if !entry.header().entry_type().is_file() { + continue; + } + let path = entry.path().map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Extraction, + format!("read local voice asset archive path: {error}"), + ) + })?; + let Some(relative) = path + .strip_prefix(archive_directory) + .ok() + .and_then(|path| path.to_str()) + else { + continue; + }; + let Some(index) = remaining.iter().position(|(path, _, _)| *path == relative) else { + continue; + }; + let (_, expected_size, expected_hash) = remaining.remove(index); + let target = destination.join(relative); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Extraction, + format!("create extracted local voice asset directory: {error}"), + ) + })?; + } + let mut output = File::create(target).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Extraction, + format!("create extracted local voice asset: {error}"), + ) + })?; + let mut hasher = Sha256::new(); + let mut size = 0_u64; + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = entry.read(&mut buffer).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Extraction, + format!("read extracted local voice asset: {error}"), + ) + })?; + if read == 0 { + break; + } + size = size.checked_add(read as u64).ok_or_else(|| { + LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "extracted local voice asset size overflow", + ) + })?; + if size > expected_size { + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "extracted local voice asset exceeded its pinned size", + )); + } + hasher.update(&buffer[..read]); + output.write_all(&buffer[..read]).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Extraction, + format!("write extracted local voice asset: {error}"), + ) + })?; + } + if size != expected_size || format!("{:x}", hasher.finalize()) != expected_hash { + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "extracted local voice asset did not match its pinned size and checksum", + )); + } + } + if !remaining.is_empty() { + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "local voice asset archive omitted a pinned file", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + recover_interrupted_publication_with, try_lock_for_mutation, try_lock_for_read, + CombinedPublication, DownloadSpec, LocalAssetLockError, LocalAssetRoots, + LocalInstallErrorKind, + }; + use sha2::{Digest, Sha256}; + use std::fs; + use std::process::Command; + use std::time::Duration; + use tokio::io::AsyncWriteExt; + + fn roots(parent: &std::path::Path) -> LocalAssetRoots { + LocalAssetRoots::new( + parent, + parent.join("native-voice-v2"), + parent.join("native-voice-v2/stt"), + ) + .expect("asset roots") + } + + #[test] + fn process_lock_excludes_readers_and_mutators_until_release() { + if let Ok(path) = std::env::var("BERD_ASSET_LOCK_CHILD") { + let roots = roots(std::path::Path::new(&path)); + assert!(matches!( + try_lock_for_mutation(&roots), + Err(LocalAssetLockError::Busy) + )); + return; + } + let root = tempfile::tempdir().expect("temporary directory"); + let roots = roots(root.path()); + let first = try_lock_for_mutation(&roots).expect("first mutation lock"); + assert!(matches!( + try_lock_for_mutation(&roots), + Err(LocalAssetLockError::Busy) + )); + assert!(matches!( + try_lock_for_read(&roots), + Err(LocalAssetLockError::Busy) + )); + let child = Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "--exact", + "local_assets::tests::process_lock_excludes_readers_and_mutators_until_release", + "--nocapture", + ]) + .env("BERD_ASSET_LOCK_CHILD", root.path()) + .status() + .expect("run lock contender process"); + assert!(child.success()); + drop(first); + try_lock_for_read(&roots).expect("read lock after release"); + } + + #[test] + fn mutation_processes_serialize_and_recheck_after_the_lock() { + if let Ok(path) = std::env::var("BERD_ASSET_SERIAL_CHILD") { + let roots = roots(std::path::Path::new(&path)); + let _mutation = + super::lock_for_mutation_blocking(&roots).expect("wait for preceding mutation"); + assert_eq!( + fs::read(roots.pocket_bundle_root().join("winner")) + .expect("recheck preceding publication"), + b"first" + ); + fs::write(roots.coordination_root().join("second-observed"), b"yes") + .expect("record second mutation"); + return; + } + let root = tempfile::tempdir().expect("temporary directory"); + let roots = roots(root.path()); + let first = try_lock_for_mutation(&roots).expect("first mutation lock"); + let mut child = Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "--exact", + "local_assets::tests::mutation_processes_serialize_and_recheck_after_the_lock", + "--nocapture", + ]) + .env("BERD_ASSET_SERIAL_CHILD", root.path()) + .spawn() + .expect("spawn second mutation process"); + std::thread::sleep(Duration::from_millis(50)); + fs::create_dir_all(roots.pocket_bundle_root()).expect("create first publication"); + fs::write(roots.pocket_bundle_root().join("winner"), b"first") + .expect("write first publication"); + drop(first); + assert!(child.wait().expect("wait for second process").success()); + assert!(roots.coordination_root().join("second-observed").is_file()); + } + + #[test] + fn next_process_recovers_the_single_valid_interrupted_backup() { + if let Ok(path) = std::env::var("BERD_ASSET_CRASH_CHILD") { + let roots = roots(std::path::Path::new(&path)); + let _mutation = try_lock_for_mutation(&roots).expect("child mutation lock"); + fs::create_dir_all(roots.pocket_bundle_root()).expect("create live tree"); + fs::write(roots.pocket_bundle_root().join("valid"), b"yes").expect("write fixture"); + fs::rename( + roots.pocket_bundle_root(), + roots.coordination_root().join(".voice-backup-crashed"), + ) + .expect("inject exit between publication renames"); + return; + } + let root = tempfile::tempdir().expect("temporary directory"); + let child = Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "--exact", + "local_assets::tests::next_process_recovers_the_single_valid_interrupted_backup", + "--nocapture", + ]) + .env("BERD_ASSET_CRASH_CHILD", root.path()) + .status() + .expect("run interrupted publisher process"); + assert!(child.success()); + + let roots = roots(root.path()); + let _mutation = try_lock_for_mutation(&roots).expect("mutation lock"); + recover_interrupted_publication_with(&roots, |candidate| candidate.join("valid").is_file()) + .expect("recover interrupted publication"); + assert_eq!( + fs::read(roots.pocket_bundle_root().join("valid")).expect("restored fixture"), + b"yes" + ); + assert!(!roots + .coordination_root() + .join(".voice-backup-crashed") + .exists()); + } + + #[test] + fn failed_final_verification_restores_the_previous_tree() { + let root = tempfile::tempdir().expect("temporary directory"); + let roots = roots(root.path()); + fs::create_dir(roots.pocket_bundle_root()).expect("create old bundle"); + fs::write(roots.pocket_bundle_root().join("old"), b"old").expect("old bundle"); + let publication = CombinedPublication::prepare(&roots).expect("publication"); + fs::write(publication.root().join("new"), b"new").expect("new bundle"); + let error = publication + .publish(|_| false) + .expect_err("verification failure"); + assert_eq!(error.kind, LocalInstallErrorKind::Integrity); + assert_eq!( + fs::read(roots.pocket_bundle_root().join("old")).expect("restored old bundle"), + b"old" + ); + assert!(!roots.pocket_bundle_root().join("new").exists()); + } + + #[test] + fn combined_publication_preserves_each_counterpart_direction() { + for target in ["pocket", "parakeet"] { + let root = tempfile::tempdir().expect("temporary directory"); + let roots = roots(root.path()); + fs::create_dir_all(roots.parakeet_bundle_root()).expect("create combined bundle"); + fs::write(roots.pocket_bundle_root().join("pocket"), b"old-pocket") + .expect("write Pocket counterpart"); + fs::write( + roots.parakeet_bundle_root().join("parakeet"), + b"old-parakeet", + ) + .expect("write Parakeet counterpart"); + let publication = CombinedPublication::prepare(&roots).expect("publication"); + fs::create_dir_all(publication.root().join("stt")).expect("create stage"); + let pocket = if target == "pocket" { + b"new-pocket" + } else { + b"old-pocket" + }; + let parakeet = if target == "parakeet" { + b"new-parakeet" + } else { + b"old-parakeet" + }; + fs::write(publication.root().join("pocket"), pocket).expect("stage Pocket"); + fs::write(publication.root().join("stt/parakeet"), parakeet).expect("stage Parakeet"); + publication + .publish(|candidate| { + candidate.join("pocket").is_file() && candidate.join("stt/parakeet").is_file() + }) + .expect("publish combined bundle"); + assert_eq!( + fs::read(roots.pocket_bundle_root().join("pocket")).expect("Pocket final"), + pocket + ); + assert_eq!( + fs::read(roots.parakeet_bundle_root().join("parakeet")).expect("Parakeet final"), + parakeet + ); + } + } + + #[test] + fn failed_final_verification_reports_rollback_failure_and_preserves_evidence() { + let root = tempfile::tempdir().expect("temporary directory"); + let roots = roots(root.path()); + fs::create_dir(roots.pocket_bundle_root()).expect("create old bundle"); + fs::write(roots.pocket_bundle_root().join("old"), b"old").expect("old bundle"); + let publication = CombinedPublication::prepare(&roots).expect("publication"); + fs::write(publication.root().join("new"), b"new").expect("new bundle"); + let backup = publication.backup_root.clone(); + let error = publication + .publish(|_| { + fs::remove_dir_all(&backup).expect("inject missing rollback source"); + false + }) + .expect_err("rollback failure"); + assert_eq!(error.kind, LocalInstallErrorKind::Rollback); + assert!(!error.recovery_paths.is_empty()); + assert!(error.recovery_paths.iter().any(|path| path.exists())); + } + + #[test] + fn exact_copy_preserves_only_pinned_regular_files() { + let root = tempfile::tempdir().expect("temporary directory"); + let source = root.path().join("source"); + let destination = root.path().join("destination"); + fs::create_dir(&source).expect("create source"); + fs::write(source.join("keep"), b"keep").expect("write pinned file"); + fs::write(source.join("unknown"), b"unknown").expect("write unknown file"); + super::copy_exact_files( + &source, + &destination, + [( + "keep", + 4, + "6ca7ea2feefc88ecb5ed6356ed963f47dc9137f82526fdd25d618ea626d0803f", + )], + ) + .expect("copy exact manifest"); + assert_eq!( + fs::read(destination.join("keep")).expect("pinned copy"), + b"keep" + ); + assert!(!destination.join("unknown").exists()); + } + + #[cfg(unix)] + #[test] + fn exact_copy_rejects_a_pinned_symlink() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().expect("temporary directory"); + let source = root.path().join("source"); + fs::create_dir(&source).expect("create source"); + fs::write(source.join("target"), b"keep").expect("write target"); + symlink(source.join("target"), source.join("keep")).expect("create symlink"); + let error = super::copy_exact_files( + &source, + &root.path().join("destination"), + [( + "keep", + 4, + "6ca7ea2feefc88ecb5ed6356ed963f47dc9137f82526fdd25d618ea626d0803f", + )], + ) + .expect_err("symlink must be rejected"); + assert_eq!(error.kind, LocalInstallErrorKind::Integrity); + } + + #[test] + fn archive_extraction_writes_only_exact_regular_manifest_entries() { + use bzip2::write::BzEncoder; + use bzip2::Compression; + + let root = tempfile::tempdir().expect("temporary directory"); + let archive_path = root.path().join("fixture.tar.bz2"); + let archive_file = fs::File::create(&archive_path).expect("create archive"); + let encoder = BzEncoder::new(archive_file, Compression::fast()); + let mut archive = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_size(4); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "bundle/keep", &b"keep"[..]) + .expect("append pinned entry"); + let mut unknown = tar::Header::new_gnu(); + unknown.set_size(7); + unknown.set_mode(0o644); + unknown.set_cksum(); + archive + .append_data(&mut unknown, "bundle/unknown", &b"unknown"[..]) + .expect("append unknown entry"); + let encoder = archive.into_inner().expect("finish tar"); + encoder.finish().expect("finish compression"); + + let destination = root.path().join("output"); + super::extract_exact_tar_bz2( + &archive_path, + &destination, + "bundle", + &[( + "keep", + 4, + "6ca7ea2feefc88ecb5ed6356ed963f47dc9137f82526fdd25d618ea626d0803f", + )], + ) + .expect("extract exact manifest"); + assert_eq!( + fs::read(destination.join("keep")).expect("pinned output"), + b"keep" + ); + assert!(!destination.join("unknown").exists()); + } + + #[tokio::test] + async fn bounded_download_rejects_a_response_larger_than_the_manifest() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind fixture server"); + let address = listener.local_addr().expect("fixture address"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept request"); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nlarge") + .await + .expect("write response"); + }); + let root = tempfile::tempdir().expect("temporary directory"); + let error = super::download( + &reqwest::Client::new(), + root.path(), + DownloadSpec { + source_url: Box::leak(format!("http://{address}/asset").into_boxed_str()), + relative_path: "asset", + size_bytes: 4, + sha256: Box::leak(format!("{:x}", Sha256::digest(b"nope")).into_boxed_str()), + }, + |_| {}, + ) + .await + .expect_err("oversized response"); + assert_eq!(error.kind, LocalInstallErrorKind::Integrity); + server.await.expect("fixture server"); + } + + #[tokio::test] + async fn download_honors_the_clients_read_timeout_for_a_stalled_body() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind fixture server"); + let address = listener.local_addr().expect("fixture address"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept request"); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nx") + .await + .expect("write partial response"); + tokio::time::sleep(Duration::from_secs(1)).await; + }); + let client = reqwest::Client::builder() + .read_timeout(Duration::from_millis(50)) + .build() + .expect("timeout client"); + let root = tempfile::tempdir().expect("temporary directory"); + let error = super::download( + &client, + root.path(), + DownloadSpec { + source_url: Box::leak(format!("http://{address}/asset").into_boxed_str()), + relative_path: "asset", + size_bytes: 2, + sha256: Box::leak(format!("{:x}", Sha256::digest(b"xx")).into_boxed_str()), + }, + |_| {}, + ) + .await + .expect_err("stalled response"); + assert_eq!(error.kind, LocalInstallErrorKind::Download); + server.abort(); + } +} diff --git a/src-tauri/crates/berd-voice/src/mac_speech.rs b/src-tauri/crates/berd-voice/src/mac_speech.rs new file mode 100644 index 000000000..f5135d9cf --- /dev/null +++ b/src-tauri/crates/berd-voice/src/mac_speech.rs @@ -0,0 +1,330 @@ +//! Safe macOS SpeechTranscriber model and recognition primitives. + +use std::{ + ffi::{c_char, c_void, CStr}, + ptr, + sync::atomic::{AtomicBool, Ordering}, +}; + +use serde::Deserialize; +use tokio::sync::mpsc; + +use crate::MAC_SPEECH_RECOGNITION_FINISH_TIMEOUT_SECONDS; + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct MacSpeechEngineStatus { + pub supported: bool, + pub locale: Option, + pub locale_supported: bool, + pub model_status: String, + pub ready: bool, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum MacSpeechRecognitionEvent { + Final(String), + Finished, + Failed(String), +} + +unsafe extern "C" { + fn berd_macos_stt_is_supported() -> bool; + fn berd_macos_stt_status_json( + locale: *const c_char, + error_out: *mut *mut c_char, + ) -> *mut c_char; + fn berd_macos_stt_install_model( + locale: *const c_char, + progress: Option, + context: *mut c_void, + error_out: *mut *mut c_char, + ) -> bool; + fn berd_macos_stt_create( + locale: *const c_char, + event: Option, + context: *mut c_void, + error_out: *mut *mut c_char, + ) -> *mut c_void; + fn berd_macos_stt_push( + handle: *mut c_void, + samples: *const f32, + count: isize, + sample_rate: f64, + error_out: *mut *mut c_char, + ) -> bool; + fn berd_macos_stt_finish( + handle: *mut c_void, + timeout_seconds: f64, + error_out: *mut *mut c_char, + ) -> bool; + fn berd_macos_stt_cancel(handle: *mut c_void); + fn berd_macos_stt_release(handle: *mut c_void); + fn berd_macos_stt_free_string(value: *mut c_char); +} + +fn take_string(value: *mut c_char) -> Option { + if value.is_null() { + return None; + } + let result = unsafe { CStr::from_ptr(value) } + .to_string_lossy() + .into_owned(); + unsafe { berd_macos_stt_free_string(value) }; + Some(result) +} + +fn take_error(value: *mut c_char, fallback: &str) -> String { + take_string(value).unwrap_or_else(|| fallback.to_string()) +} + +pub fn mac_speech_is_supported() -> bool { + unsafe { berd_macos_stt_is_supported() } +} + +/// Reads the engine status for the current system locale. +pub fn mac_speech_status() -> Result { + let mut error = ptr::null_mut(); + let json = unsafe { berd_macos_stt_status_json(ptr::null(), &mut error) }; + let json = take_string(json) + .ok_or_else(|| take_error(error, "Could not read the macOS speech recognition status."))?; + decode_status(&json) +} + +fn decode_status(json: &str) -> Result { + serde_json::from_str(json).map_err(|error| format!("decode macOS speech status: {error}")) +} + +struct ProgressContext { + callback: Box, + panicked: AtomicBool, +} + +unsafe extern "C" fn install_progress(value: f64, context: *mut c_void) { + if context.is_null() { + return; + } + let context = unsafe { &*context.cast::() }; + if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + (context.callback)(value); + })) + .is_err() + { + context.panicked.store(true, Ordering::Release); + } +} + +/// Blocks while installing the current locale's on-device speech model. +/// +/// The progress callback remains alive for the entire native operation and is +/// never retained after this function returns. +pub fn install_mac_speech_model( + progress: impl Fn(f64) + Send + Sync + 'static, +) -> Result<(), String> { + let context = Box::into_raw(Box::new(ProgressContext { + callback: Box::new(progress), + panicked: AtomicBool::new(false), + })); + let mut error = ptr::null_mut(); + let installed = unsafe { + berd_macos_stt_install_model( + ptr::null(), + Some(install_progress), + context.cast(), + &mut error, + ) + }; + let callback_panicked = unsafe { (*context).panicked.load(Ordering::Acquire) }; + unsafe { drop(Box::from_raw(context)) }; + let install_error = (!installed).then(|| { + take_error( + error, + "Could not install the macOS speech recognition model.", + ) + }); + if callback_panicked { + Err("macOS speech model progress callback panicked.".to_string()) + } else if let Some(error) = install_error { + Err(error) + } else { + Ok(()) + } +} + +struct RecognitionContext { + events: mpsc::UnboundedSender, +} + +fn map_recognition_event(code: i32, text: Option) -> Option { + match code { + 1 => Some(MacSpeechRecognitionEvent::Final(text.unwrap_or_default())), + 2 => Some(MacSpeechRecognitionEvent::Finished), + 3 => Some(MacSpeechRecognitionEvent::Failed(text.unwrap_or_else( + || "macOS speech recognition failed.".to_string(), + ))), + _ => None, + } +} + +unsafe extern "C" fn recognition_event(code: i32, text: *const c_char, context: *mut c_void) { + if context.is_null() { + return; + } + let context = unsafe { &*context.cast::() }; + let text = + (!text.is_null()).then(|| unsafe { CStr::from_ptr(text).to_string_lossy().into_owned() }); + if let Some(event) = map_recognition_event(code, text) { + let _ = context.events.send(event); + } +} + +/// A concrete macOS SpeechTranscriber session for mono Float32 PCM. +pub struct MacSpeechRecognizer { + handle: *mut c_void, + context: *mut RecognitionContext, +} + +impl MacSpeechRecognizer { + /// Creates a recognizer for the current system locale. + pub fn new() -> Result<(Self, mpsc::UnboundedReceiver), String> { + let (events, receiver) = mpsc::unbounded_channel(); + let context = Box::into_raw(Box::new(RecognitionContext { events })); + let mut error = ptr::null_mut(); + let handle = unsafe { + berd_macos_stt_create( + ptr::null(), + Some(recognition_event), + context.cast(), + &mut error, + ) + }; + if handle.is_null() { + unsafe { drop(Box::from_raw(context)) }; + return Err(take_error( + error, + "Could not start macOS speech recognition.", + )); + } + Ok((Self { handle, context }, receiver)) + } + + /// Synchronously copies one batch of 48 kHz mono Float32 PCM into the + /// recognizer's native bounded input stream. + pub fn push_48khz_mono_f32(&mut self, samples: &[f32]) -> Result<(), String> { + let mut error = ptr::null_mut(); + let pushed = unsafe { + berd_macos_stt_push( + self.handle, + samples.as_ptr(), + samples.len() as isize, + 48_000.0, + &mut error, + ) + }; + if pushed { + Ok(()) + } else { + Err(take_error( + error, + "Could not send audio to macOS speech recognition.", + )) + } + } + + /// Finalizes input and waits up to five seconds for native completion. + pub fn finish(&mut self) -> Result<(), String> { + let mut error = ptr::null_mut(); + let finished = unsafe { + berd_macos_stt_finish( + self.handle, + MAC_SPEECH_RECOGNITION_FINISH_TIMEOUT_SECONDS as f64, + &mut error, + ) + }; + if finished { + Ok(()) + } else { + Err(take_error( + error, + "macOS speech recognition did not finish.", + )) + } + } + + /// Stops recognition and suppresses any later callbacks. Idempotent. + pub fn cancel(&mut self) { + unsafe { berd_macos_stt_cancel(self.handle) }; + } +} + +impl Drop for MacSpeechRecognizer { + fn drop(&mut self) { + unsafe { + berd_macos_stt_release(self.handle); + drop(Box::from_raw(self.context)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + + #[test] + fn native_event_codes_map_without_provider_types_escaping() { + assert_eq!( + map_recognition_event(1, Some("hello".to_string())), + Some(MacSpeechRecognitionEvent::Final("hello".to_string())) + ); + assert_eq!( + map_recognition_event(2, None), + Some(MacSpeechRecognitionEvent::Finished) + ); + assert_eq!( + map_recognition_event(3, None), + Some(MacSpeechRecognitionEvent::Failed( + "macOS speech recognition failed.".to_string() + )) + ); + assert_eq!(map_recognition_event(99, None), None); + } + + #[test] + fn native_status_maps_without_tauri_projection() { + assert_eq!( + decode_status( + r#"{"supported":true,"locale":"en-US","localeSupported":true,"modelStatus":"downloading","ready":false}"# + ) + .unwrap(), + MacSpeechEngineStatus { + supported: true, + locale: Some("en-US".to_string()), + locale_supported: true, + model_status: "downloading".to_string(), + ready: false, + } + ); + } + + #[test] + fn callback_is_safe_after_the_event_receiver_is_dropped() { + let (events, receiver) = mpsc::unbounded_channel(); + drop(receiver); + let context = Box::into_raw(Box::new(RecognitionContext { events })); + let text = CString::new("ignored").unwrap(); + unsafe { recognition_event(1, text.as_ptr(), context.cast()) }; + unsafe { drop(Box::from_raw(context)) }; + } + + #[test] + fn progress_callback_cannot_unwind_across_swift() { + let context = Box::into_raw(Box::new(ProgressContext { + callback: Box::new(|_| panic!("test panic")), + panicked: AtomicBool::new(false), + })); + unsafe { install_progress(0.5, context.cast()) }; + assert!(unsafe { (*context).panicked.load(Ordering::Acquire) }); + unsafe { drop(Box::from_raw(context)) }; + } +} diff --git a/src-tauri/src/commands/pocket_audio_player.rs b/src-tauri/crates/berd-voice/src/macos_audio_output.rs similarity index 69% rename from src-tauri/src/commands/pocket_audio_player.rs rename to src-tauri/crates/berd-voice/src/macos_audio_output.rs index bda7831ec..f48a6eeaa 100644 --- a/src-tauri/src/commands/pocket_audio_player.rs +++ b/src-tauri/crates/berd-voice/src/macos_audio_output.rs @@ -1,8 +1,8 @@ -//! Safe ownership wrapper for the macOS AVAudioUnitTimePitch Pocket player. +//! Safe ownership wrapper for the shared macOS AVAudioUnitTimePitch PCM player. use std::ffi::{c_char, c_void, CStr}; -const MAX_POCKET_PLAYBACK_SPEED: f32 = 2.0; +use crate::PcmAudioOutput; unsafe extern "C" { fn berd_pocket_audio_player_create( @@ -17,11 +17,6 @@ unsafe extern "C" { frame_count: u32, error_out: *mut *mut c_char, ) -> bool; - fn berd_pocket_audio_player_set_rate( - player: *mut c_void, - rate: f32, - error_out: *mut *mut c_char, - ) -> bool; fn berd_pocket_audio_player_completed_source_frames(player: *mut c_void) -> u64; fn berd_pocket_audio_player_pending_buffers(player: *mut c_void) -> u64; fn berd_pocket_audio_player_failed(player: *mut c_void) -> bool; @@ -30,13 +25,13 @@ unsafe extern "C" { fn berd_siri_tts_free_string(value: *mut c_char); } -pub(super) struct PocketAudioPlayer { +pub struct PocketAudioPlayer { raw: *mut c_void, delivery_safety_frames: u64, } impl PocketAudioPlayer { - pub(super) fn new( + pub fn new( sample_rate: u32, rate: f32, output_device_name: Option<&str>, @@ -55,35 +50,20 @@ impl PocketAudioPlayer { berd_pocket_audio_player_create(sample_rate, rate, output_device_id, &mut error) }; if raw.is_null() { - return Err(take_error(error, "Could not start native Pocket playback")); + return Err(take_error(error, "Could not start native PCM playback")); } Ok(Self { raw, - delivery_safety_frames: delivery_safety_frames(sample_rate, MAX_POCKET_PLAYBACK_SPEED), + delivery_safety_frames: delivery_safety_frames(sample_rate, rate), }) } - pub(super) fn set_rate(&self, rate: f32) -> Result<(), String> { - let mut error = std::ptr::null_mut(); - // SAFETY: `self.raw` is a live retained player and the bridge validates - // the rate before updating the connected time-pitch unit. - let updated = unsafe { berd_pocket_audio_player_set_rate(self.raw, rate, &mut error) }; - if updated { - Ok(()) - } else { - Err(take_error( - error, - "Could not update native Pocket playback speed", - )) - } - } - - pub(super) fn enqueue(&self, samples: &[f32]) -> Result<(), String> { + pub fn enqueue(&self, samples: &[f32]) -> Result<(), String> { if samples.is_empty() { return Ok(()); } let frame_count = u32::try_from(samples.len()) - .map_err(|_| "Pocket audio chunk is too large to queue".to_string())?; + .map_err(|_| "PCM audio chunk is too large to queue".to_string())?; let mut error = std::ptr::null_mut(); // SAFETY: The bridge copies `frame_count` samples before returning and // `self.raw` remains retained for this wrapper's lifetime. @@ -93,37 +73,59 @@ impl PocketAudioPlayer { if enqueued { Ok(()) } else { - Err(take_error(error, "Could not queue native Pocket audio")) + Err(take_error(error, "Could not queue native PCM audio")) } } - pub(super) fn played_frames(&self) -> u64 { + pub fn played_frames(&self) -> u64 { // SAFETY: `self.raw` is a live retained player. The bridge counts only // source buffers confirmed played back, so idle queue gaps add nothing. apply_delivery_safety(self.completed_source_frames(), self.delivery_safety_frames) } - pub(super) fn completed_source_frames(&self) -> u64 { + pub fn completed_source_frames(&self) -> u64 { // SAFETY: `self.raw` is a live retained player. unsafe { berd_pocket_audio_player_completed_source_frames(self.raw) } } - pub(super) fn is_empty(&self) -> bool { + pub fn is_empty(&self) -> bool { // SAFETY: `self.raw` is a live retained player. unsafe { berd_pocket_audio_player_pending_buffers(self.raw) == 0 } } - pub(super) fn ensure_healthy(&self) -> Result<(), String> { + pub fn check_health(&self) -> Result<(), String> { // SAFETY: `self.raw` is a live retained player. playback_health(unsafe { berd_pocket_audio_player_failed(self.raw) }) } - pub(super) fn stop(&self) { + pub fn stop(&self) { // SAFETY: `self.raw` is a live retained player and stop is idempotent. unsafe { berd_pocket_audio_player_stop(self.raw) }; } } +impl PcmAudioOutput for PocketAudioPlayer { + fn write(&self, samples: &[f32]) -> Result<(), String> { + self.enqueue(samples) + } + + fn cancel(&self) { + self.stop(); + } + + fn is_drained(&self) -> bool { + self.is_empty() + } + + fn check_health(&self) -> Result<(), String> { + PocketAudioPlayer::check_health(self) + } + + fn played_frames(&self) -> u64 { + PocketAudioPlayer::played_frames(self) + } +} + fn delivery_safety_frames(sample_rate: u32, rate: f32) -> u64 { (f64::from(sample_rate) * 0.1 * f64::from(rate)).ceil() as u64 } @@ -134,7 +136,7 @@ fn apply_delivery_safety(completed_source_frames: u64, safety_frames: u64) -> u6 fn playback_health(failed: bool) -> Result<(), String> { if failed { - Err("Pocket audio output stopped unexpectedly".to_string()) + Err("PCM audio output stopped unexpectedly".to_string()) } else { Ok(()) } @@ -163,7 +165,12 @@ fn take_error(error: *mut c_char, fallback: &str) -> String { #[cfg(test)] mod tests { use super::{ - apply_delivery_safety, delivery_safety_frames, playback_health, MAX_POCKET_PLAYBACK_SPEED, + apply_delivery_safety, delivery_safety_frames, playback_health, PocketAudioPlayer, + }; + use crate::wait_until_drained; + use std::{ + sync::atomic::AtomicBool, + time::{Duration, Instant}, }; #[test] @@ -189,19 +196,37 @@ mod tests { ); } - #[test] - fn live_rate_changes_reserve_maximum_delivery_safety() { - let safety = delivery_safety_frames(24_000, MAX_POCKET_PLAYBACK_SPEED); - assert_eq!(safety, 4_800); - assert_eq!(apply_delivery_safety(10_000, safety), 5_200); - } - #[test] fn unexpected_output_stops_fail_playback() { assert!(playback_health(false).is_ok()); assert_eq!( playback_health(true).expect_err("unexpected stop must fail"), - "Pocket audio output stopped unexpectedly" + "PCM audio output stopped unexpectedly" ); } + + #[test] + #[ignore = "opens the default CoreAudio output and queues silent PCM"] + fn cancelling_queued_silence_returns_promptly() { + let output = PocketAudioPlayer::new(48_000, 1.0, None).unwrap(); + output.enqueue(&vec![0.0; 48_000 * 30]).unwrap(); + let started = Instant::now(); + output.stop(); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(output.is_empty()); + } + + #[test] + #[ignore = "requires BERD_MULTICHANNEL_TEST_OUTPUT_DEVICE naming a multi-channel CoreAudio output"] + fn configured_multichannel_output_plays_pcm() { + let device = std::env::var("BERD_MULTICHANNEL_TEST_OUTPUT_DEVICE").unwrap(); + for frame_count in [4_800, 48_000 * 3] { + let output = PocketAudioPlayer::new(48_000, 1.0, Some(&device)).unwrap(); + output.enqueue(&vec![0.0; frame_count]).unwrap(); + assert!( + wait_until_drained(&output, &AtomicBool::new(true), Duration::from_millis(5),) + .unwrap() + ); + } + } } diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs new file mode 100644 index 000000000..f8d4a5eac --- /dev/null +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -0,0 +1,6649 @@ +use std::io::{self, BufWriter, Read, Write}; +use std::os::fd::RawFd; +use std::path::{Path, PathBuf}; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, Receiver, SyncSender}, + Arc, +}; +use std::thread; +use std::time::{Duration, Instant}; + +use berd_voice::benchmark::{ + benchmark_stt, benchmark_tts, benchmark_tts_manifest, load_bundled_stt_fixture_pack, + load_bundled_tts_prompt_manifest, SttBenchmarkEnvironment, SttBenchmarkMode, + SttBenchmarkTarget, TtsBenchmarkMode, TtsBenchmarkPromptManifest, TtsBenchmarkTarget, +}; +use berd_voice::input::{ + AssistantActivityGuard, InputDuringTtsSlot, InputDuringTtsSnapshot, VoiceInputConfig, + VoiceInputControls, VoiceInputEngineConfig, VoiceInputEvent, VoiceInputFrame, + VoiceInputRuntime, INPUT_FRAME_SAMPLES, +}; +use berd_voice::protocol::{ + CancelOutcome, InputDuringTtsOutcome, NotAdmittedReason, OutputReadyOutcome, SessionMessage, + SessionRequest, TtsSettingsOutcome, VoiceSessionSnapshot, +}; +use berd_voice::session::{PrepareOutcome, PrepareRequest, SessionCore}; +use berd_voice::{ + estimated_spoken_through_utf8, + local_assets::{ + LocalAssetLockError, LocalAssetRoots, LocalInstallError, LocalInstallErrorKind, + LocalInstallPhase, LocalInstallProgress, + }, + ConfiguredTtsSlot, DeliveryProgress, TtsBackend, TtsConfiguration, TtsConfigurationLease, + TtsConfigurationRejection, TtsConfigurationRejectionKind, WavSynthesisErrorKind, +}; +use serde::Serialize; + +mod session_audio; + +use session_audio::{ + AudioHostAck, AudioOutputControlRequest, AudioPipeTransport, RemotePcmAudioOutput, + AUDIO_CANCELLED, +}; + +const WIRE_MARKER: u32 = 2; +const MAX_LINE_BYTES: usize = 1024 * 1024; +const FRAME_MAGIC: [u8; 2] = *b"BV"; +const JSON_FRAME_KIND: u8 = 1; +const PCM_FRAME_KIND: u8 = 2; +const FRAME_HEADER_BYTES: usize = 8; +const PCM_FRAME_BYTES: usize = INPUT_FRAME_SAMPLES * std::mem::size_of::(); +const MAX_FINAL_TEXT_BYTES: usize = 64 * 1024; +const MAX_SPEAK_TEXT_BYTES: usize = 16 * 1024; +const INPUT_QUEUE_CAPACITY: usize = 32; +const INPUT_STARTUP_TIMEOUT: Duration = Duration::from_secs(60); +const SHUTDOWN_PLAYBACK_TIMEOUT: Duration = Duration::from_secs(3); +const TTS_CONFIGURATION_TIMEOUT: Duration = Duration::from_secs(120); +const MAX_OPENAI_BENCHMARK_REQUESTS: usize = 20; +const MAX_OPENAI_BENCHMARK_TEXT_BYTES: usize = 64 * 1024; +const MAX_OPENAI_STT_BENCHMARK_SECONDS: f64 = 120.0; + +enum Input { + Request(SessionRequest), + Pcm(Box), + Invalid(String), + Eof, +} + +struct OrderedControl { + after_pcm: u64, + input: Input, +} + +#[derive(Debug)] +enum PlaybackEvent { + #[cfg(test)] + Started(u64), + Completed(u64), + Interrupted(u64, u64), + Failed(u64, String, bool), +} + +#[derive(Debug)] +struct PlaybackFailure { + message: String, + output_quiescent: bool, +} + +struct TtsConfigurationEvent { + attempt: u64, + id: u64, + result: Result, +} + +#[derive(Clone, Copy, Debug)] +struct ActiveTtsConfigurationUpdate { + attempt: u64, + id: u64, + deadline: Instant, +} + +struct ActivePlayback { + prepare_id: u64, + speech_id: u64, + text: String, + output: Option>, + active: Option>, + ready_deadline: Instant, + assistant_activity: Option, + input_during_tts: InputDuringTtsSnapshot, + tts: TtsConfigurationLease, + suspension_requested: bool, +} + +#[derive(Clone, Debug, PartialEq)] +enum TtsBackendConfig { + OpenAi { + rate: f32, + }, + Siri { + voice: String, + language: String, + rate: f32, + }, + Pocket { + model_dir: PathBuf, + voice: String, + rate: f32, + }, +} + +#[derive(Clone, Debug, PartialEq)] +enum SttBackendConfig { + Macos, + Parakeet { model_dir: PathBuf }, + OpenAi, +} + +#[derive(Clone, Debug, PartialEq)] +struct SessionConfig { + tts: TtsBackendConfig, + stt: SttBackendConfig, +} + +#[derive(Clone, Debug, PartialEq)] +struct TtsBenchmarkConfig { + tts: TtsBackendConfig, + prompts: TtsBenchmarkPrompts, + mode: TtsBenchmarkMode, +} + +#[derive(Clone, Debug, PartialEq)] +enum TtsBenchmarkPrompts { + ExactRepeat { text: String, runs: usize }, + Manifest(TtsBenchmarkPromptManifest), +} + +#[derive(Clone, Debug, PartialEq)] +struct SttBenchmarkConfig { + stt: SttBackendConfig, + runs: usize, + mode: SttBenchmarkMode, + allow_paid_openai: bool, +} + +#[derive(Clone, Debug, PartialEq)] +enum SynthesisTtsConfig { + OpenAi { + model: String, + voice: String, + rate: f32, + }, + Local(TtsBackendConfig), +} + +#[derive(Clone, Debug, PartialEq)] +struct SynthesisConfig { + tts: SynthesisTtsConfig, + text: String, + output: PathBuf, +} + +impl SynthesisConfig { + fn backend(&self) -> &'static str { + match &self.tts { + SynthesisTtsConfig::OpenAi { .. } => "openai", + SynthesisTtsConfig::Local(TtsBackendConfig::Siri { .. }) => "siri", + SynthesisTtsConfig::Local(TtsBackendConfig::Pocket { .. }) => "pocket", + SynthesisTtsConfig::Local(TtsBackendConfig::OpenAi { .. }) => { + unreachable!("OpenAI synthesis carries explicit identity") + } + } + } +} + +const MANAGEMENT_SCHEMA_VERSION: u32 = 1; + +#[derive(Clone, Debug, PartialEq, Eq)] +enum ManagementCommand { + ListVoices { + language: Option, + }, + DownloadVoice { + identity: berd_voice::siri::SiriVoiceIdentity, + availability_wait: berd_voice::siri::SiriDownloadAvailabilityWait, + }, + MacosModelStatus, + InstallMacosModel, + PocketModelStatus { + roots: LocalAssetRoots, + }, + InstallPocketModel { + roots: LocalAssetRoots, + }, + ListPocketVoices, + ParakeetModelStatus { + roots: LocalAssetRoots, + }, + InstallParakeetModel { + roots: LocalAssetRoots, + }, +} + +impl ManagementCommand { + fn operation(&self) -> &'static str { + match self { + Self::ListVoices { .. } => "voices.list", + Self::DownloadVoice { .. } => "voices.download", + Self::MacosModelStatus => "models.macos.status", + Self::InstallMacosModel => "models.macos.install", + Self::PocketModelStatus { .. } => "models.pocket.status", + Self::InstallPocketModel { .. } => "models.pocket.install", + Self::ListPocketVoices => "models.pocket.voices", + Self::ParakeetModelStatus { .. } => "models.parakeet.status", + Self::InstallParakeetModel { .. } => "models.parakeet.install", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum LocalModelKind { + Pocket, + Parakeet, +} + +impl LocalModelKind { + fn backend(self) -> &'static str { + match self { + Self::Pocket => "pocket", + Self::Parakeet => "parakeet", + } + } + + fn model_id(self) -> &'static str { + match self { + Self::Pocket => berd_voice::pocket_assets::MODEL_ID, + Self::Parakeet => berd_voice::parakeet_assets::MODEL_ID, + } + } + + fn total_download_bytes(self) -> u64 { + match self { + Self::Pocket => berd_voice::pocket_assets::download_bytes(), + Self::Parakeet => berd_voice::parakeet_assets::download_bytes(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum LocalModelState { + Missing, + Invalid, + Ready { verified_bytes: u64 }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +struct LocalModelStatusResult { + backend: &'static str, + model_id: &'static str, + state: &'static str, + ready: bool, + verified_bytes: Option, + total_download_bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +struct LocalModelInstallResult { + backend: &'static str, + model_id: &'static str, + outcome: &'static str, + ready: bool, + verified_bytes: u64, + cleanup_pending: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +struct PocketVoicesResult { + backend: &'static str, + model_id: &'static str, + voice_license_id: &'static str, + voices: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +struct PocketVoiceResult { + id: &'static str, + name: &'static str, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +struct VoicesListResult { + backend: &'static str, + supported: bool, + language_filter: Option, + available_languages: Vec, + voices: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +struct VoiceDownloadResult { + backend: &'static str, + voice: berd_voice::siri::SiriVoiceIdentity, + installed: bool, + availability_wait_seconds: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +struct MacosModelStatus { + supported: bool, + locale: Option, + locale_supported: bool, + model_status: String, + ready: bool, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ManagementResultEnvelope { + schema_version: u32, + operation: &'static str, + event: &'static str, + result: T, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ManagementProgressEnvelope { + schema_version: u32, + operation: &'static str, + event: &'static str, + fraction: f64, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct LocalModelProgressEnvelope { + schema_version: u32, + operation: &'static str, + event: &'static str, + phase: &'static str, + downloaded_bytes: u64, + total_download_bytes: u64, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ManagementErrorEnvelope { + schema_version: u32, + operation: &'static str, + event: &'static str, + error: ManagementErrorBody, +} + +#[derive(Serialize)] +struct ManagementErrorBody { + code: &'static str, + message: &'static str, +} + +#[derive(Debug)] +struct ManagementFailure { + code: &'static str, + public_message: &'static str, + detail: String, +} + +#[derive(Debug)] +struct SynthesisFailure { + code: &'static str, + public_message: &'static str, + detail: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct SynthesisResult { + backend: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + model: Option, + voice: String, + #[serde(skip_serializing_if = "Option::is_none")] + language: Option, + rate: f32, + wav: SynthesisWavResult, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct SynthesisWavResult { + encoding: &'static str, + sample_rate: u32, + channels: u16, + bits_per_sample: u16, + frames: u64, + duration_ms: f64, + bytes: u64, +} + +fn main() { + let args: Vec<_> = std::env::args().collect(); + match args.get(1).map(String::as_str) { + Some("session") => { + let pcm_output_fd = + parse_pcm_output_fd(&args).unwrap_or_else(|error| usage_error(&error)); + let config = parse_args(&args).unwrap_or_else(|error| usage_error(&error)); + if let Err(error) = run_session(config, pcm_output_fd) { + eprintln!("berd-voice session failed: {error}"); + std::process::exit(1); + } + } + Some("benchmark") if args.get(2).map(String::as_str) == Some("tts") => { + let config = + parse_tts_benchmark_args(&args).unwrap_or_else(|error| usage_error(&error)); + if let Err(error) = run_tts_benchmark(config) { + eprintln!("berd-voice benchmark tts failed: {error}"); + std::process::exit(1); + } + } + Some("benchmark") if args.get(2).map(String::as_str) == Some("stt") => { + let config = + parse_stt_benchmark_args(&args).unwrap_or_else(|error| usage_error(&error)); + if let Err(error) = run_stt_benchmark(config) { + eprintln!("berd-voice benchmark stt failed: {error}"); + std::process::exit(1); + } + } + Some("synthesize") => { + let config = + parse_synthesis_args(&args).unwrap_or_else(|error| usage_error(&error)); + if let Err(failure) = run_synthesis_command(config) { + if failure.code != "output_failed" { + let envelope = ManagementErrorEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "synthesize", + event: "error", + error: ManagementErrorBody { + code: failure.code, + message: failure.public_message, + }, + }; + if let Err(error) = write_json_line(io::stdout().lock(), &envelope) { + eprintln!("berd-voice could not write synthesis error: {error}"); + } + } + eprintln!("berd-voice synthesize failed: {}", failure.detail); + std::process::exit(1); + } + } + Some("voices" | "models") => { + let command = parse_management_args(&args).unwrap_or_else(|error| usage_error(&error)); + let operation = command.operation(); + if let Err(failure) = run_management_command(command) { + if failure.code == "output_failed" { + eprintln!("berd-voice {operation} failed: {}", failure.detail); + std::process::exit(1); + } + let envelope = management_error_envelope(operation, &failure); + if let Err(error) = write_json_line(io::stdout().lock(), &envelope) { + eprintln!("berd-voice could not write management error: {error}"); + } + eprintln!("berd-voice {operation} failed: {}", failure.detail); + std::process::exit(1); + } + } + _ => usage_error( + "supported commands are session, synthesize, voices, models, benchmark tts, and benchmark stt", + ), + } +} + +fn usage_error(error: &str) -> ! { + eprintln!("{error}"); + eprintln!( + "usage:\n berd-voice session --pcm-output-fd FD [--tts-backend siri|openai|pocket] \ + [--model-dir PATH] [--voice ID] [--language BCP47] [--rate FLOAT] \ + [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH]\n \ + berd-voice synthesize --tts-backend siri|openai|pocket --voice ID \ + [--language BCP47] [--model MODEL] [--model-dir ABSOLUTE_PATH] [--rate FLOAT] \ + [--allow-paid-openai] --text TEXT --output PATH\n \ + berd-voice benchmark tts --tts-backend openai|siri|pocket \ + [--model-dir PATH] [--voice ID] [--language BCP47] [--rate FLOAT] \ + (--text TEXT --runs COUNT | --prompt-manifest english-short-v1) \ + --mode fresh-backend|warm [--allow-paid-openai]\n \ + berd-voice benchmark stt --stt-backend macos|parakeet|openai \ + [--stt-model-dir PATH] --runs COUNT --mode cold|warm \ + [--allow-paid-openai]\n \ + berd-voice voices list [--language BCP47]\n \ + berd-voice voices download --voice NAME --language BCP47 \ + [--availability-wait-seconds 1..1800]\n \ + berd-voice models macos status\n \ + berd-voice models macos install\n \ + berd-voice models pocket status|install --store-root ABSOLUTE_PATH\n \ + berd-voice models pocket voices\n \ + berd-voice models parakeet status|install --store-root ABSOLUTE_PATH" + ); + std::process::exit(2); +} + +fn parse_management_args(args: &[String]) -> Result { + match ( + args.get(1).map(String::as_str), + args.get(2).map(String::as_str), + args.get(3).map(String::as_str), + ) { + (Some("voices"), Some("list"), _) => { + let mut language = None; + let mut index = 3; + while index < args.len() { + let flag = args[index].as_str(); + let value = args + .get(index + 1) + .ok_or_else(|| format!("{flag} requires a value"))?; + match flag { + "--language" if language.is_none() => language = Some(value.clone()), + "--language" => return Err("--language may be provided only once".into()), + _ => return Err(format!("unknown voices list argument: {flag}")), + } + index += 2; + } + let language = language + .as_deref() + .map(berd_voice::siri::normalize_language) + .transpose()?; + Ok(ManagementCommand::ListVoices { language }) + } + (Some("voices"), Some("download"), _) => { + let mut voice = None; + let mut language = None; + let mut availability_wait = berd_voice::siri::SiriDownloadAvailabilityWait::default(); + let mut wait_seen = false; + let mut index = 3; + while index < args.len() { + let flag = args[index].as_str(); + let value = args + .get(index + 1) + .ok_or_else(|| format!("{flag} requires a value"))?; + match flag { + "--voice" if voice.is_none() => voice = Some(value.clone()), + "--language" if language.is_none() => language = Some(value.clone()), + "--availability-wait-seconds" if !wait_seen => { + let seconds = value.parse::().map_err(|_| { + "--availability-wait-seconds must be an integer from 1 to 1800" + .to_string() + })?; + availability_wait = + berd_voice::siri::SiriDownloadAvailabilityWait::from_seconds(seconds)?; + wait_seen = true; + } + "--voice" | "--language" | "--availability-wait-seconds" => { + return Err(format!("{flag} may be provided only once")) + } + _ => return Err(format!("unknown voices download argument: {flag}")), + } + index += 2; + } + let voice = voice.ok_or_else(|| "--voice is required".to_string())?; + let language = language.ok_or_else(|| "--language is required".to_string())?; + Ok(ManagementCommand::DownloadVoice { + identity: berd_voice::siri::SiriVoiceIdentity::new(voice, &language)?, + availability_wait, + }) + } + (Some("models"), Some("macos"), Some("status")) if args.len() == 4 => { + Ok(ManagementCommand::MacosModelStatus) + } + (Some("models"), Some("macos"), Some("install")) if args.len() == 4 => { + Ok(ManagementCommand::InstallMacosModel) + } + (Some("models"), Some("pocket"), Some("status")) => { + Ok(ManagementCommand::PocketModelStatus { + roots: parse_local_model_roots(args)?, + }) + } + (Some("models"), Some("pocket"), Some("install")) => { + Ok(ManagementCommand::InstallPocketModel { + roots: parse_local_model_roots(args)?, + }) + } + (Some("models"), Some("pocket"), Some("voices")) if args.len() == 4 => { + Ok(ManagementCommand::ListPocketVoices) + } + (Some("models"), Some("parakeet"), Some("status")) => { + Ok(ManagementCommand::ParakeetModelStatus { + roots: parse_local_model_roots(args)?, + }) + } + (Some("models"), Some("parakeet"), Some("install")) => { + Ok(ManagementCommand::InstallParakeetModel { + roots: parse_local_model_roots(args)?, + }) + } + (Some("voices"), _, _) => Err("expected voices list or voices download".into()), + (Some("models"), _, _) => Err("expected a supported models command".into()), + _ => Err("expected a management command".into()), + } +} + +fn parse_local_model_roots(args: &[String]) -> Result { + if args.len() != 6 || args.get(4).map(String::as_str) != Some("--store-root") { + return Err("local model status/install requires --store-root exactly once".into()); + } + let value = &args[5]; + if value + .split(['/', '\\']) + .any(|component| matches!(component, "." | "..")) + { + return Err("--store-root must not contain . or .. components".into()); + } + local_model_roots(std::path::Path::new(value)) +} + +fn local_model_roots(store_root: &std::path::Path) -> Result { + LocalAssetRoots::new( + store_root, + store_root.join(berd_voice::pocket_assets::MODEL_ID), + store_root + .join(berd_voice::pocket_assets::MODEL_ID) + .join("stt"), + ) + .map_err(|error| error.to_string()) +} + +fn voices_list_report( + supported: bool, + language_filter: Option, + catalog: berd_voice::siri::SiriVoiceCatalog, +) -> VoicesListResult { + VoicesListResult { + backend: "siri", + supported, + language_filter, + available_languages: catalog.available_languages, + voices: catalog.voices, + } +} + +fn voice_download_report( + identity: &berd_voice::siri::SiriVoiceIdentity, + availability_wait: berd_voice::siri::SiriDownloadAvailabilityWait, +) -> VoiceDownloadResult { + VoiceDownloadResult { + backend: "siri", + voice: identity.clone(), + installed: true, + availability_wait_seconds: availability_wait.seconds(), + } +} + +fn pocket_voices_report() -> PocketVoicesResult { + PocketVoicesResult { + backend: "pocket", + model_id: berd_voice::pocket_assets::MODEL_ID, + voice_license_id: berd_voice::pocket_assets::VOICE_LICENSE_ID, + voices: berd_voice::pocket_assets::voices() + .iter() + .map(|voice| PocketVoiceResult { + id: voice.id, + name: voice.name, + }) + .collect(), + } +} + +fn local_model_status_report( + model: LocalModelKind, + state: LocalModelState, +) -> LocalModelStatusResult { + let (state_name, verified_bytes) = match state { + LocalModelState::Missing => ("missing", None), + LocalModelState::Invalid => ("invalid", None), + LocalModelState::Ready { verified_bytes } => ("ready", Some(verified_bytes)), + }; + LocalModelStatusResult { + backend: model.backend(), + model_id: model.model_id(), + state: state_name, + ready: matches!(state, LocalModelState::Ready { .. }), + verified_bytes, + total_download_bytes: model.total_download_bytes(), + } +} + +fn read_local_model_status( + model: LocalModelKind, + roots: &LocalAssetRoots, +) -> Result { + match std::fs::symlink_metadata(roots.coordination_root()) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(local_model_status_report(model, LocalModelState::Missing)); + } + Err(error) => { + return Err(management_failure( + "io_failed", + "Could not inspect the local model store", + error.to_string(), + )); + } + Ok(_) => {} + } + let _assets = + berd_voice::local_assets::try_lock_for_read(roots).map_err(local_model_lock_failure)?; + let state = match model { + LocalModelKind::Pocket => match berd_voice::pocket_assets::inspect( + roots.pocket_bundle_root(), + ) + .map_err(|error| { + management_failure( + "integrity_failed", + "Could not inspect the Pocket model", + error, + ) + })? { + berd_voice::pocket_assets::PocketAssetStatus::Missing => LocalModelState::Missing, + berd_voice::pocket_assets::PocketAssetStatus::Invalid => LocalModelState::Invalid, + berd_voice::pocket_assets::PocketAssetStatus::Ready { verified_bytes } => { + LocalModelState::Ready { verified_bytes } + } + }, + LocalModelKind::Parakeet => { + match berd_voice::parakeet_assets::inspect(roots.parakeet_bundle_root()).map_err( + |error| { + management_failure( + "integrity_failed", + "Could not inspect the Parakeet model", + error, + ) + }, + )? { + berd_voice::parakeet_assets::ParakeetAssetStatus::Missing => { + LocalModelState::Missing + } + berd_voice::parakeet_assets::ParakeetAssetStatus::Invalid => { + LocalModelState::Invalid + } + berd_voice::parakeet_assets::ParakeetAssetStatus::Ready { verified_bytes } => { + LocalModelState::Ready { verified_bytes } + } + } + } + }; + Ok(local_model_status_report(model, state)) +} + +fn local_model_lock_failure(error: LocalAssetLockError) -> ManagementFailure { + match error { + LocalAssetLockError::Busy => management_failure( + "busy", + "The local model store is being updated", + error.to_string(), + ), + LocalAssetLockError::InvalidRoot(_) => management_failure( + "invalid_root", + "The local model store root is invalid", + error.to_string(), + ), + LocalAssetLockError::Io(_) => management_failure( + "io_failed", + "Could not access the local model store", + error.to_string(), + ), + } +} + +#[cfg(any(test, not(target_os = "macos")))] +fn unsupported_macos_model_status() -> MacosModelStatus { + MacosModelStatus { + supported: false, + locale: None, + locale_supported: false, + model_status: "unsupported".into(), + ready: false, + } +} + +#[cfg(target_os = "macos")] +fn current_macos_model_status() -> Result { + let status = berd_voice::mac_speech::mac_speech_status()?; + Ok(MacosModelStatus { + supported: status.supported, + locale: status.locale, + locale_supported: status.locale_supported, + model_status: status.model_status, + ready: status.ready, + }) +} + +#[cfg(not(target_os = "macos"))] +fn current_macos_model_status() -> Result { + Ok(unsupported_macos_model_status()) +} + +fn macos_install_needs_mutation(status: &MacosModelStatus) -> Result { + if !status.supported { + return Err(management_failure( + "unsupported", + "macOS SpeechTranscriber is unavailable on this system", + "macOS SpeechTranscriber is unavailable on this system", + )); + } + if !status.locale_supported { + return Err(management_failure( + "unsupported_locale", + "macOS SpeechTranscriber does not support the current locale", + "macOS SpeechTranscriber does not support the current locale", + )); + } + Ok(!status.ready) +} + +#[cfg(target_os = "macos")] +fn install_macos_model_platform() -> Result<(), String> { + berd_voice::mac_speech::install_mac_speech_model(write_management_progress) +} + +#[cfg(not(target_os = "macos"))] +fn install_macos_model_platform() -> Result<(), String> { + Err("macOS speech model installation is available only on macOS".into()) +} + +fn normalized_install_progress(value: f64) -> Option { + value.is_finite().then(|| value.clamp(0.0, 1.0)) +} + +fn write_json_line(mut writer: impl Write, value: &impl Serialize) -> Result<(), String> { + serde_json::to_writer(&mut writer, value).map_err(|error| error.to_string())?; + writer + .write_all(b"\n") + .and_then(|_| writer.flush()) + .map_err(|error| error.to_string()) +} + +fn write_management_result(operation: &'static str, result: T) -> Result<(), String> { + write_json_line( + io::stdout().lock(), + &ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation, + event: "result", + result, + }, + ) +} + +fn write_management_progress(progress: f64) { + let Some(fraction) = normalized_install_progress(progress) else { + eprintln!("berd-voice ignored invalid macOS model install progress: {progress}"); + return; + }; + let envelope = ManagementProgressEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "models.macos.install", + event: "progress", + fraction, + }; + if let Err(error) = write_json_line(io::stdout().lock(), &envelope) { + eprintln!("berd-voice could not write install progress: {error}"); + } +} + +fn local_install_phase_name(phase: LocalInstallPhase) -> &'static str { + match phase { + LocalInstallPhase::Downloading => "downloading", + LocalInstallPhase::Extracting => "extracting", + LocalInstallPhase::Verifying => "verifying", + LocalInstallPhase::Publishing => "publishing", + LocalInstallPhase::Complete => "complete", + } +} + +fn write_local_model_progress(operation: &'static str, progress: LocalInstallProgress) { + let envelope = LocalModelProgressEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation, + event: "progress", + phase: local_install_phase_name(progress.phase), + downloaded_bytes: progress.downloaded_bytes, + total_download_bytes: progress.total_download_bytes, + }; + if let Err(error) = write_json_line(io::stdout().lock(), &envelope) { + eprintln!("berd-voice could not write local model install progress: {error}"); + } +} + +fn local_install_failure(error: LocalInstallError) -> ManagementFailure { + let (code, message) = match error.kind { + LocalInstallErrorKind::Busy => ("busy", "The local model store is being updated"), + LocalInstallErrorKind::InvalidRoot => { + ("invalid_root", "The local model store root is invalid") + } + LocalInstallErrorKind::Download => ("download_failed", "Could not download the model"), + LocalInstallErrorKind::Integrity => { + ("integrity_failed", "The local model failed verification") + } + LocalInstallErrorKind::Extraction => { + ("extraction_failed", "Could not extract the local model") + } + LocalInstallErrorKind::Io => ("io_failed", "Could not access the local model store"), + LocalInstallErrorKind::Publish => ("publish_failed", "Could not publish the local model"), + LocalInstallErrorKind::Rollback => ( + "rollback_failed", + "Could not restore the prior local model store", + ), + LocalInstallErrorKind::Recovery => { + ("recovery_failed", "The local model store needs recovery") + } + LocalInstallErrorKind::Cleanup => { + ("cleanup_failed", "Could not clean the local model store") + } + }; + let mut detail = error.to_string(); + if !error.recovery_paths.is_empty() { + detail.push_str("; recovery data remains at "); + detail.push_str( + &error + .recovery_paths + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", "), + ); + } + management_failure(code, message, detail) +} + +fn run_local_model_install( + model: LocalModelKind, + roots: LocalAssetRoots, + operation: &'static str, +) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| { + management_failure( + "operation_failed", + "Could not start the local model installer", + error.to_string(), + ) + })?; + let (outcome, verified_bytes, cleanup_pending) = match model { + LocalModelKind::Pocket => { + match runtime.block_on(berd_voice::pocket_assets::install(&roots, |progress| { + write_local_model_progress(operation, progress); + })) { + Ok(berd_voice::pocket_assets::PocketInstallOutcome::AlreadyReady { + verified_bytes, + }) => ("alreadyReady", verified_bytes, None), + Ok(berd_voice::pocket_assets::PocketInstallOutcome::Installed { + verified_bytes, + cleanup_pending, + }) => ("installed", verified_bytes, cleanup_pending), + Err(error) => return Err(local_install_failure(error)), + } + } + LocalModelKind::Parakeet => { + match runtime.block_on(berd_voice::parakeet_assets::install(&roots, |progress| { + write_local_model_progress(operation, progress); + })) { + Ok(berd_voice::parakeet_assets::ParakeetInstallOutcome::AlreadyReady { + verified_bytes, + }) => ("alreadyReady", verified_bytes, None), + Ok(berd_voice::parakeet_assets::ParakeetInstallOutcome::Installed { + verified_bytes, + cleanup_pending, + }) => ("installed", verified_bytes, cleanup_pending), + Err(error) => return Err(local_install_failure(error)), + } + } + }; + if let Some(path) = cleanup_pending.as_ref() { + eprintln!( + "berd-voice installed the {} model; prior backup cleanup remains at {}", + model.backend(), + path.display() + ); + } + Ok(LocalModelInstallResult { + backend: model.backend(), + model_id: model.model_id(), + outcome, + ready: true, + verified_bytes, + cleanup_pending: cleanup_pending.is_some(), + }) +} + +fn management_failure( + code: &'static str, + public_message: &'static str, + detail: impl Into, +) -> ManagementFailure { + ManagementFailure { + code, + public_message, + detail: detail.into(), + } +} + +fn management_error_envelope( + operation: &'static str, + failure: &ManagementFailure, +) -> ManagementErrorEnvelope { + ManagementErrorEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation, + event: "error", + error: ManagementErrorBody { + code: failure.code, + message: failure.public_message, + }, + } +} + +#[cfg(any(test, target_os = "macos"))] +fn voice_download_failure(error: berd_voice::siri::SiriVoiceDownloadError) -> ManagementFailure { + match error { + berd_voice::siri::SiriVoiceDownloadError::NotFound(_) => management_failure( + "voice_not_found", + "The requested Siri voice was not found", + error.to_string(), + ), + berd_voice::siri::SiriVoiceDownloadError::Operation(_) => management_failure( + "operation_failed", + "Could not make the requested Siri voice available", + error.to_string(), + ), + } +} + +fn run_management_command(command: ManagementCommand) -> Result<(), ManagementFailure> { + let operation = command.operation(); + match command { + ManagementCommand::ListVoices { language } => { + let catalog = + berd_voice::siri::load_voice_catalog(language.as_deref()).map_err(|error| { + management_failure("operation_failed", "Could not list Siri voices", error) + })?; + write_management_result( + operation, + voices_list_report(cfg!(target_os = "macos"), language, catalog), + ) + .map_err(|error| { + management_failure("output_failed", "Could not write command result", error) + }) + } + ManagementCommand::DownloadVoice { + identity, + availability_wait, + } => { + #[cfg(not(target_os = "macos"))] + { + let _ = (identity, availability_wait); + return Err(management_failure( + "unsupported", + "Siri voice download is available only on macOS", + "Siri voice download is available only on macOS", + )); + } + #[cfg(target_os = "macos")] + { + let identity = berd_voice::siri::download_voice(&identity, availability_wait) + .map_err(voice_download_failure)?; + let result = voice_download_report(&identity, availability_wait); + write_management_result(operation, result).map_err(|error| { + management_failure("output_failed", "Could not write command result", error) + }) + } + } + ManagementCommand::MacosModelStatus => { + let status = current_macos_model_status().map_err(|error| { + management_failure( + "operation_failed", + "Could not read macOS speech model status", + error, + ) + })?; + write_management_result(operation, status).map_err(|error| { + management_failure("output_failed", "Could not write command result", error) + }) + } + ManagementCommand::InstallMacosModel => { + let initial_status = current_macos_model_status().map_err(|error| { + management_failure( + "operation_failed", + "Could not read macOS speech model status", + error, + ) + })?; + let needs_mutation = macos_install_needs_mutation(&initial_status)?; + let status = if needs_mutation { + install_macos_model_platform().map_err(|error| { + management_failure( + "operation_failed", + "Could not install the macOS speech model", + error, + ) + })?; + current_macos_model_status().map_err(|error| { + management_failure( + "operation_failed", + "The model installed but its status could not be read", + error, + ) + })? + } else { + initial_status + }; + write_management_result(operation, status).map_err(|error| { + management_failure("output_failed", "Could not write command result", error) + }) + } + ManagementCommand::PocketModelStatus { roots } => { + let status = read_local_model_status(LocalModelKind::Pocket, &roots)?; + write_management_result(operation, status).map_err(|error| { + management_failure("output_failed", "Could not write command result", error) + }) + } + ManagementCommand::InstallPocketModel { roots } => { + let result = run_local_model_install(LocalModelKind::Pocket, roots, operation)?; + write_management_result(operation, result).map_err(|error| { + management_failure("output_failed", "Could not write command result", error) + }) + } + ManagementCommand::ListPocketVoices => { + write_management_result(operation, pocket_voices_report()).map_err(|error| { + management_failure("output_failed", "Could not write command result", error) + }) + } + ManagementCommand::ParakeetModelStatus { roots } => { + let status = read_local_model_status(LocalModelKind::Parakeet, &roots)?; + write_management_result(operation, status).map_err(|error| { + management_failure("output_failed", "Could not write command result", error) + }) + } + ManagementCommand::InstallParakeetModel { roots } => { + let result = run_local_model_install(LocalModelKind::Parakeet, roots, operation)?; + write_management_result(operation, result).map_err(|error| { + management_failure("output_failed", "Could not write command result", error) + }) + } + } +} + +fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String> { + let (control_tx, control_rx) = mpsc::channel(); + let (pcm_tx, pcm_rx) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); + thread::spawn(move || read_framed_requests(io::stdin().lock(), control_tx, pcm_tx)); + let audio_transport = Arc::new(unsafe { AudioPipeTransport::from_raw_fd(pcm_output_fd)? }); + let (playback_tx, playback_rx) = mpsc::channel(); + let (audio_control_tx, audio_control_rx) = mpsc::channel(); + let (tts_configuration_tx, tts_configuration_rx) = mpsc::channel::(); + let stdout = io::stdout(); + let mut writer = BufWriter::new(stdout.lock()); + let mut core = SessionCore::default(); + let mut initialized = false; + let mut tts_slot: Option> = None; + let mut input_during_tts_slot: Option = None; + let mut tts_update: Option = None; + let mut next_tts_update_attempt = 1_u64; + let mut input_runtime: Option = None; + let mut input_events: Option> = None; + let mut input_controls: Option = None; + let mut next_input_token = 1_u64; + let mut pending_control = None; + let mut processed_pcm = 0_u64; + let mut held: Option = None; + let mut active: Option = None; + + loop { + if let Some(events) = input_events.as_mut() { + while let Ok(event) = events.try_recv() { + handle_voice_input_event( + event, + &mut core, + &mut active, + &mut next_input_token, + &mut writer, + )?; + } + } + while let Ok(request) = audio_control_rx.try_recv() { + write_audio_control_request(request, active.as_ref(), &mut writer)?; + } + while let Ok(event) = playback_rx.try_recv() { + handle_playback_event(event, &mut core, &mut active, &mut writer)?; + } + if let Some(output) = active.as_ref().and_then(|current| current.output.as_ref()) { + if let Err(message) = output.check_suspension_deadline(Instant::now()) { + if let Some(flag) = active.as_ref().and_then(|current| current.active.as_ref()) { + flag.store(false, Ordering::SeqCst); + output.notify_cancel_requested(); + } + let current = active + .take() + .expect("audio control requires active playback"); + core.finish(current.speech_id); + write_message( + &mut writer, + &SessionMessage::SpeechFailed { + id: current.prepare_id, + speech_id: current.speech_id, + message: message.clone(), + }, + )?; + return Err(message); + } + } + poll_tts_configuration_update( + Instant::now(), + &tts_configuration_rx, + tts_slot.as_deref(), + &mut tts_update, + &mut writer, + )?; + reevaluate_held( + &mut held, + &mut core, + tts_slot.as_deref(), + input_during_tts_slot.as_ref(), + &mut active, + &mut writer, + )?; + if active.as_ref().is_some_and(|current| { + current.active.is_none() && current.ready_deadline <= Instant::now() + }) { + let current = active.take().expect("waiting output exists"); + core.finish(current.speech_id); + write_message( + &mut writer, + &SessionMessage::SpeechFailed { + id: current.prepare_id, + speech_id: current.speech_id, + message: "output readiness timed out".into(), + }, + )?; + } + + let Some(input) = receive_session_input( + &control_rx, + &pcm_rx, + &mut pending_control, + &mut processed_pcm, + ) else { + continue; + }; + match input { + Input::Invalid(message) => { + write_protocol_fatal(&mut writer, "invalid session input", &message)?; + abort_active(&active); + if let Some(runtime) = input_runtime.as_ref() { + runtime.cancel(); + } + return Ok(()); + } + Input::Eof => { + if let Some(current) = active.as_mut() { + if let Some(flag) = ¤t.active { + flag.store(false, Ordering::SeqCst); + } + } + if let Some(runtime) = input_runtime.as_ref() { + runtime.cancel(); + } + return Ok(()); + } + Input::Pcm(frame) if !initialized => { + let _ = frame; + write_message( + &mut writer, + &SessionMessage::Fatal { + message: "PCM input requires an initialized session".into(), + }, + )?; + if let Some(runtime) = input_runtime.as_ref() { + runtime.cancel(); + } + return Ok(()); + } + Input::Pcm(frame) => { + if let Err(message) = input_runtime + .as_ref() + .expect("hello initializes input before PCM") + .try_push_frame(*frame) + { + write_protocol_fatal(&mut writer, "voice input frame was rejected", &message)?; + input_runtime + .as_ref() + .expect("hello initialized input runtime") + .cancel(); + return Ok(()); + } + } + Input::Request(SessionRequest::Shutdown) => { + reject_tts_configuration_update( + &mut tts_update, + tts_slot.as_deref(), + "session is shutting down", + &mut writer, + )?; + if let Some(held) = held.take() { + write_message( + &mut writer, + &SessionMessage::NotAdmitted { + id: held.id, + reason: NotAdmittedReason::Cancelled, + }, + )?; + } + interrupt_active(&mut core, &mut active, &mut writer)?; + finish_shutdown_playback( + &playback_rx, + &mut core, + &mut active, + &mut writer, + SHUTDOWN_PLAYBACK_TIMEOUT, + )?; + if let (Some(runtime), Some(events)) = (input_runtime.take(), input_events.as_mut()) + { + finish_input_runtime( + runtime, + events, + &mut core, + &mut active, + &mut next_input_token, + &mut writer, + )?; + } + return Ok(()); + } + Input::Request(SessionRequest::Hello { + id, + input_during_tts, + }) => { + if initialized { + write_message( + &mut writer, + &SessionMessage::Fatal { + message: "hello may only be sent once".into(), + }, + )?; + abort_active(&active); + return Ok(()); + } + let slot = match create_tts_slot(&config.tts) { + Ok(slot) => Arc::new(slot), + Err(message) => { + write_protocol_fatal( + &mut writer, + &public_tts_startup_error(&config.tts), + &format!("TTS startup failed: {message}"), + )?; + return Ok(()); + } + }; + let (runtime, mut events) = match create_input_runtime(&config.stt) { + Ok(runtime) => runtime, + Err(message) => { + write_protocol_fatal( + &mut writer, + &public_stt_startup_error(&config.stt), + &format!("STT startup failed: {message}"), + )?; + return Ok(()); + } + }; + let readiness = wait_for_input_ready(&mut events, INPUT_STARTUP_TIMEOUT); + if let Err(message) = readiness { + runtime.cancel(); + let write_result = write_protocol_fatal( + &mut writer, + &public_stt_startup_error(&config.stt), + &format!("STT readiness failed: {message}"), + ); + let finish_result = finish_unready_input_runtime(runtime); + write_result?; + finish_result?; + return Ok(()); + } + input_controls = Some(runtime.controls()); + input_runtime = Some(runtime); + input_events = Some(events); + initialized = true; + let input_policy = InputDuringTtsSlot::new(input_during_tts); + let session = VoiceSessionSnapshot { + tts: slot.snapshot()?, + input_during_tts: input_policy.snapshot()?, + }; + tts_slot = Some(slot); + input_during_tts_slot = Some(input_policy); + write_message( + &mut writer, + &SessionMessage::Ready { + id, + protocol: WIRE_MARKER, + session, + }, + )?; + } + Input::Request(request) if !initialized => { + let _ = request; + write_message( + &mut writer, + &SessionMessage::Fatal { + message: "hello must be the first request".into(), + }, + )?; + return Ok(()); + } + Input::Request(SessionRequest::SetInputMuted { id, active: muted }) => { + handle_input_muted( + id, + muted, + input_controls + .as_ref() + .expect("hello initialized input controls"), + &mut core, + &mut active, + &mut writer, + )?; + } + Input::Request(SessionRequest::SetTtsSettings { + id, + expected_revision, + settings, + }) => { + let slot = Arc::clone(tts_slot.as_ref().expect("hello initialized TTS")); + if tts_update.is_some() { + write_message( + &mut writer, + &SessionMessage::TtsSettingsResult { + id, + outcome: TtsSettingsOutcome::Rejected, + snapshot: slot.snapshot()?, + message: Some("another TTS configuration update is in progress".into()), + }, + )?; + } else { + let attempt = next_tts_update_attempt; + next_tts_update_attempt = + next_tts_update_attempt.checked_add(1).ok_or_else(|| { + "TTS configuration attempt space is exhausted".to_string() + })?; + tts_update = Some(ActiveTtsConfigurationUpdate { + attempt, + id, + deadline: Instant::now() + TTS_CONFIGURATION_TIMEOUT, + }); + let sender = tts_configuration_tx.clone(); + thread::spawn(move || { + let result = slot.prepare_replacement(expected_revision, settings); + let _ = sender.send(TtsConfigurationEvent { + attempt, + id, + result, + }); + }); + } + } + Input::Request(SessionRequest::SetInputDuringTts { + id, + expected_revision, + policy, + }) => { + let slot = input_during_tts_slot + .as_ref() + .expect("hello initialized input-during-TTS policy"); + let (outcome, snapshot) = match slot.update(expected_revision, policy) { + Ok(snapshot) => (InputDuringTtsOutcome::Applied, snapshot), + Err(snapshot) => (InputDuringTtsOutcome::Rejected, snapshot), + }; + write_message( + &mut writer, + &SessionMessage::InputDuringTtsResult { + id, + outcome, + snapshot, + }, + )?; + } + Input::Request(SessionRequest::ResetInput { id }) => { + handle_reset_input( + id, + input_controls + .as_ref() + .expect("hello initialized input controls"), + &mut core, + &mut active, + &mut writer, + )?; + } + Input::Request(SessionRequest::SetPaused { active: paused }) => { + if core.set_paused(paused) { + interrupt_active(&mut core, &mut active, &mut writer)?; + } + } + Input::Request(SessionRequest::PrepareSpeak { + id, + acknowledgement, + text, + }) => { + let request = PrepareRequest { + id, + acknowledgement, + text, + }; + if held.is_some() { + write_message( + &mut writer, + &SessionMessage::NotAdmitted { + id, + reason: NotAdmittedReason::InProgress, + }, + )?; + } else { + process_prepare( + request, + &mut core, + tts_slot.as_deref().expect("hello initialized TTS"), + input_during_tts_slot + .as_ref() + .expect("hello initialized input-during-TTS policy"), + &mut active, + &mut held, + &mut writer, + )?; + } + } + Input::Request(SessionRequest::OutputReady { id, speech_id }) => { + if let Some(current) = active.as_mut().filter(|current| { + current.prepare_id == id + && current.speech_id == speech_id + && current.active.is_none() + }) { + acknowledge_output_ready(current, input_controls.as_ref(), &mut writer)?; + let playback_active = Arc::new(AtomicBool::new(true)); + let output = Arc::new(RemotePcmAudioOutput::new( + speech_id, + current.tts.backend().pcm_spec(), + Arc::clone(&audio_transport), + Arc::clone(&playback_active), + audio_control_tx.clone(), + )?); + if current.suspension_requested { + output.request_suspend()?; + } + current.output = Some(Arc::clone(&output)); + current.active = Some(Arc::clone(&playback_active)); + spawn_playback( + speech_id, + current.text.clone(), + Arc::clone(current.tts.backend()), + output, + playback_active, + playback_tx.clone(), + ); + } else { + write_message( + &mut writer, + &SessionMessage::OutputReadyResult { + id, + speech_id, + outcome: OutputReadyOutcome::Stale, + }, + )?; + } + } + Input::Request(SessionRequest::AudioBeginAccepted { speech_id }) => { + if let Err(message) = + handle_audio_ack(speech_id, AudioHostAck::BeginAccepted, active.as_ref()) + { + write_protocol_fatal( + &mut writer, + "invalid host audio acknowledgement", + &message, + )?; + abort_active(&active); + return Ok(()); + } + } + Input::Request(SessionRequest::AudioBeginFailed { + speech_id, + played_frames, + message, + }) => { + eprintln!("host audio begin failed: {message}"); + if let Err(message) = handle_audio_ack( + speech_id, + AudioHostAck::BeginFailed { + played_frames, + message, + }, + active.as_ref(), + ) { + write_protocol_fatal( + &mut writer, + "invalid host audio acknowledgement", + &message, + )?; + abort_active(&active); + return Ok(()); + } + } + Input::Request(SessionRequest::AudioChunkAccepted { + speech_id, + sequence, + }) => { + match handle_audio_ack( + speech_id, + AudioHostAck::ChunkAccepted { sequence }, + active.as_ref(), + ) { + Ok(true) => { + publish_speech_started(speech_id, &mut core, active.as_ref(), &mut writer)? + } + Ok(false) => {} + Err(message) => { + write_protocol_fatal( + &mut writer, + "invalid host audio acknowledgement", + &message, + )?; + abort_active(&active); + return Ok(()); + } + } + } + Input::Request(SessionRequest::AudioPlayed { + speech_id, + played_frames, + }) => { + if let Err(message) = handle_audio_ack( + speech_id, + AudioHostAck::Played { played_frames }, + active.as_ref(), + ) { + write_protocol_fatal( + &mut writer, + "invalid host audio acknowledgement", + &message, + )?; + abort_active(&active); + return Ok(()); + } + } + Input::Request(SessionRequest::AudioSuspended { + speech_id, + played_frames, + }) => { + if let Err(message) = handle_audio_ack( + speech_id, + AudioHostAck::Suspended { played_frames }, + active.as_ref(), + ) { + write_protocol_fatal( + &mut writer, + "invalid host audio acknowledgement", + &message, + )?; + abort_active(&active); + return Ok(()); + } + } + Input::Request(SessionRequest::AudioResumed { + speech_id, + played_frames, + }) => { + if let Err(message) = handle_audio_ack( + speech_id, + AudioHostAck::Resumed { played_frames }, + active.as_ref(), + ) { + write_protocol_fatal( + &mut writer, + "invalid host audio acknowledgement", + &message, + )?; + abort_active(&active); + return Ok(()); + } + } + Input::Request(SessionRequest::AudioDrained { + speech_id, + sequence, + played_frames, + }) => { + if let Err(message) = handle_audio_ack( + speech_id, + AudioHostAck::Drained { + sequence, + played_frames, + }, + active.as_ref(), + ) { + write_protocol_fatal( + &mut writer, + "invalid host audio acknowledgement", + &message, + )?; + abort_active(&active); + return Ok(()); + } + } + Input::Request(SessionRequest::AudioFailed { + speech_id, + played_frames, + message, + }) => { + eprintln!("host audio output failed: {message}"); + if let Err(message) = handle_audio_ack( + speech_id, + AudioHostAck::Failed { + played_frames, + message, + }, + active.as_ref(), + ) { + write_protocol_fatal( + &mut writer, + "invalid host audio acknowledgement", + &message, + )?; + abort_active(&active); + return Ok(()); + } + } + Input::Request(SessionRequest::AudioCancelled { + speech_id, + played_frames, + }) => { + if let Err(message) = handle_audio_ack( + speech_id, + AudioHostAck::Cancelled { played_frames }, + active.as_ref(), + ) { + write_protocol_fatal( + &mut writer, + "invalid host audio acknowledgement", + &message, + )?; + abort_active(&active); + return Ok(()); + } + } + Input::Request(SessionRequest::QueryState { id, after }) => { + write_state(&mut writer, id, after, &core)? + } + Input::Request(SessionRequest::Cancel { id }) => { + handle_cancel(id, &mut held, &mut core, &mut active, &mut writer)?; + } + } + } +} + +fn acknowledge_output_ready( + current: &mut ActivePlayback, + input_controls: Option<&VoiceInputControls>, + writer: &mut impl Write, +) -> Result<(), String> { + current.assistant_activity = input_controls.map(|controls| { + controls + .begin_assistant_activity(0.65, current.input_during_tts.policy) + .expect("balanced assistant threshold is valid") + }); + write_message( + writer, + &SessionMessage::OutputReadyResult { + id: current.prepare_id, + speech_id: current.speech_id, + outcome: OutputReadyOutcome::Accepted, + }, + ) +} + +fn handle_audio_ack( + speech_id: u64, + ack: AudioHostAck, + active: Option<&ActivePlayback>, +) -> Result { + let current = active + .filter(|current| current.speech_id == speech_id) + .ok_or_else(|| "audio acknowledgement does not target the active speech".to_string())?; + current + .output + .as_ref() + .ok_or_else(|| "audio acknowledgement arrived before remote output began".to_string())? + .handle_ack(ack) +} + +fn write_audio_control_request( + request: AudioOutputControlRequest, + active: Option<&ActivePlayback>, + writer: &mut impl Write, +) -> Result<(), String> { + let (speech_id, message) = match request { + AudioOutputControlRequest::Suspend { speech_id } => { + (speech_id, SessionMessage::AudioSuspend { speech_id }) + } + AudioOutputControlRequest::Resume { speech_id } => { + (speech_id, SessionMessage::AudioResume { speech_id }) + } + }; + if active.is_none_or(|current| current.speech_id != speech_id) { + return Err("audio control request does not target the active speech".into()); + } + if !active + .and_then(|current| current.output.as_ref()) + .is_some_and(|output| output.control_request_is_outstanding(request)) + { + return Ok(()); + } + write_message(writer, &message) +} + +fn wait_for_input_ready( + events: &mut tokio::sync::mpsc::Receiver, + timeout: Duration, +) -> Result<(), String> { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .map_err(|error| format!("initialize voice input readiness wait: {error}"))?; + match runtime.block_on(async { tokio::time::timeout(timeout, events.recv()).await }) { + Ok(Some(VoiceInputEvent::Ready)) => Ok(()), + Ok(Some(VoiceInputEvent::Failed(message))) => Err(message), + Ok(Some(_)) => Err("voice input emitted data before readiness".into()), + Ok(None) => Err("voice input stopped before readiness".into()), + Err(_) => Err("voice input readiness timed out".into()), + } +} + +fn finish_unready_input_runtime(runtime: VoiceInputRuntime) -> Result<(), String> { + tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .map_err(|error| format!("initialize voice input startup cleanup: {error}"))? + .block_on(runtime.finish()) + .map_err(|error| format!("finish unready voice input runtime: {error}")) +} + +fn parse_args(args: &[String]) -> Result { + if args.get(1).map(String::as_str) != Some("session") { + return Err("the only supported command is session".into()); + } + let mut backend = "siri"; + let mut voice = None; + let mut language = None; + let mut model_dir = None; + let mut rate = None; + let mut stt_backend = "macos"; + let mut stt_model_dir = None; + let mut index = 2; + while index < args.len() { + let flag = args[index].as_str(); + let value = args + .get(index + 1) + .ok_or_else(|| format!("{flag} requires a value"))?; + match flag { + "--tts-backend" => backend = value, + "--voice" => voice = Some(value.clone()), + "--language" => language = Some(value.clone()), + "--model-dir" => model_dir = Some(PathBuf::from(value)), + "--rate" => { + rate = Some( + value + .parse::() + .map_err(|_| "--rate must be a number".to_string())?, + ) + } + "--stt-backend" => stt_backend = value, + "--stt-model-dir" => stt_model_dir = Some(PathBuf::from(value)), + "--pcm-output-fd" => {} + _ => return Err(format!("unknown argument: {flag}")), + } + index += 2; + } + let tts = build_tts_backend_config(backend, voice, language, model_dir, rate)?; + let stt = build_stt_backend_config(stt_backend, stt_model_dir)?; + Ok(SessionConfig { tts, stt }) +} + +fn parse_pcm_output_fd(args: &[String]) -> Result { + let mut value = None; + let mut index = 2; + while index < args.len() { + if args[index] == "--pcm-output-fd" { + if value.is_some() { + return Err("--pcm-output-fd may be provided only once".into()); + } + value = args.get(index + 1).cloned(); + } + index += 2; + } + let fd = value + .ok_or("--pcm-output-fd is required")? + .parse::() + .map_err(|_| "--pcm-output-fd must be an integer file descriptor".to_string())?; + if fd < 3 { + return Err("--pcm-output-fd must be at least 3".into()); + } + Ok(fd) +} + +fn parse_synthesis_args(args: &[String]) -> Result { + if args.get(1).map(String::as_str) != Some("synthesize") { + return Err("expected synthesize".into()); + } + let mut backend = None; + let mut model = None; + let mut voice = None; + let mut language = None; + let mut model_dir = None; + let mut rate = None; + let mut text = None; + let mut output = None; + let mut allow_paid_openai = false; + let mut index = 2; + while index < args.len() { + let flag = args[index].as_str(); + if flag == "--allow-paid-openai" { + if allow_paid_openai { + return Err("--allow-paid-openai may be provided only once".into()); + } + allow_paid_openai = true; + index += 1; + continue; + } + let value = args + .get(index + 1) + .ok_or_else(|| format!("{flag} requires a value"))?; + let destination = match flag { + "--tts-backend" => &mut backend, + "--model" => &mut model, + "--voice" => &mut voice, + "--language" => &mut language, + "--model-dir" => &mut model_dir, + "--text" => &mut text, + "--output" => &mut output, + "--rate" => { + if rate.is_some() { + return Err("--rate may be provided only once".into()); + } + rate = Some( + value + .parse::() + .map_err(|_| "--rate must be a number".to_string())?, + ); + index += 2; + continue; + } + _ => return Err(format!("unknown synthesize argument: {flag}")), + }; + if destination.is_some() { + return Err(format!("{flag} may be provided only once")); + } + *destination = Some(value.clone()); + index += 2; + } + + let backend = backend.ok_or_else(|| "--tts-backend is required".to_string())?; + let text = text.ok_or_else(|| "--text is required".to_string())?; + if text.trim().is_empty() { + return Err("--text must be nonempty".into()); + } + if text.len() > MAX_SPEAK_TEXT_BYTES { + return Err(format!("--text exceeds {MAX_SPEAK_TEXT_BYTES} UTF-8 bytes")); + } + let output = PathBuf::from(output.ok_or_else(|| "--output is required".to_string())?); + if output.as_os_str().is_empty() || output == Path::new("-") { + return Err("--output must name a WAV file; stdout is not supported".into()); + } + + let tts = match backend.as_str() { + "openai" => { + if language.is_some() || model_dir.is_some() { + return Err("--language and --model-dir are not valid with OpenAI".into()); + } + if !allow_paid_openai { + return Err( + "OpenAI synthesis requires explicit --allow-paid-openai consent".into(), + ); + } + let model = model + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "--model is required with OpenAI".to_string())?; + let voice = voice + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "--voice is required with OpenAI".to_string())?; + let rate = rate.unwrap_or(1.0); + if !rate.is_finite() || !(0.75..=2.0).contains(&rate) { + return Err("--rate must be between 0.75 and 2.0 for OpenAI".into()); + } + SynthesisTtsConfig::OpenAi { model, voice, rate } + } + "siri" | "pocket" => { + if model.is_some() { + return Err("--model is only valid with OpenAI".into()); + } + if allow_paid_openai { + return Err("--allow-paid-openai is only valid with OpenAI".into()); + } + let mut local = build_tts_backend_config( + &backend, + voice, + language, + model_dir.map(PathBuf::from), + rate, + )?; + if let TtsBackendConfig::Siri { + voice, language, .. + } = &mut local + { + let identity = berd_voice::siri::SiriVoiceIdentity::new(voice.clone(), language)?; + *voice = identity.name().to_string(); + *language = identity.language().to_string(); + } + if matches!(local, TtsBackendConfig::Pocket { rate, .. } if rate != 1.0) { + return Err("Pocket WAV synthesis supports only --rate 1.0".into()); + } + SynthesisTtsConfig::Local(local) + } + value => return Err(format!("unsupported TTS backend: {value}")), + }; + Ok(SynthesisConfig { tts, text, output }) +} + +fn build_stt_backend_config( + stt_backend: &str, + stt_model_dir: Option, +) -> Result { + match stt_backend { + "macos" => { + if stt_model_dir.is_some() { + return Err("--stt-model-dir is only valid with Parakeet STT".into()); + } + Ok(SttBackendConfig::Macos) + } + "parakeet" => { + let model_dir = stt_model_dir + .filter(|path| !path.as_os_str().is_empty()) + .ok_or_else(|| "--stt-model-dir is required with Parakeet STT".to_string())?; + if !model_dir.is_absolute() { + return Err("--stt-model-dir must be an absolute path".into()); + } + Ok(SttBackendConfig::Parakeet { model_dir }) + } + "openai" => { + if stt_model_dir.is_some() { + return Err("--stt-model-dir is only valid with Parakeet STT".into()); + } + Ok(SttBackendConfig::OpenAi) + } + value => Err(format!("unsupported STT backend: {value}")), + } +} + +fn build_tts_backend_config( + backend: &str, + voice: Option, + language: Option, + model_dir: Option, + rate: Option, +) -> Result { + match backend { + "openai" => { + if voice.is_some() || language.is_some() || model_dir.is_some() { + return Err( + "--voice, --language, and --model-dir require a non-OpenAI backend".into(), + ); + } + let rate = rate.unwrap_or(1.0); + if !rate.is_finite() || !(0.75..=2.0).contains(&rate) { + return Err("--rate must be between 0.75 and 2.0 for OpenAI".into()); + } + Ok(TtsBackendConfig::OpenAi { rate }) + } + "siri" => { + if model_dir.is_some() { + return Err("--model-dir is only valid with Pocket".into()); + } + let voice = voice + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + "Siri TTS is the default; select an installed voice with --voice NAME and --language BCP47" + .to_string() + })?; + let language = language + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + "Siri TTS is the default; select an installed voice with --voice NAME and --language BCP47" + .to_string() + })?; + let rate = rate.unwrap_or(1.0); + if !rate.is_finite() || !(0.5..=2.0).contains(&rate) { + return Err("--rate must be between 0.5 and 2.0".into()); + } + Ok(TtsBackendConfig::Siri { + voice, + language, + rate, + }) + } + "pocket" => { + if language.is_some() { + return Err("--language is only valid with Siri".into()); + } + let model_dir = model_dir + .filter(|value| !value.as_os_str().is_empty()) + .ok_or_else(|| "--model-dir is required with Pocket".to_string())?; + if !model_dir.is_absolute() { + return Err("--model-dir must be an absolute path".into()); + } + let voice = voice + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "--voice is required with Pocket".to_string())?; + let rate = rate.unwrap_or(1.0); + if !rate.is_finite() || !(0.75..=2.0).contains(&rate) { + return Err("--rate must be between 0.75 and 2.0 for Pocket".into()); + } + Ok(TtsBackendConfig::Pocket { + model_dir, + voice, + rate, + }) + } + value => Err(format!("unsupported TTS backend: {value}")), + } +} + +fn parse_tts_benchmark_args(args: &[String]) -> Result { + if args.get(1).map(String::as_str) != Some("benchmark") + || args.get(2).map(String::as_str) != Some("tts") + { + return Err("expected benchmark tts".into()); + } + let mut backend = None; + let mut voice = None; + let mut language = None; + let mut model_dir = None; + let mut rate = None; + let mut text = None; + let mut prompt_manifest = None; + let mut runs = None; + let mut mode = None; + let mut allow_paid_openai = false; + let mut index = 3; + while index < args.len() { + let flag = args[index].as_str(); + if flag == "--allow-paid-openai" { + allow_paid_openai = true; + index += 1; + continue; + } + let value = args + .get(index + 1) + .ok_or_else(|| format!("{flag} requires a value"))?; + match flag { + "--tts-backend" => backend = Some(value.as_str()), + "--voice" => voice = Some(value.clone()), + "--language" => language = Some(value.clone()), + "--model-dir" => model_dir = Some(PathBuf::from(value)), + "--rate" => { + rate = Some( + value + .parse::() + .map_err(|_| "--rate must be a number".to_string())?, + ) + } + "--text" => text = Some(value.clone()), + "--prompt-manifest" => prompt_manifest = Some(value.clone()), + "--runs" => { + let parsed = value + .parse::() + .map_err(|_| "--runs must be a positive integer".to_string())?; + if !(1..=100).contains(&parsed) { + return Err("--runs must be between 1 and 100".into()); + } + runs = Some(parsed); + } + "--mode" => { + mode = Some(match value.as_str() { + "fresh-backend" => TtsBenchmarkMode::FreshBackend, + "warm" => TtsBenchmarkMode::Warm, + _ => return Err("--mode must be fresh-backend or warm".into()), + }) + } + _ => return Err(format!("unknown argument: {flag}")), + } + index += 2; + } + let mode = mode.ok_or_else(|| "--mode is required".to_string())?; + let prompts = match (text, prompt_manifest, runs) { + (Some(text), None, Some(runs)) => { + if text.trim().is_empty() { + return Err("--text must be nonempty".into()); + } + if text.len() > MAX_SPEAK_TEXT_BYTES { + return Err(format!("--text exceeds {MAX_SPEAK_TEXT_BYTES} UTF-8 bytes")); + } + TtsBenchmarkPrompts::ExactRepeat { text, runs } + } + (None, Some(id), None) => { + TtsBenchmarkPrompts::Manifest(load_bundled_tts_prompt_manifest(&id)?) + } + (Some(_), Some(_), _) => { + return Err("--text and --prompt-manifest are mutually exclusive".into()) + } + (None, Some(_), Some(_)) => { + return Err("--runs is fixed by --prompt-manifest and must be omitted".into()) + } + (Some(_), None, None) => return Err("--runs is required with --text".into()), + (None, None, _) => return Err("either --text or --prompt-manifest is required".into()), + }; + let tts = build_tts_backend_config( + backend.ok_or_else(|| "--tts-backend is required".to_string())?, + voice, + language, + model_dir, + rate, + )?; + if let (TtsBackendConfig::Siri { language, .. }, TtsBenchmarkPrompts::Manifest(manifest)) = + (&tts, &prompts) + { + if language != &manifest.language { + return Err(format!( + "TTS prompt manifest {} requires Siri language {}", + manifest.id, manifest.language + )); + } + } + let (request_count, total_text_bytes) = match &prompts { + TtsBenchmarkPrompts::ExactRepeat { text, runs } => { + let requests = runs.saturating_add(usize::from(mode == TtsBenchmarkMode::Warm)); + let bytes = text + .len() + .checked_mul(requests) + .ok_or_else(|| "TTS benchmark workload is too large".to_string())?; + (requests, bytes) + } + TtsBenchmarkPrompts::Manifest(manifest) => { + let requests = manifest.prompts.len() + usize::from(mode == TtsBenchmarkMode::Warm); + let measured_bytes = manifest.prompts.iter().try_fold(0_usize, |total, prompt| { + total.checked_add(prompt.text.len()) + }); + let bytes = measured_bytes + .and_then(|total| { + total.checked_add(if mode == TtsBenchmarkMode::Warm { + manifest.warmup.text.len() + } else { + 0 + }) + }) + .ok_or_else(|| "TTS benchmark workload is too large".to_string())?; + (requests, bytes) + } + }; + if matches!(tts, TtsBackendConfig::OpenAi { .. }) { + if !allow_paid_openai { + return Err("OpenAI benchmarks require explicit --allow-paid-openai consent".into()); + } + if request_count > MAX_OPENAI_BENCHMARK_REQUESTS { + return Err(format!( + "OpenAI benchmark would make {request_count} requests; maximum is {MAX_OPENAI_BENCHMARK_REQUESTS}" + )); + } + if total_text_bytes > MAX_OPENAI_BENCHMARK_TEXT_BYTES { + return Err(format!( + "OpenAI benchmark would submit {total_text_bytes} total UTF-8 text bytes; maximum is {MAX_OPENAI_BENCHMARK_TEXT_BYTES}" + )); + } + } else if allow_paid_openai { + return Err("--allow-paid-openai is only valid with OpenAI".into()); + } + Ok(TtsBenchmarkConfig { tts, prompts, mode }) +} + +fn run_tts_benchmark(config: TtsBenchmarkConfig) -> Result<(), String> { + let target = tts_benchmark_target(&config.tts, std::env::var_os("OPENAI_BASE_URL").is_some()); + let report = match &config.prompts { + TtsBenchmarkPrompts::ExactRepeat { text, runs } => { + benchmark_tts(target, text, *runs, config.mode, || { + create_tts_backend(&config.tts) + }) + } + TtsBenchmarkPrompts::Manifest(manifest) => { + benchmark_tts_manifest(target, manifest, config.mode, || { + create_tts_backend(&config.tts) + }) + } + }; + let succeeded = report.succeeded(); + serde_json::to_writer(io::stdout().lock(), &report).map_err(|error| error.to_string())?; + println!(); + if succeeded { + Ok(()) + } else { + Err("one or more benchmark runs failed; see JSON output".into()) + } +} + +fn tts_benchmark_target( + config: &TtsBackendConfig, + openai_endpoint_from_environment: bool, +) -> TtsBenchmarkTarget { + match config { + TtsBackendConfig::OpenAi { rate, .. } => TtsBenchmarkTarget { + backend: "openai".into(), + model: Some( + std::env::var("OPENAI_TTS_MODEL").unwrap_or_else(|_| "gpt-4o-mini-tts".into()), + ), + voice: Some(std::env::var("OPENAI_TTS_VOICE").unwrap_or_else(|_| "marin".into())), + language: None, + rate: Some(*rate), + endpoint_source: Some( + if openai_endpoint_from_environment { + "OPENAI_BASE_URL_environment" + } else { + "built_in_default" + } + .into(), + ), + }, + TtsBackendConfig::Siri { + voice, + language, + rate, + } => TtsBenchmarkTarget { + backend: "siri".into(), + model: None, + voice: Some(voice.clone()), + language: Some(language.clone()), + rate: Some(*rate), + endpoint_source: None, + }, + TtsBackendConfig::Pocket { + model_dir, + voice, + rate, + } => TtsBenchmarkTarget { + backend: "pocket".into(), + model: model_dir + .file_name() + .map(|name| name.to_string_lossy().into_owned()), + voice: Some(voice.clone()), + language: None, + rate: Some(*rate), + endpoint_source: None, + }, + } +} + +fn parse_stt_benchmark_args(args: &[String]) -> Result { + if args.get(1).map(String::as_str) != Some("benchmark") + || args.get(2).map(String::as_str) != Some("stt") + { + return Err("expected benchmark stt".into()); + } + let mut backend = None; + let mut model_dir = None; + let mut runs = None; + let mut mode = None; + let mut allow_paid_openai = false; + let mut index = 3; + while index < args.len() { + let flag = args[index].as_str(); + if flag == "--allow-paid-openai" { + allow_paid_openai = true; + index += 1; + continue; + } + let value = args + .get(index + 1) + .ok_or_else(|| format!("{flag} requires a value"))?; + match flag { + "--stt-backend" => backend = Some(value.as_str()), + "--stt-model-dir" => model_dir = Some(PathBuf::from(value)), + "--runs" => { + let parsed = value + .parse::() + .map_err(|_| "--runs must be a positive integer".to_string())?; + if !(1..=100).contains(&parsed) { + return Err("--runs must be between 1 and 100".into()); + } + runs = Some(parsed); + } + "--mode" => { + mode = Some(match value.as_str() { + "cold" => SttBenchmarkMode::Cold, + "warm" => SttBenchmarkMode::Warm, + _ => return Err("--mode must be cold or warm".into()), + }) + } + _ => return Err(format!("unknown argument: {flag}")), + } + index += 2; + } + let stt = build_stt_backend_config( + backend.ok_or_else(|| "--stt-backend is required".to_string())?, + model_dir, + )?; + let runs = runs.ok_or_else(|| "--runs is required".to_string())?; + let mode = mode.ok_or_else(|| "--mode is required".to_string())?; + if matches!(stt, SttBackendConfig::OpenAi) && !allow_paid_openai { + return Err("OpenAI benchmarks require explicit --allow-paid-openai consent".into()); + } + if !matches!(stt, SttBackendConfig::OpenAi) && allow_paid_openai { + return Err("--allow-paid-openai is only valid with OpenAI".into()); + } + Ok(SttBenchmarkConfig { + stt, + runs, + mode, + allow_paid_openai, + }) +} + +fn validate_stt_benchmark_workload( + config: &SttBenchmarkConfig, + workload: &berd_voice::benchmark::SttBenchmarkWorkload, +) -> Result<(), String> { + if !matches!(config.stt, SttBackendConfig::OpenAi) { + return Ok(()); + } + debug_assert!(config.allow_paid_openai); + if workload.recognition_commits > MAX_OPENAI_BENCHMARK_REQUESTS { + return Err(format!( + "OpenAI benchmark would make {} recognition commits; maximum is {MAX_OPENAI_BENCHMARK_REQUESTS}", + workload.recognition_commits + )); + } + if workload.streamed_audio_seconds > MAX_OPENAI_STT_BENCHMARK_SECONDS { + return Err(format!( + "OpenAI benchmark would stream {:.2} seconds of audio; maximum is {MAX_OPENAI_STT_BENCHMARK_SECONDS:.0}", + workload.streamed_audio_seconds + )); + } + Ok(()) +} + +fn run_stt_benchmark(config: SttBenchmarkConfig) -> Result<(), String> { + let report = create_stt_benchmark_report(&config)?; + let succeeded = report.succeeded(); + serde_json::to_writer(io::stdout().lock(), &report).map_err(|error| error.to_string())?; + println!(); + if succeeded { + Ok(()) + } else { + Err("one or more benchmark runs failed; see JSON output".into()) + } +} + +fn create_stt_benchmark_report( + config: &SttBenchmarkConfig, +) -> Result { + let pack = load_bundled_stt_fixture_pack()?; + let workload = pack.workload(config.runs, config.mode); + validate_stt_benchmark_workload(config, &workload)?; + let target = stt_benchmark_target(&config.stt)?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("initialize STT benchmark runtime: {error}"))?; + Ok(runtime.block_on(benchmark_stt( + target, + SttBenchmarkEnvironment::default(), + &pack, + config.runs, + config.mode, + || create_input_runtime(&config.stt), + ))) +} + +fn stt_benchmark_target(config: &SttBackendConfig) -> Result { + match config { + SttBackendConfig::Parakeet { model_dir } => Ok(SttBenchmarkTarget { + backend: "parakeet".into(), + model: model_dir + .file_name() + .map(|name| name.to_string_lossy().into_owned()), + locale: None, + vad_threshold: 0.5, + endpoint_source: None, + model_source: Some("explicit --stt-model-dir".into()), + credential_source: None, + }), + SttBackendConfig::Macos => { + #[cfg(target_os = "macos")] + { + let status = berd_voice::mac_speech::mac_speech_status()?; + Ok(SttBenchmarkTarget { + backend: "macos".into(), + model: Some(status.model_status), + locale: status.locale, + vad_threshold: 0.5, + endpoint_source: None, + model_source: Some("installed current-locale model".into()), + credential_source: None, + }) + } + #[cfg(not(target_os = "macos"))] + { + Err("macOS speech recognition is only available on macOS".into()) + } + } + SttBackendConfig::OpenAi => { + let (model, model_source) = if let Some(model) = + std::env::var("OPENAI_TRANSCRIPTION_MODEL") + .ok() + .filter(|value| !value.trim().is_empty()) + { + (model, "OPENAI_TRANSCRIPTION_MODEL environment variable") + } else if let Some(model) = std::env::var("OPENAI_STT_MODEL") + .ok() + .filter(|value| !value.trim().is_empty()) + { + (model, "OPENAI_STT_MODEL environment variable") + } else { + ("gpt-live-transcribe".into(), "built-in default") + }; + let endpoint_source = std::env::var("OPENAI_REALTIME_ENDPOINT") + .ok() + .filter(|value| !value.trim().is_empty()) + .map(|_| "OPENAI_REALTIME_ENDPOINT environment variable") + .unwrap_or("built-in default"); + Ok(SttBenchmarkTarget { + backend: "openai".into(), + model: Some(model), + locale: None, + vad_threshold: 0.5, + endpoint_source: Some(endpoint_source.into()), + model_source: Some(model_source.into()), + credential_source: Some("OPENAI_API_KEY environment variable".into()), + }) + } + } +} + +fn create_tts_configuration(config: &TtsBackendConfig) -> Result { + match config { + TtsBackendConfig::OpenAi { rate } => create_openai_tts_configuration( + *rate, + std::env::var("OPENAI_TTS_MODEL").unwrap_or_else(|_| "gpt-4o-mini-tts".into()), + std::env::var("OPENAI_TTS_VOICE").unwrap_or_else(|_| "marin".into()), + ), + TtsBackendConfig::Siri { + voice, + language, + rate, + } => Ok(TtsConfiguration::siri( + voice.clone(), + language.clone(), + *rate, + )), + TtsBackendConfig::Pocket { + model_dir, + voice, + rate, + } => Ok(TtsConfiguration::pocket( + model_dir.clone(), + berd_voice::pocket_assets::MODEL_ID.into(), + voice.clone(), + *rate, + )), + } +} + +fn create_openai_tts_configuration( + rate: f32, + model: String, + voice: String, +) -> Result { + let api_key = std::env::var("OPENAI_API_KEY") + .ok() + .filter(|key| !key.trim().is_empty()) + .ok_or_else(|| "OPENAI_API_KEY is required".to_string())?; + let base = + std::env::var("OPENAI_BASE_URL").unwrap_or_else(|_| "https://api.openai.com/v1".into()); + Ok(TtsConfiguration::openai( + format!("{}/audio/speech", base.trim_end_matches('/')), + api_key, + model, + voice, + rate, + )) +} + +fn create_tts_slot(config: &TtsBackendConfig) -> Result { + #[cfg(not(target_os = "macos"))] + if matches!(config, TtsBackendConfig::Siri { .. }) { + return Err( + "Siri TTS is the default but is only available on macOS; explicitly select --tts-backend openai or --tts-backend pocket on this platform" + .into(), + ); + } + ConfiguredTtsSlot::new(create_tts_configuration(config)?).map_err(|error| match config { + TtsBackendConfig::Siri { + voice, language, .. + } => format!( + "Siri TTS voice {voice:?} ({language}) is unavailable: {error}. Download it in Berd Voice settings or select another installed voice with --voice and --language" + ), + _ => error, + }) +} + +fn create_tts_backend(config: &TtsBackendConfig) -> Result, String> { + let slot = create_tts_slot(config)?; + Ok(Arc::clone(slot.lease()?.backend())) +} + +fn create_synthesis_backend(config: &SynthesisTtsConfig) -> Result, String> { + match config { + SynthesisTtsConfig::OpenAi { model, voice, rate } => { + let slot = ConfiguredTtsSlot::new(create_openai_tts_configuration( + *rate, + model.clone(), + voice.clone(), + )?)?; + Ok(Arc::clone(slot.lease()?.backend())) + } + SynthesisTtsConfig::Local(config) => create_tts_backend(config), + } +} + +fn synthesis_failure( + code: &'static str, + public_message: &'static str, + detail: impl Into, +) -> SynthesisFailure { + SynthesisFailure { + code, + public_message, + detail: detail.into(), + } +} + +fn prepare_synthesis_output(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => { + return Err(synthesis_failure( + "output_unavailable", + "The output file already exists", + format!("output already exists: {}", path.display()), + )) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(synthesis_failure( + "output_unavailable", + "The output path could not be inspected", + error.to_string(), + )) + } + } + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + tempfile::Builder::new() + .prefix(".berd-voice-synthesize-") + .suffix(".wav.tmp") + .tempfile_in(parent) + .map_err(|error| { + synthesis_failure( + "output_unavailable", + "A temporary output file could not be created", + error.to_string(), + ) + }) +} + +fn synthesis_identity(config: &SynthesisConfig) -> (Option, String, Option, f32) { + match &config.tts { + SynthesisTtsConfig::OpenAi { model, voice, rate } => { + (Some(model.clone()), voice.clone(), None, *rate) + } + SynthesisTtsConfig::Local(TtsBackendConfig::Siri { + voice, + language, + rate, + }) => (None, voice.clone(), Some(language.clone()), *rate), + SynthesisTtsConfig::Local(TtsBackendConfig::Pocket { voice, rate, .. }) => ( + Some(berd_voice::pocket_assets::MODEL_ID.into()), + voice.clone(), + None, + *rate, + ), + SynthesisTtsConfig::Local(TtsBackendConfig::OpenAi { .. }) => { + unreachable!("OpenAI synthesis carries explicit identity") + } + } +} + +fn run_synthesis_with_factory( + config: &SynthesisConfig, + factory: impl FnOnce(&SynthesisTtsConfig) -> Result, String>, +) -> Result { + // Establish that a no-clobber output is possible before constructing a backend. For OpenAI, + // this keeps ordinary path failures at zero paid requests. + let mut temporary = prepare_synthesis_output(&config.output)?; + let backend = factory(&config.tts).map_err(|error| { + synthesis_failure( + "backend_unavailable", + "The selected TTS backend is unavailable", + error, + ) + })?; + let wav = + berd_voice::synthesize_pcm16_wav(backend.as_ref(), &config.text, temporary.as_file_mut()) + .map_err(|error| { + let (code, message) = match error.kind { + WavSynthesisErrorKind::Backend => ( + "synthesis_failed", + "The TTS backend could not synthesize the text", + ), + WavSynthesisErrorKind::Cancelled => { + ("synthesis_cancelled", "TTS synthesis was cancelled") + } + WavSynthesisErrorKind::Empty => { + ("invalid_audio", "TTS synthesis produced no audio") + } + WavSynthesisErrorKind::InvalidPcm => { + ("invalid_audio", "TTS synthesis produced invalid audio") + } + WavSynthesisErrorKind::TooLong => { + ("audio_too_long", "TTS synthesis exceeded ten minutes") + } + WavSynthesisErrorKind::Output => { + ("output_unavailable", "The WAV output could not be written") + } + }; + synthesis_failure(code, message, error.detail) + })?; + temporary.as_file().sync_all().map_err(|error| { + synthesis_failure( + "output_unavailable", + "The WAV output could not be synchronized", + error.to_string(), + ) + })?; + let bytes = temporary + .as_file() + .metadata() + .map_err(|error| { + synthesis_failure( + "output_unavailable", + "The WAV output could not be inspected", + error.to_string(), + ) + })? + .len(); + temporary + .persist_noclobber(&config.output) + .map_err(|error| { + synthesis_failure( + "output_unavailable", + "The output file appeared before synthesis completed", + error.error.to_string(), + ) + })?; + let (model, voice, language, rate) = synthesis_identity(config); + Ok(SynthesisResult { + backend: config.backend(), + model, + voice, + language, + rate, + wav: SynthesisWavResult { + encoding: "pcm_s16le", + sample_rate: wav.sample_rate, + channels: 1, + bits_per_sample: 16, + frames: wav.frames, + duration_ms: wav.frames as f64 * 1_000.0 / f64::from(wav.sample_rate), + bytes, + }, + }) +} + +fn run_synthesis_command(config: SynthesisConfig) -> Result<(), SynthesisFailure> { + let result = run_synthesis_with_factory(&config, create_synthesis_backend)?; + write_json_line( + io::stdout().lock(), + &ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "synthesize", + event: "result", + result, + }, + ) + .map_err(|error| synthesis_failure("output_failed", "Could not write command result", error)) +} + +fn create_input_runtime( + config: &SttBackendConfig, +) -> Result< + ( + VoiceInputRuntime, + tokio::sync::mpsc::Receiver, + ), + String, +> { + let engine = match config { + SttBackendConfig::Parakeet { model_dir } => VoiceInputEngineConfig::Parakeet { + model_dir: model_dir.clone(), + }, + SttBackendConfig::Macos => { + #[cfg(target_os = "macos")] + { + let status = berd_voice::mac_speech::mac_speech_status().map_err(|error| { + format!( + "Could not check the default macOS speech recognition engine: {error}. Open Berd Voice settings to verify or install the current-locale model" + ) + })?; + validate_macos_stt_status(&status)?; + VoiceInputEngineConfig::MacSpeech + } + #[cfg(not(target_os = "macos"))] + { + return Err( + "macOS speech recognition is the default but is only available on macOS; explicitly select --stt-backend parakeet or --stt-backend openai on this platform" + .into(), + ); + } + } + SttBackendConfig::OpenAi => { + let api_key = std::env::var("OPENAI_API_KEY") + .ok() + .filter(|key| !key.trim().is_empty()) + .ok_or_else(|| "OPENAI_API_KEY is required for OpenAI STT".to_string())?; + let endpoint = std::env::var("OPENAI_REALTIME_ENDPOINT") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| { + "wss://api.openai.com/v1/realtime?intent=transcription".to_string() + }); + let model = std::env::var("OPENAI_TRANSCRIPTION_MODEL") + .ok() + .filter(|value| !value.trim().is_empty()) + .or_else(|| { + std::env::var("OPENAI_STT_MODEL") + .ok() + .filter(|value| !value.trim().is_empty()) + }) + .unwrap_or_else(|| "gpt-live-transcribe".to_string()); + VoiceInputEngineConfig::OpenAi { + endpoint, + api_key, + model, + } + } + }; + VoiceInputRuntime::start(VoiceInputConfig { + engine, + speech_vad_threshold: 0.5, + controls: VoiceInputControls::default(), + }) + .map_err(|error| match config { + SttBackendConfig::Macos => format!( + "Could not start the default macOS speech recognition engine: {error}. Open Berd Voice settings to verify or install the current-locale model" + ), + _ => error, + }) +} + +#[cfg(target_os = "macos")] +fn validate_macos_stt_status( + status: &berd_voice::mac_speech::MacSpeechEngineStatus, +) -> Result<(), String> { + if status.ready { + return Ok(()); + } + if !status.supported { + return Err( + "The default macOS speech engine requires macOS 26 or later with SpeechTranscriber available. Upgrade macOS or verify SpeechTranscriber availability, or explicitly select --stt-backend parakeet or --stt-backend openai" + .into(), + ); + } + if !status.locale_supported { + return Err( + "The default macOS SpeechTranscriber engine does not support the current system locale. Select a supported macOS language and locale, or explicitly select --stt-backend parakeet or --stt-backend openai" + .into(), + ); + } + let action = match status.model_status.as_str() { + "downloading" => "Wait for the download to finish in Berd Voice settings", + "available" => "Download the current-locale model in Berd Voice settings", + _ => "Open Berd Voice settings to verify or install the current-locale model", + }; + Err(format!( + "The default macOS SpeechTranscriber model is not ready (model status: {}). {action}, or explicitly select --stt-backend parakeet or --stt-backend openai", + status.model_status + )) +} + +fn poll_tts_configuration_update( + now: Instant, + receiver: &Receiver, + tts_slot: Option<&ConfiguredTtsSlot>, + active: &mut Option, + writer: &mut impl Write, +) -> Result<(), String> { + if active.is_some_and(|update| update.deadline <= now) { + reject_tts_configuration_update( + active, + tts_slot, + "TTS configuration update timed out", + writer, + )?; + } + while let Ok(event) = receiver.try_recv() { + if active.is_none_or(|update| update.attempt != event.attempt || update.id != event.id) { + continue; + } + active.take(); + let slot = tts_slot.expect("TTS update requires initialized slot"); + let result = event + .result + .and_then(|replacement| slot.commit_replacement(replacement)); + let (outcome, snapshot, message) = match result { + Ok(snapshot) => (TtsSettingsOutcome::Applied, snapshot, None), + Err(rejection) => { + eprintln!("TTS configuration update failed: {}", rejection.message); + let message = public_tts_rejection_message(rejection.kind); + ( + TtsSettingsOutcome::Rejected, + rejection.snapshot, + Some(message.into()), + ) + } + }; + write_message( + writer, + &SessionMessage::TtsSettingsResult { + id: event.id, + outcome, + snapshot, + message, + }, + )?; + } + Ok(()) +} + +fn public_tts_rejection_message(kind: TtsConfigurationRejectionKind) -> &'static str { + match kind { + TtsConfigurationRejectionKind::StaleRevision => { + "TTS settings revision is stale; retry with the authoritative snapshot" + } + TtsConfigurationRejectionKind::BackendMismatch => { + "TTS backend cannot be changed in a live session" + } + TtsConfigurationRejectionKind::InvalidSettings => { + "TTS settings are invalid; the previous configuration remains active" + } + TtsConfigurationRejectionKind::Initialization => { + "TTS settings could not be initialized; the previous configuration remains active" + } + TtsConfigurationRejectionKind::Internal => { + "TTS settings could not be applied; the previous configuration remains active" + } + } +} + +fn public_tts_startup_error(config: &TtsBackendConfig) -> String { + match config { + TtsBackendConfig::OpenAi { .. } => { + "OpenAI TTS could not initialize; verify OPENAI_API_KEY and the selected model and voice" + .into() + } + TtsBackendConfig::Siri { .. } => + "Siri TTS could not initialize; download the selected voice in Berd Voice settings or select another installed voice" + .into(), + TtsBackendConfig::Pocket { .. } => + "Pocket TTS could not initialize; verify the selected Pocket bundle and voice".into(), + } +} + +fn public_stt_startup_error(config: &SttBackendConfig) -> String { + match config { + SttBackendConfig::Macos => { + "macOS speech recognition could not initialize; verify SpeechTranscriber availability, locale support, and the installed model in Berd Voice settings" + .into() + } + SttBackendConfig::Parakeet { .. } => { + "Parakeet speech recognition could not initialize; verify the selected model bundle" + .into() + } + SttBackendConfig::OpenAi => { + "OpenAI speech recognition could not initialize; verify OPENAI_API_KEY and the selected transcription model" + .into() + } + } +} + +fn write_protocol_fatal( + writer: &mut impl Write, + public_message: &str, + diagnostic: &str, +) -> Result<(), String> { + eprintln!("{diagnostic}"); + write_message( + writer, + &SessionMessage::Fatal { + message: public_message.into(), + }, + ) +} + +fn reject_tts_configuration_update( + active: &mut Option, + tts_slot: Option<&ConfiguredTtsSlot>, + message: &str, + writer: &mut impl Write, +) -> Result<(), String> { + let Some(update) = active.take() else { + return Ok(()); + }; + write_message( + writer, + &SessionMessage::TtsSettingsResult { + id: update.id, + outcome: TtsSettingsOutcome::Rejected, + snapshot: tts_slot + .expect("TTS update requires initialized slot") + .snapshot()?, + message: Some(message.into()), + }, + ) +} + +#[allow(clippy::too_many_arguments)] +fn process_prepare( + request: PrepareRequest, + core: &mut SessionCore, + tts_slot: &ConfiguredTtsSlot, + input_during_tts_slot: &InputDuringTtsSlot, + active: &mut Option, + held: &mut Option, + writer: &mut impl Write, +) -> Result<(), String> { + let id = request.id; + match core.prepare(request.clone()) { + PrepareOutcome::Hold => { + *held = Some(request); + } + PrepareOutcome::Pending(utterances) => { + write_message(writer, &SessionMessage::Pending { id, utterances })?; + } + PrepareOutcome::NotAdmitted(reason) => { + write_message(writer, &SessionMessage::NotAdmitted { id, reason })?; + } + PrepareOutcome::Admitted { + speech_id, + confirmed_token, + text, + } => { + let tts = tts_slot.lease()?; + let input_during_tts = input_during_tts_slot.snapshot()?; + *active = Some(ActivePlayback { + prepare_id: id, + speech_id, + text, + output: None, + active: None, + ready_deadline: Instant::now() + Duration::from_secs(2), + assistant_activity: None, + input_during_tts, + tts, + suspension_requested: false, + }); + write_message( + writer, + &SessionMessage::Admitted { + id, + speech_id, + confirmed_token, + }, + )?; + } + } + Ok(()) +} + +fn reevaluate_held( + held: &mut Option, + core: &mut SessionCore, + tts_slot: Option<&ConfiguredTtsSlot>, + input_during_tts_slot: Option<&InputDuringTtsSlot>, + active: &mut Option, + writer: &mut impl Write, +) -> Result<(), String> { + if !core.user_speaking() && !core.recognition_pending() && active.is_none() { + if let Some(pending_prepare) = held.take() { + process_prepare( + pending_prepare, + core, + tts_slot.expect("held prepare requires initialized TTS"), + input_during_tts_slot + .expect("held prepare requires initialized input-during-TTS policy"), + active, + held, + writer, + )?; + } + } + Ok(()) +} + +fn handle_voice_input_event( + event: VoiceInputEvent, + core: &mut SessionCore, + active: &mut Option, + next_token: &mut u64, + writer: &mut impl Write, +) -> Result<(), String> { + match event { + VoiceInputEvent::Ready => { + return Err("voice input emitted a duplicate readiness event".into()) + } + VoiceInputEvent::SpeakingChanged(speaking) => { + core.set_user_speaking(speaking); + write_message(writer, &SessionMessage::InputSpeaking { active: speaking })?; + update_provisional_suspension(core, active)?; + } + VoiceInputEvent::RecognitionPendingChanged(pending) => { + core.set_recognition_pending(pending); + write_message( + writer, + &SessionMessage::RecognitionPending { active: pending }, + )?; + update_provisional_suspension(core, active)?; + } + VoiceInputEvent::FinalTranscript { + text, + storage_receipt, + } => store_and_publish_voice_final( + text, + || storage_receipt.stored(), + core, + active, + next_token, + writer, + )?, + VoiceInputEvent::Failed(message) => { + write_protocol_fatal(writer, "voice input runtime failed", &message)?; + abort_active(active); + return Err(message); + } + } + Ok(()) +} + +fn update_provisional_suspension( + core: &SessionCore, + active: &mut Option, +) -> Result<(), String> { + let Some(current) = active.as_mut() else { + return Ok(()); + }; + if current + .active + .as_ref() + .is_some_and(|authority| !authority.load(Ordering::SeqCst)) + { + return Ok(()); + } + let requested = core.user_speaking() || core.recognition_pending(); + if current.suspension_requested == requested { + return Ok(()); + } + current.suspension_requested = requested; + if let Some(output) = ¤t.output { + if requested { + output.request_suspend()?; + } else { + output.request_resume()?; + } + } + Ok(()) +} + +fn store_and_publish_voice_final( + text: String, + mark_stored: impl FnOnce(), + core: &mut SessionCore, + active: &mut Option, + next_token: &mut u64, + writer: &mut impl Write, +) -> Result<(), String> { + if text.len() > MAX_FINAL_TEXT_BYTES { + let message = "final text exceeds 64 KiB".to_string(); + write_message( + writer, + &SessionMessage::Fatal { + message: message.clone(), + }, + )?; + return Err(message); + } + let token = *next_token; + let Some(next) = token.checked_add(1) else { + let message = "voice input token space is exhausted".to_string(); + write_message( + writer, + &SessionMessage::Fatal { + message: message.clone(), + }, + )?; + return Err(message); + }; + *next_token = next; + core.add_final(token, text.clone())?; + mark_stored(); + write_message(writer, &SessionMessage::UserFinal { token, text })?; + interrupt_active(core, active, writer) +} + +fn finish_input_runtime( + runtime: VoiceInputRuntime, + events: &mut tokio::sync::mpsc::Receiver, + core: &mut SessionCore, + active: &mut Option, + next_token: &mut u64, + writer: &mut impl Write, +) -> Result<(), String> { + let (done_tx, done_rx) = mpsc::sync_channel(1); + thread::spawn(move || { + let result = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .map_err(|error| format!("initialize voice input shutdown: {error}")) + .and_then(|runtime_handle| { + runtime_handle + .block_on(runtime.finish()) + .map_err(|error| error.to_string()) + }); + let _ = done_tx.send(result); + }); + loop { + while let Ok(event) = events.try_recv() { + handle_voice_input_event(event, core, active, next_token, writer)?; + } + match done_rx.try_recv() { + Ok(result) => { + while let Ok(event) = events.try_recv() { + handle_voice_input_event(event, core, active, next_token, writer)?; + } + return result; + } + Err(mpsc::TryRecvError::Empty) => thread::sleep(Duration::from_millis(10)), + Err(mpsc::TryRecvError::Disconnected) => { + return Err("voice input shutdown worker disconnected".into()) + } + } + } +} + +fn handle_playback_event( + event: PlaybackEvent, + core: &mut SessionCore, + active: &mut Option, + writer: &mut impl Write, +) -> Result<(), String> { + match event { + #[cfg(test)] + PlaybackEvent::Started(speech_id) => { + publish_speech_started(speech_id, core, active.as_ref(), writer)? + } + PlaybackEvent::Completed(speech_id) => { + let id = active.as_ref().map_or(0, |current| current.prepare_id); + finish_playback( + core, + active, + speech_id, + SessionMessage::SpeechCompleted { id, speech_id }, + writer, + )? + } + PlaybackEvent::Interrupted(speech_id, spoken_through_utf8) => { + let id = active.as_ref().map_or(0, |current| current.prepare_id); + finish_playback( + core, + active, + speech_id, + SessionMessage::SpeechInterrupted { + id, + speech_id, + spoken_through_utf8, + }, + writer, + )? + } + PlaybackEvent::Failed(speech_id, message, output_quiescent) => { + let id = active.as_ref().map_or(0, |current| current.prepare_id); + finish_playback( + core, + active, + speech_id, + SessionMessage::SpeechFailed { + id, + speech_id, + message, + }, + writer, + )?; + if !output_quiescent { + return Err("remote PCM output did not reach a quiescent terminal".into()); + } + } + } + Ok(()) +} + +fn publish_speech_started( + speech_id: u64, + core: &mut SessionCore, + active: Option<&ActivePlayback>, + writer: &mut impl Write, +) -> Result<(), String> { + if core.mark_started(speech_id) { + let id = active.map_or(0, |current| current.prepare_id); + write_message(writer, &SessionMessage::SpeechStarted { id, speech_id })?; + } + Ok(()) +} + +fn finish_playback( + core: &mut SessionCore, + active: &mut Option, + speech_id: u64, + message: SessionMessage, + writer: &mut impl Write, +) -> Result<(), String> { + if core.finish(speech_id) { + if active + .as_ref() + .is_some_and(|current| current.speech_id == speech_id) + { + *active = None; + } + write_message(writer, &message)?; + } + Ok(()) +} + +fn interrupt_active( + core: &mut SessionCore, + active: &mut Option, + writer: &mut impl Write, +) -> Result<(), String> { + let Some(current) = active.as_mut() else { + return Ok(()); + }; + if let Some(flag) = ¤t.active { + flag.store(false, Ordering::SeqCst); + if let Some(output) = ¤t.output { + output.notify_cancel_requested(); + } + } else { + let id = current.prepare_id; + let speech_id = current.speech_id; + core.finish(speech_id); + *active = None; + write_message( + writer, + &SessionMessage::SpeechInterrupted { + id, + speech_id, + spoken_through_utf8: 0, + }, + )?; + } + Ok(()) +} + +fn discard_provisional_active( + core: &mut SessionCore, + active: &mut Option, + writer: &mut impl Write, +) -> Result<(), String> { + if active + .as_ref() + .is_some_and(|current| current.suspension_requested) + { + interrupt_active(core, active, writer)?; + } + Ok(()) +} + +fn handle_input_muted( + id: u64, + muted: bool, + controls: &VoiceInputControls, + core: &mut SessionCore, + active: &mut Option, + writer: &mut impl Write, +) -> Result<(), String> { + controls.set_host_muted(muted); + write_message( + writer, + &SessionMessage::InputMuteApplied { id, active: muted }, + )?; + if muted { + discard_provisional_active(core, active, writer)?; + } + Ok(()) +} + +fn handle_reset_input( + id: u64, + controls: &VoiceInputControls, + core: &mut SessionCore, + active: &mut Option, + writer: &mut impl Write, +) -> Result<(), String> { + controls.reset(); + write_message(writer, &SessionMessage::InputResetApplied { id })?; + discard_provisional_active(core, active, writer) +} + +fn handle_cancel( + id: u64, + held: &mut Option, + core: &mut SessionCore, + active: &mut Option, + writer: &mut impl Write, +) -> Result<(), String> { + if held.as_ref().is_some_and(|held| held.id == id) { + held.take(); + write_message( + writer, + &SessionMessage::CancelResult { + id, + outcome: CancelOutcome::Cancelled, + speech_id: None, + }, + )?; + write_message( + writer, + &SessionMessage::NotAdmitted { + id, + reason: NotAdmittedReason::Cancelled, + }, + )?; + } else if active + .as_ref() + .is_some_and(|current| current.prepare_id == id) + { + let speech_id = active.as_ref().map(|current| current.speech_id); + write_message( + writer, + &SessionMessage::CancelResult { + id, + outcome: CancelOutcome::Cancelled, + speech_id, + }, + )?; + interrupt_active(core, active, writer)?; + } else { + write_message( + writer, + &SessionMessage::CancelResult { + id, + outcome: CancelOutcome::Stale, + speech_id: None, + }, + )?; + } + Ok(()) +} + +fn abort_active(active: &Option) { + if let Some(flag) = active.as_ref().and_then(|current| current.active.as_ref()) { + flag.store(false, Ordering::SeqCst); + } +} + +fn finish_shutdown_playback( + playback_rx: &Receiver, + core: &mut SessionCore, + active: &mut Option, + writer: &mut impl Write, + timeout: Duration, +) -> Result<(), String> { + let deadline = Instant::now() + timeout; + while active.is_some() { + let remaining = deadline.saturating_duration_since(Instant::now()); + let event = if remaining.is_zero() { + Err(mpsc::RecvTimeoutError::Timeout) + } else { + playback_rx.recv_timeout(remaining) + }; + match event { + Ok(event) => handle_playback_event(event, core, active, writer)?, + Err(error) => { + let current = active.take().expect("active playback exists"); + core.finish(current.speech_id); + let message = match error { + mpsc::RecvTimeoutError::Timeout => { + "playback cancellation timed out during shutdown" + } + mpsc::RecvTimeoutError::Disconnected => { + "playback worker disconnected during shutdown" + } + }; + write_message( + writer, + &SessionMessage::SpeechFailed { + id: current.prepare_id, + speech_id: current.speech_id, + message: message.into(), + }, + )?; + } + } + } + Ok(()) +} + +fn write_state( + writer: &mut impl Write, + id: u64, + after: u64, + core: &SessionCore, +) -> Result<(), String> { + write_message( + writer, + &SessionMessage::State { + id, + confirmed_token: core.confirmed_token(), + utterances_after: core.utterances_after(after), + }, + ) +} + +fn write_message(writer: &mut impl Write, message: &SessionMessage) -> Result<(), String> { + serde_json::to_writer(&mut *writer, message).map_err(|error| error.to_string())?; + writer + .write_all(b"\n") + .and_then(|_| writer.flush()) + .map_err(|error| error.to_string()) +} + +fn receive_session_input( + control_rx: &Receiver, + pcm_rx: &Receiver>, + pending_control: &mut Option, + processed_pcm: &mut u64, +) -> Option { + if pending_control.is_none() { + *pending_control = control_rx.try_recv().ok(); + } + if pending_control + .as_ref() + .is_some_and(|control| control.after_pcm <= *processed_pcm) + { + return pending_control.take().map(|control| control.input); + } + match pcm_rx.recv_timeout(Duration::from_millis(10)) { + Ok(frame) => { + *processed_pcm = processed_pcm.saturating_add(1); + Some(Input::Pcm(frame)) + } + Err(mpsc::RecvTimeoutError::Timeout) => { + if pending_control.is_none() { + *pending_control = control_rx.try_recv().ok(); + } + if pending_control + .as_ref() + .is_some_and(|control| control.after_pcm <= *processed_pcm) + { + pending_control.take().map(|control| control.input) + } else { + None + } + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + if pending_control.is_none() { + *pending_control = control_rx.try_recv().ok(); + } + pending_control + .take() + .map(|control| control.input) + .or(Some(Input::Eof)) + } + } +} + +fn read_framed_requests( + mut reader: impl Read, + control_sender: mpsc::Sender, + pcm_sender: SyncSender>, +) { + let mut sent_pcm = 0_u64; + loop { + let mut header = [0_u8; FRAME_HEADER_BYTES]; + let input = match reader.read(&mut header[..1]) { + Ok(0) => break, + Ok(1) => match reader.read_exact(&mut header[1..]) { + Ok(()) => decode_framed_input(&mut reader, header), + Err(error) => Input::Invalid(format!("truncated session frame header: {error}")), + }, + Ok(_) => unreachable!("one-byte read"), + Err(error) => Input::Invalid(format!("could not read stdin: {error}")), + }; + match input { + Input::Pcm(frame) => match pcm_sender.try_send(frame) { + Ok(()) => sent_pcm = sent_pcm.saturating_add(1), + Err(mpsc::TrySendError::Full(_)) => { + let _ = control_sender.send(OrderedControl { + after_pcm: sent_pcm, + input: Input::Invalid("session PCM input queue is full".into()), + }); + return; + } + Err(mpsc::TrySendError::Disconnected(_)) => return, + }, + input => { + let terminal = matches!(input, Input::Invalid(_)); + if control_sender + .send(OrderedControl { + after_pcm: sent_pcm, + input, + }) + .is_err() + || terminal + { + return; + } + } + } + } + let _ = control_sender.send(OrderedControl { + after_pcm: sent_pcm, + input: Input::Eof, + }); +} + +fn decode_framed_input(reader: &mut impl Read, header: [u8; FRAME_HEADER_BYTES]) -> Input { + if header[..2] != FRAME_MAGIC { + return Input::Invalid("invalid session frame magic".into()); + } + if header[2] != WIRE_MARKER as u8 { + return Input::Invalid(format!("invalid session frame marker: {}", header[2])); + } + let kind = header[3]; + let length = u32::from_le_bytes(header[4..8].try_into().expect("four-byte length")) as usize; + match kind { + JSON_FRAME_KIND if length > MAX_LINE_BYTES => { + return Input::Invalid("request exceeds 1 MiB".into()) + } + PCM_FRAME_KIND if length != PCM_FRAME_BYTES => { + return Input::Invalid(format!( + "PCM frame has {length} bytes; expected {PCM_FRAME_BYTES}" + )) + } + JSON_FRAME_KIND | PCM_FRAME_KIND => {} + _ => return Input::Invalid(format!("unknown session frame kind: {kind}")), + } + let mut payload = vec![0_u8; length]; + if let Err(error) = reader.read_exact(&mut payload) { + return Input::Invalid(format!("truncated session frame payload: {error}")); + } + if kind == JSON_FRAME_KIND { + String::from_utf8(payload) + .map_err(|error| format!("invalid request UTF-8: {error}")) + .and_then(|json| { + serde_json::from_str(&json).map_err(|error| format!("invalid request: {error}")) + }) + .and_then(validate_request) + .map(Input::Request) + .unwrap_or_else(Input::Invalid) + } else { + let samples = payload + .chunks_exact(std::mem::size_of::()) + .map(|sample| f32::from_le_bytes(sample.try_into().expect("four-byte sample"))) + .collect::>(); + VoiceInputFrame::try_from_samples(&samples) + .map(|frame| Input::Pcm(Box::new(frame))) + .unwrap_or_else(Input::Invalid) + } +} + +fn validate_request(request: SessionRequest) -> Result { + let id = match &request { + SessionRequest::Hello { id, .. } + | SessionRequest::SetInputMuted { id, .. } + | SessionRequest::SetTtsSettings { id, .. } + | SessionRequest::SetInputDuringTts { id, .. } + | SessionRequest::ResetInput { id } + | SessionRequest::PrepareSpeak { id, .. } + | SessionRequest::OutputReady { id, .. } + | SessionRequest::QueryState { id, .. } + | SessionRequest::Cancel { id } => Some(*id), + SessionRequest::SetPaused { .. } + | SessionRequest::AudioBeginAccepted { .. } + | SessionRequest::AudioBeginFailed { .. } + | SessionRequest::AudioChunkAccepted { .. } + | SessionRequest::AudioPlayed { .. } + | SessionRequest::AudioSuspended { .. } + | SessionRequest::AudioResumed { .. } + | SessionRequest::AudioDrained { .. } + | SessionRequest::AudioFailed { .. } + | SessionRequest::AudioCancelled { .. } + | SessionRequest::Shutdown => None, + }; + if id == Some(0) { + return Err("request id must be positive".into()); + } + match &request { + SessionRequest::PrepareSpeak { text, .. } if text.len() > MAX_SPEAK_TEXT_BYTES => { + return Err("speak text exceeds 16 KiB".into()) + } + SessionRequest::OutputReady { speech_id: 0, .. } => { + return Err("speech id must be positive".into()) + } + SessionRequest::SetTtsSettings { + expected_revision: 0, + .. + } => return Err("expected TTS revision must be positive".into()), + SessionRequest::SetInputDuringTts { + expected_revision: 0, + .. + } => return Err("expected input-during-TTS revision must be positive".into()), + SessionRequest::AudioBeginAccepted { speech_id: 0 } + | SessionRequest::AudioBeginFailed { speech_id: 0, .. } + | SessionRequest::AudioChunkAccepted { speech_id: 0, .. } + | SessionRequest::AudioPlayed { speech_id: 0, .. } + | SessionRequest::AudioSuspended { speech_id: 0, .. } + | SessionRequest::AudioResumed { speech_id: 0, .. } + | SessionRequest::AudioDrained { speech_id: 0, .. } + | SessionRequest::AudioFailed { speech_id: 0, .. } + | SessionRequest::AudioCancelled { speech_id: 0, .. } => { + return Err("audio speech id must be positive".into()) + } + SessionRequest::AudioChunkAccepted { sequence: 0, .. } + | SessionRequest::AudioDrained { sequence: 0, .. } => { + return Err("audio sequence must be positive".into()) + } + SessionRequest::AudioBeginFailed { message, .. } + | SessionRequest::AudioFailed { message, .. } + if message.len() > 4096 => + { + return Err("audio failure message exceeds 4 KiB".into()) + } + _ => {} + } + Ok(request) +} + +fn spawn_playback( + speech_id: u64, + text: String, + backend: Arc, + output: Arc, + active: Arc, + sender: mpsc::Sender, +) { + thread::spawn(move || { + let terminal = match play_tts(&text, backend.as_ref(), &output, &active) { + Ok((true, _)) => PlaybackEvent::Completed(speech_id), + Ok((false, delivery)) => PlaybackEvent::Interrupted( + speech_id, + u64::try_from(estimated_spoken_through_utf8(&text, &delivery)) + .expect("speech text is bounded well below u64"), + ), + Err(failure) => { + PlaybackEvent::Failed(speech_id, failure.message, failure.output_quiescent) + } + }; + let _ = sender.send(terminal); + }); +} + +fn play_tts( + text: &str, + backend: &dyn TtsBackend, + output: &RemotePcmAudioOutput, + active: &AtomicBool, +) -> Result<(bool, DeliveryProgress), PlaybackFailure> { + if let Err(message) = output.start() { + if message == AUDIO_CANCELLED { + return Ok(( + false, + DeliveryProgress { + sample_rate: backend.pcm_spec().sample_rate, + segments: Vec::new(), + }, + )); + } + return Err(PlaybackFailure { + message, + output_quiescent: output.failure_is_quiescent(), + }); + } + synthesize_to_output_with_finish( + text, + backend, + output, + active, + &mut || output.finish_writes(), + &mut || Ok(()), + ) +} + +#[cfg(test)] +fn synthesize_to_output( + speech_id: u64, + text: &str, + backend: &dyn TtsBackend, + output: &dyn berd_voice::PcmAudioOutput, + active: &AtomicBool, + sender: &mpsc::Sender, +) -> Result { + synthesize_to_output_with_finish(text, backend, output, active, &mut || Ok(()), &mut || { + let _ = sender.send(PlaybackEvent::Started(speech_id)); + Ok(()) + }) + .map(|(completed, _)| completed) +} + +fn synthesize_to_output_with_finish( + text: &str, + backend: &dyn TtsBackend, + output: &dyn berd_voice::PcmAudioOutput, + active: &AtomicBool, + finish_writes: &mut dyn FnMut() -> Result<(), String>, + on_started: &mut dyn FnMut() -> Result<(), String>, +) -> Result<(bool, DeliveryProgress), PlaybackFailure> { + use berd_voice::{DrainPolicy, DrainTimeoutOutcome, OutboundOutcome, OutboundPlayback}; + + let spec = backend.pcm_spec(); + let initial_frames = usize::try_from(spec.sample_rate / 5).map_err(|_| PlaybackFailure { + message: "TTS sample rate is too large".into(), + output_quiescent: false, + })?; + let mut playback = OutboundPlayback::new(output, active, spec.sample_rate, initial_frames) + .map_err(|message| PlaybackFailure { + message, + output_quiescent: false, + })?; + if playback + .synthesize_segment(backend, text, &mut |_| Ok(()), on_started, &mut |_| Ok(())) + .map_err(|failure| PlaybackFailure { + message: failure.message, + output_quiescent: failure.output_quiescent, + })? + == OutboundOutcome::Interrupted + { + return Ok((false, playback.snapshot())); + } + if let Err(message) = finish_writes() { + let output_quiescent = output.cancel_and_snapshot().is_ok(); + return Err(PlaybackFailure { + message, + output_quiescent, + }); + } + let outcome = playback + .finish( + DrainPolicy { + timeout: Some(Duration::from_secs(2)), + timeout_outcome: DrainTimeoutOutcome::Fail, + ..DrainPolicy::default() + }, + &mut |_| Ok(()), + ) + .map_err(|failure| PlaybackFailure { + message: failure.message, + output_quiescent: failure.output_quiescent, + })?; + Ok((outcome == OutboundOutcome::Completed, playback.snapshot())) +} + +#[cfg(test)] +mod tests { + use super::*; + use berd_voice::input::InputDuringTtsPolicy; + use berd_voice::{PcmAudioOutput, TtsOutcome, TtsPcmSpec}; + use serde_json::{json, Value}; + use std::io::{Cursor, Read, Write}; + use std::os::fd::IntoRawFd; + use std::os::unix::net::UnixStream; + use std::sync::Mutex; + + fn synthesis_config(tts: SynthesisTtsConfig, output: PathBuf) -> SynthesisConfig { + SynthesisConfig { + tts, + text: "A bounded test sentence.".into(), + output, + } + } + + #[test] + fn parses_closed_synthesis_surface_for_each_backend() { + let siri = parse_synthesis_args(&args(&[ + "berd-voice", + "synthesize", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en_US", + "--rate", + "2", + "--text", + "hello", + "--output", + "voice.wav", + ])) + .unwrap(); + assert!(matches!( + siri.tts, + SynthesisTtsConfig::Local(TtsBackendConfig::Siri { + language, + rate: 2.0, + .. + }) if language == "en-US" + )); + + let pocket = parse_synthesis_args(&args(&[ + "berd-voice", + "synthesize", + "--tts-backend", + "pocket", + "--model-dir", + "/models/pocket", + "--voice", + "mary", + "--rate", + "1", + "--text", + "hello", + "--output", + "voice.wav", + ])) + .unwrap(); + assert!(matches!( + pocket.tts, + SynthesisTtsConfig::Local(TtsBackendConfig::Pocket { rate: 1.0, .. }) + )); + + let openai = parse_synthesis_args(&args(&[ + "berd-voice", + "synthesize", + "--tts-backend", + "openai", + "--model", + "gpt-test", + "--voice", + "marin", + "--rate", + "1.5", + "--allow-paid-openai", + "--text", + "hello", + "--output", + "voice.wav", + ])) + .unwrap(); + assert!(matches!( + openai.tts, + SynthesisTtsConfig::OpenAi { rate: 1.5, .. } + )); + } + + #[test] + fn synthesis_parser_rejects_unsafe_or_untruthful_combinations() { + let cases = [ + vec![ + "--tts-backend", + "openai", + "--model", + "gpt-test", + "--voice", + "marin", + ], + vec![ + "--tts-backend", + "openai", + "--model", + "gpt-test", + "--voice", + "marin", + "--allow-paid-openai", + "--language", + "en-US", + ], + vec![ + "--tts-backend", + "pocket", + "--model-dir", + "/models/pocket", + "--voice", + "mary", + "--rate", + "2", + ], + vec![ + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-US", + "--allow-paid-openai", + ], + ]; + for mut flags in cases { + let mut values = vec!["berd-voice", "synthesize"]; + values.append(&mut flags); + values.extend(["--text", "hello", "--output", "voice.wav"]); + assert!(parse_synthesis_args(&args(&values)).is_err(), "{values:?}"); + } + assert!(parse_synthesis_args(&args(&[ + "berd-voice", + "synthesize", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-US", + "--text", + "hello", + "--output", + "-", + ])) + .is_err()); + } + + #[test] + fn output_preflight_precedes_backend_construction_and_never_clobbers() { + let directory = tempfile::tempdir().unwrap(); + let output = directory.path().join("voice.wav"); + std::fs::write(&output, b"owned").unwrap(); + let config = synthesis_config( + SynthesisTtsConfig::OpenAi { + model: "gpt-test".into(), + voice: "marin".into(), + rate: 1.0, + }, + output.clone(), + ); + let constructed = AtomicBool::new(false); + let error = run_synthesis_with_factory(&config, |_| { + constructed.store(true, Ordering::SeqCst); + Ok(Arc::new(FakeTts { frames: vec![0.1] })) + }) + .unwrap_err(); + assert_eq!(error.code, "output_unavailable"); + assert!(!constructed.load(Ordering::SeqCst)); + assert_eq!(std::fs::read(output).unwrap(), b"owned"); + } + + #[test] + fn synthesis_publishes_valid_wav_and_reports_only_public_identity() { + let directory = tempfile::tempdir().unwrap(); + let output = directory.path().join("voice.wav"); + let config = synthesis_config( + SynthesisTtsConfig::Local(TtsBackendConfig::Pocket { + model_dir: PathBuf::from("/private/model/path"), + voice: "mary".into(), + rate: 1.0, + }), + output.clone(), + ); + let result = run_synthesis_with_factory(&config, |_| { + Ok(Arc::new(FakeTts { + frames: vec![0.25, -0.25], + })) + }) + .unwrap(); + assert_eq!(&std::fs::read(&output).unwrap()[..4], b"RIFF"); + let value = serde_json::to_value(ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "synthesize", + event: "result", + result, + }) + .unwrap(); + assert_eq!(value["result"]["backend"], "pocket"); + assert_eq!( + value["result"]["model"], + berd_voice::pocket_assets::MODEL_ID + ); + assert_eq!(value["result"]["voice"], "mary"); + let serialized = value.to_string(); + assert!(!serialized.contains("/private")); + assert!(!serialized.contains("bounded test")); + } + + #[test] + fn synthesis_publish_race_preserves_the_competing_target() { + let directory = tempfile::tempdir().unwrap(); + let output = directory.path().join("voice.wav"); + let config = synthesis_config( + SynthesisTtsConfig::Local(TtsBackendConfig::Siri { + voice: "Aaron".into(), + language: "en-US".into(), + rate: 1.0, + }), + output.clone(), + ); + let error = run_synthesis_with_factory(&config, |_| { + std::fs::write(&output, b"race winner").unwrap(); + Ok(Arc::new(FakeTts { frames: vec![0.1] })) + }) + .unwrap_err(); + assert_eq!(error.code, "output_unavailable"); + assert_eq!(std::fs::read(&output).unwrap(), b"race winner"); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1); + } + + #[test] + fn synthesis_failure_after_partial_pcm_leaves_no_artifact() { + let directory = tempfile::tempdir().unwrap(); + let output = directory.path().join("voice.wav"); + let config = synthesis_config( + SynthesisTtsConfig::Local(TtsBackendConfig::Siri { + voice: "Aaron".into(), + language: "en-US".into(), + rate: 1.0, + }), + output.clone(), + ); + let error = + run_synthesis_with_factory(&config, |_| Ok(Arc::new(PartialFailureTts))).unwrap_err(); + assert_eq!(error.code, "synthesis_failed"); + assert!(!output.exists()); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + } + + struct FakeTts { + frames: Vec, + } + + struct PartialFailureTts; + + struct LongRemoteTts; + + impl TtsBackend for LongRemoteTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + } + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + on_frames(&vec![0.25; session_audio::MAX_AUDIO_CHUNK_FRAMES * 8])?; + Ok(TtsOutcome::Completed) + } + } + + impl TtsBackend for PartialFailureTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + } + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + on_frames(&[0.25, -0.25])?; + Err("provider stopped".into()) + } + } + + impl TtsBackend for FakeTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 10, + playback_rate: 1.0, + } + } + + fn synthesize( + &self, + _text: &str, + active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + if !active.load(Ordering::SeqCst) { + return Ok(TtsOutcome::Cancelled); + } + on_frames(&self.frames)?; + Ok(TtsOutcome::Completed) + } + } + + #[derive(Default)] + struct FakeOutput { + frames: Mutex>, + cancelled: AtomicBool, + } + + struct BlockingOutput { + cancelled: AtomicBool, + } + + struct InputStateWriter<'a> { + controls: &'a VoiceInputControls, + expected_muted: bool, + bytes: Vec, + } + + impl Write for InputStateWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + assert_eq!(self.controls.is_muted(), self.expected_muted); + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + assert_eq!(self.controls.is_muted(), self.expected_muted); + Ok(()) + } + } + + impl PcmAudioOutput for BlockingOutput { + fn write(&self, _samples: &[f32]) -> Result<(), String> { + Ok(()) + } + fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + } + fn is_drained(&self) -> bool { + self.cancelled.load(Ordering::SeqCst) + } + fn check_health(&self) -> Result<(), String> { + Ok(()) + } + fn played_frames(&self) -> u64 { + 0 + } + } + + impl PcmAudioOutput for FakeOutput { + fn write(&self, samples: &[f32]) -> Result<(), String> { + self.frames.lock().unwrap().extend_from_slice(samples); + Ok(()) + } + fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + } + fn is_drained(&self) -> bool { + true + } + fn check_health(&self) -> Result<(), String> { + Ok(()) + } + fn played_frames(&self) -> u64 { + self.frames.lock().unwrap().len() as u64 + } + } + + fn test_tts_slot() -> ConfiguredTtsSlot { + ConfiguredTtsSlot::new(TtsConfiguration::openai( + "https://example.invalid/audio/speech".into(), + "test-key".into(), + "test-model".into(), + "test-voice".into(), + 1.0, + )) + .unwrap() + } + + fn test_tts_lease() -> TtsConfigurationLease { + test_tts_slot().lease().unwrap() + } + + fn test_input_policy_slot() -> InputDuringTtsSlot { + InputDuringTtsSlot::new(InputDuringTtsPolicy::AllowBargeIn) + } + + fn test_input_policy() -> InputDuringTtsSnapshot { + test_input_policy_slot().snapshot().unwrap() + } + + fn active_playback(core: &mut SessionCore) -> ActivePlayback { + let PrepareOutcome::Admitted { + speech_id, text, .. + } = core.prepare(PrepareRequest { + id: 7, + acknowledgement: None, + text: "reply".into(), + }) + else { + panic!("test speech must be admitted") + }; + ActivePlayback { + prepare_id: 7, + speech_id, + text, + output: None, + active: Some(Arc::new(AtomicBool::new(true))), + ready_deadline: Instant::now() + Duration::from_secs(2), + assistant_activity: None, + input_during_tts: test_input_policy(), + tts: test_tts_lease(), + suspension_requested: false, + } + } + + fn read_audio_record(reader: &mut impl Read) -> (u8, Vec) { + let mut header = [0; session_audio::AUDIO_FRAME_HEADER_BYTES]; + reader.read_exact(&mut header).unwrap(); + assert_eq!(header[..2], session_audio::AUDIO_FRAME_MAGIC); + assert_eq!(header[2], session_audio::AUDIO_FRAME_MARKER); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + let mut payload = vec![0; length]; + reader.read_exact(&mut payload).unwrap(); + (header[3], payload) + } + + #[test] + fn false_barge_quiesces_and_resumes_one_remote_speech_without_a_terminal() { + let mut core = SessionCore::default(); + let mut current = active_playback(&mut core); + let speech_id = current.speech_id; + let (child, mut host) = UnixStream::pair().unwrap(); + let transport = unsafe { AudioPipeTransport::from_raw_fd(child.into_raw_fd()) }.unwrap(); + let authority = current.active.as_ref().unwrap().clone(); + let (control_sender, control_receiver) = mpsc::channel(); + let output = Arc::new( + RemotePcmAudioOutput::new( + speech_id, + LongRemoteTts.pcm_spec(), + Arc::new(transport), + Arc::clone(&authority), + control_sender, + ) + .unwrap(), + ); + current.output = Some(Arc::clone(&output)); + let mut active = Some(current); + let (playback_sender, playback_receiver) = mpsc::channel(); + spawn_playback( + speech_id, + "a long remote reply".into(), + Arc::new(LongRemoteTts), + Arc::clone(&output), + authority, + playback_sender, + ); + + let (kind, _) = read_audio_record(&mut host); + assert_eq!(kind, session_audio::AUDIO_BEGIN_KIND); + assert!( + !handle_audio_ack(speech_id, AudioHostAck::BeginAccepted, active.as_ref()).unwrap() + ); + let (kind, first_chunk) = read_audio_record(&mut host); + assert_eq!(kind, session_audio::AUDIO_CHUNK_KIND); + let first_frames = u64::try_from((first_chunk.len() - 16) / 4).unwrap(); + let mut output_messages = Vec::new(); + let mut next_token = 1; + + handle_voice_input_event( + VoiceInputEvent::SpeakingChanged(true), + &mut core, + &mut active, + &mut next_token, + &mut output_messages, + ) + .unwrap(); + let suspend = control_receiver + .recv_timeout(Duration::from_secs(1)) + .unwrap(); + write_audio_control_request(suspend, active.as_ref(), &mut output_messages).unwrap(); + assert!(handle_audio_ack( + speech_id, + AudioHostAck::ChunkAccepted { sequence: 1 }, + active.as_ref(), + ) + .unwrap()); + publish_speech_started(speech_id, &mut core, active.as_ref(), &mut output_messages) + .unwrap(); + handle_audio_ack( + speech_id, + AudioHostAck::Played { + played_frames: first_frames, + }, + active.as_ref(), + ) + .unwrap(); + handle_audio_ack( + speech_id, + AudioHostAck::Suspended { + played_frames: first_frames, + }, + active.as_ref(), + ) + .unwrap(); + host.set_read_timeout(Some(Duration::from_millis(30))) + .unwrap(); + let mut byte = [0]; + assert!(host.read(&mut byte).is_err()); + + for event in [ + VoiceInputEvent::RecognitionPendingChanged(true), + VoiceInputEvent::SpeakingChanged(false), + ] { + handle_voice_input_event( + event, + &mut core, + &mut active, + &mut next_token, + &mut output_messages, + ) + .unwrap(); + } + assert!(control_receiver.try_recv().is_err()); + handle_voice_input_event( + VoiceInputEvent::RecognitionPendingChanged(false), + &mut core, + &mut active, + &mut next_token, + &mut output_messages, + ) + .unwrap(); + let resume = control_receiver + .recv_timeout(Duration::from_secs(1)) + .unwrap(); + write_audio_control_request(resume, active.as_ref(), &mut output_messages).unwrap(); + handle_audio_ack( + speech_id, + AudioHostAck::Resumed { + played_frames: first_frames, + }, + active.as_ref(), + ) + .unwrap(); + + host.set_read_timeout(None).unwrap(); + let mut played_frames = first_frames; + let mut last_sequence = 1; + loop { + let (kind, payload) = read_audio_record(&mut host); + match kind { + session_audio::AUDIO_CHUNK_KIND => { + let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); + let frames = u64::try_from((payload.len() - 16) / 4).unwrap(); + assert_eq!(sequence, last_sequence + 1); + last_sequence = sequence; + handle_audio_ack( + speech_id, + AudioHostAck::ChunkAccepted { sequence }, + active.as_ref(), + ) + .unwrap(); + played_frames += frames; + handle_audio_ack( + speech_id, + AudioHostAck::Played { played_frames }, + active.as_ref(), + ) + .unwrap(); + } + session_audio::AUDIO_END_KIND => { + let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); + let total_frames = u64::from_le_bytes(payload[16..24].try_into().unwrap()); + assert_eq!(sequence, last_sequence); + assert_eq!(total_frames, played_frames); + handle_audio_ack( + speech_id, + AudioHostAck::Drained { + sequence, + played_frames, + }, + active.as_ref(), + ) + .unwrap(); + break; + } + other => panic!("unexpected audio record kind {other}"), + } + } + let terminal = playback_receiver + .recv_timeout(Duration::from_secs(1)) + .unwrap(); + handle_playback_event(terminal, &mut core, &mut active, &mut output_messages).unwrap(); + + let emitted = messages(&output_messages); + assert!(!emitted + .iter() + .any(|message| message["type"] == "speech_interrupted")); + assert_eq!( + emitted + .iter() + .filter(|message| message["type"] == "speech_completed") + .count(), + 1 + ); + assert!(active.is_none()); + } + + fn messages(output: &[u8]) -> Vec { + std::str::from_utf8(output) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } + + #[test] + fn public_tts_protocol_messages_never_expose_private_paths() { + let private_path = "/Users/alice/private/native-voice-v2"; + let snapshot = berd_voice::TtsConfigurationSnapshot { + revision: 1, + settings: berd_voice::TtsSettings::Pocket { + model: berd_voice::pocket_assets::MODEL_ID.into(), + voice: "mary".into(), + rate: 1.0, + }, + }; + let ready = serde_json::to_string(&SessionMessage::Ready { + id: 1, + protocol: WIRE_MARKER, + session: VoiceSessionSnapshot { + tts: snapshot.clone(), + input_during_tts: test_input_policy(), + }, + }) + .unwrap(); + let rejection = TtsConfigurationRejection { + kind: TtsConfigurationRejectionKind::Initialization, + message: format!("could not load {private_path}/model.onnx"), + snapshot: snapshot.clone(), + }; + let result = serde_json::to_string(&SessionMessage::TtsSettingsResult { + id: 2, + outcome: TtsSettingsOutcome::Rejected, + snapshot: rejection.snapshot, + message: Some(public_tts_rejection_message(rejection.kind).into()), + }) + .unwrap(); + let fatal = serde_json::to_string(&SessionMessage::Fatal { + message: public_tts_startup_error(&TtsBackendConfig::Pocket { + model_dir: PathBuf::from(private_path), + voice: private_path.into(), + rate: 1.0, + }), + }) + .unwrap(); + + for message in [ready, result, fatal] { + assert!(!message.contains(private_path)); + assert!(!message.contains("/Users/alice")); + } + } + + fn args(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_string()).collect() + } + + #[test] + fn management_cli_parses_only_the_closed_command_shapes() { + assert_eq!( + parse_management_args(&args(&["berd-voice", "voices", "list"])).unwrap(), + ManagementCommand::ListVoices { language: None } + ); + assert_eq!( + parse_management_args(&args(&[ + "berd-voice", + "voices", + "list", + "--language", + "en_US" + ])) + .unwrap(), + ManagementCommand::ListVoices { + language: Some("en-US".into()) + } + ); + assert_eq!( + parse_management_args(&args(&[ + "berd-voice", + "voices", + "download", + "--voice", + "Aaron", + "--language", + "en_US" + ])) + .unwrap(), + ManagementCommand::DownloadVoice { + identity: berd_voice::siri::SiriVoiceIdentity::new("Aaron", "en-US").unwrap(), + availability_wait: berd_voice::siri::SiriDownloadAvailabilityWait::default(), + } + ); + assert_eq!( + parse_management_args(&args(&[ + "berd-voice", + "voices", + "download", + "--voice", + "Aaron", + "--language", + "en-US", + "--availability-wait-seconds", + "12" + ])) + .unwrap(), + ManagementCommand::DownloadVoice { + identity: berd_voice::siri::SiriVoiceIdentity::new("Aaron", "en-US").unwrap(), + availability_wait: berd_voice::siri::SiriDownloadAvailabilityWait::from_seconds(12) + .unwrap(), + } + ); + assert_eq!( + parse_management_args(&args(&["berd-voice", "models", "macos", "status"])).unwrap(), + ManagementCommand::MacosModelStatus + ); + assert_eq!( + parse_management_args(&args(&["berd-voice", "models", "macos", "install"])).unwrap(), + ManagementCommand::InstallMacosModel + ); + let store = std::env::temp_dir().join("berd-voice-management-parser"); + let roots = local_model_roots(&store).unwrap(); + assert_eq!( + parse_management_args(&args(&[ + "berd-voice", + "models", + "pocket", + "status", + "--store-root", + store.to_str().unwrap(), + ])) + .unwrap(), + ManagementCommand::PocketModelStatus { + roots: roots.clone() + } + ); + assert_eq!( + parse_management_args(&args(&[ + "berd-voice", + "models", + "pocket", + "install", + "--store-root", + store.to_str().unwrap(), + ])) + .unwrap(), + ManagementCommand::InstallPocketModel { + roots: roots.clone() + } + ); + assert_eq!( + parse_management_args(&args(&[ + "berd-voice", + "models", + "parakeet", + "status", + "--store-root", + store.to_str().unwrap(), + ])) + .unwrap(), + ManagementCommand::ParakeetModelStatus { + roots: roots.clone() + } + ); + assert_eq!( + parse_management_args(&args(&[ + "berd-voice", + "models", + "parakeet", + "install", + "--store-root", + store.to_str().unwrap(), + ])) + .unwrap(), + ManagementCommand::InstallParakeetModel { roots } + ); + assert_eq!( + parse_management_args(&args(&["berd-voice", "models", "pocket", "voices"])).unwrap(), + ManagementCommand::ListPocketVoices + ); + + for invalid in [ + vec!["berd-voice", "voices", "list", "--language"], + vec!["berd-voice", "voices", "list", "--unknown", "en-US"], + vec!["berd-voice", "voices", "download", "--voice", "Aaron"], + vec![ + "berd-voice", + "voices", + "download", + "--voice", + "Aaron", + "--language", + "en-US", + "--availability-wait-seconds", + "0", + ], + vec![ + "berd-voice", + "voices", + "download", + "--voice", + "Aaron", + "--language", + "en-US", + "--availability-wait-seconds", + "1801", + ], + vec![ + "berd-voice", + "voices", + "download", + "--voice", + "aaron ", + "--language", + "en-US", + ], + vec!["berd-voice", "models", "macos", "status", "extra"], + vec!["berd-voice", "models", "pocket", "status"], + vec![ + "berd-voice", + "models", + "pocket", + "status", + "--store-root", + "relative", + ], + vec![ + "berd-voice", + "models", + "parakeet", + "install", + "--store-root", + "/tmp/../outside", + ], + vec![ + "berd-voice", + "models", + "pocket", + "status", + "--store-root", + "/tmp/./store", + ], + vec!["berd-voice", "models", "pocket", "voices", "extra"], + ] { + assert!( + parse_management_args(&args(&invalid)).is_err(), + "{invalid:?}" + ); + } + assert_eq!( + parse_management_args(&args(&["berd-voice", "models", "pocket", "typo"])).unwrap_err(), + "expected a supported models command" + ); + } + + #[test] + fn management_json_schemas_are_stable_and_sanitized() { + let list = voices_list_report( + true, + Some("en-US".into()), + berd_voice::siri::SiriVoiceCatalog { + available_languages: vec!["en-US".into()], + voices: vec![berd_voice::siri::SiriVoice { + name: "Aaron".into(), + language: "en-US".into(), + size_bytes: 42, + installed: true, + }], + }, + ); + assert_eq!( + serde_json::to_value(ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "voices.list", + event: "result", + result: list, + }) + .unwrap(), + json!({ + "schemaVersion": 1, + "operation": "voices.list", + "event": "result", + "result": { + "supported": true, + "backend": "siri", + "languageFilter": "en-US", + "availableLanguages": ["en-US"], + "voices": [{ + "name": "Aaron", + "language": "en-US", + "sizeBytes": 42, + "installed": true + }] + } + }) + ); + + assert_eq!( + serde_json::to_value(ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "models.pocket.status", + event: "result", + result: local_model_status_report(LocalModelKind::Pocket, LocalModelState::Missing), + }) + .unwrap(), + json!({ + "schemaVersion": 1, + "operation": "models.pocket.status", + "event": "result", + "result": { + "backend": "pocket", + "modelId": "native-voice-v2", + "state": "missing", + "ready": false, + "verifiedBytes": null, + "totalDownloadBytes": berd_voice::pocket_assets::download_bytes() + } + }) + ); + assert_eq!( + serde_json::to_value(ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "models.pocket.voices", + event: "result", + result: pocket_voices_report(), + }) + .unwrap()["result"]["voices"][0], + json!({"id": "anna", "name": "Anna"}) + ); + let voices = serde_json::to_value(ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "models.pocket.voices", + event: "result", + result: pocket_voices_report(), + }) + .unwrap(); + assert_eq!(voices["result"]["backend"], "pocket"); + assert_eq!(voices["result"]["modelId"], "native-voice-v2"); + assert_eq!(voices["result"]["voiceLicenseId"], "CC-BY-4.0"); + assert_eq!(voices["result"]["voices"].as_array().unwrap().len(), 12); + let voices = voices.to_string(); + for private_field in [ + "relativePath", + "sizeBytes", + "sha256", + "sourceUrl", + "https://", + ] { + assert!(!voices.contains(private_field)); + } + + assert_eq!( + serde_json::to_value(ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "models.parakeet.install", + event: "result", + result: LocalModelInstallResult { + backend: "parakeet", + model_id: berd_voice::parakeet_assets::MODEL_ID, + outcome: "installed", + ready: true, + verified_bytes: 123, + cleanup_pending: true, + }, + }) + .unwrap(), + json!({ + "schemaVersion": 1, + "operation": "models.parakeet.install", + "event": "result", + "result": { + "backend": "parakeet", + "modelId": "parakeet-tdt-ctc-110m-en-int8", + "outcome": "installed", + "ready": true, + "verifiedBytes": 123, + "cleanupPending": true + } + }) + ); + + let identity = berd_voice::siri::SiriVoiceIdentity::new("Aaron", "en_US").unwrap(); + assert_eq!( + serde_json::to_value(ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "voices.download", + event: "result", + result: voice_download_report( + &identity, + berd_voice::siri::SiriDownloadAvailabilityWait::default(), + ), + }) + .unwrap(), + json!({ + "schemaVersion": 1, + "operation": "voices.download", + "event": "result", + "result": { + "backend": "siri", + "voice": {"name": "Aaron", "language": "en-US"}, + "installed": true, + "availabilityWaitSeconds": 300 + } + }) + ); + + let status = MacosModelStatus { + supported: true, + locale: Some("en-US".into()), + locale_supported: true, + model_status: "installed".into(), + ready: true, + }; + assert_eq!( + serde_json::to_value(ManagementResultEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "models.macos.status", + event: "result", + result: status, + }) + .unwrap(), + json!({ + "schemaVersion": 1, + "operation": "models.macos.status", + "event": "result", + "result": { + "supported": true, + "locale": "en-US", + "localeSupported": true, + "modelStatus": "installed", + "ready": true + } + }) + ); + } + + #[test] + fn macos_model_install_progress_is_honest_and_bounded() { + assert_eq!(normalized_install_progress(0.0), Some(0.0)); + assert_eq!(normalized_install_progress(0.427), Some(0.427)); + assert_eq!(normalized_install_progress(1.0), Some(1.0)); + assert_eq!(normalized_install_progress(-0.1), Some(0.0)); + assert_eq!(normalized_install_progress(1.1), Some(1.0)); + assert_eq!(normalized_install_progress(f64::NAN), None); + assert_eq!( + serde_json::to_value(ManagementProgressEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "models.macos.install", + event: "progress", + fraction: 0.427, + }) + .unwrap(), + json!({ + "schemaVersion": 1, + "operation": "models.macos.install", + "event": "progress", + "fraction": 0.427 + }) + ); + assert_eq!( + serde_json::to_value(LocalModelProgressEnvelope { + schema_version: MANAGEMENT_SCHEMA_VERSION, + operation: "models.pocket.install", + event: "progress", + phase: local_install_phase_name(LocalInstallPhase::Verifying), + downloaded_bytes: 42, + total_download_bytes: 100, + }) + .unwrap(), + json!({ + "schemaVersion": 1, + "operation": "models.pocket.install", + "event": "progress", + "phase": "verifying", + "downloadedBytes": 42, + "totalDownloadBytes": 100 + }) + ); + } + + #[test] + fn management_operation_errors_are_structured_without_details() { + let failure = management_failure( + "operation_failed", + "Could not make the requested Siri voice available", + "private native detail at /Users/alice/private", + ); + let envelope = management_error_envelope("voices.download", &failure); + let json = serde_json::to_string(&envelope).unwrap(); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + json!({ + "schemaVersion": 1, + "operation": "voices.download", + "event": "error", + "error": { + "code": "operation_failed", + "message": "Could not make the requested Siri voice available" + } + }) + ); + assert!(!json.contains("/Users/alice/private")); + + let missing = voice_download_failure(berd_voice::siri::SiriVoiceDownloadError::NotFound( + berd_voice::siri::SiriVoiceIdentity::new("Missing", "en-US").unwrap(), + )); + assert_eq!(missing.code, "voice_not_found"); + + let local = local_install_failure(LocalInstallError { + kind: LocalInstallErrorKind::Rollback, + message: "private rollback detail".into(), + recovery_paths: vec![PathBuf::from("/Users/alice/private-backup")], + }); + assert_eq!(local.code, "rollback_failed"); + assert!(local.detail.contains("/Users/alice/private-backup")); + let envelope = + serde_json::to_string(&management_error_envelope("models.pocket.install", &local)) + .unwrap(); + assert!(!envelope.contains("/Users/alice")); + assert!(!envelope.contains("private rollback detail")); + } + + #[test] + fn unsupported_platform_status_has_the_same_schema() { + assert_eq!( + unsupported_macos_model_status(), + MacosModelStatus { + supported: false, + locale: None, + locale_supported: false, + model_status: "unsupported".into(), + ready: false, + } + ); + } + + #[test] + fn macos_model_install_is_idempotent_and_rejects_unsupported_states_before_mutation() { + let status = |supported: bool, locale_supported: bool, ready: bool| MacosModelStatus { + supported, + locale: locale_supported.then(|| "en-US".into()), + locale_supported, + model_status: if ready { "installed" } else { "available" }.into(), + ready, + }; + + assert!(!macos_install_needs_mutation(&status(true, true, true)).unwrap()); + assert!(macos_install_needs_mutation(&status(true, true, false)).unwrap()); + assert_eq!( + macos_install_needs_mutation(&status(false, false, false)) + .unwrap_err() + .code, + "unsupported" + ); + assert_eq!( + macos_install_needs_mutation(&status(true, false, false)) + .unwrap_err() + .code, + "unsupported_locale" + ); + } + + #[test] + fn cli_defaults_to_exact_siri_and_macos_without_cloud_fallback() { + let missing_voice = parse_args(&args(&["berd-voice", "session"])).unwrap_err(); + assert!(missing_voice.contains("Siri TTS is the default")); + assert!(missing_voice.contains("--voice NAME and --language BCP47")); + + assert_eq!( + parse_args(&args(&[ + "berd-voice", + "session", + "--voice", + "Aaron", + "--language", + "en-US" + ])) + .unwrap(), + SessionConfig { + tts: TtsBackendConfig::Siri { + voice: "Aaron".into(), + language: "en-US".into(), + rate: 1.0, + }, + stt: SttBackendConfig::Macos, + } + ); + + assert_eq!( + parse_args(&args(&["berd-voice", "session", "--tts-backend", "openai"])).unwrap(), + SessionConfig { + tts: TtsBackendConfig::OpenAi { rate: 1.0 }, + stt: SttBackendConfig::Macos, + } + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn native_default_availability_errors_are_actionable() { + let unavailable_siri = create_tts_backend(&TtsBackendConfig::Siri { + voice: "__berd_voice_does_not_exist__".into(), + language: "en-US".into(), + rate: 1.0, + }) + .err() + .unwrap(); + assert!(unavailable_siri.contains("is unavailable")); + assert!(unavailable_siri.contains("Download it in Berd Voice settings")); + + let status = |supported: bool, locale_supported: bool, model_status: &str, ready: bool| { + berd_voice::mac_speech::MacSpeechEngineStatus { + supported, + locale: locale_supported.then(|| "en-US".into()), + locale_supported, + model_status: model_status.into(), + ready, + } + }; + for (status, expected) in [ + ( + status(false, false, "unsupported", false), + "requires macOS 26 or later with SpeechTranscriber available", + ), + ( + status(true, false, "unsupported", false), + "does not support the current system locale", + ), + ( + status(true, true, "downloading", false), + "Wait for the download to finish", + ), + ( + status(true, true, "available", false), + "Download the current-locale model", + ), + ] { + let error = validate_macos_stt_status(&status).unwrap_err(); + assert!(error.contains(expected), "{error}"); + assert!(error.contains("explicitly select --stt-backend")); + } + + let ready = status(true, true, "installed", true); + assert_eq!(validate_macos_stt_status(&ready), Ok(())); + } + + #[test] + fn cli_requires_exact_siri_selection_and_bounds_rate() { + assert_eq!( + parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-US" + ])) + .unwrap(), + SessionConfig { + tts: TtsBackendConfig::Siri { + voice: "Aaron".into(), + language: "en-US".into(), + rate: 1.0, + }, + stt: SttBackendConfig::Macos, + } + ); + assert!(parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-US", + "--rate", + "2.1" + ])) + .is_err()); + } + + #[test] + fn cli_accepts_openai_rate_two_and_rejects_out_of_range_rates() { + assert_eq!( + parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "openai", + "--rate", + "2.0" + ])) + .unwrap() + .tts, + TtsBackendConfig::OpenAi { rate: 2.0 } + ); + assert!(parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "openai", + "--rate", + "2.1" + ])) + .unwrap_err() + .contains("0.75 and 2.0")); + } + + #[test] + fn cli_requires_explicit_pocket_bundle_and_voice() { + assert_eq!( + parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "pocket", + "--model-dir", + "/models/native-voice-v2", + "--voice", + "george" + ])) + .unwrap(), + SessionConfig { + tts: TtsBackendConfig::Pocket { + model_dir: PathBuf::from("/models/native-voice-v2"), + voice: "george".into(), + rate: 1.0, + }, + stt: SttBackendConfig::Macos, + } + ); + assert!(parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "pocket", + "--voice", + "george" + ])) + .unwrap_err() + .contains("--model-dir is required")); + assert!(parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "pocket", + "--model-dir", + "/models", + "--voice", + "george", + "--rate", + "0.5" + ])) + .unwrap_err() + .contains("0.75 and 2.0")); + assert!(parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "pocket", + "--model-dir", + "relative/model", + "--voice", + "george" + ])) + .unwrap_err() + .contains("absolute path")); + } + + #[test] + fn cli_stt_selection_is_closed_and_parakeet_owns_only_an_explicit_bundle() { + assert_eq!( + parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "openai", + "--stt-backend", + "parakeet", + "--stt-model-dir", + "/models/parakeet" + ])) + .unwrap(), + SessionConfig { + tts: TtsBackendConfig::OpenAi { rate: 1.0 }, + stt: SttBackendConfig::Parakeet { + model_dir: PathBuf::from("/models/parakeet") + } + } + ); + assert!(parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "openai", + "--stt-backend", + "parakeet" + ])) + .unwrap_err() + .contains("--stt-model-dir is required")); + assert!(parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "openai", + "--stt-backend", + "macos", + "--stt-model-dir", + "/models/parakeet" + ])) + .unwrap_err() + .contains("only valid with Parakeet")); + assert!(parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "openai", + "--stt-backend", + "parakeet", + "--stt-model-dir", + "relative" + ])) + .unwrap_err() + .contains("absolute path")); + } + + #[test] + fn benchmark_cli_requires_explicit_comparable_inputs() { + assert_eq!( + parse_tts_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-US", + "--text", + "A fixed benchmark sentence.", + "--runs", + "3", + "--mode", + "warm" + ])) + .unwrap(), + TtsBenchmarkConfig { + tts: TtsBackendConfig::Siri { + voice: "Aaron".into(), + language: "en-US".into(), + rate: 1.0, + }, + prompts: TtsBenchmarkPrompts::ExactRepeat { + text: "A fixed benchmark sentence.".into(), + runs: 3, + }, + mode: TtsBenchmarkMode::Warm, + } + ); + assert!(parse_tts_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "openai", + "--text", + "hello", + "--mode", + "fresh-backend" + ])) + .unwrap_err() + .contains("--runs is required")); + assert!(parse_tts_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "openai", + "--text", + "hello", + "--runs", + "0", + "--mode", + "fresh-backend" + ])) + .unwrap_err() + .contains("between 1 and 100")); + } + + #[test] + fn benchmark_cli_reuses_backend_specific_validation() { + assert!(parse_tts_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "pocket", + "--model-dir", + "relative", + "--voice", + "mary", + "--text", + "hello", + "--runs", + "1", + "--mode", + "fresh-backend" + ])) + .unwrap_err() + .contains("absolute path")); + assert!(parse_tts_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "openai", + "--text", + "hello", + "--runs", + "1", + "--mode", + "fresh-backend", + "--stt-backend", + "macos" + ])) + .unwrap_err() + .contains("unknown argument")); + } + + #[test] + fn benchmark_cli_selects_fixed_distinct_prompt_manifest() { + let config = parse_tts_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-US", + "--prompt-manifest", + "english-short-v1", + "--mode", + "warm", + ])) + .unwrap(); + let TtsBenchmarkPrompts::Manifest(manifest) = config.prompts else { + panic!("expected prompt manifest") + }; + assert_eq!(manifest.id, "english-short-v1"); + assert_eq!(manifest.prompts.len(), 5); + + assert!(parse_tts_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-CA", + "--prompt-manifest", + "english-short-v1", + "--mode", + "warm", + ])) + .unwrap_err() + .contains("requires Siri language en-US")); + assert!(parse_tts_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-US", + "--prompt-manifest", + "english-short-v1", + "--runs", + "5", + "--mode", + "warm", + ])) + .unwrap_err() + .contains("fixed by --prompt-manifest")); + } + + #[test] + fn openai_tts_target_reports_rate_and_endpoint_source() { + let target = tts_benchmark_target(&TtsBackendConfig::OpenAi { rate: 1.75 }, false); + assert_eq!(target.rate, Some(1.75)); + assert_eq!(target.endpoint_source.as_deref(), Some("built_in_default")); + assert_eq!( + tts_benchmark_target(&TtsBackendConfig::OpenAi { rate: 1.0 }, true) + .endpoint_source + .as_deref(), + Some("OPENAI_BASE_URL_environment") + ); + } + + #[test] + fn benchmark_cli_requires_and_bounds_paid_openai_consent() { + let base = [ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "openai", + "--text", + "hello", + "--runs", + "1", + "--mode", + "fresh-backend", + ]; + assert!(parse_tts_benchmark_args(&args(&base)) + .unwrap_err() + .contains("--allow-paid-openai")); + + let mut consented = args(&base); + consented.push("--allow-paid-openai".into()); + assert!(parse_tts_benchmark_args(&consented).is_ok()); + + let warm_limit = args(&[ + "berd-voice", + "benchmark", + "tts", + "--tts-backend", + "openai", + "--text", + "hello", + "--runs", + "20", + "--mode", + "warm", + "--allow-paid-openai", + ]); + assert!(parse_tts_benchmark_args(&warm_limit) + .unwrap_err() + .contains("21 requests")); + + let oversized_text = "a".repeat(4_000); + let oversized_workload = vec![ + "berd-voice".into(), + "benchmark".into(), + "tts".into(), + "--tts-backend".into(), + "openai".into(), + "--text".into(), + oversized_text, + "--runs".into(), + "20".into(), + "--mode".into(), + "fresh-backend".into(), + "--allow-paid-openai".into(), + ]; + assert!(parse_tts_benchmark_args(&oversized_workload) + .unwrap_err() + .contains("80000 total UTF-8 text bytes")); + } + + #[test] + fn stt_benchmark_cli_is_explicit_and_reuses_engine_validation() { + assert_eq!( + parse_stt_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "stt", + "--stt-backend", + "macos", + "--runs", + "2", + "--mode", + "cold", + ])) + .unwrap(), + SttBenchmarkConfig { + stt: SttBackendConfig::Macos, + runs: 2, + mode: SttBenchmarkMode::Cold, + allow_paid_openai: false, + } + ); + assert!(parse_stt_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "stt", + "--stt-backend", + "parakeet", + "--runs", + "1", + "--mode", + "warm", + ])) + .unwrap_err() + .contains("--stt-model-dir is required")); + assert!(parse_stt_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "stt", + "--stt-backend", + "parakeet", + "--stt-model-dir", + "relative", + "--runs", + "1", + "--mode", + "warm", + ])) + .unwrap_err() + .contains("absolute path")); + } + + #[test] + fn stt_benchmark_paid_openai_consent_bounds_full_streamed_workload() { + let base = [ + "berd-voice", + "benchmark", + "stt", + "--stt-backend", + "openai", + "--runs", + "1", + "--mode", + "cold", + ]; + assert!(parse_stt_benchmark_args(&args(&base)) + .unwrap_err() + .contains("--allow-paid-openai")); + + let pack = load_bundled_stt_fixture_pack().unwrap(); + let allowed = parse_stt_benchmark_args(&args(&[ + "berd-voice", + "benchmark", + "stt", + "--stt-backend", + "openai", + "--runs", + "2", + "--mode", + "warm", + "--allow-paid-openai", + ])) + .unwrap(); + validate_stt_benchmark_workload(&allowed, &pack.workload(2, SttBenchmarkMode::Warm)) + .unwrap(); + + let too_many_seconds = SttBenchmarkConfig { + runs: 6, + mode: SttBenchmarkMode::Cold, + ..allowed.clone() + }; + assert!(validate_stt_benchmark_workload( + &too_many_seconds, + &pack.workload(6, SttBenchmarkMode::Cold) + ) + .unwrap_err() + .contains("232.92 seconds")); + + let too_many_commits = SttBenchmarkConfig { + runs: 7, + mode: SttBenchmarkMode::Cold, + ..allowed + }; + assert!(validate_stt_benchmark_workload( + &too_many_commits, + &pack.workload(7, SttBenchmarkMode::Cold) + ) + .unwrap_err() + .contains("21 recognition commits")); + } + + #[cfg(target_os = "macos")] + #[test] + #[ignore = "requires the installed current-locale macOS SpeechTranscriber model"] + fn local_macos_stt_benchmark_uses_the_production_runtime() { + let report = create_stt_benchmark_report(&SttBenchmarkConfig { + stt: SttBackendConfig::Macos, + runs: 1, + mode: SttBenchmarkMode::Cold, + allow_paid_openai: false, + }) + .unwrap(); + assert!(report.succeeded()); + assert_eq!(report.runs[0].utterances.len(), 3); + } + + #[test] + #[ignore = "requires BERD_PARAKEET_TEST_MODEL_DIR with a complete Parakeet bundle"] + fn local_parakeet_stt_benchmark_uses_the_production_runtime() { + let model_dir = PathBuf::from(std::env::var("BERD_PARAKEET_TEST_MODEL_DIR").unwrap()); + let report = create_stt_benchmark_report(&SttBenchmarkConfig { + stt: SttBackendConfig::Parakeet { model_dir }, + runs: 1, + mode: SttBenchmarkMode::Cold, + allow_paid_openai: false, + }) + .unwrap(); + assert!(report.succeeded()); + assert_eq!(report.runs[0].utterances.len(), 3); + } + + #[test] + fn siri_tts_and_openai_stt_selection_are_orthogonal() { + assert_eq!( + parse_args(&args(&[ + "berd-voice", + "session", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-US", + "--stt-backend", + "openai" + ])) + .unwrap(), + SessionConfig { + tts: TtsBackendConfig::Siri { + voice: "Aaron".into(), + language: "en-US".into(), + rate: 1.0 + }, + stt: SttBackendConfig::OpenAi + } + ); + } + + #[test] + fn session_requires_one_inherited_pcm_output_descriptor() { + assert_eq!( + parse_pcm_output_fd(&args(&["berd-voice", "session", "--pcm-output-fd", "9"])).unwrap(), + 9 + ); + assert_eq!( + parse_pcm_output_fd(&args(&["berd-voice", "session"])).unwrap_err(), + "--pcm-output-fd is required" + ); + assert!( + parse_pcm_output_fd(&args(&["berd-voice", "session", "--pcm-output-fd", "2"])).is_err() + ); + assert!(parse_pcm_output_fd(&args(&[ + "berd-voice", + "session", + "--pcm-output-fd", + "7", + "--pcm-output-fd", + "8" + ])) + .is_err()); + } + + fn framed(kind: u8, payload: &[u8]) -> Vec { + let mut frame = Vec::from([b'B', b'V', WIRE_MARKER as u8, kind]); + frame.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + frame.extend_from_slice(payload); + frame + } + + #[test] + fn framing_decodes_json_and_exact_pcm_without_line_ambiguity() { + let json = br#"{"type":"hello","id":1,"input_during_tts":"allow_barge_in"}"#; + let pcm = [0_u8; PCM_FRAME_BYTES]; + let mut bytes = framed(JSON_FRAME_KIND, json); + bytes.extend_from_slice(&framed(PCM_FRAME_KIND, &pcm)); + let (control_sender, control_receiver) = mpsc::channel(); + let (pcm_sender, pcm_receiver) = mpsc::sync_channel(3); + + read_framed_requests(Cursor::new(bytes), control_sender, pcm_sender); + + assert!(matches!( + control_receiver.recv().unwrap().input, + Input::Request(SessionRequest::Hello { id: 1, .. }) + )); + assert!(pcm_receiver.recv().is_ok()); + assert!(matches!(control_receiver.recv().unwrap().input, Input::Eof)); + } + + #[test] + fn disconnected_pcm_channel_does_not_overtake_queued_control() { + let (control_sender, control_receiver) = mpsc::channel(); + let (pcm_sender, pcm_receiver) = mpsc::sync_channel(1); + control_sender + .send(OrderedControl { + after_pcm: 0, + input: Input::Request(SessionRequest::Shutdown), + }) + .unwrap(); + drop(control_sender); + drop(pcm_sender); + + let mut pending = None; + let mut processed = 0; + assert!(matches!( + receive_session_input( + &control_receiver, + &pcm_receiver, + &mut pending, + &mut processed + ), + Some(Input::Request(SessionRequest::Shutdown)) + )); + } + + #[test] + fn framing_rejects_oversized_json_and_wrong_pcm_before_payload_allocation() { + for (kind, length, expected) in [ + (JSON_FRAME_KIND, MAX_LINE_BYTES + 1, "request exceeds 1 MiB"), + (PCM_FRAME_KIND, PCM_FRAME_BYTES - 1, "PCM frame has"), + ] { + let mut header = Vec::from([b'B', b'V', WIRE_MARKER as u8, kind]); + header.extend_from_slice(&(length as u32).to_le_bytes()); + let (control_sender, control_receiver) = mpsc::channel(); + let (pcm_sender, _pcm_receiver) = mpsc::sync_channel(1); + read_framed_requests(Cursor::new(header), control_sender, pcm_sender); + let Input::Invalid(message) = control_receiver.recv().unwrap().input else { + panic!("invalid frame must be terminal") + }; + assert!(message.contains(expected)); + assert!(control_receiver.try_recv().is_err()); + } + } + + #[test] + fn first_pcm_queue_overflow_is_terminal_without_blocking_the_reader() { + let pcm = [0_u8; PCM_FRAME_BYTES]; + let mut bytes = framed(PCM_FRAME_KIND, &pcm); + bytes.extend_from_slice(&framed(PCM_FRAME_KIND, &pcm)); + let (control_sender, control_receiver) = mpsc::channel(); + let (pcm_sender, pcm_receiver) = mpsc::sync_channel(1); + + read_framed_requests(Cursor::new(bytes), control_sender, pcm_sender); + + assert!(pcm_receiver.recv().is_ok()); + let control = control_receiver.recv().unwrap(); + assert_eq!(control.after_pcm, 1); + let Input::Invalid(message) = control.input else { + panic!("queue discontinuity must be terminal") + }; + assert_eq!(message, "session PCM input queue is full"); + assert!(control_receiver.try_recv().is_err()); + } + + #[test] + fn input_policy_update_requires_a_positive_expected_revision() { + let request = SessionRequest::SetInputDuringTts { + id: 9, + expected_revision: 0, + policy: InputDuringTtsPolicy::SuppressInput, + }; + + assert_eq!( + validate_request(request).unwrap_err(), + "expected input-during-TTS revision must be positive" + ); + } + + #[test] + fn ready_requires_the_runtime_ready_event() { + let (sender, mut receiver) = tokio::sync::mpsc::channel(1); + sender.blocking_send(VoiceInputEvent::Ready).unwrap(); + assert_eq!( + wait_for_input_ready(&mut receiver, Duration::from_secs(1)), + Ok(()) + ); + + let (sender, mut receiver) = tokio::sync::mpsc::channel(1); + sender + .blocking_send(VoiceInputEvent::Failed("not ready".into())) + .unwrap(); + assert_eq!( + wait_for_input_ready(&mut receiver, Duration::from_secs(1)), + Err("not ready".into()) + ); + } + + #[test] + fn stalled_input_startup_reaches_a_bounded_terminal_failure() { + let (_sender, mut receiver) = tokio::sync::mpsc::channel(1); + + assert_eq!( + wait_for_input_ready(&mut receiver, Duration::from_millis(10)), + Err("voice input readiness timed out".into()) + ); + } + + #[test] + fn held_prepare_waits_for_pending_to_clear_without_a_timeout() { + let mut core = SessionCore::default(); + core.set_recognition_pending(true); + let input_policy = test_input_policy_slot(); + let mut active = None; + let mut held = None; + let mut output = Vec::new(); + process_prepare( + PrepareRequest { + id: 4, + acknowledgement: None, + text: "reply".into(), + }, + &mut core, + &test_tts_slot(), + &input_policy, + &mut active, + &mut held, + &mut output, + ) + .unwrap(); + assert!(held.is_some()); + assert!(output.is_empty()); + + input_policy + .update(1, InputDuringTtsPolicy::SuppressInput) + .unwrap(); + core.set_recognition_pending(false); + reevaluate_held( + &mut held, + &mut core, + Some(&test_tts_slot()), + Some(&input_policy), + &mut active, + &mut output, + ) + .unwrap(); + assert_eq!(messages(&output)[0]["type"], "admitted"); + assert_eq!( + active.as_ref().unwrap().input_during_tts.policy, + InputDuringTtsPolicy::SuppressInput + ); + } + + #[test] + fn admission_leases_configuration_before_a_later_atomic_update() { + let slot = test_tts_slot(); + let input_policy = test_input_policy_slot(); + let mut core = SessionCore::default(); + let mut active = None; + let mut held = None; + let mut output = Vec::new(); + process_prepare( + PrepareRequest { + id: 4, + acknowledgement: None, + text: "old voice".into(), + }, + &mut core, + &slot, + &input_policy, + &mut active, + &mut held, + &mut output, + ) + .unwrap(); + let old_revision = active.as_ref().unwrap().tts.snapshot().revision; + let leased_input_policy = active.as_ref().unwrap().input_during_tts; + let replacement = slot + .prepare_replacement( + 1, + berd_voice::TtsSettings::OpenAi { + model: "test-model".into(), + voice: "next-voice".into(), + rate: 2.0, + }, + ) + .unwrap(); + let applied = slot.commit_replacement(replacement).unwrap(); + let applied_input_policy = input_policy + .update(1, InputDuringTtsPolicy::SuppressInput) + .unwrap(); + + assert_eq!(old_revision, 1); + assert_eq!(active.as_ref().unwrap().tts.snapshot().revision, 1); + assert_eq!(leased_input_policy.revision, 1); + assert_eq!( + leased_input_policy.policy, + InputDuringTtsPolicy::AllowBargeIn + ); + assert_eq!( + active.as_ref().unwrap().input_during_tts, + leased_input_policy + ); + assert_eq!(applied_input_policy.revision, 2); + assert_eq!( + active.as_ref().unwrap().tts.snapshot().settings.voice(), + "test-voice" + ); + assert_eq!(applied.revision, 2); + assert_eq!( + slot.lease().unwrap().snapshot().settings.voice(), + "next-voice" + ); + } + + fn prepared_tts_event( + slot: &ConfiguredTtsSlot, + attempt: u64, + id: u64, + voice: &str, + ) -> TtsConfigurationEvent { + TtsConfigurationEvent { + attempt, + id, + result: slot.prepare_replacement( + 1, + berd_voice::TtsSettings::OpenAi { + model: "test-model".into(), + voice: voice.into(), + rate: 2.0, + }, + ), + } + } + + #[test] + fn tts_update_before_deadline_applies_once() { + let slot = test_tts_slot(); + let (sender, receiver) = mpsc::channel(); + let now = Instant::now(); + let mut active = Some(ActiveTtsConfigurationUpdate { + attempt: 9, + id: 4, + deadline: now + Duration::from_secs(1), + }); + sender + .send(prepared_tts_event(&slot, 9, 4, "next")) + .unwrap(); + let mut output = Vec::new(); + + poll_tts_configuration_update(now, &receiver, Some(&slot), &mut active, &mut output) + .unwrap(); + + assert!(active.is_none()); + assert_eq!(slot.snapshot().unwrap().revision, 2); + assert_eq!(messages(&output).len(), 1); + assert_eq!(messages(&output)[0]["outcome"], "applied"); + } + + #[test] + fn tts_update_at_deadline_rejects_once_and_ignores_late_attempt() { + let slot = test_tts_slot(); + let (sender, receiver) = mpsc::channel(); + let deadline = Instant::now(); + let mut active = Some(ActiveTtsConfigurationUpdate { + attempt: 9, + id: 4, + deadline, + }); + sender + .send(prepared_tts_event(&slot, 9, 4, "too-late")) + .unwrap(); + let mut output = Vec::new(); + + poll_tts_configuration_update(deadline, &receiver, Some(&slot), &mut active, &mut output) + .unwrap(); + poll_tts_configuration_update( + deadline + Duration::from_secs(1), + &receiver, + Some(&slot), + &mut active, + &mut output, + ) + .unwrap(); + + assert!(active.is_none()); + assert_eq!(slot.snapshot().unwrap().revision, 1); + assert_eq!(messages(&output).len(), 1); + assert_eq!(messages(&output)[0]["outcome"], "rejected"); + assert_eq!( + messages(&output)[0]["message"], + "TTS configuration update timed out" + ); + } + + #[test] + fn shutdown_rejects_once_and_generation_blocks_a_reused_client_id() { + let slot = test_tts_slot(); + let (sender, receiver) = mpsc::channel(); + let mut active = Some(ActiveTtsConfigurationUpdate { + attempt: 9, + id: 4, + deadline: Instant::now() + Duration::from_secs(1), + }); + sender + .send(prepared_tts_event(&slot, 9, 4, "old-attempt")) + .unwrap(); + let mut output = Vec::new(); + + reject_tts_configuration_update( + &mut active, + Some(&slot), + "session is shutting down", + &mut output, + ) + .unwrap(); + active = Some(ActiveTtsConfigurationUpdate { + attempt: 10, + id: 4, + deadline: Instant::now() + Duration::from_secs(1), + }); + poll_tts_configuration_update( + Instant::now(), + &receiver, + Some(&slot), + &mut active, + &mut output, + ) + .unwrap(); + + assert_eq!(slot.snapshot().unwrap().revision, 1); + assert_eq!(messages(&output).len(), 1); + assert_eq!(messages(&output)[0]["outcome"], "rejected"); + assert_eq!(messages(&output)[0]["message"], "session is shutting down"); + assert_eq!(active.unwrap().attempt, 10); + } + + #[test] + fn targeted_cancel_orders_result_before_terminal_and_repeats_as_stale() { + let mut core = SessionCore::default(); + let PrepareOutcome::Admitted { + speech_id, text, .. + } = core.prepare(PrepareRequest { + id: 7, + acknowledgement: None, + text: "reply".into(), + }) + else { + panic!("test speech must be admitted") + }; + let mut active = Some(ActivePlayback { + prepare_id: 7, + speech_id, + text, + output: None, + active: None, + ready_deadline: Instant::now() + Duration::from_secs(2), + assistant_activity: None, + input_during_tts: test_input_policy(), + tts: test_tts_lease(), + suspension_requested: false, + }); + let mut held = None; + let mut output = Vec::new(); + + handle_cancel(7, &mut held, &mut core, &mut active, &mut output).unwrap(); + handle_cancel(7, &mut held, &mut core, &mut active, &mut output).unwrap(); + + assert_eq!( + messages(&output), + [ + json!({"type":"cancel_result","id":7,"outcome":"cancelled","speech_id":1}), + json!({"type":"speech_interrupted","id":7,"speech_id":1,"spoken_through_utf8":0}), + json!({"type":"cancel_result","id":7,"outcome":"stale","speech_id":null}), + ] + ); + } + + #[test] + fn output_ready_installs_leased_suppression_before_acknowledgement() { + let mut core = SessionCore::default(); + let mut current = active_playback(&mut core); + current.active = None; + current.input_during_tts = InputDuringTtsSnapshot { + revision: 2, + policy: InputDuringTtsPolicy::SuppressInput, + }; + let controls = VoiceInputControls::default(); + let mut writer = InputStateWriter { + controls: &controls, + expected_muted: true, + bytes: Vec::new(), + }; + + acknowledge_output_ready(&mut current, Some(&controls), &mut writer).unwrap(); + + assert!(current.assistant_activity.is_some()); + assert_eq!( + messages(&writer.bytes), + [json!({ + "type":"output_ready_result", + "id":7, + "speech_id":1, + "outcome":"accepted" + })] + ); + } + + #[test] + fn terminal_clears_suppression_before_publishing_completion() { + let mut core = SessionCore::default(); + let mut current = active_playback(&mut core); + current.input_during_tts = InputDuringTtsSnapshot { + revision: 2, + policy: InputDuringTtsPolicy::SuppressInput, + }; + let controls = VoiceInputControls::default(); + let mut ignored = Vec::new(); + acknowledge_output_ready(&mut current, Some(&controls), &mut ignored).unwrap(); + assert!(controls.is_muted()); + let speech_id = current.speech_id; + let mut active = Some(current); + let mut writer = InputStateWriter { + controls: &controls, + expected_muted: false, + bytes: Vec::new(), + }; + + handle_playback_event( + PlaybackEvent::Completed(speech_id), + &mut core, + &mut active, + &mut writer, + ) + .unwrap(); + + assert!(active.is_none()); + assert_eq!(messages(&writer.bytes)[0]["type"], "speech_completed"); + } + + #[test] + fn unquiesced_output_failure_terminates_the_session_after_its_speech_terminal() { + let mut core = SessionCore::default(); + let current = active_playback(&mut core); + let speech_id = current.speech_id; + let mut active = Some(current); + let mut output = Vec::new(); + + let error = handle_playback_event( + PlaybackEvent::Failed(speech_id, "output failed".into(), false), + &mut core, + &mut active, + &mut output, + ) + .unwrap_err(); + + assert_eq!( + error, + "remote PCM output did not reach a quiescent terminal" + ); + assert!(active.is_none()); + assert_eq!(messages(&output)[0]["type"], "speech_failed"); + } + + #[test] + fn query_state_returns_authoritative_confirmation_and_order() { + let mut core = SessionCore::default(); + core.add_final(4, "one".into()).unwrap(); + core.add_final(9, "two".into()).unwrap(); + assert!(matches!( + core.prepare(PrepareRequest { + id: 5, + acknowledgement: Some(9), + text: "reply".into(), + }), + PrepareOutcome::Admitted { .. } + )); + let mut output = Vec::new(); + + write_state(&mut output, 6, 4, &core).unwrap(); + + assert_eq!( + messages(&output), + [json!({ + "type":"state", + "id":6, + "confirmed_token":9, + "utterances_after":[{"token":9,"text":"two"}] + })] + ); + } + + #[test] + fn runtime_final_is_stored_then_published_then_interrupts_output() { + let mut core = SessionCore::default(); + let PrepareOutcome::Admitted { + speech_id, text, .. + } = core.prepare(PrepareRequest { + id: 7, + acknowledgement: None, + text: "reply".into(), + }) + else { + panic!("test speech admitted") + }; + let mut active = Some(ActivePlayback { + prepare_id: 7, + speech_id, + text, + output: None, + active: None, + ready_deadline: Instant::now() + Duration::from_secs(2), + assistant_activity: None, + input_during_tts: test_input_policy(), + tts: test_tts_lease(), + suspension_requested: false, + }); + let mut next_token = 1; + let mut output = Vec::new(); + let stored = AtomicBool::new(false); + core.set_recognition_pending(true); + active.as_mut().unwrap().suspension_requested = true; + + store_and_publish_voice_final( + "hello".into(), + || stored.store(true, Ordering::SeqCst), + &mut core, + &mut active, + &mut next_token, + &mut output, + ) + .unwrap(); + + assert!(stored.load(Ordering::SeqCst)); + assert_eq!( + messages(&output) + .iter() + .map(|message| message["type"].as_str().unwrap()) + .collect::>(), + ["user_final", "speech_interrupted"] + ); + assert_eq!(core.utterances_after(0)[0].token, 1); + } + + #[test] + fn runtime_pending_provisionally_holds_reserved_output_without_a_terminal() { + let mut core = SessionCore::default(); + let PrepareOutcome::Admitted { + speech_id, text, .. + } = core.prepare(PrepareRequest { + id: 7, + acknowledgement: None, + text: "reply".into(), + }) + else { + panic!("test speech admitted") + }; + let mut active = Some(ActivePlayback { + prepare_id: 7, + speech_id, + text, + output: None, + active: None, + ready_deadline: Instant::now() + Duration::from_secs(2), + assistant_activity: None, + input_during_tts: test_input_policy(), + tts: test_tts_lease(), + suspension_requested: false, + }); + let mut next_token = 1; + let mut output = Vec::new(); + + handle_voice_input_event( + VoiceInputEvent::RecognitionPendingChanged(true), + &mut core, + &mut active, + &mut next_token, + &mut output, + ) + .unwrap(); + + assert_eq!( + messages(&output) + .iter() + .map(|message| message["type"].as_str().unwrap()) + .collect::>(), + ["recognition_pending"] + ); + assert!(core.recognition_pending()); + assert!(active.unwrap().suspension_requested); + } + + #[test] + fn final_then_pending_settlement_never_requests_resume() { + let mut core = SessionCore::default(); + let mut current = active_playback(&mut core); + let speech_id = current.speech_id; + let (child, _host) = UnixStream::pair().unwrap(); + let transport = unsafe { AudioPipeTransport::from_raw_fd(child.into_raw_fd()) }.unwrap(); + let (control_sender, control_receiver) = mpsc::channel(); + current.output = Some(Arc::new( + RemotePcmAudioOutput::new( + speech_id, + LongRemoteTts.pcm_spec(), + Arc::new(transport), + current.active.as_ref().unwrap().clone(), + control_sender, + ) + .unwrap(), + )); + let mut active = Some(current); + let mut next_token = 1; + let mut output = Vec::new(); + handle_voice_input_event( + VoiceInputEvent::RecognitionPendingChanged(true), + &mut core, + &mut active, + &mut next_token, + &mut output, + ) + .unwrap(); + assert!(matches!( + control_receiver.recv_timeout(Duration::from_millis(30)), + Ok(AudioOutputControlRequest::Suspend { .. }) + )); + + store_and_publish_voice_final( + "real words".into(), + || {}, + &mut core, + &mut active, + &mut next_token, + &mut output, + ) + .unwrap(); + handle_voice_input_event( + VoiceInputEvent::RecognitionPendingChanged(false), + &mut core, + &mut active, + &mut next_token, + &mut output, + ) + .unwrap(); + + assert!(control_receiver.try_recv().is_err()); + assert!(!active + .as_ref() + .unwrap() + .active + .as_ref() + .unwrap() + .load(Ordering::SeqCst)); + } + + #[test] + fn host_mute_and_reset_discard_only_a_provisional_hold() { + for reset in [false, true] { + let mut core = SessionCore::default(); + let mut active = Some(active_playback(&mut core)); + active.as_mut().unwrap().active = None; + active.as_mut().unwrap().suspension_requested = true; + let controls = VoiceInputControls::default(); + let mut output = Vec::new(); + if reset { + handle_reset_input(9, &controls, &mut core, &mut active, &mut output).unwrap(); + } else { + handle_input_muted(9, true, &controls, &mut core, &mut active, &mut output) + .unwrap(); + } + assert!(active.is_none()); + assert_eq!( + messages(&output) + .iter() + .map(|message| message["type"].as_str().unwrap()) + .collect::>(), + if reset { + vec!["input_reset_applied", "speech_interrupted"] + } else { + vec!["input_mute_applied", "speech_interrupted"] + } + ); + } + + let mut core = SessionCore::default(); + let mut active = Some(active_playback(&mut core)); + let controls = VoiceInputControls::default(); + let mut output = Vec::new(); + handle_input_muted(10, true, &controls, &mut core, &mut active, &mut output).unwrap(); + assert!(active.is_some()); + assert_eq!(messages(&output)[0]["type"], "input_mute_applied"); + } + + #[test] + fn input_control_acknowledgements_are_exact() { + assert_eq!( + serde_json::to_value(SessionMessage::InputMuteApplied { + id: 8, + active: true + }) + .unwrap(), + serde_json::json!({"type":"input_mute_applied","id":8,"active":true}) + ); + assert_eq!( + serde_json::to_value(SessionMessage::InputResetApplied { id: 9 }).unwrap(), + serde_json::json!({"type":"input_reset_applied","id":9}) + ); + } + + #[test] + fn backend_neutral_playback_starts_only_after_initial_pcm_is_accepted() { + let backend = FakeTts { + frames: vec![0.1, 0.2], + }; + let output = FakeOutput::default(); + let active = AtomicBool::new(true); + let (sender, receiver) = mpsc::channel(); + assert!(synthesize_to_output(9, "hi", &backend, &output, &active, &sender).unwrap()); + assert!(matches!(receiver.try_recv(), Ok(PlaybackEvent::Started(9)))); + assert_eq!(*output.frames.lock().unwrap(), [0.1, 0.2]); + } + + #[test] + fn backend_neutral_playback_cancels_without_start_when_authority_is_absent() { + let backend = FakeTts { frames: vec![0.1] }; + let output = FakeOutput::default(); + let active = AtomicBool::new(false); + let (sender, receiver) = mpsc::channel(); + assert!(!synthesize_to_output(9, "hi", &backend, &output, &active, &sender).unwrap()); + assert!(receiver.try_recv().is_err()); + assert!(output.cancelled.load(Ordering::SeqCst)); + assert!(output.frames.lock().unwrap().is_empty()); + } + + #[test] + fn cancellation_during_output_drain_returns_an_interruption_promptly() { + let backend = FakeTts { + frames: vec![0.1, 0.2], + }; + let output = BlockingOutput { + cancelled: AtomicBool::new(false), + }; + let active = AtomicBool::new(true); + let (sender, receiver) = mpsc::channel(); + std::thread::scope(|scope| { + let active_ref = &active; + scope.spawn(move || { + assert!(matches!(receiver.recv(), Ok(PlaybackEvent::Started(9)))); + active_ref.store(false, Ordering::SeqCst); + }); + assert!(!synthesize_to_output(9, "hi", &backend, &output, &active, &sender).unwrap()); + }); + assert!(output.cancelled.load(Ordering::SeqCst)); + } + + #[test] + fn shutdown_drains_started_before_interrupted_terminal() { + let mut core = SessionCore::default(); + let mut active = Some(active_playback(&mut core)); + let speech_id = active.as_ref().unwrap().speech_id; + let mut output = Vec::new(); + interrupt_active(&mut core, &mut active, &mut output).unwrap(); + let (sender, receiver) = mpsc::channel(); + sender.send(PlaybackEvent::Started(speech_id)).unwrap(); + sender + .send(PlaybackEvent::Interrupted(speech_id, 0)) + .unwrap(); + + finish_shutdown_playback( + &receiver, + &mut core, + &mut active, + &mut output, + Duration::from_millis(10), + ) + .unwrap(); + + assert!(active.is_none()); + assert_eq!( + messages(&output) + .iter() + .map(|message| message["type"].as_str().unwrap()) + .collect::>(), + ["speech_started", "speech_interrupted"] + ); + } + + #[test] + fn shutdown_timeout_emits_terminal_failure_and_clears_state() { + let mut core = SessionCore::default(); + let mut active = Some(active_playback(&mut core)); + let (_sender, receiver) = mpsc::channel(); + let mut output = Vec::new(); + finish_shutdown_playback( + &receiver, + &mut core, + &mut active, + &mut output, + Duration::ZERO, + ) + .unwrap(); + assert!(active.is_none()); + assert_eq!(messages(&output)[0]["type"], "speech_failed"); + } + + #[test] + fn shutdown_worker_disconnect_emits_terminal_failure_and_clears_state() { + let mut core = SessionCore::default(); + let mut active = Some(active_playback(&mut core)); + let (sender, receiver) = mpsc::channel(); + drop(sender); + let mut output = Vec::new(); + finish_shutdown_playback( + &receiver, + &mut core, + &mut active, + &mut output, + Duration::from_secs(1), + ) + .unwrap(); + assert!(active.is_none()); + let failure = &messages(&output)[0]; + assert_eq!(failure["type"], "speech_failed"); + assert_eq!( + failure["message"], + "playback worker disconnected during shutdown" + ); + } +} diff --git a/src-tauri/crates/berd-voice/src/openai.rs b/src-tauri/crates/berd-voice/src/openai.rs new file mode 100644 index 000000000..ab3d78caa --- /dev/null +++ b/src-tauri/crates/berd-voice/src/openai.rs @@ -0,0 +1,143 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use futures_util::StreamExt; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE}; +use serde_json::json; + +#[derive(Clone, Debug)] +pub struct OpenAiSpeechConfig { + pub endpoint: String, + pub api_key: String, + pub model: String, + pub voice: String, + pub speed: f32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OpenAiPcmOutcome { + Completed, + Cancelled, +} + +/// Streams OpenAI's 24 kHz mono PCM response as unit-scale `f32` frames. +/// +/// This operation owns HTTP and PCM framing only. Its caller remains +/// responsible for buffering, playback, device selection, and delivery policy. +/// The callback receives an empty slice on idle polls so a host can update +/// playback guards while the network stream is temporarily quiet. +pub async fn stream_openai_pcm( + client: &reqwest::Client, + config: &OpenAiSpeechConfig, + input: &str, + active: &AtomicBool, + mut on_frames: F, +) -> Result +where + F: FnMut(&[f32]) -> Result<(), String>, +{ + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", config.api_key)) + .map_err(|_| "OpenAI API key is not a valid header value".to_string())?, + ); + let request = client + .post(&config.endpoint) + .headers(headers) + .header(CONTENT_TYPE, "application/json") + .json(&json!({ + "model": config.model, + "voice": config.voice, + "input": input, + "speed": config.speed, + "response_format": "pcm", + "stream_format": "audio" + })) + .send(); + tokio::pin!(request); + let response = loop { + tokio::select! { + response = &mut request => break response.map_err(|error| format_request_error("start speech audio", error))?, + _ = tokio::time::sleep(Duration::from_millis(20)) => { + if !active.load(Ordering::SeqCst) { return Ok(OpenAiPcmOutcome::Cancelled); } + } + } + }; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(format_response_error("start speech audio", status, &body)); + } + + let mut stream = response.bytes_stream(); + let mut remainder = Vec::new(); + loop { + if !active.load(Ordering::SeqCst) { + return Ok(OpenAiPcmOutcome::Cancelled); + } + let item = match tokio::time::timeout(Duration::from_millis(50), stream.next()).await { + Ok(item) => item, + Err(_) => { + on_frames(&[])?; + continue; + } + }; + let Some(item) = item else { break }; + let item = item.map_err(|error| format_request_error("stream speech audio", error))?; + remainder.extend_from_slice(&item); + let sample_bytes = remainder.len() / 2 * 2; + if sample_bytes != 0 { + let samples = pcm16le_to_f32(&remainder[..sample_bytes]); + remainder.drain(..sample_bytes); + on_frames(&samples)?; + } + } + if !remainder.is_empty() { + return Err("OpenAI speech returned an incomplete PCM sample".to_string()); + } + Ok(OpenAiPcmOutcome::Completed) +} + +fn format_request_error(action: &str, error: reqwest::Error) -> String { + if error.is_timeout() { + format!("OpenAI voice could not {action}: the request timed out") + } else if error.is_connect() { + format!("OpenAI voice could not {action}: check your network connection") + } else { + format!("OpenAI voice could not {action}: {error}") + } +} + +fn format_response_error(action: &str, status: reqwest::StatusCode, body: &str) -> String { + let preview: String = body.chars().take(500).collect(); + format!("OpenAI voice could not {action}: HTTP {status}: {preview}") +} + +fn pcm16le_to_f32(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(2) + .map(|sample| { + let sample = i16::from_le_bytes([sample[0], sample[1]]); + if sample < 0 { + sample as f32 / 32_768.0 + } else { + sample as f32 / i16::MAX as f32 + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::pcm16le_to_f32; + + #[test] + fn decodes_little_endian_pcm_without_changing_frame_units() { + let samples = pcm16le_to_f32(&[0, 0, 0xff, 0x7f, 0, 0x80]); + assert_eq!(samples.len(), 3); + assert_eq!(samples[0], 0.0); + assert_eq!(samples[1], 1.0); + assert_eq!(samples[2], -1.0); + } +} diff --git a/src-tauri/crates/berd-voice/src/openai_realtime.rs b/src-tauri/crates/berd-voice/src/openai_realtime.rs new file mode 100644 index 000000000..ede600fca --- /dev/null +++ b/src-tauri/crates/berd-voice/src/openai_realtime.rs @@ -0,0 +1,470 @@ +use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use futures_util::{SinkExt, StreamExt}; +use tokio::net::TcpStream; +use tokio_tungstenite::{ + tungstenite::{client::IntoClientRequest, Message}, + MaybeTlsStream, WebSocketStream, +}; + +/// Explicit connection settings for OpenAI Realtime transcription. +/// +/// This type deliberately does not implement `Debug` because it contains an +/// API key. +pub struct OpenAiRealtimeTranscriptionConfig { + endpoint: String, + api_key: String, + model: String, +} + +impl OpenAiRealtimeTranscriptionConfig { + pub fn new(endpoint: String, api_key: String, model: String) -> Self { + Self { + endpoint, + api_key, + model, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub enum OpenAiRealtimeTranscriptionEvent { + Committed { item_id: String }, + Completed { item_id: String, transcript: String }, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum OpenAiRealtimeTranscriptionError { + TranscriptionFailed { item_id: String, message: String }, + Provider(String), + Disconnected, + Socket(String), +} + +impl std::fmt::Display for OpenAiRealtimeTranscriptionError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TranscriptionFailed { message, .. } | Self::Provider(message) => { + formatter.write_str(message) + } + Self::Disconnected => { + formatter.write_str("OpenAI realtime transcription disconnected.") + } + Self::Socket(message) => { + write!(formatter, "OpenAI realtime transcription failed: {message}") + } + } + } +} + +impl std::error::Error for OpenAiRealtimeTranscriptionError {} + +/// A connected OpenAI Realtime transcription websocket. +pub struct OpenAiRealtimeTranscriptionClient { + socket: WebSocketStream>, + model: String, +} + +impl OpenAiRealtimeTranscriptionClient { + /// Connects and authenticates without configuring the transcription session. + pub async fn connect(config: OpenAiRealtimeTranscriptionConfig) -> Result { + if let Err(existing) = rustls::crypto::aws_lc_rs::default_provider().install_default() { + // Another dependency may have installed the same process-wide provider first. + drop(existing); + } + let mut request = config + .endpoint + .into_client_request() + .map_err(|error| format!("prepare OpenAI realtime connection: {error}"))?; + let authorization = format!("Bearer {}", config.api_key) + .parse() + .map_err(|_| "OpenAI API key is not a valid header value".to_string())?; + request.headers_mut().insert("Authorization", authorization); + let (socket, _) = tokio_tungstenite::connect_async(request) + .await + .map_err(|error| error.to_string())?; + Ok(Self { + socket, + model: config.model, + }) + } + + /// Configures 24 kHz PCM transcription with provider turn detection disabled. + pub async fn configure(&mut self) -> Result<(), String> { + self.send(serde_json::json!({ + "type": "session.update", + "session": { + "type": "transcription", + "audio": { "input": { + "format": { "type": "audio/pcm", "rate": 24000 }, + "transcription": { "model": self.model, "delay": "low" }, + "turn_detection": null + }} + } + })) + .await?; + + loop { + let value = self.next_json().await.map_err(|error| error.to_string())?; + match value.get("type").and_then(|value| value.as_str()) { + Some("session.updated") => return Ok(()), + Some("error") => { + return Err(provider_message(&value).to_string()); + } + _ => {} + } + } + } + + pub async fn append_pcm16le_24khz(&mut self, pcm: &[u8]) -> Result<(), String> { + self.send(serde_json::json!({ + "type": "input_audio_buffer.append", + "audio": BASE64.encode(pcm), + })) + .await + } + + pub async fn clear(&mut self) -> Result<(), String> { + self.send(serde_json::json!({"type": "input_audio_buffer.clear"})) + .await + } + + pub async fn commit(&mut self) -> Result<(), String> { + self.send(serde_json::json!({"type": "input_audio_buffer.commit"})) + .await + } + + /// Returns the next recognized provider event, ignoring unrelated or + /// malformed messages while preserving terminal provider/socket failures. + pub async fn next_event( + &mut self, + ) -> Result { + loop { + let value = self.next_json().await?; + match value.get("type").and_then(|value| value.as_str()) { + Some("input_audio_buffer.committed") => { + if let Some(item_id) = value.get("item_id").and_then(|value| value.as_str()) { + return Ok(OpenAiRealtimeTranscriptionEvent::Committed { + item_id: item_id.to_string(), + }); + } + } + Some("conversation.item.input_audio_transcription.completed") => { + if let (Some(item_id), Some(transcript)) = ( + value.get("item_id").and_then(|value| value.as_str()), + value.get("transcript").and_then(|value| value.as_str()), + ) { + return Ok(OpenAiRealtimeTranscriptionEvent::Completed { + item_id: item_id.to_string(), + transcript: transcript.trim().to_string(), + }); + } + } + Some("conversation.item.input_audio_transcription.failed") => { + if let Some(item_id) = value.get("item_id").and_then(|value| value.as_str()) { + return Err(OpenAiRealtimeTranscriptionError::TranscriptionFailed { + item_id: item_id.to_string(), + message: provider_message(&value).to_string(), + }); + } + } + Some("error") => { + return Err(OpenAiRealtimeTranscriptionError::Provider( + provider_message(&value).to_string(), + )); + } + _ => {} + } + } + } + + async fn next_json(&mut self) -> RealtimeJsonResult { + loop { + let message = match self.socket.next().await { + Some(Ok(Message::Text(text))) => text, + Some(Ok(Message::Close(_))) | None => { + return Err(OpenAiRealtimeTranscriptionError::Disconnected); + } + Some(Ok(_)) => continue, + Some(Err(error)) => { + return Err(OpenAiRealtimeTranscriptionError::Socket(error.to_string())); + } + }; + if let Ok(value) = serde_json::from_str(&message) { + return Ok(value); + } + } + } + + async fn send(&mut self, value: serde_json::Value) -> Result<(), String> { + self.socket + .send(Message::Text(value.to_string().into())) + .await + .map_err(|error| error.to_string()) + } +} + +type RealtimeJson = serde_json::Value; +type RealtimeJsonResult = Result; + +fn provider_message(value: &serde_json::Value) -> &str { + value + .pointer("/error/message") + .and_then(|value| value.as_str()) + .unwrap_or("OpenAI realtime transcription failed.") +} + +#[cfg(test)] +mod tests { + use super::{ + OpenAiRealtimeTranscriptionClient, OpenAiRealtimeTranscriptionConfig, + OpenAiRealtimeTranscriptionError, OpenAiRealtimeTranscriptionEvent, + }; + use futures_util::{SinkExt, StreamExt}; + use serde_json::{json, Value}; + use tokio::net::TcpListener; + use tokio_tungstenite::{ + accept_hdr_async, + tungstenite::{ + handshake::server::{ErrorResponse, Request, Response}, + Message, + }, + WebSocketStream, + }; + + #[allow(clippy::result_large_err)] // Signature is fixed by tungstenite's handshake callback. + fn require_test_authorization( + request: &Request, + response: Response, + ) -> Result { + assert_eq!( + request + .headers() + .get("Authorization") + .and_then(|value| value.to_str().ok()), + Some("Bearer test-key") + ); + Ok(response) + } + + async fn receive_json(socket: &mut WebSocketStream) -> Value { + let Message::Text(text) = socket.next().await.unwrap().unwrap() else { + panic!("expected JSON text") + }; + serde_json::from_str(&text).unwrap() + } + + async fn fake_server() -> (String, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("ws://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_hdr_async(stream, require_test_authorization) + .await + .unwrap(); + + let configured = receive_json(&mut socket).await; + assert_eq!(configured["type"], "session.update"); + assert_eq!( + configured.pointer("/session/audio/input/transcription/model"), + Some(&json!("test-model")) + ); + assert_eq!( + configured.pointer("/session/audio/input/format/rate"), + Some(&json!(24_000)) + ); + assert_eq!( + configured.pointer("/session/audio/input/turn_detection"), + Some(&Value::Null) + ); + socket + .send(Message::Text( + json!({"type":"session.updated"}).to_string().into(), + )) + .await + .unwrap(); + + let appended = receive_json(&mut socket).await; + assert_eq!( + appended, + json!({"type":"input_audio_buffer.append","audio":"AQID"}) + ); + assert_eq!( + receive_json(&mut socket).await, + json!({"type":"input_audio_buffer.clear"}) + ); + assert_eq!( + receive_json(&mut socket).await, + json!({"type":"input_audio_buffer.commit"}) + ); + + socket.send(Message::Text("not json".into())).await.unwrap(); + socket + .send(Message::Text( + json!({"type":"input_audio_buffer.committed","item_id":"item-1"}) + .to_string() + .into(), + )) + .await + .unwrap(); + socket + .send(Message::Text( + json!({ + "type":"conversation.item.input_audio_transcription.completed", + "item_id":"item-1", + "transcript":" hello " + }) + .to_string() + .into(), + )) + .await + .unwrap(); + socket + .send(Message::Text( + json!({ + "type":"conversation.item.input_audio_transcription.failed", + "item_id":"item-1", + "error":{"message":"turn failed"} + }) + .to_string() + .into(), + )) + .await + .unwrap(); + }); + (endpoint, server) + } + + #[tokio::test] + async fn client_owns_auth_configuration_framing_and_provider_events() { + let (endpoint, server) = fake_server().await; + let config = OpenAiRealtimeTranscriptionConfig::new( + endpoint, + "test-key".into(), + "test-model".into(), + ); + let mut client = OpenAiRealtimeTranscriptionClient::connect(config) + .await + .unwrap(); + client.configure().await.unwrap(); + client.append_pcm16le_24khz(&[1, 2, 3]).await.unwrap(); + client.clear().await.unwrap(); + client.commit().await.unwrap(); + assert_eq!( + client.next_event().await.unwrap(), + OpenAiRealtimeTranscriptionEvent::Committed { + item_id: "item-1".into() + } + ); + assert_eq!( + client.next_event().await.unwrap(), + OpenAiRealtimeTranscriptionEvent::Completed { + item_id: "item-1".into(), + transcript: "hello".into() + } + ); + assert_eq!( + client.next_event().await.unwrap_err(), + OpenAiRealtimeTranscriptionError::TranscriptionFailed { + item_id: "item-1".into(), + message: "turn failed".into(), + } + ); + server.await.unwrap(); + } + + #[tokio::test] + async fn configure_waits_for_provider_acknowledgement() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("ws://{}", listener.local_addr().unwrap()); + let (acknowledge, acknowledged) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + assert_eq!(receive_json(&mut socket).await["type"], "session.update"); + acknowledged.await.unwrap(); + socket + .send(Message::Text( + json!({"type":"session.updated"}).to_string().into(), + )) + .await + .unwrap(); + }); + let config = OpenAiRealtimeTranscriptionConfig::new( + endpoint, + "test-key".into(), + "test-model".into(), + ); + let mut client = OpenAiRealtimeTranscriptionClient::connect(config) + .await + .unwrap(); + let configuring = client.configure(); + tokio::pin!(configuring); + let deadline = std::time::Duration::from_millis(20); + let early = tokio::time::timeout(deadline, &mut configuring).await; + assert!(early.is_err()); + acknowledge.send(()).unwrap(); + configuring.await.unwrap(); + server.await.unwrap(); + } + + #[tokio::test] + async fn close_is_a_terminal_disconnect() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("ws://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + socket.close(None).await.unwrap(); + }); + let config = OpenAiRealtimeTranscriptionConfig::new( + endpoint, + "test-key".into(), + "test-model".into(), + ); + let mut client = OpenAiRealtimeTranscriptionClient::connect(config) + .await + .unwrap(); + assert_eq!( + client.next_event().await.unwrap_err(), + OpenAiRealtimeTranscriptionError::Disconnected + ); + server.await.unwrap(); + } + + #[tokio::test] + async fn eof_is_a_terminal_socket_failure() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("ws://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + drop(socket); + }); + let config = OpenAiRealtimeTranscriptionConfig::new( + endpoint, + "test-key".into(), + "test-model".into(), + ); + let mut client = OpenAiRealtimeTranscriptionClient::connect(config) + .await + .unwrap(); + let error = client.next_event().await.unwrap_err(); + assert!(matches!(error, OpenAiRealtimeTranscriptionError::Socket(_))); + server.await.unwrap(); + } + + #[tokio::test] + async fn invalid_api_key_error_does_not_echo_the_secret() { + let config = OpenAiRealtimeTranscriptionConfig::new( + "ws://127.0.0.1:1".into(), + "secret\nvalue".into(), + "test-model".into(), + ); + let error = OpenAiRealtimeTranscriptionClient::connect(config) + .await + .err() + .expect("invalid header must fail before connecting"); + assert_eq!(error, "OpenAI API key is not a valid header value"); + assert!(!error.contains("secret")); + } +} diff --git a/src-tauri/crates/berd-voice/src/outbound.rs b/src-tauri/crates/berd-voice/src/outbound.rs new file mode 100644 index 000000000..7e4f3a3ae --- /dev/null +++ b/src-tauri/crates/berd-voice/src/outbound.rs @@ -0,0 +1,951 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use serde::Serialize; + +use crate::{PcmAudioOutput, TtsBackend, TtsOutcome, TtsSynthesisEvent}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DeliverySegment { + pub text: String, + pub played_frames: u64, + pub total_frames: u64, + pub synthesis_complete: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct DeliveryProgress { + #[serde(rename = "sampleRate")] + pub sample_rate: u32, + pub segments: Vec, +} + +/// Estimate a conservative UTF-8 byte boundary through the last fully played +/// word. Hosts can render the remaining suffix as not spoken without owning a +/// second delivery estimator. +pub fn estimated_spoken_through_utf8(text: &str, delivery: &DeliveryProgress) -> usize { + let Some(segment) = delivery.segments.first() else { + return 0; + }; + if delivery.segments.len() != 1 || segment.text != text || segment.total_frames == 0 { + return 0; + } + if segment.synthesis_complete && segment.played_frames >= segment.total_frames { + return text.len(); + } + + let character_count = text.chars().count(); + let generated_cutoff = ((character_count as u128 * segment.played_frames as u128) + / segment.total_frames as u128) as usize; + let approximate_cutoff = if segment.synthesis_complete { + generated_cutoff + } else { + let duration_cutoff = ((segment.played_frames as u128 * 6) + / u128::from(delivery.sample_rate.max(1))) as usize; + generated_cutoff.min(duration_cutoff) + }; + + let mut character_index = 0; + let mut last_word_end = 0; + let mut in_word = false; + for (byte_index, character) in text.char_indices() { + if character_index >= approximate_cutoff { + break; + } + let is_word = character.is_alphanumeric(); + if in_word && !is_word { + last_word_end = byte_index; + } + in_word = is_word; + character_index += 1; + } + if character_index >= character_count && in_word { + text.len() + } else { + last_word_end + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OutboundOutcome { + Completed, + Interrupted, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DrainTimeoutOutcome { + Fail, + Complete, +} + +#[derive(Clone, Copy, Debug)] +pub struct DrainPolicy { + pub poll_interval: Duration, + pub timeout: Option, + pub timeout_outcome: DrainTimeoutOutcome, + /// Keeps cancellation and health polling active after native source-frame + /// completion while downstream route latency can still be audible. + pub post_drain: Duration, +} + +impl Default for DrainPolicy { + fn default() -> Self { + Self { + poll_interval: Duration::from_millis(10), + timeout: None, + timeout_outcome: DrainTimeoutOutcome::Fail, + post_drain: Duration::ZERO, + } + } +} + +#[derive(Debug)] +pub struct OutboundFailure { + pub message: String, + pub delivery: DeliveryProgress, + /// False means output cancellation did not prove a quiescent terminal and + /// the host must not admit another speech on the same data plane. + pub output_quiescent: bool, +} + +#[derive(Debug)] +struct DeliveryLedger { + sample_rate: u32, + segments: Vec, +} + +#[derive(Debug)] +struct LedgerSegment { + text: String, + total_frames: u64, + synthesis_complete: bool, +} + +impl DeliveryLedger { + fn new(sample_rate: u32) -> Self { + Self { + sample_rate, + segments: Vec::new(), + } + } + + fn begin_segment(&mut self, text: String) { + self.segments.push(LedgerSegment { + text, + total_frames: 0, + synthesis_complete: false, + }); + } + + fn append_frames(&mut self, frames: usize) { + if let Some(segment) = self.segments.last_mut() { + segment.total_frames = segment.total_frames.saturating_add(frames as u64); + } + } + + fn complete_segment(&mut self) { + if let Some(segment) = self.segments.last_mut() { + segment.synthesis_complete = true; + } + } + + fn snapshot(&self, played_frames: u64) -> DeliveryProgress { + let mut segment_start = 0_u64; + let segments = self + .segments + .iter() + .map(|segment| { + let played_frames = played_frames + .saturating_sub(segment_start) + .min(segment.total_frames); + segment_start = segment_start.saturating_add(segment.total_frames); + DeliverySegment { + text: segment.text.clone(), + played_frames, + total_frames: segment.total_frames, + synthesis_complete: segment.synthesis_complete, + } + }) + .collect(); + DeliveryProgress { + sample_rate: self.sample_rate, + segments, + } + } +} + +/// Coordinates backend-neutral TTS PCM with one host-provided audio output. +/// +/// Text accumulation, device selection, host events, admission, and assistant +/// activity remain outside this type. A coordinator is single-use after a +/// terminal outcome. +pub struct OutboundPlayback<'a> { + output: &'a dyn PcmAudioOutput, + active: &'a AtomicBool, + initial_buffer_frames: usize, + initial: Vec, + started: bool, + terminal: bool, + terminal_delivery: Option, + ledger: DeliveryLedger, +} + +impl<'a> OutboundPlayback<'a> { + pub fn new( + output: &'a dyn PcmAudioOutput, + active: &'a AtomicBool, + sample_rate: u32, + initial_buffer_frames: usize, + ) -> Result { + if sample_rate == 0 { + return Err("TTS sample rate must be positive".into()); + } + Ok(Self { + output, + active, + initial_buffer_frames, + initial: Vec::new(), + started: false, + terminal: false, + terminal_delivery: None, + ledger: DeliveryLedger::new(sample_rate), + }) + } + + pub fn started(&self) -> bool { + self.started + } + + pub fn snapshot(&self) -> DeliveryProgress { + self.terminal_delivery + .clone() + .unwrap_or_else(|| self.ledger.snapshot(self.output.played_frames())) + } + + /// Checks cancellation authority and asynchronous output health while the + /// host is waiting for more text. Returns `false` after interruption. + pub fn poll(&mut self) -> Result { + self.ensure_live()?; + if !self.active.load(Ordering::SeqCst) { + self.interrupt()?; + return Ok(false); + } + self.output + .check_health() + .map_err(|message| self.fail(message))?; + Ok(true) + } + + pub fn synthesize_segment( + &mut self, + backend: &dyn TtsBackend, + text: &str, + before_write: &mut dyn FnMut(bool) -> Result<(), String>, + on_started: &mut dyn FnMut() -> Result<(), String>, + on_progress: &mut dyn FnMut(&DeliveryProgress) -> Result<(), String>, + ) -> Result { + self.ensure_live()?; + if !self.active.load(Ordering::SeqCst) { + return self.interrupt(); + } + self.ledger.begin_segment(text.to_string()); + let outcome = backend.synthesize_with_poll(text, self.active, &mut |event| match event { + TtsSynthesisEvent::Frames(samples) => { + if samples.is_empty() { + return Ok(()); + } + if !self.active.load(Ordering::SeqCst) { + return Ok(()); + } + self.output.check_health()?; + self.ledger.append_frames(samples.len()); + if self.started { + before_write(false)?; + self.output.write(samples)?; + on_progress(&self.snapshot()) + } else { + self.initial.extend_from_slice(samples); + if self.initial.len() >= self.initial_buffer_frames.max(1) { + self.flush_initial(before_write, on_started)?; + on_progress(&self.snapshot()) + } else { + Ok(()) + } + } + } + TtsSynthesisEvent::Poll => { + if self.active.load(Ordering::SeqCst) { + self.output.check_health()?; + on_progress(&self.snapshot())?; + } + Ok(()) + } + }); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(_) if !self.active.load(Ordering::SeqCst) => return self.interrupt(), + Err(message) => return Err(self.fail(message)), + }; + if outcome == TtsOutcome::Cancelled || !self.active.load(Ordering::SeqCst) { + return self.interrupt(); + } + self.ledger.complete_segment(); + if !self.initial.is_empty() { + if let Err(message) = self.flush_initial(before_write, on_started) { + return Err(self.fail(message)); + } + if let Err(message) = on_progress(&self.snapshot()) { + return Err(self.fail(message)); + } + } + Ok(OutboundOutcome::Completed) + } + + pub fn finish( + &mut self, + policy: DrainPolicy, + on_poll: &mut dyn FnMut(&DeliveryProgress) -> Result<(), String>, + ) -> Result { + self.ensure_live()?; + let started_at = Instant::now(); + let mut forced_complete = false; + while !self.output.is_drained() { + if !self.active.load(Ordering::SeqCst) { + return self.interrupt(); + } + if policy + .timeout + .is_some_and(|timeout| started_at.elapsed() >= timeout) + { + if policy.timeout_outcome == DrainTimeoutOutcome::Complete { + self.remember_delivery_and_cancel()?; + forced_complete = true; + break; + } + return Err(self.fail("TTS playback did not drain before its deadline".into())); + } + self.output + .check_health() + .map_err(|message| self.fail(message))?; + on_poll(&self.snapshot()).map_err(|message| self.fail(message))?; + std::thread::sleep(policy.poll_interval); + } + if !self.active.load(Ordering::SeqCst) { + return self.interrupt(); + } + if !forced_complete { + self.output + .check_health() + .map_err(|message| self.fail(message))?; + } + let post_drain_started = Instant::now(); + while post_drain_started.elapsed() < policy.post_drain { + if !self.active.load(Ordering::SeqCst) { + return self.interrupt(); + } + if !forced_complete { + self.output + .check_health() + .map_err(|message| self.fail(message))?; + } + on_poll(&self.snapshot()).map_err(|message| self.fail(message))?; + std::thread::sleep(policy.poll_interval); + } + self.terminal = true; + Ok(OutboundOutcome::Completed) + } + + pub fn interrupt(&mut self) -> Result { + if !self.terminal { + self.remember_delivery_and_cancel()?; + self.terminal = true; + } + Ok(OutboundOutcome::Interrupted) + } + + fn flush_initial( + &mut self, + before_write: &mut dyn FnMut(bool) -> Result<(), String>, + on_started: &mut dyn FnMut() -> Result<(), String>, + ) -> Result<(), String> { + if self.initial.is_empty() { + return Ok(()); + } + before_write(true)?; + self.output.write(&self.initial)?; + self.initial.clear(); + self.started = true; + on_started()?; + Ok(()) + } + + fn ensure_live(&mut self) -> Result<(), OutboundFailure> { + if self.terminal { + Err(OutboundFailure { + message: "TTS playback is already terminal".into(), + delivery: self.snapshot(), + output_quiescent: true, + }) + } else { + Ok(()) + } + } + + fn fail(&mut self, message: String) -> OutboundFailure { + self.terminal_delivery = Some(self.snapshot()); + let cancellation = self.output.cancel_and_snapshot(); + let output_quiescent = cancellation.is_ok(); + if let Ok(played_frames) = cancellation { + self.terminal_delivery = Some(self.ledger.snapshot(played_frames)); + } + self.terminal = true; + let delivery = self + .terminal_delivery + .clone() + .expect("failure records terminal delivery"); + let message = if output_quiescent { + message + } else { + format!("{message}; output cancellation did not reach a quiescent terminal") + }; + OutboundFailure { + message, + delivery, + output_quiescent, + } + } + + fn remember_delivery_and_cancel(&mut self) -> Result<(), OutboundFailure> { + if self.terminal_delivery.is_some() { + self.output.cancel(); + return Ok(()); + } + let played_frames = self + .output + .cancel_and_snapshot() + .map_err(|message| self.fail_without_cancel(message))?; + self.terminal_delivery = Some(self.ledger.snapshot(played_frames)); + Ok(()) + } + + fn fail_without_cancel(&mut self, message: String) -> OutboundFailure { + let delivery = self.snapshot(); + self.terminal_delivery = Some(delivery.clone()); + self.terminal = true; + OutboundFailure { + message, + delivery, + output_quiescent: false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TtsPcmSpec; + use std::sync::atomic::{AtomicU64, AtomicUsize}; + use std::sync::Mutex; + + #[test] + fn spoken_through_estimate_uses_completed_word_utf8_boundary() { + let text = "Hello café world."; + let delivery = DeliveryProgress { + sample_rate: 24_000, + segments: vec![DeliverySegment { + text: text.into(), + played_frames: 14_000, + total_frames: 24_000, + synthesis_complete: true, + }], + }; + + let cutoff = estimated_spoken_through_utf8(text, &delivery); + assert_eq!(&text[..cutoff], "Hello"); + assert!(text.is_char_boundary(cutoff)); + } + + #[test] + fn spoken_through_estimate_is_conservative_for_incomplete_synthesis() { + let text = "One two three four five six seven eight."; + let delivery = DeliveryProgress { + sample_rate: 24_000, + segments: vec![DeliverySegment { + text: text.into(), + played_frames: 24_000, + total_frames: 24_000, + synthesis_complete: false, + }], + }; + + assert_eq!( + &text[..estimated_spoken_through_utf8(text, &delivery)], + "One" + ); + } + + struct FakeTts { + chunks: Vec>, + cancel_after_first: bool, + } + + struct PollThenFramesTts; + + struct UnquiescedOutput; + + impl PcmAudioOutput for UnquiescedOutput { + fn write(&self, _samples: &[f32]) -> Result<(), String> { + Err("remote write failed".into()) + } + + fn cancel(&self) {} + + fn cancel_and_snapshot(&self) -> Result { + Err("remote cancel timed out".into()) + } + + fn is_drained(&self) -> bool { + false + } + + fn check_health(&self) -> Result<(), String> { + Ok(()) + } + + fn played_frames(&self) -> u64 { + 0 + } + } + + impl TtsBackend for PollThenFramesTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 10, + playback_rate: 1.0, + } + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + _on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + unreachable!("test backend exercises the polling synthesis seam") + } + + fn synthesize_with_poll( + &self, + _text: &str, + _active: &AtomicBool, + on_event: &mut dyn FnMut(TtsSynthesisEvent<'_>) -> Result<(), String>, + ) -> Result { + on_event(TtsSynthesisEvent::Poll)?; + on_event(TtsSynthesisEvent::Frames(&[0.3]))?; + Ok(TtsOutcome::Completed) + } + } + + impl TtsBackend for FakeTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 10, + playback_rate: 1.0, + } + } + + fn synthesize( + &self, + _text: &str, + active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + for (index, chunk) in self.chunks.iter().enumerate() { + on_frames(chunk)?; + if self.cancel_after_first && index == 0 { + active.store(false, Ordering::SeqCst); + return Ok(TtsOutcome::Cancelled); + } + } + Ok(TtsOutcome::Completed) + } + } + + struct FakeOutput { + writes: Mutex>>, + played: AtomicU64, + drain_polls: AtomicUsize, + drain_after: AtomicUsize, + cancelled: AtomicBool, + fail_health: AtomicBool, + } + + impl FakeOutput { + fn new(drain_after: usize) -> Self { + Self { + writes: Mutex::new(Vec::new()), + played: AtomicU64::new(0), + drain_polls: AtomicUsize::new(0), + drain_after: AtomicUsize::new(drain_after), + cancelled: AtomicBool::new(false), + fail_health: AtomicBool::new(false), + } + } + } + + impl PcmAudioOutput for FakeOutput { + fn write(&self, samples: &[f32]) -> Result<(), String> { + self.writes.lock().unwrap().push(samples.to_vec()); + Ok(()) + } + fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + self.played.store(0, Ordering::SeqCst); + } + fn is_drained(&self) -> bool { + self.drain_polls.fetch_add(1, Ordering::SeqCst) + >= self.drain_after.load(Ordering::SeqCst) + } + fn check_health(&self) -> Result<(), String> { + if self.fail_health.load(Ordering::SeqCst) { + Err("fake output failed".into()) + } else { + Ok(()) + } + } + fn played_frames(&self) -> u64 { + self.played.load(Ordering::SeqCst) + } + } + + #[test] + fn buffers_initial_pcm_and_starts_exactly_once() { + let active = AtomicBool::new(true); + let output = FakeOutput::new(0); + let backend = FakeTts { + chunks: vec![vec![0.1, 0.2], vec![0.3, 0.4], vec![0.5]], + cancel_after_first: false, + }; + let mut playback = OutboundPlayback::new(&output, &active, 10, 4).unwrap(); + let mut starting = 0; + let mut started = 0; + assert_eq!( + playback + .synthesize_segment( + &backend, + "hello", + &mut |first| { + starting += usize::from(first); + Ok(()) + }, + &mut || { + started += 1; + Ok(()) + }, + &mut |_| Ok(()), + ) + .unwrap(), + OutboundOutcome::Completed + ); + assert_eq!(starting, 1); + assert_eq!(started, 1); + assert_eq!( + output.writes.lock().unwrap().as_slice(), + &[vec![0.1, 0.2, 0.3, 0.4], vec![0.5]] + ); + } + + #[test] + fn write_failure_without_a_cancel_barrier_is_not_reusable() { + let active = AtomicBool::new(true); + let backend = FakeTts { + chunks: vec![vec![0.2]], + cancel_after_first: false, + }; + let mut playback = OutboundPlayback::new(&UnquiescedOutput, &active, 10, 1).unwrap(); + let failure = playback + .synthesize_segment( + &backend, + "failure", + &mut |_| Ok(()), + &mut || Ok(()), + &mut |_| Ok(()), + ) + .unwrap_err(); + + assert!(!failure.output_quiescent); + assert!(failure + .message + .contains("did not reach a quiescent terminal")); + } + + #[test] + fn delivery_maps_confirmed_frames_across_segments() { + let active = AtomicBool::new(true); + let output = FakeOutput::new(0); + let backend = FakeTts { + chunks: vec![vec![0.1, 0.2, 0.3]], + cancel_after_first: false, + }; + let mut playback = OutboundPlayback::new(&output, &active, 10, 0).unwrap(); + for text in ["one", "two"] { + playback + .synthesize_segment(&backend, text, &mut |_| Ok(()), &mut || Ok(()), &mut |_| { + Ok(()) + }) + .unwrap(); + } + output.played.store(4, Ordering::SeqCst); + let snapshot = playback.snapshot(); + assert_eq!(snapshot.segments[0].played_frames, 3); + assert_eq!(snapshot.segments[1].played_frames, 1); + assert!(snapshot + .segments + .iter() + .all(|segment| segment.synthesis_complete)); + } + + #[test] + fn synthesis_poll_can_release_and_next_pcm_reacquires_host_guard() { + use std::cell::{Cell, RefCell}; + + let active = AtomicBool::new(true); + let output = FakeOutput::new(0); + let first = FakeTts { + chunks: vec![vec![0.1, 0.2]], + cancel_after_first: false, + }; + let mut playback = OutboundPlayback::new(&output, &active, 10, 0).unwrap(); + let guard_active = Cell::new(false); + let lifecycle = RefCell::new(Vec::new()); + playback + .synthesize_segment( + &first, + "first", + &mut |_| { + guard_active.set(true); + Ok(()) + }, + &mut || Ok(()), + &mut |_| Ok(()), + ) + .unwrap(); + assert!(guard_active.get()); + + playback + .synthesize_segment( + &PollThenFramesTts, + "second", + &mut |_| { + if !guard_active.replace(true) { + lifecycle.borrow_mut().push("reacquired"); + } + output.drain_after.store(usize::MAX, Ordering::SeqCst); + Ok(()) + }, + &mut || Ok(()), + &mut |_| { + if output.is_drained() && guard_active.replace(false) { + lifecycle.borrow_mut().push("released"); + } + Ok(()) + }, + ) + .unwrap(); + + assert_eq!(&*lifecycle.borrow(), &["released", "reacquired"]); + assert!(guard_active.get()); + } + + #[test] + fn cancellation_snapshots_before_stopping_and_is_terminal() { + let active = AtomicBool::new(true); + let output = FakeOutput::new(0); + output.played.store(1, Ordering::SeqCst); + let backend = FakeTts { + chunks: vec![vec![0.1, 0.2]], + cancel_after_first: true, + }; + let mut playback = OutboundPlayback::new(&output, &active, 10, 0).unwrap(); + assert_eq!( + playback + .synthesize_segment( + &backend, + "cancel", + &mut |_| Ok(()), + &mut || Ok(()), + &mut |_| Ok(()), + ) + .unwrap(), + OutboundOutcome::Interrupted + ); + assert!(output.cancelled.load(Ordering::SeqCst)); + assert_eq!(playback.snapshot().segments[0].played_frames, 1); + assert!(playback + .synthesize_segment( + &backend, + "again", + &mut |_| Ok(()), + &mut || Ok(()), + &mut |_| Ok(()), + ) + .is_err()); + } + + #[test] + fn drain_checks_health_and_has_a_bounded_failure() { + let active = AtomicBool::new(true); + let output = FakeOutput::new(usize::MAX); + let mut playback = OutboundPlayback::new(&output, &active, 10, 0).unwrap(); + let failure = playback + .finish( + DrainPolicy { + poll_interval: Duration::ZERO, + timeout: Some(Duration::ZERO), + timeout_outcome: DrainTimeoutOutcome::Fail, + post_drain: Duration::ZERO, + }, + &mut |_| Ok(()), + ) + .unwrap_err(); + assert!(failure.message.contains("deadline")); + assert!(output.cancelled.load(Ordering::SeqCst)); + + let output = FakeOutput::new(1); + output.fail_health.store(true, Ordering::SeqCst); + let mut playback = OutboundPlayback::new(&output, &active, 10, 0).unwrap(); + assert_eq!( + playback + .finish(DrainPolicy::default(), &mut |_| Ok(())) + .unwrap_err() + .message, + "fake output failed" + ); + } + + #[test] + fn forced_complete_snapshots_before_cancel_and_runs_post_drain() { + let active = AtomicBool::new(true); + let output = FakeOutput::new(usize::MAX); + output.played.store(1, Ordering::SeqCst); + let backend = FakeTts { + chunks: vec![vec![0.1, 0.2]], + cancel_after_first: false, + }; + let mut playback = OutboundPlayback::new(&output, &active, 10, 0).unwrap(); + playback + .synthesize_segment( + &backend, + "forced", + &mut |_| Ok(()), + &mut || Ok(()), + &mut |_| Ok(()), + ) + .unwrap(); + let mut post_drain_polls = 0; + assert_eq!( + playback + .finish( + DrainPolicy { + poll_interval: Duration::ZERO, + timeout: Some(Duration::ZERO), + timeout_outcome: DrainTimeoutOutcome::Complete, + post_drain: Duration::from_millis(10), + }, + &mut |delivery| { + post_drain_polls += 1; + assert_eq!(delivery.segments[0].played_frames, 1); + Ok(()) + }, + ) + .unwrap(), + OutboundOutcome::Completed + ); + assert!(post_drain_polls > 0); + assert!(output.cancelled.load(Ordering::SeqCst)); + assert_eq!(playback.snapshot().segments[0].played_frames, 1); + } + + #[test] + fn cancellation_during_forced_complete_grace_is_interrupted() { + let active = AtomicBool::new(true); + let output = FakeOutput::new(usize::MAX); + output.played.store(1, Ordering::SeqCst); + let backend = FakeTts { + chunks: vec![vec![0.1, 0.2]], + cancel_after_first: false, + }; + let mut playback = OutboundPlayback::new(&output, &active, 10, 0).unwrap(); + playback + .synthesize_segment( + &backend, + "forced cancel", + &mut |_| Ok(()), + &mut || Ok(()), + &mut |_| Ok(()), + ) + .unwrap(); + assert_eq!( + playback + .finish( + DrainPolicy { + poll_interval: Duration::ZERO, + timeout: Some(Duration::ZERO), + timeout_outcome: DrainTimeoutOutcome::Complete, + post_drain: Duration::from_secs(1), + }, + &mut |_| { + active.store(false, Ordering::SeqCst); + Ok(()) + }, + ) + .unwrap(), + OutboundOutcome::Interrupted + ); + assert!(output.cancelled.load(Ordering::SeqCst)); + assert_eq!(playback.snapshot().segments[0].played_frames, 1); + } + + #[test] + fn cancellation_during_post_drain_is_interrupted_and_keeps_delivery() { + let active = AtomicBool::new(true); + let output = FakeOutput::new(0); + output.played.store(1, Ordering::SeqCst); + let backend = FakeTts { + chunks: vec![vec![0.1, 0.2]], + cancel_after_first: false, + }; + let mut playback = OutboundPlayback::new(&output, &active, 10, 0).unwrap(); + playback + .synthesize_segment( + &backend, + "tail", + &mut |_| Ok(()), + &mut || Ok(()), + &mut |_| Ok(()), + ) + .unwrap(); + let mut polls = 0; + assert_eq!( + playback + .finish( + DrainPolicy { + poll_interval: Duration::ZERO, + post_drain: Duration::from_secs(1), + ..DrainPolicy::default() + }, + &mut |_| { + polls += 1; + active.store(false, Ordering::SeqCst); + Ok(()) + }, + ) + .unwrap(), + OutboundOutcome::Interrupted + ); + assert_eq!(polls, 1); + assert!(output.cancelled.load(Ordering::SeqCst)); + assert_eq!(playback.snapshot().segments[0].played_frames, 1); + } +} diff --git a/src-tauri/crates/berd-voice/src/parakeet.rs b/src-tauri/crates/berd-voice/src/parakeet.rs new file mode 100644 index 000000000..7299936bb --- /dev/null +++ b/src-tauri/crates/berd-voice/src/parakeet.rs @@ -0,0 +1,75 @@ +use std::path::Path; + +use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; + +/// A loaded Parakeet recognizer for complete 16 kHz mono utterances. +pub struct ParakeetRecognizer { + recognizer: OfflineRecognizer, +} + +impl ParakeetRecognizer { + /// Loads a compatible Parakeet model bundle from an explicit directory. + pub fn load(model_dir: &Path) -> Result { + let config = recognizer_config(model_dir); + let recognizer = OfflineRecognizer::create(&config) + .ok_or_else(|| "Could not load the Parakeet speech model.".to_string())?; + Ok(Self { recognizer }) + } + + /// Recognizes one complete utterance of 16 kHz mono Float32 PCM. + pub fn recognize_utterance(&self, samples: &[f32]) -> String { + let stream = self.recognizer.create_stream(); + stream.accept_waveform(16_000, samples); + self.recognizer.decode(&stream); + stream + .get_result() + .map(|result| result.text.trim().to_string()) + .unwrap_or_default() + } +} + +fn recognizer_config(model_dir: &Path) -> OfflineRecognizerConfig { + let mut config = OfflineRecognizerConfig::default(); + config.model_config.nemo_ctc.model = Some( + model_dir + .join("model.int8.onnx") + .to_string_lossy() + .into_owned(), + ); + config.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); + config.model_config.num_threads = 1; + config.model_config.debug = false; + config +} + +#[cfg(test)] +mod tests { + use super::{recognizer_config, ParakeetRecognizer}; + use std::path::Path; + + #[test] + fn parakeet_config_uses_the_portable_bundle_layout() { + let model_dir = Path::new("models").join("parakeet"); + let config = recognizer_config(&model_dir); + let model = model_dir.join("model.int8.onnx"); + let tokens = model_dir.join("tokens.txt"); + assert_eq!( + config.model_config.nemo_ctc.model.as_deref(), + Some(model.to_string_lossy().as_ref()) + ); + assert_eq!( + config.model_config.tokens.as_deref(), + Some(tokens.to_string_lossy().as_ref()) + ); + assert_eq!(config.model_config.num_threads, 1); + assert!(!config.model_config.debug); + } + + #[test] + #[ignore = "requires BERD_PARAKEET_TEST_MODEL_DIR with a complete Parakeet bundle"] + fn installed_model_loads_and_decodes_silence() { + let model_dir = std::env::var("BERD_PARAKEET_TEST_MODEL_DIR").unwrap(); + let recognizer = ParakeetRecognizer::load(Path::new(&model_dir)).unwrap(); + assert_eq!(recognizer.recognize_utterance(&[0.0; 16_000]), ""); + } +} diff --git a/src-tauri/crates/berd-voice/src/parakeet_assets.rs b/src-tauri/crates/berd-voice/src/parakeet_assets.rs new file mode 100644 index 000000000..c8e3ff357 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/parakeet_assets.rs @@ -0,0 +1,717 @@ +//! Pinned, portable Parakeet model assets. + +use crate::asset_verification::{inspect_assets, AssetInspection, PinnedAsset}; +use crate::local_assets::{ + self, CombinedPublication, DownloadSpec, LocalAssetRoots, LocalInstallError, + LocalInstallErrorKind, LocalInstallPhase, LocalInstallProgress, TemporaryDirectory, +}; +use std::path::Path; + +/// Stable public identity of Berd's pinned Parakeet model. +pub const MODEL_ID: &str = "parakeet-tdt-ctc-110m-en-int8"; +/// License identifier for the upstream model and conversion. +pub const LICENSE_ID: &str = "CC-BY-4.0"; +/// Directory inside the pinned upstream archive. +pub const ARCHIVE_DIRECTORY: &str = "sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8"; + +/// One immutable downloadable archive. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParakeetArchive { + pub filename: &'static str, + pub size_bytes: u64, + pub sha256: &'static str, + pub source_url: &'static str, +} + +/// One immutable file in the portable Parakeet bundle root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ParakeetAsset { + pub relative_path: &'static str, + pub size_bytes: u64, + pub sha256: &'static str, +} + +/// Installation state for one explicit portable Parakeet bundle root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ParakeetAssetStatus { + Missing, + Invalid, + Ready { verified_bytes: u64 }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ParakeetInstallOutcome { + AlreadyReady { + verified_bytes: u64, + }, + Installed { + verified_bytes: u64, + cleanup_pending: Option, + }, +} + +pub const ARCHIVE: ParakeetArchive = ParakeetArchive { + filename: "parakeet.tar.bz2", + size_bytes: 104_337_827, + sha256: "17f945007b52ccd8b7200ffc7c5652e9e8e961dfdf479cefcabd06cf5703630b", + source_url: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2", +}; + +const LICENSE_TEXT: &str = "\ +NVIDIA Parakeet TDT-CTC 110M (English)\n\ +© NVIDIA Corporation.\n\ +\n\ +Licensed under the Creative Commons Attribution 4.0 International License:\n\ +https://creativecommons.org/licenses/by/4.0/\n\ +\n\ +Original model: https://huggingface.co/nvidia/parakeet-tdt_ctc-110m\n\ +ONNX conversion: https://github.com/k2-fsa/sherpa-onnx\n"; + +const PUBLISHED_ASSETS: &[ParakeetAsset] = &[ + ParakeetAsset { + relative_path: "model.int8.onnx", + size_bytes: 131_652_171, + sha256: "9177a9146cf32ee0cc8152276ef95116f312018d316be37ccf57f7efea81fc1a", + }, + ParakeetAsset { + relative_path: "tokens.txt", + size_bytes: 9_953, + sha256: "450e56bd2f036fe5b6aa821865838cc5aa9d8b0106134ce9a9ba0664abe6cd10", + }, + ParakeetAsset { + relative_path: "MODEL_LICENSE.txt", + size_bytes: 307, + sha256: "7ac2cc80a2b55558dabcdb73bb75ffd6f75dcc854b029f955023a38fb08b337b", + }, +]; + +pub fn published_assets() -> &'static [ParakeetAsset] { + PUBLISHED_ASSETS +} + +pub fn license_text() -> &'static str { + LICENSE_TEXT +} + +pub fn download_bytes() -> u64 { + ARCHIVE.size_bytes +} + +pub fn published_bytes() -> u64 { + PUBLISHED_ASSETS.iter().map(|asset| asset.size_bytes).sum() +} + +pub fn inspect(root: &Path) -> Result { + inspect_manifest(root, PUBLISHED_ASSETS) +} + +pub async fn install( + roots: &LocalAssetRoots, + on_progress: impl FnMut(LocalInstallProgress), +) -> Result { + let client = local_assets::default_client()?; + let archive = DownloadSpec { + source_url: ARCHIVE.source_url, + relative_path: ARCHIVE.filename, + size_bytes: ARCHIVE.size_bytes, + sha256: ARCHIVE.sha256, + }; + let published = exact_files() + .into_iter() + .map(|(relative_path, size_bytes, sha256)| DownloadSpec { + source_url: "", + relative_path, + size_bytes, + sha256, + }) + .collect::>(); + install_with_client( + roots, + &client, + ParakeetInstallPlan { + archive, + archive_directory: ARCHIVE_DIRECTORY, + runtime_specs: &published[..2], + published_specs: &published, + license_text: LICENSE_TEXT.as_bytes(), + }, + None, + on_progress, + ) + .await +} + +#[derive(Clone, Copy)] +struct ParakeetInstallPlan<'a> { + archive: DownloadSpec<'static>, + archive_directory: &'static str, + runtime_specs: &'a [DownloadSpec<'static>], + published_specs: &'a [DownloadSpec<'static>], + license_text: &'static [u8], +} + +async fn install_with_client( + roots: &LocalAssetRoots, + client: &reqwest::Client, + plan: ParakeetInstallPlan<'_>, + preparation_barrier: Option<&tokio::sync::Barrier>, + mut on_progress: impl FnMut(LocalInstallProgress), +) -> Result { + let ParakeetInstallPlan { + archive: archive_spec, + archive_directory, + runtime_specs, + published_specs, + license_text, + } = plan; + let total_download_bytes = archive_spec.size_bytes; + { + let lock = local_assets::lock_for_mutation(roots) + .await + .map_err(LocalInstallError::from)?; + lock.recover_interrupted_publication()?; + if let crate::asset_verification::AssetInspection::Ready { verified_bytes } = + local_assets::inspect_download_specs(roots.parakeet_bundle_root(), published_specs) + .map_err(|message| LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message, + recovery_paths: Vec::new(), + })? + { + return Ok(ParakeetInstallOutcome::AlreadyReady { verified_bytes }); + } + } + if let Some(barrier) = preparation_barrier { + barrier.wait().await; + } + + let prepared = TemporaryDirectory::create(roots.coordination_root(), "parakeet-download")?; + let mut downloaded_bytes = 0_u64; + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Downloading, + downloaded_bytes, + total_download_bytes, + }); + local_assets::download(client, prepared.path(), archive_spec, |increment| { + downloaded_bytes = downloaded_bytes.saturating_add(increment); + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Downloading, + downloaded_bytes, + total_download_bytes, + }); + }) + .await?; + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Extracting, + downloaded_bytes, + total_download_bytes, + }); + let archive = prepared.path().join(archive_spec.relative_path); + let destination = prepared.path().to_path_buf(); + let runtime_specs = runtime_specs.to_vec(); + let extraction_manifest = published_specs.to_vec(); + let (prepared, extraction) = tokio::task::spawn_blocking(move || { + let expected = runtime_specs + .iter() + .map(|asset| (asset.relative_path, asset.size_bytes, asset.sha256)) + .collect::>(); + let result = local_assets::extract_exact_tar_bz2( + &archive, + &destination, + archive_directory, + &expected, + ) + .and_then(|()| { + let license = extraction_manifest.last().ok_or_else(|| { + LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "Parakeet published manifest omitted its license", + ) + })?; + std::fs::write(destination.join(license.relative_path), license_text).map_err(|error| { + LocalInstallError::new( + LocalInstallErrorKind::Extraction, + format!("write Parakeet attribution: {error}"), + ) + }) + }); + (prepared, result) + }) + .await + .map_err(|error| LocalInstallError { + kind: LocalInstallErrorKind::Extraction, + message: format!("Parakeet extraction task failed: {error}"), + recovery_paths: Vec::new(), + })?; + extraction?; + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Verifying, + downloaded_bytes, + total_download_bytes, + }); + if !matches!( + local_assets::inspect_download_specs(prepared.path(), published_specs), + Ok(crate::asset_verification::AssetInspection::Ready { .. }) + ) { + return Err(LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message: "prepared Parakeet bundle failed pinned-file verification".to_string(), + recovery_paths: Vec::new(), + }); + } + + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Publishing, + downloaded_bytes, + total_download_bytes, + }); + let lock = local_assets::lock_for_mutation(roots) + .await + .map_err(LocalInstallError::from)?; + lock.recover_interrupted_publication()?; + if let crate::asset_verification::AssetInspection::Ready { verified_bytes } = + local_assets::inspect_download_specs(roots.parakeet_bundle_root(), published_specs) + .map_err(|message| LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message, + recovery_paths: Vec::new(), + })? + { + drop(lock); + report_complete(&mut on_progress, downloaded_bytes, total_download_bytes); + return Ok(ParakeetInstallOutcome::AlreadyReady { verified_bytes }); + } + let preserve_pocket = matches!( + crate::pocket_assets::inspect(roots.pocket_bundle_root()), + Ok(crate::pocket_assets::PocketAssetStatus::Ready { .. }) + ); + let publication = CombinedPublication::prepare(roots)?; + if preserve_pocket { + local_assets::copy_exact_files( + roots.pocket_bundle_root(), + publication.root(), + crate::pocket_assets::exact_files(), + )?; + } + local_assets::copy_exact_files( + prepared.path(), + &publication.root().join("stt"), + published_specs + .iter() + .map(|spec| (spec.relative_path, spec.size_bytes, spec.sha256)), + )?; + let target_ready = |root: &Path| { + matches!( + local_assets::inspect_download_specs(&root.join("stt"), published_specs), + Ok(crate::asset_verification::AssetInspection::Ready { .. }) + ) + }; + let combined_ready = |root: &Path| { + target_ready(root) + && (!preserve_pocket + || matches!( + crate::pocket_assets::inspect(root), + Ok(crate::pocket_assets::PocketAssetStatus::Ready { .. }) + )) + }; + if !combined_ready(publication.root()) { + return Err(LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message: "staged Parakeet bundle failed pinned-file verification".to_string(), + recovery_paths: Vec::new(), + }); + } + let cleanup_pending = publication.publish(combined_ready)?; + let verified_bytes = + match local_assets::inspect_download_specs(roots.parakeet_bundle_root(), published_specs) { + Ok(crate::asset_verification::AssetInspection::Ready { verified_bytes }) => { + verified_bytes + } + _ => { + return Err(LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message: "published Parakeet bundle was not ready".to_string(), + recovery_paths: Vec::new(), + }) + } + }; + drop(lock); + report_complete(&mut on_progress, downloaded_bytes, total_download_bytes); + Ok(ParakeetInstallOutcome::Installed { + verified_bytes, + cleanup_pending, + }) +} + +fn report_complete( + on_progress: &mut impl FnMut(LocalInstallProgress), + downloaded_bytes: u64, + total_download_bytes: u64, +) { + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Complete, + downloaded_bytes, + total_download_bytes, + }); +} + +pub(crate) fn exact_files() -> Vec<(&'static str, u64, &'static str)> { + PUBLISHED_ASSETS + .iter() + .map(|asset| (asset.relative_path, asset.size_bytes, asset.sha256)) + .collect() +} + +/// Copy one verified Parakeet bundle into a host-owned transaction staging root. +pub fn stage_verified_bundle( + mutation: &crate::local_assets::LocalAssetMutationGuard, + source: &Path, + destination: &Path, +) -> Result<(), LocalInstallError> { + mutation.validate_staging_paths( + source, + mutation.roots().parakeet_bundle_root(), + destination, + )?; + local_assets::copy_exact_files(source, destination, exact_files())?; + if !matches!(inspect(destination), Ok(ParakeetAssetStatus::Ready { .. })) { + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "staged Parakeet bundle failed pinned-file verification", + )); + } + Ok(()) +} + +fn inspect_manifest(root: &Path, assets: &[ParakeetAsset]) -> Result { + let manifest: Vec<_> = assets + .iter() + .map(|asset| PinnedAsset { + relative_path: asset.relative_path, + size_bytes: asset.size_bytes, + sha256: asset.sha256, + }) + .collect(); + Ok(match inspect_assets(root, &manifest)? { + AssetInspection::Missing => ParakeetAssetStatus::Missing, + AssetInspection::Invalid => ParakeetAssetStatus::Invalid, + AssetInspection::Ready { verified_bytes } => ParakeetAssetStatus::Ready { verified_bytes }, + }) +} + +#[cfg(test)] +mod tests { + use super::{ + inspect_manifest, license_text, published_assets, published_bytes, ParakeetAssetStatus, + ARCHIVE, LICENSE_ID, MODEL_ID, + }; + use sha2::{Digest, Sha256}; + use std::collections::HashSet; + use std::fs; + use std::sync::{Arc, Mutex}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + #[test] + fn pinned_catalog_includes_exact_attribution() { + assert_eq!(MODEL_ID, "parakeet-tdt-ctc-110m-en-int8"); + assert_eq!(LICENSE_ID, "CC-BY-4.0"); + assert!(ARCHIVE.source_url.starts_with("https://")); + assert_eq!(ARCHIVE.sha256.len(), 64); + assert_eq!( + published_assets() + .iter() + .map(|asset| asset.relative_path) + .collect::>(), + ["model.int8.onnx", "tokens.txt", "MODEL_LICENSE.txt"] + ); + assert!(license_text().contains("Creative Commons Attribution 4.0")); + let license = published_assets() + .iter() + .find(|asset| asset.relative_path == "MODEL_LICENSE.txt") + .expect("license asset"); + assert_eq!(license.size_bytes, license_text().len() as u64); + assert_eq!( + license.sha256, + format!("{:x}", Sha256::digest(license_text().as_bytes())) + ); + assert_eq!( + published_bytes(), + published_assets() + .iter() + .map(|asset| asset.size_bytes) + .sum::() + ); + assert_eq!( + published_assets() + .iter() + .map(|asset| asset.relative_path) + .collect::>() + .len(), + published_assets().len() + ); + } + + #[test] + fn exact_license_text_is_part_of_bundle_readiness() { + let root = + std::env::temp_dir().join(format!("berd-parakeet-license-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir(&root).expect("create temporary directory"); + let license = &published_assets()[2..]; + fs::write(root.join("MODEL_LICENSE.txt"), license_text()).expect("write license"); + assert_eq!( + inspect_manifest(&root, license).expect("inspect exact license"), + ParakeetAssetStatus::Ready { + verified_bytes: license_text().len() as u64 + } + ); + + let mut corrupt = license_text().as_bytes().to_vec(); + corrupt[0] ^= 1; + fs::write(root.join("MODEL_LICENSE.txt"), corrupt).expect("corrupt license"); + assert_eq!( + inspect_manifest(&root, license).expect("inspect corrupt license"), + ParakeetAssetStatus::Invalid + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn late_ready_race_reports_terminal_complete_progress() { + let mut progress = Vec::new(); + super::report_complete( + &mut |event| progress.push(event), + super::download_bytes(), + super::download_bytes(), + ); + assert_eq!(progress.len(), 1); + assert_eq!(progress[0].phase, super::LocalInstallPhase::Complete); + assert_eq!(progress[0].downloaded_bytes, super::download_bytes()); + } + + #[tokio::test] + async fn concrete_installs_serialize_extract_exactly_and_complete_the_late_racer() { + let encoder = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::fast()); + let mut archive = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_size(4); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "bundle/model", &b"keep"[..]) + .expect("append model"); + let encoder = archive.into_inner().expect("finish tar"); + let archive_bytes = encoder.finish().expect("finish compression"); + let archive_hash = + Box::leak(format!("{:x}", Sha256::digest(&archive_bytes)).into_boxed_str()); + let archive_size = archive_bytes.len() as u64; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind fixture server"); + let address = listener.local_addr().expect("fixture address"); + let server = tokio::spawn(async move { + for _ in 0..2 { + let (mut socket, _) = listener.accept().await.expect("accept request"); + let mut request = [0_u8; 1024]; + let bytes_read = socket.read(&mut request).await.expect("read request"); + assert!(bytes_read > 0, "fixture request was empty"); + socket + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + archive_bytes.len() + ) + .as_bytes(), + ) + .await + .expect("write response headers"); + socket + .write_all(&archive_bytes) + .await + .expect("write archive"); + } + }); + let root = tempfile::tempdir().expect("temporary directory"); + let roots = super::LocalAssetRoots::new( + root.path(), + root.path().join("native-voice-v2"), + root.path().join("native-voice-v2/stt"), + ) + .expect("asset roots"); + let archive_spec = super::DownloadSpec { + source_url: Box::leak(format!("http://{address}/archive").into_boxed_str()), + relative_path: "archive.tar.bz2", + size_bytes: archive_size, + sha256: archive_hash, + }; + let published = [ + super::DownloadSpec { + source_url: "", + relative_path: "model", + size_bytes: 4, + sha256: "6ca7ea2feefc88ecb5ed6356ed963f47dc9137f82526fdd25d618ea626d0803f", + }, + super::DownloadSpec { + source_url: "", + relative_path: "LICENSE", + size_bytes: 7, + sha256: "cc1d3b0234846714b0aeda6cc34b057b4305bb83dd447fb88f816efeb59a4e96", + }, + ]; + let client = reqwest::Client::new(); + let first_progress = Arc::new(Mutex::new(Vec::new())); + let second_progress = Arc::new(Mutex::new(Vec::new())); + let first_events = Arc::clone(&first_progress); + let second_events = Arc::clone(&second_progress); + let barrier = tokio::sync::Barrier::new(2); + let (first, second) = tokio::join!( + super::install_with_client( + &roots, + &client, + super::ParakeetInstallPlan { + archive: archive_spec, + archive_directory: "bundle", + runtime_specs: &published[..1], + published_specs: &published, + license_text: b"license", + }, + Some(&barrier), + move |event| first_events.lock().expect("first progress").push(event), + ), + super::install_with_client( + &roots, + &client, + super::ParakeetInstallPlan { + archive: archive_spec, + archive_directory: "bundle", + runtime_specs: &published[..1], + published_specs: &published, + license_text: b"license", + }, + Some(&barrier), + move |event| second_events.lock().expect("second progress").push(event), + ), + ); + let outcomes = [ + first.expect("first install"), + second.expect("second install"), + ]; + assert!(outcomes + .iter() + .any(|outcome| matches!(outcome, super::ParakeetInstallOutcome::Installed { .. }))); + assert!(outcomes + .iter() + .any(|outcome| matches!(outcome, super::ParakeetInstallOutcome::AlreadyReady { .. }))); + for events in [first_progress, second_progress] { + assert_eq!( + events + .lock() + .expect("progress") + .last() + .map(|event| event.phase), + Some(super::LocalInstallPhase::Complete) + ); + } + assert!(!roots + .parakeet_bundle_root() + .join("archive.tar.bz2") + .exists()); + server.await.expect("fixture server"); + } + + #[test] + fn staging_rejects_a_mutation_guard_from_another_store() { + let first = tempfile::tempdir().expect("first store"); + let second = tempfile::tempdir().expect("second store"); + let first_roots = super::LocalAssetRoots::new( + first.path(), + first.path().join("native-voice-v2"), + first.path().join("native-voice-v2/stt"), + ) + .expect("first roots"); + let second_roots = super::LocalAssetRoots::new( + second.path(), + second.path().join("native-voice-v2"), + second.path().join("native-voice-v2/stt"), + ) + .expect("second roots"); + let guard = crate::local_assets::try_lock_for_mutation(&first_roots).expect("first lock"); + let error = super::stage_verified_bundle( + &guard, + second_roots.parakeet_bundle_root(), + first.path().join("stage/stt").as_path(), + ) + .expect_err("cross-store guard"); + assert_eq!(error.kind, super::LocalInstallErrorKind::InvalidRoot); + + for destination in [ + first.path().join("../outside/stt"), + first.path().to_path_buf(), + first_roots.pocket_bundle_root().join("nested-stage/stt"), + ] { + let error = super::stage_verified_bundle( + &guard, + first_roots.parakeet_bundle_root(), + &destination, + ) + .expect_err("unsafe staging destination"); + assert_eq!(error.kind, super::LocalInstallErrorKind::InvalidRoot); + } + } + + #[tokio::test] + async fn initial_ready_preflight_makes_no_request_and_reports_already_ready() { + let root = tempfile::tempdir().expect("temporary directory"); + let roots = super::LocalAssetRoots::new( + root.path(), + root.path().join("native-voice-v2"), + root.path().join("native-voice-v2/stt"), + ) + .expect("asset roots"); + std::fs::create_dir_all(roots.parakeet_bundle_root()).expect("create ready bundle"); + std::fs::write(roots.parakeet_bundle_root().join("model"), b"keep").expect("write model"); + std::fs::write(roots.parakeet_bundle_root().join("LICENSE"), b"license") + .expect("write license"); + let published = [ + super::DownloadSpec { + source_url: "", + relative_path: "model", + size_bytes: 4, + sha256: "6ca7ea2feefc88ecb5ed6356ed963f47dc9137f82526fdd25d618ea626d0803f", + }, + super::DownloadSpec { + source_url: "", + relative_path: "LICENSE", + size_bytes: 7, + sha256: "cc1d3b0234846714b0aeda6cc34b057b4305bb83dd447fb88f816efeb59a4e96", + }, + ]; + let archive = super::DownloadSpec { + source_url: "http://127.0.0.1:1/must-not-be-called", + relative_path: "archive", + size_bytes: 1, + sha256: "00", + }; + let mut progress = Vec::new(); + let outcome = super::install_with_client( + &roots, + &reqwest::Client::new(), + super::ParakeetInstallPlan { + archive, + archive_directory: "bundle", + runtime_specs: &published[..1], + published_specs: &published, + license_text: b"license", + }, + None, + |event| progress.push(event), + ) + .await + .expect("already ready"); + assert!(matches!( + outcome, + super::ParakeetInstallOutcome::AlreadyReady { .. } + )); + assert!(progress.is_empty()); + } +} diff --git a/src-tauri/crates/berd-voice/src/pocket.rs b/src-tauri/crates/berd-voice/src/pocket.rs index 0d120a093..29789c91c 100644 --- a/src-tauri/crates/berd-voice/src/pocket.rs +++ b/src-tauri/crates/berd-voice/src/pocket.rs @@ -124,6 +124,27 @@ pub fn load_voice_style(path: &Path) -> Result { }) } +/// Resolve and load an exact Pocket voice ID from a self-contained model +/// bundle. Voice IDs are deliberately path-safe; callers choose the bundle +/// root and this function owns the stable `voices/.wav` layout. +pub fn load_pocket_voice_style(model_dir: &Path, voice_id: &str) -> Result { + if voice_id.is_empty() + || !voice_id + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + { + return Err(format!("invalid Pocket voice ID: {voice_id}")); + } + let path = model_dir.join("voices").join(format!("{voice_id}.wav")); + if !path.is_file() { + return Err(format!( + "Pocket voice {voice_id} is not installed at {}", + path.display() + )); + } + load_voice_style(&path) +} + /// Resident April INT8 Pocket TTS engine. pub struct PocketTts { inner: Mutex, @@ -256,4 +277,15 @@ mod tests { "Pocket TTS synthesis callback panicked" ); } + + #[test] + fn pocket_voice_ids_cannot_escape_the_bundle() { + let root = Path::new("/tmp/model"); + assert!(load_pocket_voice_style(root, "../mary").is_err()); + assert!(load_pocket_voice_style(root, "Mary").is_err()); + assert!(load_pocket_voice_style(root, "").is_err()); + assert!(load_pocket_voice_style(root, "mary") + .unwrap_err() + .contains("is not installed")); + } } diff --git a/src-tauri/crates/berd-voice/src/pocket_assets.rs b/src-tauri/crates/berd-voice/src/pocket_assets.rs new file mode 100644 index 000000000..f0cf62328 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/pocket_assets.rs @@ -0,0 +1,549 @@ +//! Pinned, portable Pocket TTS model and voice assets. + +use crate::asset_verification::{inspect_assets, AssetInspection, PinnedAsset}; +use crate::local_assets::{ + self, CombinedPublication, DownloadSpec, LocalAssetRoots, LocalInstallError, + LocalInstallErrorKind, LocalInstallPhase, LocalInstallProgress, TemporaryDirectory, +}; +use std::path::Path; + +/// Stable public identity of Berd's pinned Pocket TTS model. +pub const MODEL_ID: &str = "native-voice-v2"; +pub const MODEL_LICENSE_ID: &str = "CC-BY-4.0"; +pub const VOICE_LICENSE_ID: &str = "CC-BY-4.0"; + +/// One immutable Pocket model file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PocketModelAsset { + pub relative_path: &'static str, + pub size_bytes: u64, + pub sha256: &'static str, + pub source_url: &'static str, +} + +/// One immutable Pocket reference voice. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PocketVoiceDescriptor { + pub id: &'static str, + pub name: &'static str, + pub relative_path: &'static str, + pub size_bytes: u64, + pub sha256: &'static str, + pub source_url: &'static str, +} + +/// Installation state for one explicit portable Pocket bundle root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PocketAssetStatus { + Missing, + Invalid, + Ready { verified_bytes: u64 }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PocketInstallOutcome { + AlreadyReady { + verified_bytes: u64, + }, + Installed { + verified_bytes: u64, + cleanup_pending: Option, + }, +} + +const MODEL_ARTIFACTS: &[PocketModelAsset] = &[ + PocketModelAsset { relative_path: "bundle.json", size_bytes: 24_381, sha256: "bab643150f437f37df080a710520ff39ed9ebd9a339f8ebdc739f7eddfc28b3f", source_url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/bundle.json" }, + PocketModelAsset { relative_path: "bos_before_voice.npy", size_bytes: 4_224, sha256: "f46edf4f7007b7ba4ea58831f49d003e59e167b4641c44bb3addfe9231a780b1", source_url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/bos_before_voice.npy" }, + PocketModelAsset { relative_path: "tokenizer.model", size_bytes: 59_339, sha256: "d461765ae179566678c93091c5fa6f2984c31bbe990bf1aa62d92c64d91bc3f6", source_url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/tokenizer.model" }, + PocketModelAsset { relative_path: "flow_lm_main_int8.onnx", size_bytes: 76_341_079, sha256: "f9bd8106b79a0192c1c43399ab938fb24900a95c1c599870d75a884e99000116", source_url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/flow_lm_main_int8.onnx" }, + PocketModelAsset { relative_path: "flow_lm_flow_int8.onnx", size_bytes: 9_962_530, sha256: "3dd781ee5abee9e195320bf0106bebd6372a852b3b36352524ee78b40554635d", source_url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/flow_lm_flow_int8.onnx" }, + PocketModelAsset { relative_path: "mimi_decoder_int8.onnx", size_bytes: 22_684_077, sha256: "3630450a3297a101792a6ac66619ebc70ab916b265e6220c2afaef8b1673f925", source_url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/mimi_decoder_int8.onnx" }, + PocketModelAsset { relative_path: "mimi_encoder.onnx", size_bytes: 39_768_446, sha256: "853e2ca623b8782d94c3745ec6133bfdff7ce33d9b11128bd29ea03f28d76e3d", source_url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/mimi_encoder.onnx" }, + PocketModelAsset { relative_path: "text_conditioner.onnx", size_bytes: 16_388_344, sha256: "4ecee995fb69f85c7a7493d11f7b5ee15d9950facc7ab3f5c9c49ef1e03847bb", source_url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/text_conditioner.onnx" }, + PocketModelAsset { relative_path: "LICENSE", size_bytes: 18_655, sha256: "fe7b4ce83b8381cc5b216bbb4af73c570688d1b819c73bbaed8ca401f4677cd6", source_url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/LICENSE" }, +]; + +const VOICES: &[PocketVoiceDescriptor] = &[ + PocketVoiceDescriptor { id: "anna", name: "Anna", relative_path: "voices/anna.wav", size_bytes: 804_630, sha256: "0a6de25cf12bf1540beb85979f306a92be81fecc051c547c5395e7e5237a3856", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p228_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "vera", name: "Vera", relative_path: "voices/vera.wav", size_bytes: 691_416, sha256: "309cf91a895830f15842b398f69a4962cb1f7e0bfab10e25dd27838e826c204b", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p229_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "fantine", name: "Fantine", relative_path: "voices/fantine.wav", size_bytes: 674_852, sha256: "5f07d4e2a3f20a15572aae885156b43ef3fc12ef3812996fd135680d9956448b", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p244_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "charles", name: "Charles", relative_path: "voices/charles.wav", size_bytes: 639_272, sha256: "6b681a429198f16e378d53bccb08d06939da7b00144a7696111d4f8f76be7756", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p254_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "paul", name: "Paul", relative_path: "voices/paul.wav", size_bytes: 717_182, sha256: "7aba504fe0b3b16478b69eb27ce6007e3cb42b0c1915b5f1c6a6024ae37d679b", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p259_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "eponine", name: "Eponine", relative_path: "voices/eponine.wav", size_bytes: 716_330, sha256: "a13c27fb47627b05223691a0ef2974358a18c886e6c2f9d2762ff1d02c20926b", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p262_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "azelma", name: "Azelma", relative_path: "voices/azelma.wav", size_bytes: 823_852, sha256: "60e3d26cdf2efdec5df712152c839928f4d5522821e6554ae11fd96c57ab1026", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p303_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "george", name: "George", relative_path: "voices/george.wav", size_bytes: 642_692, sha256: "29a41f93bf5236e5b21501091d7774c255d5f3d4e62fa4f9fdf0a92a793c84ae", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p315_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "mary", name: "Mary", relative_path: "voices/mary.wav", size_bytes: 639_084, sha256: "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p333_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "jane", name: "Jane", relative_path: "voices/jane.wav", size_bytes: 759_340, sha256: "2f12e7f155eb3118f55425394f1b049e5b1b67bdc9b3932c8ba4521420aeb84a", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p339_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "michael", name: "Michael", relative_path: "voices/michael.wav", size_bytes: 751_140, sha256: "b6743e9195e5e3fd34fe9d1633ae93f7ffab787b249e45f6467d7d6f7a6ee6ad", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p360_023_enhanced.wav" }, + PocketVoiceDescriptor { id: "eve", name: "Eve", relative_path: "voices/eve.wav", size_bytes: 671_872, sha256: "396e7cbd066b0f3fb6d67fa26e7904076958239d736d4390f15b5fe88feb14cd", source_url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p361_023_enhanced.wav" }, +]; + +pub fn model_artifacts() -> &'static [PocketModelAsset] { + MODEL_ARTIFACTS +} + +pub fn voices() -> &'static [PocketVoiceDescriptor] { + VOICES +} + +pub fn download_bytes() -> u64 { + MODEL_ARTIFACTS + .iter() + .map(|asset| asset.size_bytes) + .chain(VOICES.iter().map(|voice| voice.size_bytes)) + .sum() +} + +pub fn inspect(root: &Path) -> Result { + let manifest: Vec<_> = MODEL_ARTIFACTS + .iter() + .map(|asset| PinnedAsset { + relative_path: asset.relative_path, + size_bytes: asset.size_bytes, + sha256: asset.sha256, + }) + .chain(VOICES.iter().map(|voice| PinnedAsset { + relative_path: voice.relative_path, + size_bytes: voice.size_bytes, + sha256: voice.sha256, + })) + .collect(); + Ok(match inspect_assets(root, &manifest)? { + AssetInspection::Missing => PocketAssetStatus::Missing, + AssetInspection::Invalid => PocketAssetStatus::Invalid, + AssetInspection::Ready { verified_bytes } => PocketAssetStatus::Ready { verified_bytes }, + }) +} + +pub async fn install( + roots: &LocalAssetRoots, + on_progress: impl FnMut(LocalInstallProgress), +) -> Result { + let client = local_assets::default_client()?; + install_with_client(roots, &client, &pocket_download_specs(), None, on_progress).await +} + +async fn install_with_client( + roots: &LocalAssetRoots, + client: &reqwest::Client, + specs: &[DownloadSpec<'static>], + preparation_barrier: Option<&tokio::sync::Barrier>, + mut on_progress: impl FnMut(LocalInstallProgress), +) -> Result { + let total_download_bytes = specs.iter().try_fold(0_u64, |total, spec| { + total.checked_add(spec.size_bytes).ok_or_else(|| { + LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "Pocket download byte total overflow", + ) + }) + })?; + { + let lock = local_assets::lock_for_mutation(roots) + .await + .map_err(LocalInstallError::from)?; + lock.recover_interrupted_publication()?; + if let crate::asset_verification::AssetInspection::Ready { verified_bytes } = + local_assets::inspect_download_specs(roots.pocket_bundle_root(), specs).map_err( + |message| LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message, + recovery_paths: Vec::new(), + }, + )? + { + return Ok(PocketInstallOutcome::AlreadyReady { verified_bytes }); + } + } + if let Some(barrier) = preparation_barrier { + barrier.wait().await; + } + + let prepared = TemporaryDirectory::create(roots.coordination_root(), "pocket-download")?; + let mut downloaded_bytes = 0_u64; + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Downloading, + downloaded_bytes, + total_download_bytes, + }); + for spec in specs.iter().copied() { + local_assets::download(client, prepared.path(), spec, |increment| { + downloaded_bytes = downloaded_bytes.saturating_add(increment); + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Downloading, + downloaded_bytes, + total_download_bytes, + }); + }) + .await?; + } + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Verifying, + downloaded_bytes, + total_download_bytes, + }); + if !matches!( + local_assets::inspect_download_specs(prepared.path(), specs), + Ok(crate::asset_verification::AssetInspection::Ready { .. }) + ) { + return Err(LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message: "downloaded Pocket bundle failed pinned-file verification".to_string(), + recovery_paths: Vec::new(), + }); + } + + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Publishing, + downloaded_bytes, + total_download_bytes, + }); + let lock = local_assets::lock_for_mutation(roots) + .await + .map_err(LocalInstallError::from)?; + lock.recover_interrupted_publication()?; + if let crate::asset_verification::AssetInspection::Ready { verified_bytes } = + local_assets::inspect_download_specs(roots.pocket_bundle_root(), specs).map_err( + |message| LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message, + recovery_paths: Vec::new(), + }, + )? + { + drop(lock); + report_complete(&mut on_progress, downloaded_bytes, total_download_bytes); + return Ok(PocketInstallOutcome::AlreadyReady { verified_bytes }); + } + let preserve_parakeet = matches!( + crate::parakeet_assets::inspect(roots.parakeet_bundle_root()), + Ok(crate::parakeet_assets::ParakeetAssetStatus::Ready { .. }) + ); + let publication = CombinedPublication::prepare(roots)?; + local_assets::copy_exact_files( + prepared.path(), + publication.root(), + specs + .iter() + .map(|spec| (spec.relative_path, spec.size_bytes, spec.sha256)), + )?; + if preserve_parakeet { + local_assets::copy_exact_files( + roots.parakeet_bundle_root(), + &publication.root().join("stt"), + crate::parakeet_assets::exact_files(), + )?; + } + let target_ready = |root: &Path| { + matches!( + local_assets::inspect_download_specs(root, specs), + Ok(crate::asset_verification::AssetInspection::Ready { .. }) + ) + }; + let combined_ready = |root: &Path| { + target_ready(root) + && (!preserve_parakeet + || matches!( + crate::parakeet_assets::inspect(&root.join("stt")), + Ok(crate::parakeet_assets::ParakeetAssetStatus::Ready { .. }) + )) + }; + if !combined_ready(publication.root()) { + return Err(LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message: "staged Pocket bundle failed pinned-file verification".to_string(), + recovery_paths: Vec::new(), + }); + } + let cleanup_pending = publication.publish(combined_ready)?; + let verified_bytes = + match local_assets::inspect_download_specs(roots.pocket_bundle_root(), specs) { + Ok(crate::asset_verification::AssetInspection::Ready { verified_bytes }) => { + verified_bytes + } + _ => { + return Err(LocalInstallError { + kind: LocalInstallErrorKind::Integrity, + message: "published Pocket bundle was not ready".to_string(), + recovery_paths: Vec::new(), + }) + } + }; + drop(lock); + report_complete(&mut on_progress, downloaded_bytes, total_download_bytes); + Ok(PocketInstallOutcome::Installed { + verified_bytes, + cleanup_pending, + }) +} + +fn report_complete( + on_progress: &mut impl FnMut(LocalInstallProgress), + downloaded_bytes: u64, + total_download_bytes: u64, +) { + on_progress(LocalInstallProgress { + phase: LocalInstallPhase::Complete, + downloaded_bytes, + total_download_bytes, + }); +} + +fn pocket_download_specs() -> Vec> { + MODEL_ARTIFACTS + .iter() + .map(|asset| DownloadSpec { + source_url: asset.source_url, + relative_path: asset.relative_path, + size_bytes: asset.size_bytes, + sha256: asset.sha256, + }) + .chain(VOICES.iter().map(|voice| DownloadSpec { + source_url: voice.source_url, + relative_path: voice.relative_path, + size_bytes: voice.size_bytes, + sha256: voice.sha256, + })) + .collect() +} + +pub(crate) fn exact_files() -> Vec<(&'static str, u64, &'static str)> { + pocket_exact_files() +} + +/// Copy one verified Pocket bundle into a host-owned transaction staging root. +pub fn stage_verified_bundle( + mutation: &crate::local_assets::LocalAssetMutationGuard, + source: &Path, + destination: &Path, +) -> Result<(), LocalInstallError> { + mutation.validate_staging_paths(source, mutation.roots().pocket_bundle_root(), destination)?; + local_assets::copy_exact_files(source, destination, pocket_exact_files())?; + if !matches!(inspect(destination), Ok(PocketAssetStatus::Ready { .. })) { + return Err(LocalInstallError::new( + LocalInstallErrorKind::Integrity, + "staged Pocket bundle failed pinned-file verification", + )); + } + Ok(()) +} + +fn pocket_exact_files() -> Vec<(&'static str, u64, &'static str)> { + MODEL_ARTIFACTS + .iter() + .map(|asset| (asset.relative_path, asset.size_bytes, asset.sha256)) + .chain( + VOICES + .iter() + .map(|voice| (voice.relative_path, voice.size_bytes, voice.sha256)), + ) + .collect() +} + +#[cfg(test)] +mod tests { + use super::{ + download_bytes, model_artifacts, voices, MODEL_ID, MODEL_LICENSE_ID, VOICE_LICENSE_ID, + }; + use std::collections::HashSet; + use std::sync::{Arc, Mutex}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + #[test] + fn pinned_catalog_has_stable_identity_and_safe_unique_paths() { + assert_eq!(MODEL_ID, "native-voice-v2"); + assert_eq!(MODEL_LICENSE_ID, "CC-BY-4.0"); + assert_eq!(VOICE_LICENSE_ID, "CC-BY-4.0"); + assert_eq!(voices().len(), 12); + assert_eq!( + voices().iter().map(|voice| voice.id).collect::>(), + [ + "anna", "vera", "fantine", "charles", "paul", "eponine", "azelma", "george", + "mary", "jane", "michael", "eve" + ] + ); + let mut paths = HashSet::new(); + for artifact in model_artifacts() { + assert!(paths.insert(artifact.relative_path)); + assert!(artifact.size_bytes > 0); + assert!(artifact.source_url.starts_with("https://")); + assert_eq!(artifact.sha256.len(), 64); + } + let mut ids = HashSet::new(); + for voice in voices() { + assert!(paths.insert(voice.relative_path)); + assert!(ids.insert(voice.id)); + assert!(voice.size_bytes > 0); + assert!(voice + .id + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')); + assert_eq!(voice.relative_path, format!("voices/{}.wav", voice.id)); + assert!(voice.source_url.starts_with("https://")); + assert_eq!(voice.sha256.len(), 64); + } + assert_eq!(download_bytes(), 173_782_737); + let license = model_artifacts() + .iter() + .find(|asset| asset.relative_path == "LICENSE") + .expect("Pocket license asset"); + assert_eq!(license.size_bytes, 18_655); + assert_eq!( + license.sha256, + "fe7b4ce83b8381cc5b216bbb4af73c570688d1b819c73bbaed8ca401f4677cd6" + ); + } + + #[test] + fn late_ready_race_reports_terminal_complete_progress() { + let mut progress = Vec::new(); + super::report_complete( + &mut |event| progress.push(event), + super::download_bytes(), + super::download_bytes(), + ); + assert_eq!(progress.len(), 1); + assert_eq!(progress[0].phase, super::LocalInstallPhase::Complete); + assert_eq!(progress[0].downloaded_bytes, super::download_bytes()); + } + + #[tokio::test] + async fn concrete_installs_serialize_and_late_racer_completes_without_redownload_loss() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind fixture server"); + let address = listener.local_addr().expect("fixture address"); + let server = tokio::spawn(async move { + for _ in 0..2 { + let (mut socket, _) = listener.accept().await.expect("accept request"); + let mut request = [0_u8; 1024]; + let bytes_read = socket.read(&mut request).await.expect("read request"); + assert!(bytes_read > 0, "fixture request was empty"); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nkeep") + .await + .expect("write fixture response"); + } + }); + let root = tempfile::tempdir().expect("temporary directory"); + let roots = super::LocalAssetRoots::new( + root.path(), + root.path().join("native-voice-v2"), + root.path().join("native-voice-v2/stt"), + ) + .expect("asset roots"); + let specs = [super::DownloadSpec { + source_url: Box::leak(format!("http://{address}/asset").into_boxed_str()), + relative_path: "asset", + size_bytes: 4, + sha256: "6ca7ea2feefc88ecb5ed6356ed963f47dc9137f82526fdd25d618ea626d0803f", + }]; + let client = reqwest::Client::new(); + let first_progress = Arc::new(Mutex::new(Vec::new())); + let second_progress = Arc::new(Mutex::new(Vec::new())); + let first_events = Arc::clone(&first_progress); + let second_events = Arc::clone(&second_progress); + let barrier = tokio::sync::Barrier::new(2); + let (first, second) = tokio::join!( + super::install_with_client(&roots, &client, &specs, Some(&barrier), move |event| { + first_events.lock().expect("first progress").push(event); + }), + super::install_with_client(&roots, &client, &specs, Some(&barrier), move |event| { + second_events.lock().expect("second progress").push(event); + }), + ); + let outcomes = [ + first.expect("first install"), + second.expect("second install"), + ]; + assert!(outcomes + .iter() + .any(|outcome| matches!(outcome, super::PocketInstallOutcome::Installed { .. }))); + assert!(outcomes + .iter() + .any(|outcome| matches!(outcome, super::PocketInstallOutcome::AlreadyReady { .. }))); + for events in [first_progress, second_progress] { + assert_eq!( + events + .lock() + .expect("progress") + .last() + .map(|event| event.phase), + Some(super::LocalInstallPhase::Complete) + ); + } + server.await.expect("fixture server"); + } + + #[test] + fn staging_rejects_a_mutation_guard_from_another_store() { + let first = tempfile::tempdir().expect("first store"); + let second = tempfile::tempdir().expect("second store"); + let first_roots = super::LocalAssetRoots::new( + first.path(), + first.path().join("native-voice-v2"), + first.path().join("native-voice-v2/stt"), + ) + .expect("first roots"); + let second_roots = super::LocalAssetRoots::new( + second.path(), + second.path().join("native-voice-v2"), + second.path().join("native-voice-v2/stt"), + ) + .expect("second roots"); + let guard = crate::local_assets::try_lock_for_mutation(&first_roots).expect("first lock"); + let error = super::stage_verified_bundle( + &guard, + second_roots.pocket_bundle_root(), + first.path().join("stage").as_path(), + ) + .expect_err("cross-store guard"); + assert_eq!(error.kind, super::LocalInstallErrorKind::InvalidRoot); + + for destination in [ + first.path().join("../outside"), + first.path().to_path_buf(), + first_roots.pocket_bundle_root().join("nested-stage"), + ] { + let error = super::stage_verified_bundle( + &guard, + first_roots.pocket_bundle_root(), + &destination, + ) + .expect_err("unsafe staging destination"); + assert_eq!(error.kind, super::LocalInstallErrorKind::InvalidRoot); + } + } + + #[tokio::test] + async fn initial_ready_preflight_makes_no_request_and_reports_already_ready() { + let root = tempfile::tempdir().expect("temporary directory"); + let roots = super::LocalAssetRoots::new( + root.path(), + root.path().join("native-voice-v2"), + root.path().join("native-voice-v2/stt"), + ) + .expect("asset roots"); + std::fs::create_dir_all(roots.pocket_bundle_root()).expect("create ready bundle"); + std::fs::write(roots.pocket_bundle_root().join("asset"), b"keep") + .expect("write ready asset"); + let specs = [super::DownloadSpec { + source_url: "http://127.0.0.1:1/must-not-be-called", + relative_path: "asset", + size_bytes: 4, + sha256: "6ca7ea2feefc88ecb5ed6356ed963f47dc9137f82526fdd25d618ea626d0803f", + }]; + let mut progress = Vec::new(); + let outcome = + super::install_with_client(&roots, &reqwest::Client::new(), &specs, None, |event| { + progress.push(event) + }) + .await + .expect("already ready"); + assert!(matches!( + outcome, + super::PocketInstallOutcome::AlreadyReady { .. } + )); + assert!(progress.is_empty()); + } +} diff --git a/src-tauri/crates/berd-voice/src/protocol.rs b/src-tauri/crates/berd-voice/src/protocol.rs new file mode 100644 index 000000000..bc4f8abe8 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/protocol.rs @@ -0,0 +1,415 @@ +use serde::{Deserialize, Serialize}; + +use crate::{ + input::{InputDuringTtsPolicy, InputDuringTtsSnapshot}, + TtsConfigurationSnapshot, TtsSettings, +}; + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum SessionRequest { + Hello { + id: u64, + input_during_tts: InputDuringTtsPolicy, + }, + SetPaused { + active: bool, + }, + SetInputMuted { + id: u64, + active: bool, + }, + SetTtsSettings { + id: u64, + expected_revision: u64, + settings: TtsSettings, + }, + SetInputDuringTts { + id: u64, + expected_revision: u64, + policy: InputDuringTtsPolicy, + }, + ResetInput { + id: u64, + }, + PrepareSpeak { + id: u64, + acknowledgement: Option, + text: String, + }, + OutputReady { + id: u64, + speech_id: u64, + }, + AudioBeginAccepted { + speech_id: u64, + }, + AudioBeginFailed { + speech_id: u64, + played_frames: u64, + message: String, + }, + AudioChunkAccepted { + speech_id: u64, + sequence: u64, + }, + AudioPlayed { + speech_id: u64, + played_frames: u64, + }, + AudioSuspended { + speech_id: u64, + played_frames: u64, + }, + AudioResumed { + speech_id: u64, + played_frames: u64, + }, + AudioDrained { + speech_id: u64, + sequence: u64, + played_frames: u64, + }, + AudioFailed { + speech_id: u64, + played_frames: u64, + message: String, + }, + AudioCancelled { + speech_id: u64, + played_frames: u64, + }, + QueryState { + id: u64, + after: u64, + }, + Cancel { + id: u64, + }, + Shutdown, +} + +#[derive(Clone, Debug, Serialize, PartialEq)] +pub struct PendingUtterance { + pub token: u64, + pub text: String, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum NotAdmittedReason { + Paused, + InProgress, + Cancelled, + EmptyText, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CancelOutcome { + Cancelled, + Stale, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum OutputReadyOutcome { + Accepted, + Stale, +} + +#[derive(Clone, Debug, Serialize, PartialEq)] +pub struct VoiceSessionSnapshot { + pub tts: TtsConfigurationSnapshot, + pub input_during_tts: InputDuringTtsSnapshot, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum TtsSettingsOutcome { + Applied, + Rejected, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum InputDuringTtsOutcome { + Applied, + Rejected, +} + +#[derive(Clone, Debug, Serialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum SessionMessage { + Ready { + id: u64, + protocol: u32, + session: VoiceSessionSnapshot, + }, + TtsSettingsResult { + id: u64, + outcome: TtsSettingsOutcome, + snapshot: TtsConfigurationSnapshot, + #[serde(skip_serializing_if = "Option::is_none")] + message: Option, + }, + InputDuringTtsResult { + id: u64, + outcome: InputDuringTtsOutcome, + snapshot: InputDuringTtsSnapshot, + }, + InputMuteApplied { + id: u64, + active: bool, + }, + InputResetApplied { + id: u64, + }, + InputSpeaking { + active: bool, + }, + RecognitionPending { + active: bool, + }, + UserFinal { + token: u64, + text: String, + }, + Pending { + id: u64, + utterances: Vec, + }, + NotAdmitted { + id: u64, + reason: NotAdmittedReason, + }, + Admitted { + id: u64, + speech_id: u64, + confirmed_token: u64, + }, + State { + id: u64, + confirmed_token: u64, + utterances_after: Vec, + }, + CancelResult { + id: u64, + outcome: CancelOutcome, + speech_id: Option, + }, + OutputReadyResult { + id: u64, + speech_id: u64, + outcome: OutputReadyOutcome, + }, + AudioSuspend { + speech_id: u64, + }, + AudioResume { + speech_id: u64, + }, + SpeechStarted { + id: u64, + speech_id: u64, + }, + SpeechCompleted { + id: u64, + speech_id: u64, + }, + SpeechInterrupted { + id: u64, + speech_id: u64, + spoken_through_utf8: u64, + }, + SpeechFailed { + id: u64, + speech_id: u64, + message: String, + }, + Fatal { + message: String, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn protocol_is_stably_tagged() { + let request: SessionRequest = serde_json::from_str( + r#"{"type":"prepare_speak","id":4,"acknowledgement":0,"text":"hi"}"#, + ) + .unwrap(); + assert_eq!( + request, + SessionRequest::PrepareSpeak { + id: 4, + acknowledgement: Some(0), + text: "hi".into() + } + ); + assert_eq!( + serde_json::to_string(&SessionMessage::Ready { + id: 4, + protocol: 2, + session: VoiceSessionSnapshot { + tts: TtsConfigurationSnapshot { + revision: 1, + settings: TtsSettings::OpenAi { + model: "gpt-4o-mini-tts".into(), + voice: "marin".into(), + rate: 1.0, + }, + }, + input_during_tts: InputDuringTtsSnapshot { + revision: 1, + policy: InputDuringTtsPolicy::AllowBargeIn, + }, + }, + }) + .unwrap(), + r#"{"type":"ready","id":4,"protocol":2,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"# + ); + assert_eq!( + serde_json::from_str::( + r#"{"type":"set_input_during_tts","id":8,"expected_revision":1,"policy":"suppress_input"}"# + ) + .unwrap(), + SessionRequest::SetInputDuringTts { + id: 8, + expected_revision: 1, + policy: InputDuringTtsPolicy::SuppressInput, + } + ); + assert_eq!( + serde_json::to_string(&SessionMessage::InputDuringTtsResult { + id: 8, + outcome: InputDuringTtsOutcome::Applied, + snapshot: InputDuringTtsSnapshot { + revision: 2, + policy: InputDuringTtsPolicy::SuppressInput, + }, + }) + .unwrap(), + r#"{"type":"input_during_tts_result","id":8,"outcome":"applied","snapshot":{"revision":2,"policy":"suppress_input"}}"# + ); + assert_eq!( + serde_json::from_str::( + r#"{"type":"set_tts_settings","id":7,"expected_revision":1,"settings":{"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":2.0}}"# + ) + .unwrap(), + SessionRequest::SetTtsSettings { + id: 7, + expected_revision: 1, + settings: TtsSettings::OpenAi { + model: "gpt-4o-mini-tts".into(), + voice: "marin".into(), + rate: 2.0, + }, + } + ); + assert_eq!( + serde_json::to_string(&SessionMessage::TtsSettingsResult { + id: 7, + outcome: TtsSettingsOutcome::Applied, + snapshot: TtsConfigurationSnapshot { + revision: 2, + settings: TtsSettings::OpenAi { + model: "gpt-4o-mini-tts".into(), + voice: "marin".into(), + rate: 2.0, + }, + }, + message: None, + }) + .unwrap(), + r#"{"type":"tts_settings_result","id":7,"outcome":"applied","snapshot":{"revision":2,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":2.0}}"# + ); + assert_eq!( + serde_json::from_str::( + r#"{"type":"set_input_muted","id":5,"active":true}"# + ) + .unwrap(), + SessionRequest::SetInputMuted { + id: 5, + active: true + } + ); + assert_eq!( + serde_json::to_string(&SessionMessage::UserFinal { + token: 6, + text: "words".into() + }) + .unwrap(), + r#"{"type":"user_final","token":6,"text":"words"}"# + ); + assert_eq!( + serde_json::to_string(&SessionMessage::InputSpeaking { active: true }).unwrap(), + r#"{"type":"input_speaking","active":true}"# + ); + assert_eq!( + serde_json::to_string(&SessionMessage::RecognitionPending { active: false }).unwrap(), + r#"{"type":"recognition_pending","active":false}"# + ); + assert_eq!( + serde_json::to_string(&SessionMessage::SpeechInterrupted { + id: 7, + speech_id: 8, + spoken_through_utf8: 12, + }) + .unwrap(), + r#"{"type":"speech_interrupted","id":7,"speech_id":8,"spoken_through_utf8":12}"# + ); + assert_eq!( + serde_json::from_str::( + r#"{"type":"audio_chunk_accepted","speech_id":9,"sequence":3}"# + ) + .unwrap(), + SessionRequest::AudioChunkAccepted { + speech_id: 9, + sequence: 3, + } + ); + assert_eq!( + serde_json::from_str::( + r#"{"type":"audio_drained","speech_id":9,"sequence":3,"played_frames":7000}"# + ) + .unwrap(), + SessionRequest::AudioDrained { + speech_id: 9, + sequence: 3, + played_frames: 7000, + } + ); + assert_eq!( + serde_json::from_str::( + r#"{"type":"audio_suspended","speech_id":9,"played_frames":4096}"# + ) + .unwrap(), + SessionRequest::AudioSuspended { + speech_id: 9, + played_frames: 4096, + } + ); + assert_eq!( + serde_json::from_str::( + r#"{"type":"audio_resumed","speech_id":9,"played_frames":4096}"# + ) + .unwrap(), + SessionRequest::AudioResumed { + speech_id: 9, + played_frames: 4096, + } + ); + assert_eq!( + serde_json::to_string(&SessionMessage::AudioSuspend { speech_id: 9 }).unwrap(), + r#"{"type":"audio_suspend","speech_id":9}"# + ); + assert_eq!( + serde_json::to_string(&SessionMessage::AudioResume { speech_id: 9 }).unwrap(), + r#"{"type":"audio_resume","speech_id":9}"# + ); + } +} diff --git a/src-tauri/crates/berd-voice/src/session.rs b/src-tauri/crates/berd-voice/src/session.rs new file mode 100644 index 000000000..f96a260f1 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/session.rs @@ -0,0 +1,421 @@ +use std::collections::HashSet; + +use crate::protocol::{NotAdmittedReason, PendingUtterance}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PlaybackState { + Idle, + WaitingOutput, + Playing, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct PrepareRequest { + pub id: u64, + pub acknowledgement: Option, + pub text: String, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum PrepareOutcome { + Hold, + Pending(Vec), + NotAdmitted(NotAdmittedReason), + Admitted { + speech_id: u64, + confirmed_token: u64, + text: String, + }, +} + +#[derive(Debug)] +pub struct SessionCore { + utterances: Vec, + individually_confirmed: HashSet, + highest_utterance_token: u64, + confirmed_token: u64, + user_speaking: bool, + recognition_pending: bool, + paused: bool, + playback: PlaybackState, + next_speech_id: u64, + active_speech_id: Option, +} + +impl Default for SessionCore { + fn default() -> Self { + Self { + utterances: Vec::new(), + individually_confirmed: HashSet::new(), + highest_utterance_token: 0, + confirmed_token: 0, + user_speaking: false, + recognition_pending: false, + paused: false, + playback: PlaybackState::Idle, + next_speech_id: 1, + active_speech_id: None, + } + } +} + +impl SessionCore { + pub fn add_final(&mut self, token: u64, text: String) -> Result<(), String> { + let previous = self.highest_utterance_token; + if token == 0 || token <= previous { + return Err(format!( + "user_final token {token} must be greater than {previous}" + )); + } + self.highest_utterance_token = token; + self.utterances.push(PendingUtterance { token, text }); + Ok(()) + } + + /// Confirms one exact finalized-input token after a host's delivery trust + /// decision succeeds. This does not imply that earlier inputs were delivered. + pub fn confirm_exact(&mut self, token: u64) -> bool { + if token == 0 + || !self + .utterances + .iter() + .any(|utterance| utterance.token == token) + { + return false; + } + self.individually_confirmed.insert(token) + } + + /// Removes a finalized input that the host has terminally abandoned. + /// Discarding never confirms that input or any input before it. + pub fn discard_final(&mut self, token: u64) -> bool { + let previous_len = self.utterances.len(); + self.utterances.retain(|utterance| utterance.token != token); + self.individually_confirmed.remove(&token); + self.utterances.len() != previous_len + } + + /// Applies an exact causal cutoff while requiring every retained input at + /// or before it to have been individually confirmed by the host. + pub fn prepare_after_host_confirmation(&mut self, request: PrepareRequest) -> PrepareOutcome { + let text = request.text.trim().to_string(); + if text.is_empty() { + return PrepareOutcome::NotAdmitted(NotAdmittedReason::EmptyText); + } + if self.user_speaking || self.recognition_pending { + return PrepareOutcome::Hold; + } + + let cutoff = request.acknowledgement.unwrap_or(0); + let pending: Vec<_> = self + .utterances + .iter() + .filter(|utterance| { + utterance.token > cutoff || !self.individually_confirmed.contains(&utterance.token) + }) + .cloned() + .collect(); + if !pending.is_empty() { + return PrepareOutcome::Pending(pending); + } + self.reserve(text) + } + + pub fn set_user_speaking(&mut self, active: bool) -> bool { + self.user_speaking = active; + active && self.playback != PlaybackState::Idle + } + + pub fn set_recognition_pending(&mut self, active: bool) -> bool { + self.recognition_pending = active; + active && self.playback != PlaybackState::Idle + } + + pub fn set_paused(&mut self, active: bool) -> bool { + self.paused = active; + active && self.playback != PlaybackState::Idle + } + + pub fn prepare(&mut self, request: PrepareRequest) -> PrepareOutcome { + let text = request.text.trim().to_string(); + if text.is_empty() { + return PrepareOutcome::NotAdmitted(NotAdmittedReason::EmptyText); + } + if self.user_speaking || self.recognition_pending { + return PrepareOutcome::Hold; + } + + let cutoff = self.apply_acknowledgement(request.acknowledgement); + let pending: Vec<_> = self + .utterances + .iter() + .filter(|utterance| utterance.token > cutoff) + .cloned() + .collect(); + if !pending.is_empty() { + return PrepareOutcome::Pending(pending); + } + self.reserve(text) + } + + pub fn mark_started(&mut self, speech_id: u64) -> bool { + if self.playback == PlaybackState::WaitingOutput + && self.active_speech_id() == Some(speech_id) + { + self.playback = PlaybackState::Playing; + true + } else { + false + } + } + + pub fn finish(&mut self, speech_id: u64) -> bool { + if self.playback != PlaybackState::Idle && self.active_speech_id() == Some(speech_id) { + self.playback = PlaybackState::Idle; + self.active_speech_id = None; + true + } else { + false + } + } + + fn active_speech_id(&self) -> Option { + self.active_speech_id + } + fn reserve(&mut self, text: String) -> PrepareOutcome { + if self.paused { + return PrepareOutcome::NotAdmitted(NotAdmittedReason::Paused); + } + if self.playback != PlaybackState::Idle { + return PrepareOutcome::NotAdmitted(NotAdmittedReason::InProgress); + } + + let speech_id = self.next_speech_id; + self.next_speech_id = self.next_speech_id.saturating_add(1); + self.playback = PlaybackState::WaitingOutput; + self.active_speech_id = Some(speech_id); + PrepareOutcome::Admitted { + speech_id, + confirmed_token: self.confirmed_token, + text, + } + } + pub fn utterances_after(&self, token: u64) -> Vec { + self.utterances + .iter() + .filter(|item| item.token > token) + .cloned() + .collect() + } + + pub fn confirmed_token(&self) -> u64 { + self.confirmed_token + } + pub fn user_speaking(&self) -> bool { + self.user_speaking + } + pub fn recognition_pending(&self) -> bool { + self.recognition_pending + } + fn apply_acknowledgement(&mut self, acknowledgement: Option) -> u64 { + let Some(token) = acknowledgement else { + return self.confirmed_token; + }; + if token == 0 { + return 0; + } + if self + .utterances + .iter() + .any(|utterance| utterance.token == token) + { + self.confirmed_token = self.confirmed_token.max(token); + token + } else { + self.confirmed_token + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request(acknowledgement: Option) -> PrepareRequest { + PrepareRequest { + id: 7, + acknowledgement, + text: "hello".into(), + } + } + + #[test] + fn exact_ack_advances_global_cursor_and_admits() { + let mut core = SessionCore::default(); + core.add_final(2, "one".into()).unwrap(); + core.add_final(5, "two".into()).unwrap(); + assert!(matches!( + core.prepare(request(Some(5))), + PrepareOutcome::Admitted { + confirmed_token: 5, + .. + } + )); + } + + #[test] + fn zero_and_missing_ack_do_not_confirm_input() { + for acknowledgement in [None, Some(0)] { + let mut core = SessionCore::default(); + core.add_final(2, "one".into()).unwrap(); + assert!(matches!( + core.prepare(request(acknowledgement)), + PrepareOutcome::Pending(_) + )); + assert_eq!(core.confirmed_token(), 0); + } + } + + #[test] + fn existing_stale_ack_is_exact_but_future_falls_back_global() { + let mut core = SessionCore::default(); + core.add_final(2, "one".into()).unwrap(); + core.add_final(5, "two".into()).unwrap(); + assert!(matches!( + core.prepare(request(Some(5))), + PrepareOutcome::Admitted { .. } + )); + assert!(core.finish(1)); + core.add_final(9, "three".into()).unwrap(); + let PrepareOutcome::Pending(stale) = core.prepare(request(Some(2))) else { + panic!("stale existing token is an exact cutoff") + }; + assert_eq!( + stale.iter().map(|item| item.token).collect::>(), + vec![5, 9] + ); + assert_eq!(core.confirmed_token(), 5); + assert!(matches!( + core.prepare(request(Some(99))), + PrepareOutcome::Pending(_) + )); + assert_eq!(core.confirmed_token(), 5); + } + + #[test] + fn speaking_holds_before_ack_mutation() { + let mut core = SessionCore::default(); + core.add_final(3, "one".into()).unwrap(); + core.set_user_speaking(true); + assert_eq!(core.prepare(request(Some(3))), PrepareOutcome::Hold); + assert_eq!(core.confirmed_token(), 0); + } + + #[test] + fn recognition_pending_holds_before_ack_mutation_until_cleared() { + let mut core = SessionCore::default(); + core.add_final(3, "one".into()).unwrap(); + core.set_recognition_pending(true); + assert_eq!(core.prepare(request(Some(3))), PrepareOutcome::Hold); + assert_eq!(core.confirmed_token(), 0); + + core.set_recognition_pending(false); + assert!(matches!( + core.prepare(request(Some(3))), + PrepareOutcome::Admitted { + confirmed_token: 3, + .. + } + )); + } + + #[test] + fn host_confirmation_is_exact_monotonic_and_duplicate_safe() { + let mut core = SessionCore::default(); + core.add_final(2, "one".into()).unwrap(); + core.add_final(5, "two".into()).unwrap(); + + assert!(!core.confirm_exact(4)); + assert_eq!(core.confirmed_token(), 0); + assert!(core.confirm_exact(5)); + assert!(core.confirm_exact(2)); + assert!(!core.confirm_exact(2)); + assert_eq!(core.confirmed_token(), 0); + } + + #[test] + fn later_individual_confirmation_cannot_hide_an_earlier_failed_delivery() { + let mut core = SessionCore::default(); + core.add_final(1, "slow first".into()).unwrap(); + core.add_final(2, "fast second".into()).unwrap(); + assert!(core.confirm_exact(2)); + + assert!(matches!( + core.prepare_after_host_confirmation(request(Some(2))), + PrepareOutcome::Pending(items) + if items.iter().map(|item| item.token).collect::>() == vec![1] + )); + assert!(core.discard_final(1)); + assert!(matches!( + core.prepare_after_host_confirmation(request(Some(2))), + PrepareOutcome::Admitted { .. } + )); + } + + #[test] + fn discarded_final_never_confirms_and_high_water_does_not_rewind() { + let mut core = SessionCore::default(); + core.add_final(2, "one".into()).unwrap(); + assert!(core.discard_final(2)); + assert!(!core.discard_final(2)); + assert!(!core.confirm_exact(2)); + assert_eq!(core.confirmed_token(), 0); + assert_eq!( + core.add_final(2, "reused".into()).unwrap_err(), + "user_final token 2 must be greater than 2" + ); + core.add_final(3, "next".into()).unwrap(); + assert!( + matches!(core.prepare(request(None)), PrepareOutcome::Pending(items) if items.len() == 1 && items[0].token == 3) + ); + } + + #[test] + fn reservation_is_single_and_pause_cancels_active() { + let mut core = SessionCore::default(); + assert!(matches!( + core.prepare(request(None)), + PrepareOutcome::Admitted { .. } + )); + assert_eq!( + core.prepare(request(None)), + PrepareOutcome::NotAdmitted(NotAdmittedReason::InProgress) + ); + assert!(core.set_paused(true)); + assert!(core.finish(1)); + assert_eq!( + core.prepare(request(None)), + PrepareOutcome::NotAdmitted(NotAdmittedReason::Paused) + ); + } + + #[test] + fn recognition_pending_interrupts_a_reserved_speech() { + let mut core = SessionCore::default(); + assert!(matches!( + core.prepare(request(None)), + PrepareOutcome::Admitted { .. } + )); + assert!(core.set_recognition_pending(true)); + assert!(core.finish(1)); + } + + #[test] + fn final_tokens_are_positive_and_monotonic() { + let mut core = SessionCore::default(); + assert!(core.add_final(0, "bad".into()).is_err()); + core.add_final(4, "ok".into()).unwrap(); + assert!(core.add_final(4, "duplicate".into()).is_err()); + assert!(core.add_final(3, "stale".into()).is_err()); + } +} diff --git a/src-tauri/crates/berd-voice/src/session_audio.rs b/src-tauri/crates/berd-voice/src/session_audio.rs new file mode 100644 index 000000000..3b227b9c2 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/session_audio.rs @@ -0,0 +1,1763 @@ +use std::collections::VecDeque; +use std::fs::File; +use std::io; +use std::os::fd::{AsRawFd, FromRawFd, RawFd}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +use berd_voice::{PcmAudioOutput, TtsPcmSpec}; + +pub const AUDIO_FRAME_MAGIC: [u8; 2] = *b"BA"; +pub const AUDIO_FRAME_MARKER: u8 = 2; +pub const AUDIO_BEGIN_KIND: u8 = 1; +pub const AUDIO_CHUNK_KIND: u8 = 2; +pub const AUDIO_END_KIND: u8 = 3; +pub const AUDIO_CANCEL_KIND: u8 = 4; +pub const AUDIO_FRAME_HEADER_BYTES: usize = 8; +pub const MAX_AUDIO_CHUNK_FRAMES: usize = 4096; +const MIN_ACCEPTED_AUDIO_RUNWAY_MS: f64 = 400.0; +const MAX_ACCEPTED_NOT_PLAYED_CHUNKS: usize = 64; + +const AUDIO_OPERATION_TIMEOUT: Duration = Duration::from_secs(2); +pub const AUDIO_CANCELLED: &str = "remote PCM output was cancelled"; + +#[derive(Clone, Debug, PartialEq)] +pub enum AudioHostAck { + BeginAccepted, + BeginFailed { played_frames: u64, message: String }, + ChunkAccepted { sequence: u64 }, + Played { played_frames: u64 }, + Suspended { played_frames: u64 }, + Resumed { played_frames: u64 }, + Drained { sequence: u64, played_frames: u64 }, + Failed { played_frames: u64, message: String }, + Cancelled { played_frames: u64 }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AudioOutputControlRequest { + Suspend { speech_id: u64 }, + Resume { speech_id: u64 }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SuspensionPhase { + Running, + Suspending, + Suspended, + Resuming, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Phase { + New, + WaitingBegin, + Streaming, + WaitingChunk, + Ended, + Cancelling, + Drained, + Cancelled, + Failed, +} + +struct State { + phase: Phase, + begin_accepted: bool, + next_sequence: u64, + pending_sequence: Option, + total_frames: u64, + accepted_frames: u64, + first_chunk_accepted: bool, + played_frames: u64, + accepted_chunk_ends: VecDeque, + ended_sequence: Option, + failure: Option, + failure_quiescent: bool, + suspension: SuspensionPhase, + suspension_requested: bool, + suspension_deadline: Option, +} + +pub struct AudioPipeTransport { + file: Mutex, + poisoned: Mutex>, +} + +impl AudioPipeTransport { + /// Takes ownership of an inherited child-write file descriptor. + pub unsafe fn from_raw_fd(fd: RawFd) -> Result { + if fd < 3 { + return Err("PCM output file descriptor must be at least 3".into()); + } + let file = File::from_raw_fd(fd); + let flags = libc::fcntl(file.as_raw_fd(), libc::F_GETFL); + if flags < 0 { + return Err(format!( + "could not configure PCM output descriptor: {}", + io::Error::last_os_error() + )); + } + if flags & libc::O_ACCMODE == libc::O_RDONLY { + return Err("PCM output file descriptor is not writable".into()); + } + if libc::fcntl(file.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) < 0 { + return Err(format!( + "could not configure PCM output descriptor: {}", + io::Error::last_os_error() + )); + } + Ok(Self { + file: Mutex::new(file), + poisoned: Mutex::new(None), + }) + } + + fn write_record(&self, kind: u8, payload: &[u8]) -> Result<(), String> { + if let Some(message) = self.poisoned.lock().expect("audio poison lock").clone() { + return Err(message); + } + let length = u32::try_from(payload.len()) + .map_err(|_| "PCM output record is too large".to_string())?; + let mut record = Vec::with_capacity(AUDIO_FRAME_HEADER_BYTES + payload.len()); + record.extend_from_slice(&AUDIO_FRAME_MAGIC); + record.push(AUDIO_FRAME_MARKER); + record.push(kind); + record.extend_from_slice(&length.to_le_bytes()); + record.extend_from_slice(payload); + + let deadline = Instant::now() + AUDIO_OPERATION_TIMEOUT; + let file = self.file.lock().expect("audio pipe lock"); + let fd = file.as_raw_fd(); + let mut offset = 0; + while offset < record.len() { + let written = + unsafe { libc::write(fd, record[offset..].as_ptr().cast(), record.len() - offset) }; + if written > 0 { + offset += usize::try_from(written).expect("positive write fits usize"); + continue; + } + if written < 0 { + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::Interrupted { + continue; + } + if error.kind() != io::ErrorKind::WouldBlock { + return self.poison(format!("PCM output pipe write failed: {error}")); + } + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return self.poison(if offset == 0 { + "PCM output host did not read within 2 seconds".into() + } else { + "PCM output host left a partial frame unread for 2 seconds".into() + }); + } + let timeout_ms = i32::try_from(remaining.as_millis().max(1).min(i32::MAX as u128)) + .expect("bounded poll timeout"); + let mut poll_fd = libc::pollfd { + fd, + events: libc::POLLOUT, + revents: 0, + }; + let result = unsafe { libc::poll(&mut poll_fd, 1, timeout_ms) }; + if result < 0 && io::Error::last_os_error().kind() != io::ErrorKind::Interrupted { + return self.poison(format!( + "PCM output pipe polling failed: {}", + io::Error::last_os_error() + )); + } + } + Ok(()) + } + + fn poison(&self, message: String) -> Result { + *self.poisoned.lock().expect("audio poison lock") = Some(message.clone()); + Err(message) + } +} + +pub struct RemotePcmAudioOutput { + speech_id: u64, + spec: TtsPcmSpec, + transport: Arc, + active: Arc, + control_sender: mpsc::Sender, + operation_timeout: Duration, + minimum_accepted_runway_frames: u64, + pending_samples: Mutex>, + state: Mutex, + changed: Condvar, +} + +impl RemotePcmAudioOutput { + pub fn new( + speech_id: u64, + spec: TtsPcmSpec, + transport: Arc, + active: Arc, + control_sender: mpsc::Sender, + ) -> Result { + Self::new_with_timeout( + speech_id, + spec, + transport, + active, + control_sender, + AUDIO_OPERATION_TIMEOUT, + ) + } + + fn new_with_timeout( + speech_id: u64, + spec: TtsPcmSpec, + transport: Arc, + active: Arc, + control_sender: mpsc::Sender, + operation_timeout: Duration, + ) -> Result { + if speech_id == 0 + || !matches!(spec.sample_rate, 24_000 | 48_000) + || !spec.playback_rate.is_finite() + || !(0.5..=2.0).contains(&spec.playback_rate) + { + return Err("remote PCM output configuration is invalid".into()); + } + Ok(Self { + speech_id, + spec, + transport, + active, + control_sender, + operation_timeout, + minimum_accepted_runway_frames: accepted_audio_runway_frames(spec), + pending_samples: Mutex::new(Vec::with_capacity(MAX_AUDIO_CHUNK_FRAMES)), + state: Mutex::new(State { + phase: Phase::New, + begin_accepted: false, + next_sequence: 1, + pending_sequence: None, + total_frames: 0, + accepted_frames: 0, + first_chunk_accepted: false, + played_frames: 0, + accepted_chunk_ends: VecDeque::new(), + ended_sequence: None, + failure: None, + failure_quiescent: false, + suspension: SuspensionPhase::Running, + suspension_requested: false, + suspension_deadline: None, + }), + changed: Condvar::new(), + }) + } + + pub fn start(&self) -> Result<(), String> { + { + let mut state = self.state.lock().expect("remote output state"); + if state.phase != Phase::New { + return Err("remote PCM output begin is not in a new state".into()); + } + state.phase = Phase::WaitingBegin; + } + let mut payload = Vec::with_capacity(16); + payload.extend_from_slice(&self.speech_id.to_le_bytes()); + payload.extend_from_slice(&self.spec.sample_rate.to_le_bytes()); + payload.extend_from_slice(&self.spec.playback_rate.to_le_bytes()); + self.transport.write_record(AUDIO_BEGIN_KIND, &payload)?; + self.wait_for( + |state| state.phase != Phase::WaitingBegin, + "audio begin acknowledgement", + true, + )?; + self.check_health()?; + self.wait_for_running() + } + + pub fn finish_writes(&self) -> Result<(), String> { + let final_samples = std::mem::take( + &mut *self + .pending_samples + .lock() + .expect("remote pending PCM lock"), + ); + if !final_samples.is_empty() { + self.write_chunk(&final_samples)?; + } + self.wait_for_running()?; + let (last_sequence, total_frames) = { + let mut state = self.state.lock().expect("remote output state"); + if state.suspension_requested || state.suspension != SuspensionPhase::Running { + drop(state); + self.wait_for_running()?; + state = self.state.lock().expect("remote output state"); + } + if state.phase != Phase::Streaming { + return Err("remote PCM output cannot end before streaming is ready".into()); + } + if state.total_frames == 0 { + return Err("remote PCM output produced no audio frames".into()); + } + state.phase = Phase::Ended; + state.ended_sequence = Some(state.next_sequence - 1); + (state.next_sequence - 1, state.total_frames) + }; + let mut payload = Vec::with_capacity(24); + payload.extend_from_slice(&self.speech_id.to_le_bytes()); + payload.extend_from_slice(&last_sequence.to_le_bytes()); + payload.extend_from_slice(&total_frames.to_le_bytes()); + self.transport.write_record(AUDIO_END_KIND, &payload) + } + + /// Wakes the playback worker so it can serialize Cancel on the audio pipe. + /// The caller owns only the cancellation flag; pipe writes remain worker-owned. + pub fn notify_cancel_requested(&self) { + self.changed.notify_all(); + } + + pub fn request_suspend(&self) -> Result<(), String> { + let mut state = self.state.lock().expect("remote output state"); + state.suspension_requested = true; + if state.suspension == SuspensionPhase::Running + && !matches!( + state.phase, + Phase::Drained | Phase::Cancelled | Phase::Failed + ) + { + state.suspension = SuspensionPhase::Suspending; + state.suspension_deadline = Some(Instant::now() + self.operation_timeout); + self.control_sender + .send(AudioOutputControlRequest::Suspend { + speech_id: self.speech_id, + }) + .map_err(|_| "audio control receiver disconnected".to_string())?; + } + self.changed.notify_all(); + Ok(()) + } + + pub fn request_resume(&self) -> Result<(), String> { + let mut state = self.state.lock().expect("remote output state"); + if !self.active.load(Ordering::SeqCst) { + return Ok(()); + } + state.suspension_requested = false; + if state.suspension == SuspensionPhase::Suspended { + state.suspension = SuspensionPhase::Resuming; + state.suspension_deadline = Some(Instant::now() + self.operation_timeout); + self.control_sender + .send(AudioOutputControlRequest::Resume { + speech_id: self.speech_id, + }) + .map_err(|_| "audio control receiver disconnected".to_string())?; + } + self.changed.notify_all(); + Ok(()) + } + + pub fn control_request_is_outstanding(&self, request: AudioOutputControlRequest) -> bool { + let state = self.state.lock().expect("remote output state"); + matches!( + (request, state.suspension), + ( + AudioOutputControlRequest::Suspend { speech_id }, + SuspensionPhase::Suspending + ) if speech_id == self.speech_id + ) || matches!( + (request, state.suspension), + ( + AudioOutputControlRequest::Resume { speech_id }, + SuspensionPhase::Resuming + ) if speech_id == self.speech_id + ) + } + + pub fn check_suspension_deadline(&self, now: Instant) -> Result<(), String> { + let mut state = self.state.lock().expect("remote output state"); + if state + .suspension_deadline + .is_some_and(|deadline| now >= deadline) + && matches!( + state.suspension, + SuspensionPhase::Suspending | SuspensionPhase::Resuming + ) + { + state.phase = Phase::Failed; + state.failure = Some("host did not settle audio suspension before its deadline".into()); + self.changed.notify_all(); + return Err(state.failure.clone().expect("suspension deadline failure")); + } + Ok(()) + } + + pub fn failure_is_quiescent(&self) -> bool { + let state = self.state.lock().expect("remote output state"); + state.phase == Phase::Failed && state.failure_quiescent + } + + pub fn handle_ack(&self, ack: AudioHostAck) -> Result { + let mut state = self.state.lock().expect("remote output state"); + let mut started = false; + match ack { + AudioHostAck::BeginAccepted + if state.phase == Phase::WaitingBegin + || (state.phase == Phase::Cancelling && !state.begin_accepted) => + { + state.begin_accepted = true; + if state.phase == Phase::WaitingBegin { + state.phase = Phase::Streaming; + } + } + AudioHostAck::BeginFailed { + played_frames, + message, + } if matches!(state.phase, Phase::WaitingBegin | Phase::Cancelling) + && !state.begin_accepted + && played_frames == 0 => + { + if state.phase == Phase::WaitingBegin { + state.phase = Phase::Failed; + state.failure_quiescent = true; + } + state.failure = Some(public_host_failure(message)); + } + AudioHostAck::ChunkAccepted { sequence } + if matches!(state.phase, Phase::WaitingChunk | Phase::Cancelling) + && state.pending_sequence == Some(sequence) => + { + let cancelling = state.phase == Phase::Cancelling; + state.pending_sequence = None; + state.next_sequence = state + .next_sequence + .checked_add(1) + .ok_or_else(|| "audio chunk sequence space is exhausted".to_string())?; + let total_frames = state.total_frames; + state.accepted_frames = total_frames; + state.accepted_chunk_ends.push_back(total_frames); + if !state.first_chunk_accepted { + state.first_chunk_accepted = true; + started = !cancelling; + } + if !cancelling { + state.phase = Phase::Streaming; + } + } + AudioHostAck::Played { played_frames } + if matches!( + state.phase, + Phase::Streaming | Phase::WaitingChunk | Phase::Ended | Phase::Cancelling + ) && played_frames >= state.played_frames + && played_frames <= state.accepted_frames => + { + state.played_frames = played_frames; + while state + .accepted_chunk_ends + .front() + .is_some_and(|end| *end <= played_frames) + { + state.accepted_chunk_ends.pop_front(); + } + } + AudioHostAck::Suspended { played_frames } + if state.suspension == SuspensionPhase::Suspending + && !matches!(state.phase, Phase::Cancelled | Phase::Failed) + && played_frames >= state.played_frames + && played_frames <= state.accepted_frames => + { + state.played_frames = played_frames; + while state + .accepted_chunk_ends + .front() + .is_some_and(|end| *end <= played_frames) + { + state.accepted_chunk_ends.pop_front(); + } + state.suspension = SuspensionPhase::Suspended; + state.suspension_deadline = None; + if !state.suspension_requested && self.active.load(Ordering::SeqCst) { + state.suspension = SuspensionPhase::Resuming; + state.suspension_deadline = Some(Instant::now() + self.operation_timeout); + self.control_sender + .send(AudioOutputControlRequest::Resume { + speech_id: self.speech_id, + }) + .map_err(|_| "audio control receiver disconnected".to_string())?; + } + } + AudioHostAck::Resumed { played_frames } + if state.suspension == SuspensionPhase::Resuming + && played_frames == state.played_frames => + { + state.suspension = SuspensionPhase::Running; + state.suspension_deadline = None; + if state.suspension_requested && self.active.load(Ordering::SeqCst) { + state.suspension = SuspensionPhase::Suspending; + state.suspension_deadline = Some(Instant::now() + self.operation_timeout); + self.control_sender + .send(AudioOutputControlRequest::Suspend { + speech_id: self.speech_id, + }) + .map_err(|_| "audio control receiver disconnected".to_string())?; + } + } + AudioHostAck::Drained { + sequence, + played_frames, + } if matches!(state.phase, Phase::Ended | Phase::Cancelling) + && state.ended_sequence == Some(sequence) + && played_frames == state.total_frames => + { + state.played_frames = played_frames; + state.accepted_chunk_ends.clear(); + if state.phase == Phase::Ended { + state.phase = Phase::Drained; + } + if state.phase == Phase::Drained && state.suspension == SuspensionPhase::Running { + state.suspension = SuspensionPhase::Running; + state.suspension_deadline = None; + } + } + AudioHostAck::Failed { + played_frames, + message, + } if matches!( + state.phase, + Phase::Streaming | Phase::WaitingChunk | Phase::Ended | Phase::Cancelling + ) && played_frames >= state.played_frames + && played_frames <= state.accepted_frames => + { + state.played_frames = played_frames; + if state.phase != Phase::Cancelling { + state.phase = Phase::Failed; + state.failure_quiescent = true; + state.suspension = SuspensionPhase::Running; + state.suspension_deadline = None; + } + state.failure = Some(public_host_failure(message)); + } + AudioHostAck::Cancelled { played_frames } + if state.phase == Phase::Cancelling + && played_frames >= state.played_frames + && played_frames <= state.accepted_frames => + { + state.played_frames = played_frames; + state.phase = if state.failure.is_some() { + state.failure_quiescent = true; + Phase::Failed + } else { + Phase::Cancelled + }; + state.suspension = SuspensionPhase::Running; + state.suspension_deadline = None; + } + AudioHostAck::Played { played_frames } + if matches!( + state.phase, + Phase::Streaming | Phase::WaitingChunk | Phase::Ended | Phase::Cancelling + ) && played_frames == state.played_frames => + { + return Ok(false) + } + _ => { + return Err( + "audio host acknowledgement is stale, out of order, or impossible".into(), + ) + } + } + self.changed.notify_all(); + Ok(started) + } + + fn wait_for( + &self, + predicate: impl Fn(&State) -> bool, + operation: &str, + observe_cancellation: bool, + ) -> Result<(), String> { + let deadline = Instant::now() + self.operation_timeout; + let mut state = self.state.lock().expect("remote output state"); + while !predicate(&state) && state.phase != Phase::Failed { + if observe_cancellation + && !self.active.load(Ordering::SeqCst) + && state.phase != Phase::Cancelling + { + drop(state); + self.cancel_settled()?; + return Err(AUDIO_CANCELLED.into()); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.phase = Phase::Failed; + let message = format!("host did not complete {operation} before its deadline"); + state.failure = Some(message.clone()); + self.changed.notify_all(); + return Err(message); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("remote output wait"); + state = next; + } + if state.phase == Phase::Failed { + Err(state + .failure + .clone() + .unwrap_or_else(|| "host audio output failed".into())) + } else { + Ok(()) + } + } + + fn cancel_settled(&self) -> Result { + { + let mut state = self.state.lock().expect("remote output state"); + while matches!( + state.suspension, + SuspensionPhase::Suspending | SuspensionPhase::Resuming + ) && state.phase != Phase::Failed + { + let remaining = state + .suspension_deadline + .map_or(Duration::ZERO, |deadline| { + deadline.saturating_duration_since(Instant::now()) + }); + if remaining.is_zero() { + state.phase = Phase::Failed; + state.failure = + Some("host did not settle audio suspension before cancellation".into()); + self.changed.notify_all(); + return Err(state.failure.clone().expect("suspension cancel failure")); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("remote output cancellation wait"); + state = next; + } + } + let should_send = { + let mut state = self.state.lock().expect("remote output state"); + match state.phase { + Phase::Drained if state.suspension == SuspensionPhase::Running => { + return Ok(state.played_frames) + } + Phase::Cancelled => return Ok(state.played_frames), + Phase::Failed => { + if state.failure_quiescent { + return Ok(state.played_frames); + } + return Err(state + .failure + .clone() + .unwrap_or_else(|| "host audio output failed".into())); + } + Phase::Cancelling => false, + Phase::WaitingBegin + | Phase::Streaming + | Phase::WaitingChunk + | Phase::Ended + | Phase::Drained => { + state.phase = Phase::Cancelling; + true + } + Phase::New => { + return Err( + "remote PCM output cannot cancel during an unfinished record".into(), + ) + } + } + }; + if should_send { + self.transport + .write_record(AUDIO_CANCEL_KIND, &self.speech_id.to_le_bytes())?; + } + self.wait_for( + |state| state.phase == Phase::Cancelled, + "audio cancellation acknowledgement", + false, + )?; + Ok(self + .state + .lock() + .expect("remote output state") + .played_frames) + } + + fn wait_for_running(&self) -> Result<(), String> { + loop { + { + let state = self.state.lock().expect("remote output state"); + match state.phase { + Phase::Cancelled => return Err(AUDIO_CANCELLED.into()), + Phase::Failed => { + return Err(state + .failure + .clone() + .unwrap_or_else(|| "host audio output failed".into())) + } + Phase::Drained => return Err("remote PCM output is already drained".into()), + _ => {} + } + if !self.active.load(Ordering::SeqCst) { + drop(state); + self.cancel_settled()?; + return Err(AUDIO_CANCELLED.into()); + } + if let (SuspensionPhase::Running, false) = + (state.suspension, state.suspension_requested) + { + return Ok(()); + } + } + let mut state = self.state.lock().expect("remote output state"); + loop { + let settled = matches!( + (state.suspension, state.suspension_requested), + (SuspensionPhase::Running, false) + | (SuspensionPhase::Suspended, true) + | (SuspensionPhase::Suspended, false) + ); + if settled || state.phase == Phase::Failed { + break; + } + let remaining = state + .suspension_deadline + .map_or(self.operation_timeout, |deadline| { + deadline.saturating_duration_since(Instant::now()) + }); + if remaining.is_zero() { + state.phase = Phase::Failed; + state.failure = + Some("host did not settle audio suspension before its deadline".into()); + self.changed.notify_all(); + return Err(state.failure.clone().expect("suspension failure")); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("remote output suspension wait"); + state = next; + } + if state.phase == Phase::Failed { + return Err(state + .failure + .clone() + .unwrap_or_else(|| "host audio output failed".into())); + } + if state.suspension == SuspensionPhase::Suspended && state.suspension_requested { + while state.suspension_requested && self.active.load(Ordering::SeqCst) { + state = self + .changed + .wait(state) + .expect("remote output suspended wait"); + } + if !self.active.load(Ordering::SeqCst) { + drop(state); + self.cancel_settled()?; + return Err(AUDIO_CANCELLED.into()); + } + } + } + } + + fn write_chunk(&self, chunk: &[f32]) -> Result<(), String> { + debug_assert!(!chunk.is_empty() && chunk.len() <= MAX_AUDIO_CHUNK_FRAMES); + self.wait_for_running()?; + { + let deadline = Instant::now() + self.operation_timeout; + let mut state = self.state.lock().expect("remote output state"); + while (state.accepted_frames.saturating_sub(state.played_frames) + >= self.minimum_accepted_runway_frames + || state.accepted_chunk_ends.len() >= MAX_ACCEPTED_NOT_PLAYED_CHUNKS) + && state.phase == Phase::Streaming + { + if !self.active.load(Ordering::SeqCst) { + drop(state); + self.cancel_settled()?; + return Err(AUDIO_CANCELLED.into()); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.phase = Phase::Failed; + state.failure = + Some("host audio queue did not release credit before its deadline".into()); + self.changed.notify_all(); + return Err(state.failure.clone().expect("credit failure")); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("remote output credit wait"); + state = next; + if state.suspension_requested { + drop(state); + self.wait_for_running()?; + state = self.state.lock().expect("remote output state"); + } + } + if state.phase != Phase::Streaming { + return Err(state + .failure + .clone() + .unwrap_or_else(|| "remote PCM output is not streaming".into())); + } + state.pending_sequence = Some(state.next_sequence); + state.total_frames = state + .total_frames + .checked_add(u64::try_from(chunk.len()).expect("chunk length fits u64")) + .ok_or_else(|| "audio source-frame count is exhausted".to_string())?; + state.phase = Phase::WaitingChunk; + } + let sequence = self + .state + .lock() + .expect("remote output state") + .pending_sequence + .expect("pending audio sequence"); + let mut payload = Vec::with_capacity(16 + chunk.len() * 4); + payload.extend_from_slice(&self.speech_id.to_le_bytes()); + payload.extend_from_slice(&sequence.to_le_bytes()); + for sample in chunk { + payload.extend_from_slice(&sample.to_le_bytes()); + } + self.transport.write_record(AUDIO_CHUNK_KIND, &payload)?; + self.wait_for( + |state| state.pending_sequence.is_none(), + "audio chunk acknowledgement", + true, + ) + } +} + +impl PcmAudioOutput for RemotePcmAudioOutput { + fn write(&self, samples: &[f32]) -> Result<(), String> { + if samples + .iter() + .any(|sample| !sample.is_finite() || !(-1.0..=1.0).contains(sample)) + { + return Err("remote PCM output requires finite unit-scale samples".into()); + } + let complete_samples = { + let mut pending = self + .pending_samples + .lock() + .expect("remote pending PCM lock"); + pending.extend_from_slice(samples); + let complete_len = pending.len() / MAX_AUDIO_CHUNK_FRAMES * MAX_AUDIO_CHUNK_FRAMES; + let tail = pending.split_off(complete_len); + std::mem::replace(&mut *pending, tail) + }; + for chunk in complete_samples.chunks_exact(MAX_AUDIO_CHUNK_FRAMES) { + self.write_chunk(chunk)?; + } + Ok(()) + } + + fn cancel(&self) { + let _ = self.cancel_settled(); + } + + fn cancel_and_snapshot(&self) -> Result { + self.cancel_settled() + } + + fn is_drained(&self) -> bool { + let state = self.state.lock().expect("remote output state"); + state.phase == Phase::Drained && state.suspension == SuspensionPhase::Running + } + + fn check_health(&self) -> Result<(), String> { + let state = self.state.lock().expect("remote output state"); + if state.phase == Phase::Failed { + Err(state + .failure + .clone() + .unwrap_or_else(|| "host audio output failed".into())) + } else { + Ok(()) + } + } + + fn played_frames(&self) -> u64 { + self.state + .lock() + .expect("remote output state") + .played_frames + } +} + +fn accepted_audio_runway_frames(spec: TtsPcmSpec) -> u64 { + let runway_frames = + (f64::from(spec.sample_rate) * f64::from(spec.playback_rate) * MIN_ACCEPTED_AUDIO_RUNWAY_MS + / 1_000.0) + .ceil() as usize; + u64::try_from(runway_frames).expect("closed PCM runway fits u64") +} + +fn public_host_failure(_message: String) -> String { + "host audio output failed".into() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + use std::os::fd::IntoRawFd; + use std::os::unix::net::UnixStream; + use std::sync::Arc; + use std::thread; + + fn fixture() -> (Arc, UnixStream) { + fixture_with_timeout(AUDIO_OPERATION_TIMEOUT) + } + + fn fixture_with_timeout(timeout: Duration) -> (Arc, UnixStream) { + fixture_with_spec_and_timeout( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + timeout, + ) + } + + fn fixture_with_spec_and_timeout( + spec: TtsPcmSpec, + timeout: Duration, + ) -> (Arc, UnixStream) { + let (output, host, _control_receiver) = fixture_with_control(spec, timeout); + (output, host) + } + + fn fixture_with_control( + spec: TtsPcmSpec, + timeout: Duration, + ) -> ( + Arc, + UnixStream, + mpsc::Receiver, + ) { + let (child, host) = UnixStream::pair().unwrap(); + let (control_sender, control_receiver) = mpsc::channel(); + let transport = unsafe { AudioPipeTransport::from_raw_fd(child.into_raw_fd()) }.unwrap(); + let output = Arc::new( + RemotePcmAudioOutput::new_with_timeout( + 7, + spec, + Arc::new(transport), + Arc::new(AtomicBool::new(true)), + control_sender, + timeout, + ) + .unwrap(), + ); + (output, host, control_receiver) + } + + fn read_record(reader: &mut impl Read) -> (u8, Vec) { + let mut header = [0; AUDIO_FRAME_HEADER_BYTES]; + reader.read_exact(&mut header).unwrap(); + assert_eq!(header[..2], AUDIO_FRAME_MAGIC); + assert_eq!(header[2], AUDIO_FRAME_MARKER); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + let mut payload = vec![0; length]; + reader.read_exact(&mut payload).unwrap(); + (header[3], payload) + } + + fn start(output: &Arc, host: &mut UnixStream) { + let current = Arc::clone(output); + let worker = thread::spawn(move || current.start()); + let (kind, payload) = read_record(host); + assert_eq!(kind, AUDIO_BEGIN_KIND); + assert_eq!(payload.len(), 16); + assert_eq!(u64::from_le_bytes(payload[..8].try_into().unwrap()), 7); + assert_eq!( + u32::from_le_bytes(payload[8..12].try_into().unwrap()), + output.spec.sample_rate + ); + output.handle_ack(AudioHostAck::BeginAccepted).unwrap(); + worker.join().unwrap().unwrap(); + } + + #[test] + fn chunks_split_at_4096_wait_for_exact_acceptance_and_end_explicitly() { + let (output, mut host) = fixture(); + start(&output, &mut host); + let current = Arc::clone(&output); + let worker = thread::spawn(move || current.write(&vec![0.25; 5_000])); + let (kind, first) = read_record(&mut host); + assert_eq!(kind, AUDIO_CHUNK_KIND); + assert_eq!(first.len(), 16 + 4096 * 4); + let started = output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 1 }) + .unwrap(); + assert!(started); + worker.join().unwrap().unwrap(); + + let current = Arc::clone(&output); + let finisher = thread::spawn(move || current.finish_writes()); + let (kind, second) = read_record(&mut host); + assert_eq!(kind, AUDIO_CHUNK_KIND); + assert_eq!(second.len(), 16 + 904 * 4); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 2 }) + .unwrap(); + let (kind, end) = read_record(&mut host); + assert_eq!(kind, AUDIO_END_KIND); + assert_eq!(u64::from_le_bytes(end[8..16].try_into().unwrap()), 2); + assert_eq!(u64::from_le_bytes(end[16..24].try_into().unwrap()), 5_000); + finisher.join().unwrap().unwrap(); + output + .handle_ack(AudioHostAck::Drained { + sequence: 2, + played_frames: 5_000, + }) + .unwrap(); + assert!(output.is_drained()); + assert_eq!(output.played_frames(), 5_000); + } + + #[test] + fn first_chunk_starts_before_a_larger_write_waits_for_played_credit() { + let (output, mut host) = fixture(); + start(&output, &mut host); + let current = Arc::clone(&output); + let worker = thread::spawn(move || current.write(&vec![0.25; 4096 * 4])); + for sequence in 1..=3 { + let (kind, _) = read_record(&mut host); + assert_eq!(kind, AUDIO_CHUNK_KIND); + let started = output + .handle_ack(AudioHostAck::ChunkAccepted { sequence }) + .unwrap(); + assert_eq!(started, sequence == 1); + } + host.set_read_timeout(Some(Duration::from_millis(30))) + .unwrap(); + let mut byte = [0]; + assert!(host.read(&mut byte).is_err()); + output + .handle_ack(AudioHostAck::Played { + played_frames: 4096, + }) + .unwrap(); + host.set_read_timeout(None).unwrap(); + let (kind, _) = read_record(&mut host); + assert_eq!(kind, AUDIO_CHUNK_KIND); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 4 }) + .unwrap(); + worker.join().unwrap().unwrap(); + } + + #[test] + fn suspension_quiesces_and_resumes_the_same_remote_stream() { + let (output, mut host, controls) = fixture_with_control( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + AUDIO_OPERATION_TIMEOUT, + ); + start(&output, &mut host); + let current = Arc::clone(&output); + let worker = thread::spawn(move || current.write(&vec![0.25; 4096 * 2])); + let (kind, _) = read_record(&mut host); + assert_eq!(kind, AUDIO_CHUNK_KIND); + + output.request_suspend().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + assert!(output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 1 }) + .unwrap()); + output + .handle_ack(AudioHostAck::Suspended { + played_frames: 4096, + }) + .unwrap(); + host.set_read_timeout(Some(Duration::from_millis(30))) + .unwrap(); + let mut byte = [0]; + assert!(host.read(&mut byte).is_err()); + + output.request_resume().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Resume { speech_id: 7 } + ); + output + .handle_ack(AudioHostAck::Resumed { + played_frames: 4096, + }) + .unwrap(); + host.set_read_timeout(None).unwrap(); + let (kind, _) = read_record(&mut host); + assert_eq!(kind, AUDIO_CHUNK_KIND); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 2 }) + .unwrap(); + worker.join().unwrap().unwrap(); + } + + #[test] + fn suspension_before_begin_and_early_settlement_are_correlated() { + let (output, mut host, controls) = fixture_with_control( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + AUDIO_OPERATION_TIMEOUT, + ); + output.request_suspend().unwrap(); + output.request_resume().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + output + .handle_ack(AudioHostAck::Suspended { played_frames: 0 }) + .unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Resume { speech_id: 7 } + ); + output + .handle_ack(AudioHostAck::Resumed { played_frames: 0 }) + .unwrap(); + + start(&output, &mut host); + } + + #[test] + fn renewed_speaking_while_resume_is_in_flight_reissues_suspend() { + let (output, mut host, controls) = fixture_with_control( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + AUDIO_OPERATION_TIMEOUT, + ); + start(&output, &mut host); + output.request_suspend().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + output + .handle_ack(AudioHostAck::Suspended { played_frames: 0 }) + .unwrap(); + output.request_resume().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Resume { speech_id: 7 } + ); + + output.request_suspend().unwrap(); + output + .handle_ack(AudioHostAck::Resumed { played_frames: 0 }) + .unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + assert!(output + .control_request_is_outstanding(AudioOutputControlRequest::Suspend { speech_id: 7 })); + } + + #[test] + fn cancellation_waits_for_an_in_flight_suspension_barrier() { + let (output, mut host, controls) = fixture_with_control( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + AUDIO_OPERATION_TIMEOUT, + ); + start(&output, &mut host); + output.request_suspend().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + let current = Arc::clone(&output); + let canceller = thread::spawn(move || current.cancel_settled()); + host.set_read_timeout(Some(Duration::from_millis(30))) + .unwrap(); + let mut byte = [0]; + assert!(host.read(&mut byte).is_err()); + output + .handle_ack(AudioHostAck::Suspended { played_frames: 0 }) + .unwrap(); + host.set_read_timeout(None).unwrap(); + let (kind, _) = read_record(&mut host); + assert_eq!(kind, AUDIO_CANCEL_KIND); + output + .handle_ack(AudioHostAck::Cancelled { played_frames: 0 }) + .unwrap(); + assert_eq!(canceller.join().unwrap().unwrap(), 0); + } + + #[test] + fn cancellation_authority_suppresses_resume_after_a_late_suspend_ack() { + let (child, mut host) = UnixStream::pair().unwrap(); + let transport = unsafe { AudioPipeTransport::from_raw_fd(child.into_raw_fd()) }.unwrap(); + let active = Arc::new(AtomicBool::new(true)); + let (control_sender, controls) = mpsc::channel(); + let output = Arc::new( + RemotePcmAudioOutput::new_with_timeout( + 7, + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + Arc::new(transport), + Arc::clone(&active), + control_sender, + AUDIO_OPERATION_TIMEOUT, + ) + .unwrap(), + ); + start(&output, &mut host); + output.request_suspend().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + output.request_resume().unwrap(); + + active.store(false, Ordering::SeqCst); + output.notify_cancel_requested(); + let current = Arc::clone(&output); + let canceller = thread::spawn(move || current.cancel_settled()); + output + .handle_ack(AudioHostAck::Suspended { played_frames: 0 }) + .unwrap(); + assert!(controls.try_recv().is_err()); + assert_eq!(read_record(&mut host).0, AUDIO_CANCEL_KIND); + output + .handle_ack(AudioHostAck::Cancelled { played_frames: 0 }) + .unwrap(); + assert_eq!(canceller.join().unwrap().unwrap(), 0); + } + + #[test] + fn cancellation_authority_suppresses_resuspend_after_a_late_resume_ack() { + let (child, mut host) = UnixStream::pair().unwrap(); + let transport = unsafe { AudioPipeTransport::from_raw_fd(child.into_raw_fd()) }.unwrap(); + let active = Arc::new(AtomicBool::new(true)); + let (control_sender, controls) = mpsc::channel(); + let output = Arc::new( + RemotePcmAudioOutput::new_with_timeout( + 7, + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + Arc::new(transport), + Arc::clone(&active), + control_sender, + AUDIO_OPERATION_TIMEOUT, + ) + .unwrap(), + ); + start(&output, &mut host); + output.request_suspend().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + output + .handle_ack(AudioHostAck::Suspended { played_frames: 0 }) + .unwrap(); + output.request_resume().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Resume { speech_id: 7 } + ); + output.request_suspend().unwrap(); + + active.store(false, Ordering::SeqCst); + output.notify_cancel_requested(); + let current = Arc::clone(&output); + let canceller = thread::spawn(move || current.cancel_settled()); + output + .handle_ack(AudioHostAck::Resumed { played_frames: 0 }) + .unwrap(); + assert!(controls.try_recv().is_err()); + assert_eq!(read_record(&mut host).0, AUDIO_CANCEL_KIND); + output + .handle_ack(AudioHostAck::Cancelled { played_frames: 0 }) + .unwrap(); + assert_eq!(canceller.join().unwrap().unwrap(), 0); + } + + #[test] + fn cancellation_waits_for_an_in_flight_resume_barrier() { + let (output, mut host, controls) = fixture_with_control( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + AUDIO_OPERATION_TIMEOUT, + ); + start(&output, &mut host); + output.request_suspend().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + output + .handle_ack(AudioHostAck::Suspended { played_frames: 0 }) + .unwrap(); + output.request_resume().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Resume { speech_id: 7 } + ); + + let current = Arc::clone(&output); + let canceller = thread::spawn(move || current.cancel_settled()); + host.set_read_timeout(Some(Duration::from_millis(30))) + .unwrap(); + let mut byte = [0]; + assert!(host.read(&mut byte).is_err()); + output + .handle_ack(AudioHostAck::Resumed { played_frames: 0 }) + .unwrap(); + host.set_read_timeout(None).unwrap(); + assert_eq!(read_record(&mut host).0, AUDIO_CANCEL_KIND); + output + .handle_ack(AudioHostAck::Cancelled { played_frames: 0 }) + .unwrap(); + assert_eq!(canceller.join().unwrap().unwrap(), 0); + } + + #[test] + fn terminal_output_rejects_a_late_suspension_ack() { + let (output, mut host, _controls) = fixture_with_control( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + AUDIO_OPERATION_TIMEOUT, + ); + start(&output, &mut host); + output.request_suspend().unwrap(); + output + .handle_ack(AudioHostAck::Failed { + played_frames: 0, + message: "route failed".into(), + }) + .unwrap(); + assert_eq!( + output + .handle_ack(AudioHostAck::Suspended { played_frames: 0 }) + .unwrap_err(), + "audio host acknowledgement is stale, out of order, or impossible" + ); + } + + #[test] + fn drained_before_suspend_ack_stays_held_until_resume() { + let (output, mut host, controls) = fixture_with_control( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + AUDIO_OPERATION_TIMEOUT, + ); + start(&output, &mut host); + let current = Arc::clone(&output); + let writer = thread::spawn(move || current.write(&vec![0.25; 4096])); + let (kind, _) = read_record(&mut host); + assert_eq!(kind, AUDIO_CHUNK_KIND); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 1 }) + .unwrap(); + writer.join().unwrap().unwrap(); + let current = Arc::clone(&output); + let finisher = thread::spawn(move || current.finish_writes()); + let (kind, end) = read_record(&mut host); + assert_eq!(kind, AUDIO_END_KIND); + assert_eq!(u64::from_le_bytes(end[8..16].try_into().unwrap()), 1); + + output.request_suspend().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + output + .handle_ack(AudioHostAck::Drained { + sequence: 1, + played_frames: 4096, + }) + .unwrap(); + assert!(!output.is_drained()); + output + .handle_ack(AudioHostAck::Suspended { + played_frames: 4096, + }) + .unwrap(); + output.request_resume().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Resume { speech_id: 7 } + ); + output + .handle_ack(AudioHostAck::Resumed { + played_frames: 4096, + }) + .unwrap(); + assert!(output.is_drained()); + finisher.join().unwrap().unwrap(); + } + + #[test] + fn terminal_cancel_clears_a_suspended_already_drained_stream() { + let (output, mut host, controls) = fixture_with_control( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + AUDIO_OPERATION_TIMEOUT, + ); + start(&output, &mut host); + let current = Arc::clone(&output); + let writer = thread::spawn(move || current.write(&vec![0.25; 4096])); + assert_eq!(read_record(&mut host).0, AUDIO_CHUNK_KIND); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 1 }) + .unwrap(); + writer.join().unwrap().unwrap(); + let current = Arc::clone(&output); + let finisher = thread::spawn(move || current.finish_writes()); + assert_eq!(read_record(&mut host).0, AUDIO_END_KIND); + output.request_suspend().unwrap(); + assert_eq!( + controls.recv_timeout(Duration::from_millis(30)).unwrap(), + AudioOutputControlRequest::Suspend { speech_id: 7 } + ); + output + .handle_ack(AudioHostAck::Drained { + sequence: 1, + played_frames: 4096, + }) + .unwrap(); + output + .handle_ack(AudioHostAck::Suspended { + played_frames: 4096, + }) + .unwrap(); + let current = Arc::clone(&output); + let canceller = thread::spawn(move || current.cancel_settled()); + assert_eq!(read_record(&mut host).0, AUDIO_CANCEL_KIND); + output + .handle_ack(AudioHostAck::Cancelled { + played_frames: 4096, + }) + .unwrap(); + assert_eq!(canceller.join().unwrap().unwrap(), 4096); + finisher.join().unwrap().unwrap(); + } + + #[test] + fn production_rates_buffer_a_sustained_runway_before_played_credit() { + for (spec, expected_chunks) in [ + ( + TtsPcmSpec { + sample_rate: 48_000, + playback_rate: 1.5, + }, + 8, + ), + ( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 2.0, + }, + 5, + ), + ] { + let (output, mut host) = + fixture_with_spec_and_timeout(spec, Duration::from_millis(100)); + start(&output, &mut host); + let current = Arc::clone(&output); + let worker = thread::spawn(move || { + current.write(&vec![0.25; MAX_AUDIO_CHUNK_FRAMES * expected_chunks]) + }); + host.set_read_timeout(Some(Duration::from_millis(50))) + .unwrap(); + for sequence in 1..=expected_chunks as u64 { + let (kind, _) = read_record(&mut host); + assert_eq!(kind, AUDIO_CHUNK_KIND); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence }) + .unwrap(); + } + worker.join().unwrap().unwrap(); + } + } + + #[test] + fn short_backend_callbacks_coalesce_into_bounded_transport_records() { + let (output, mut host) = fixture(); + start(&output, &mut host); + for _ in 0..7 { + output.write(&vec![0.25; 512]).unwrap(); + } + host.set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let mut byte = [0]; + assert!(host.read(&mut byte).is_err()); + + let current = Arc::clone(&output); + let worker = thread::spawn(move || current.write(&vec![0.25; 512])); + host.set_read_timeout(None).unwrap(); + let (kind, chunk) = read_record(&mut host); + assert_eq!(kind, AUDIO_CHUNK_KIND); + assert_eq!(chunk.len(), 16 + MAX_AUDIO_CHUNK_FRAMES * 4); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 1 }) + .unwrap(); + worker.join().unwrap().unwrap(); + } + + #[test] + fn empty_output_fails_before_emitting_audio_end() { + let (output, mut host) = fixture(); + start(&output, &mut host); + + assert_eq!( + output.finish_writes().unwrap_err(), + "remote PCM output produced no audio frames" + ); + host.set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let mut byte = [0]; + assert!(host.read(&mut byte).is_err()); + } + + #[test] + fn cancel_waits_for_quiescence_and_returns_the_settled_snapshot() { + let (output, mut host) = fixture(); + start(&output, &mut host); + let current = Arc::clone(&output); + let worker = thread::spawn(move || current.cancel_and_snapshot()); + let (kind, payload) = read_record(&mut host); + assert_eq!(kind, AUDIO_CANCEL_KIND); + assert_eq!(u64::from_le_bytes(payload.try_into().unwrap()), 7); + output + .handle_ack(AudioHostAck::Cancelled { played_frames: 0 }) + .unwrap(); + assert_eq!(worker.join().unwrap().unwrap(), 0); + assert!(output + .handle_ack(AudioHostAck::Played { played_frames: 1 }) + .is_err()); + } + + #[test] + fn cancellation_overtakes_a_held_begin_ack_on_the_worker_pipe() { + let (output, mut host) = fixture(); + let current = Arc::clone(&output); + let worker = thread::spawn(move || current.start()); + assert_eq!(read_record(&mut host).0, AUDIO_BEGIN_KIND); + + output.active.store(false, Ordering::SeqCst); + output.notify_cancel_requested(); + assert_eq!(read_record(&mut host).0, AUDIO_CANCEL_KIND); + output.handle_ack(AudioHostAck::BeginAccepted).unwrap(); + output + .handle_ack(AudioHostAck::Cancelled { played_frames: 0 }) + .unwrap(); + assert_eq!(worker.join().unwrap().unwrap_err(), AUDIO_CANCELLED); + } + + #[test] + fn cancellation_overtakes_a_held_chunk_ack_and_uses_only_accepted_frames() { + let (output, mut host) = fixture(); + start(&output, &mut host); + let current = Arc::clone(&output); + let worker = thread::spawn(move || current.write(&[0.25; MAX_AUDIO_CHUNK_FRAMES])); + assert_eq!(read_record(&mut host).0, AUDIO_CHUNK_KIND); + + output.active.store(false, Ordering::SeqCst); + output.notify_cancel_requested(); + assert_eq!(read_record(&mut host).0, AUDIO_CANCEL_KIND); + output + .handle_ack(AudioHostAck::Cancelled { played_frames: 0 }) + .unwrap(); + assert_eq!(worker.join().unwrap().unwrap_err(), AUDIO_CANCELLED); + assert!(output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 1 }) + .is_err()); + } + + #[test] + fn pending_chunk_acceptance_may_settle_in_pipe_order_before_cancelled() { + let (output, mut host) = fixture(); + start(&output, &mut host); + let current = Arc::clone(&output); + let worker = thread::spawn(move || current.write(&[0.25; MAX_AUDIO_CHUNK_FRAMES])); + assert_eq!(read_record(&mut host).0, AUDIO_CHUNK_KIND); + + output.active.store(false, Ordering::SeqCst); + output.notify_cancel_requested(); + assert_eq!(read_record(&mut host).0, AUDIO_CANCEL_KIND); + let started = output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 1 }) + .unwrap(); + assert!(!started, "cancelling speech must not publish a late start"); + output + .handle_ack(AudioHostAck::Played { played_frames: 64 }) + .unwrap(); + output + .handle_ack(AudioHostAck::Cancelled { played_frames: 64 }) + .unwrap(); + assert_eq!(worker.join().unwrap().unwrap_err(), AUDIO_CANCELLED); + assert_eq!(output.played_frames(), 64); + } + + #[test] + fn drained_before_cancelled_still_resolves_the_authoritative_cancellation() { + let (output, mut host) = fixture(); + start(&output, &mut host); + let current = Arc::clone(&output); + let writer = thread::spawn(move || current.write(&[0.25; MAX_AUDIO_CHUNK_FRAMES])); + assert_eq!(read_record(&mut host).0, AUDIO_CHUNK_KIND); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 1 }) + .unwrap(); + writer.join().unwrap().unwrap(); + output.finish_writes().unwrap(); + assert_eq!(read_record(&mut host).0, AUDIO_END_KIND); + + let current = Arc::clone(&output); + let cancel = thread::spawn(move || current.cancel_and_snapshot()); + assert_eq!(read_record(&mut host).0, AUDIO_CANCEL_KIND); + output + .handle_ack(AudioHostAck::Drained { + sequence: 1, + played_frames: MAX_AUDIO_CHUNK_FRAMES as u64, + }) + .unwrap(); + output + .handle_ack(AudioHostAck::Cancelled { + played_frames: MAX_AUDIO_CHUNK_FRAMES as u64, + }) + .unwrap(); + assert_eq!( + cancel.join().unwrap().unwrap(), + MAX_AUDIO_CHUNK_FRAMES as u64 + ); + } + + #[test] + fn host_failure_before_cancelled_remains_a_failure_after_quiescence() { + let (output, mut host) = fixture(); + start(&output, &mut host); + let current = Arc::clone(&output); + let cancel = thread::spawn(move || current.cancel_and_snapshot()); + assert_eq!(read_record(&mut host).0, AUDIO_CANCEL_KIND); + output + .handle_ack(AudioHostAck::Failed { + played_frames: 0, + message: "private device detail".into(), + }) + .unwrap(); + output + .handle_ack(AudioHostAck::Cancelled { played_frames: 0 }) + .unwrap(); + assert_eq!( + cancel.join().unwrap().unwrap_err(), + "host audio output failed" + ); + } + + #[test] + fn impossible_progress_and_wrong_sequence_are_rejected() { + let (output, mut host) = fixture(); + start(&output, &mut host); + assert!(output + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 2 }) + .is_err()); + assert!(output + .handle_ack(AudioHostAck::Played { played_frames: 1 }) + .is_err()); + } + + #[test] + fn begin_ack_silence_is_bounded_and_terminal() { + let (output, mut host) = fixture_with_timeout(Duration::from_millis(20)); + let current = Arc::clone(&output); + let worker = thread::spawn(move || current.start()); + assert_eq!(read_record(&mut host).0, AUDIO_BEGIN_KIND); + assert!(worker + .join() + .unwrap() + .unwrap_err() + .contains("before its deadline")); + assert!(output.check_health().is_err()); + } + + #[test] + fn closed_pcm_pipe_fails_begin_without_waiting_for_an_ack() { + let (output, host) = fixture(); + drop(host); + assert!(output.start().unwrap_err().contains("pipe write failed")); + } + + #[test] + fn inherited_read_only_descriptor_is_rejected_before_session_ready() { + let mut descriptors = [-1; 2]; + assert_eq!(unsafe { libc::pipe(descriptors.as_mut_ptr()) }, 0); + unsafe { libc::close(descriptors[1]) }; + let error = match unsafe { AudioPipeTransport::from_raw_fd(descriptors[0]) } { + Ok(_) => panic!("read-only descriptor must fail preflight"), + Err(error) => error, + }; + assert_eq!(error, "PCM output file descriptor is not writable"); + } + + #[test] + fn closed_pcm_specs_accept_both_engine_rates_and_reject_other_shapes() { + for (sample_rate, playback_rate) in [(24_000, 0.5), (24_000, 2.0), (48_000, 1.0)] { + let (child, _host) = UnixStream::pair().unwrap(); + let transport = + unsafe { AudioPipeTransport::from_raw_fd(child.into_raw_fd()) }.unwrap(); + let (control_sender, _control_receiver) = mpsc::channel(); + assert!(RemotePcmAudioOutput::new( + 1, + TtsPcmSpec { + sample_rate, + playback_rate, + }, + Arc::new(transport), + Arc::new(AtomicBool::new(true)), + control_sender, + ) + .is_ok()); + } + let (child, _host) = UnixStream::pair().unwrap(); + let transport = unsafe { AudioPipeTransport::from_raw_fd(child.into_raw_fd()) }.unwrap(); + let (control_sender, _control_receiver) = mpsc::channel(); + assert!(RemotePcmAudioOutput::new( + 1, + TtsPcmSpec { + sample_rate: 44_100, + playback_rate: 1.0, + }, + Arc::new(transport), + Arc::new(AtomicBool::new(true)), + control_sender, + ) + .is_err()); + } +} diff --git a/src-tauri/crates/berd-voice/src/siri.rs b/src-tauri/crates/berd-voice/src/siri.rs new file mode 100644 index 000000000..bd89b458a --- /dev/null +++ b/src-tauri/crates/berd-voice/src/siri.rs @@ -0,0 +1,1058 @@ +//! Safe Siri voice management and device-free sirittsd synthesis primitives. + +#[cfg(any(test, target_os = "macos"))] +use std::collections::{BTreeSet, VecDeque}; +#[cfg(target_os = "macos")] +use std::ffi::{c_char, c_void, CStr, CString}; +use std::fmt; +#[cfg(target_os = "macos")] +use std::sync::atomic::{AtomicBool, Ordering}; +#[cfg(target_os = "macos")] +use std::sync::{mpsc, Mutex}; +use std::time::Duration; + +use serde::{Deserialize, Deserializer, Serialize}; + +#[cfg(target_os = "macos")] +use crate::{TtsBackend, TtsOutcome, TtsPcmSpec, TtsSynthesisEvent}; + +#[cfg(target_os = "macos")] +const SIRI_PCM_SAMPLE_RATE: u32 = 48_000; +#[cfg(target_os = "macos")] +const MAX_PENDING_PCM_FRAMES: usize = SIRI_PCM_SAMPLE_RATE as usize * 60; +#[cfg(target_os = "macos")] +const SYNTHESIS_POLL_INTERVAL: Duration = Duration::from_millis(10); +#[cfg(target_os = "macos")] +const SIRI_SYNTHESIS_STALL_TIMEOUT: Duration = Duration::from_secs(60); +const DEFAULT_SIRI_DOWNLOAD_WAIT_TIMEOUT: Duration = Duration::from_secs(300); +const MIN_SIRI_DOWNLOAD_WAIT_TIMEOUT: Duration = Duration::from_secs(1); +const MAX_SIRI_DOWNLOAD_WAIT_TIMEOUT: Duration = Duration::from_secs(30 * 60); + +/// A validated bound for polling whether a requested Siri voice became available. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SiriDownloadAvailabilityWait(Duration); + +impl SiriDownloadAvailabilityWait { + /// Creates an availability wait from an inclusive `1..=1800` second bound. + pub fn from_seconds(seconds: u64) -> Result { + let timeout = Duration::from_secs(seconds); + validate_download_wait_timeout(timeout)?; + Ok(Self(timeout)) + } + + fn duration(self) -> Duration { + self.0 + } + + /// Returns the configured availability-polling bound in whole seconds. + pub fn seconds(self) -> u64 { + self.0.as_secs() + } +} + +impl Default for SiriDownloadAvailabilityWait { + fn default() -> Self { + Self(DEFAULT_SIRI_DOWNLOAD_WAIT_TIMEOUT) + } +} + +#[cfg(target_os = "macos")] +unsafe extern "C" { + fn berd_siri_tts_catalog_json( + language: *const c_char, + error_out: *mut *mut c_char, + ) -> *mut c_char; + fn berd_siri_tts_languages_json(error_out: *mut *mut c_char) -> *mut c_char; + fn berd_siri_tts_download_voice( + language: *const c_char, + voice_name: *const c_char, + availability_wait_timeout_seconds: f64, + error_out: *mut *mut c_char, + ) -> bool; + fn berd_siri_tts_validate_voice( + language: *const c_char, + voice_name: *const c_char, + error_out: *mut *mut c_char, + ) -> bool; + fn berd_siri_tts_synthesize_pcm( + text: *const c_char, + language: *const c_char, + voice_name: *const c_char, + rate: f32, + should_stop: unsafe extern "C" fn(*mut c_void) -> bool, + pcm_frames: unsafe extern "C" fn(*const f32, u32, *mut c_void) -> bool, + context: *mut c_void, + error_out: *mut *mut c_char, + ) -> bool; + #[cfg(test)] + fn berd_siri_tts_test_closed_pcm_gate_ignores_late_callback() -> bool; + fn berd_siri_tts_free_string(value: *mut c_char); +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SiriVoiceIdentity { + name: String, + language: String, +} + +impl SiriVoiceIdentity { + pub fn new(name: impl Into, language: &str) -> Result { + let name = name.into(); + if name.is_empty() || name.trim() != name { + return Err( + "Siri voice name must be nonempty and have no surrounding whitespace".into(), + ); + } + if name.contains('\0') { + return Err("Siri voice name contains NUL".into()); + } + Ok(Self { + name, + language: normalize_language(language)?, + }) + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn language(&self) -> &str { + &self.language + } +} + +impl<'de> Deserialize<'de> for SiriVoiceIdentity { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct RawIdentity { + name: String, + language: String, + } + + let raw = RawIdentity::deserialize(deserializer)?; + Self::new(raw.name, &raw.language).map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SiriVoice { + pub name: String, + pub language: String, + pub size_bytes: u64, + pub installed: bool, +} + +impl SiriVoice { + pub fn identity(&self) -> SiriVoiceIdentity { + SiriVoiceIdentity { + name: self.name.clone(), + language: self.language.clone(), + } + } + + pub fn matches(&self, identity: &SiriVoiceIdentity) -> bool { + self.name == identity.name && self.language == identity.language + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SiriVoiceCatalog { + pub available_languages: Vec, + pub voices: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SiriVoiceDownloadError { + NotFound(SiriVoiceIdentity), + Operation(String), +} + +impl fmt::Display for SiriVoiceDownloadError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotFound(identity) => write!( + formatter, + "Siri voice {:?} ({}) was not found", + identity.name(), + identity.language() + ), + Self::Operation(message) => formatter.write_str(message), + } + } +} + +impl std::error::Error for SiriVoiceDownloadError {} + +#[cfg(any(test, target_os = "macos"))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct RawSiriVoice { + name: String, + language: String, + size_bytes: u64, + installed: bool, +} + +/// Normalizes one Siri catalog language using the shared BCP-47 identity rule. +pub fn normalize_language(value: &str) -> Result { + let value = value.trim().replace('_', "-"); + if value.is_empty() || value.contains('\0') { + return Err("Siri voice language must be a nonempty BCP-47 tag".into()); + } + let mut normalized = Vec::new(); + let mut has_script = false; + let mut has_region = false; + let mut in_extension = false; + for (index, segment) in value.split('-').enumerate() { + if segment.is_empty() || !segment.bytes().all(|byte| byte.is_ascii_alphanumeric()) { + return Err(format!("invalid Siri voice BCP-47 language: {value}")); + } + let segment = if index == 0 { + segment.to_ascii_lowercase() + } else if segment.len() == 1 { + in_extension = true; + segment.to_ascii_lowercase() + } else if in_extension { + segment.to_ascii_lowercase() + } else if !has_script + && segment.len() == 4 + && segment.bytes().all(|byte| byte.is_ascii_alphabetic()) + { + has_script = true; + let mut chars = segment.chars(); + let first = chars.next().expect("four-character segment"); + format!( + "{}{}", + first.to_ascii_uppercase(), + chars.as_str().to_ascii_lowercase() + ) + } else if !has_region + && ((segment.len() == 2 && segment.bytes().all(|byte| byte.is_ascii_alphabetic())) + || (segment.len() == 3 && segment.bytes().all(|byte| byte.is_ascii_digit()))) + { + has_region = true; + segment.to_ascii_uppercase() + } else { + segment.to_ascii_lowercase() + }; + normalized.push(segment); + } + Ok(normalized.join("-")) +} + +#[cfg(any(test, target_os = "macos"))] +fn parse_catalog_json(json: &str) -> Result, String> { + serde_json::from_str::>(json) + .map_err(|error| format!("decode Siri voice catalog: {error}"))? + .into_iter() + .map(|raw| { + let identity = SiriVoiceIdentity::new(raw.name, &raw.language)?; + Ok(SiriVoice { + name: identity.name, + language: identity.language, + size_bytes: raw.size_bytes, + installed: raw.installed, + }) + }) + .collect() +} + +#[cfg(any(test, target_os = "macos"))] +fn parse_languages_json(json: &str) -> Result, String> { + let raw = serde_json::from_str::>(json) + .map_err(|error| format!("decode Siri voice languages: {error}"))?; + let mut seen = BTreeSet::new(); + let mut languages = Vec::new(); + for language in raw { + let language = normalize_language(&language)?; + if seen.insert(language.clone()) { + languages.push(language); + } + } + Ok(languages) +} + +/// Loads the Siri voice catalog for one exact language, or all languages when +/// `None`. Voice identities use exact catalog names and normalized languages. +#[cfg(target_os = "macos")] +pub fn load_voice_catalog(language: Option<&str>) -> Result { + let language = language + .map(normalize_language) + .transpose()? + .unwrap_or_default(); + let language = CString::new(language).expect("normalized language has no NUL"); + let mut error = std::ptr::null_mut(); + // SAFETY: The bridge copies the input and returns malloc-owned strings. + let voices = unsafe { berd_siri_tts_catalog_json(language.as_ptr(), &mut error) }; + let voices = take_string(voices) + .ok_or_else(|| take_error(error, "Could not load the Siri voice catalog"))?; + + error = std::ptr::null_mut(); + // SAFETY: The bridge returns a malloc-owned string. + let languages = unsafe { berd_siri_tts_languages_json(&mut error) }; + let languages = take_string(languages) + .ok_or_else(|| take_error(error, "Could not load Siri voice languages"))?; + + Ok(SiriVoiceCatalog { + available_languages: parse_languages_json(&languages)?, + voices: parse_catalog_json(&voices)?, + }) +} + +/// Returns an empty unsupported-platform catalog outside macOS. +#[cfg(not(target_os = "macos"))] +pub fn load_voice_catalog(_language: Option<&str>) -> Result { + Ok(SiriVoiceCatalog { + available_languages: Vec::new(), + voices: Vec::new(), + }) +} + +/// Validates that one exact Siri voice identity is currently installed. +#[cfg(target_os = "macos")] +pub fn validate_installed_voice(identity: &SiriVoiceIdentity) -> Result<(), String> { + let language = + CString::new(identity.language.as_str()).expect("normalized language has no NUL"); + let name = CString::new(identity.name.as_str()).expect("validated name has no NUL"); + let mut error = std::ptr::null_mut(); + // SAFETY: Both strings remain valid for the duration of the call. + if unsafe { berd_siri_tts_validate_voice(language.as_ptr(), name.as_ptr(), &mut error) } { + Ok(()) + } else { + Err(take_error(error, "Siri voice is not installed")) + } +} + +#[cfg(not(target_os = "macos"))] +pub fn validate_installed_voice(_identity: &SiriVoiceIdentity) -> Result<(), String> { + Err("Siri TTS is only available on macOS".into()) +} + +fn validate_download_wait_timeout(timeout: Duration) -> Result<(), String> { + if !(MIN_SIRI_DOWNLOAD_WAIT_TIMEOUT..=MAX_SIRI_DOWNLOAD_WAIT_TIMEOUT).contains(&timeout) { + return Err(format!( + "Siri download availability wait must be between {} and {} seconds", + MIN_SIRI_DOWNLOAD_WAIT_TIMEOUT.as_secs(), + MAX_SIRI_DOWNLOAD_WAIT_TIMEOUT.as_secs() + )); + } + Ok(()) +} + +/// Resolves one exact catalog identity, then requests it when not already +/// installed and blocks until it is available or the bounded +/// availability-polling wait elapses. A missing identity fails before native +/// mutation. Native validation and subscription have their own bounded calls, +/// so this is not a hard whole-operation deadline. +pub fn download_voice( + identity: &SiriVoiceIdentity, + availability_wait: SiriDownloadAvailabilityWait, +) -> Result { + download_voice_with( + identity, + availability_wait, + |language| load_voice_catalog(Some(language)), + download_voice_platform, + ) +} + +fn download_voice_with( + requested: &SiriVoiceIdentity, + availability_wait: SiriDownloadAvailabilityWait, + load_catalog: impl FnOnce(&str) -> Result, + download: impl FnOnce(&SiriVoiceIdentity, Duration) -> Result<(), String>, +) -> Result { + let catalog = load_catalog(requested.language()).map_err(SiriVoiceDownloadError::Operation)?; + let voice = catalog + .voices + .iter() + .find(|voice| voice.matches(requested)) + .ok_or_else(|| SiriVoiceDownloadError::NotFound(requested.clone()))?; + let identity = voice.identity(); + if !voice.installed { + download(&identity, availability_wait.duration()) + .map_err(SiriVoiceDownloadError::Operation)?; + } + Ok(identity) +} + +#[cfg(target_os = "macos")] +fn download_voice_platform( + identity: &SiriVoiceIdentity, + availability_wait_timeout: Duration, +) -> Result<(), String> { + let language = + CString::new(identity.language.as_str()).expect("normalized language has no NUL"); + let name = CString::new(identity.name.as_str()).expect("validated name has no NUL"); + let mut error = std::ptr::null_mut(); + // SAFETY: Both strings remain live for this blocking call. + if unsafe { + berd_siri_tts_download_voice( + language.as_ptr(), + name.as_ptr(), + availability_wait_timeout.as_secs_f64(), + &mut error, + ) + } { + Ok(()) + } else { + Err(take_error(error, "Siri voice download failed")) + } +} + +#[cfg(not(target_os = "macos"))] +fn download_voice_platform( + _identity: &SiriVoiceIdentity, + _availability_wait_timeout: Duration, +) -> Result<(), String> { + Err("Siri TTS is only available on macOS".into()) +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug)] +pub struct SiriTts { + language: CString, + voice_name: CString, + rate: f32, +} + +#[cfg(target_os = "macos")] +impl SiriTts { + pub fn new(language: &str, voice_name: &str, rate: f32) -> Result { + if !rate.is_finite() || !(0.5..=2.0).contains(&rate) { + return Err("Siri rate must be between 0.5 and 2.0".into()); + } + let identity = SiriVoiceIdentity::new(voice_name, language)?; + validate_installed_voice(&identity)?; + let language = CString::new(identity.language).expect("normalized language has no NUL"); + let voice_name = CString::new(identity.name).expect("validated name has no NUL"); + Ok(Self { + language, + voice_name, + rate, + }) + } +} + +#[cfg(target_os = "macos")] +struct CallbackContext<'a> { + active: &'a AtomicBool, + callback_cancelled: &'a AtomicBool, + queue_overflowed: &'a AtomicBool, + frames: &'a Mutex>, + notification: mpsc::SyncSender<()>, +} + +#[cfg(target_os = "macos")] +unsafe extern "C" fn should_stop(context: *mut c_void) -> bool { + // SAFETY: The native call is scoped to the lifetime of this context. + let context = unsafe { &*(context.cast::>()) }; + !context.active.load(Ordering::SeqCst) || context.callback_cancelled.load(Ordering::SeqCst) +} + +#[cfg(target_os = "macos")] +unsafe extern "C" fn receive_pcm( + samples: *const f32, + frame_count: u32, + context: *mut c_void, +) -> bool { + // SAFETY: The bridge guarantees `frame_count` valid samples for this call. + let frames = unsafe { std::slice::from_raw_parts(samples, frame_count as usize) }; + // SAFETY: The native call is scoped to the lifetime of this context. + let context = unsafe { &*(context.cast::>()) }; + if !context.active.load(Ordering::SeqCst) || context.callback_cancelled.load(Ordering::SeqCst) { + return false; + } + let Ok(mut pending) = context.frames.lock() else { + context.queue_overflowed.store(true, Ordering::SeqCst); + context.callback_cancelled.store(true, Ordering::SeqCst); + return false; + }; + if pending + .len() + .checked_add(frames.len()) + .filter(|total| *total <= MAX_PENDING_PCM_FRAMES) + .is_none() + || pending.try_reserve(frames.len()).is_err() + { + context.queue_overflowed.store(true, Ordering::SeqCst); + context.callback_cancelled.store(true, Ordering::SeqCst); + return false; + } + pending.extend(frames.iter().copied()); + drop(pending); + match context.notification.try_send(()) { + Ok(()) | Err(mpsc::TrySendError::Full(())) => true, + Err(mpsc::TrySendError::Disconnected(())) => { + context.queue_overflowed.store(true, Ordering::SeqCst); + context.callback_cancelled.store(true, Ordering::SeqCst); + false + } + } +} + +#[cfg(target_os = "macos")] +impl TtsBackend for SiriTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: SIRI_PCM_SAMPLE_RATE, + playback_rate: 1.0, + } + } + + fn synthesize( + &self, + text: &str, + active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + self.synthesize_with_poll(text, active, &mut |event| match event { + TtsSynthesisEvent::Frames(frames) => on_frames(frames), + TtsSynthesisEvent::Poll => Ok(()), + }) + } + + fn synthesize_with_poll( + &self, + text: &str, + active: &AtomicBool, + on_event: &mut dyn FnMut(TtsSynthesisEvent<'_>) -> Result<(), String>, + ) -> Result { + if !active.load(Ordering::SeqCst) { + return Ok(TtsOutcome::Cancelled); + } + let text = CString::new(text).map_err(|_| "Siri text contains NUL")?; + let frames = Mutex::new(VecDeque::new()); + let (notification, receiver) = mpsc::sync_channel(1); + let callback_cancelled = AtomicBool::new(false); + let queue_overflowed = AtomicBool::new(false); + let language = self.language.clone(); + let voice_name = self.voice_name.clone(); + let rate = self.rate; + let result = std::thread::scope(|scope| { + let mut context = CallbackContext { + active, + callback_cancelled: &callback_cancelled, + queue_overflowed: &queue_overflowed, + frames: &frames, + notification, + }; + let native = scope.spawn(move || { + let mut error = std::ptr::null_mut(); + // SAFETY: All pointers remain valid until this blocking native + // call returns, and callbacks only borrow the scoped context. + let completed = unsafe { + berd_siri_tts_synthesize_pcm( + text.as_ptr(), + language.as_ptr(), + voice_name.as_ptr(), + rate, + should_stop, + receive_pcm, + (&mut context as *mut CallbackContext<'_>).cast(), + &mut error, + ) + }; + if completed { + Ok(()) + } else { + Err(take_error(error, "Siri synthesis failed")) + } + }); + let receive_result = receive_pcm_until_complete( + receiver, + &callback_cancelled, + &queue_overflowed, + &frames, + SYNTHESIS_POLL_INTERVAL, + SIRI_SYNTHESIS_STALL_TIMEOUT, + on_event, + ); + let native = native + .join() + .map_err(|_| "Siri synthesis thread panicked".to_string())?; + receive_result.and(native) + }); + result?; + Ok(if active.load(Ordering::SeqCst) { + TtsOutcome::Completed + } else { + TtsOutcome::Cancelled + }) + } +} + +#[cfg(target_os = "macos")] +fn receive_pcm_until_complete( + receiver: mpsc::Receiver<()>, + callback_cancelled: &AtomicBool, + queue_overflowed: &AtomicBool, + pending_frames: &Mutex>, + poll_interval: Duration, + stall_timeout: Duration, + on_event: &mut dyn FnMut(TtsSynthesisEvent<'_>) -> Result<(), String>, +) -> Result<(), String> { + let mut last_progress_at = std::time::Instant::now(); + loop { + if queue_overflowed.load(Ordering::SeqCst) { + return Err("Siri synthesis exceeded its bounded PCM queue".into()); + } + let frames = { + let mut pending = pending_frames + .lock() + .map_err(|_| "Siri PCM queue failed".to_string())?; + let count = pending.len().min(4_096); + pending.drain(..count).collect::>() + }; + if !frames.is_empty() { + last_progress_at = std::time::Instant::now(); + if let Err(error) = on_event(TtsSynthesisEvent::Frames(&frames)) { + callback_cancelled.store(true, Ordering::SeqCst); + return Err(error); + } + continue; + } + match receiver.recv_timeout(poll_interval) { + Ok(()) => continue, + Err(mpsc::RecvTimeoutError::Timeout) => { + if let Err(error) = on_event(TtsSynthesisEvent::Poll) { + callback_cancelled.store(true, Ordering::SeqCst); + return Err(error); + } + if last_progress_at.elapsed() >= stall_timeout { + callback_cancelled.store(true, Ordering::SeqCst); + return Err("Siri synthesis stopped making progress".into()); + } + } + Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(()), + } + } +} + +#[cfg(target_os = "macos")] +fn take_error(error: *mut c_char, fallback: &str) -> String { + if error.is_null() { + return fallback.to_string(); + } + // SAFETY: Bridge errors are malloc strings paired with this free function. + let message = unsafe { CStr::from_ptr(error) } + .to_string_lossy() + .into_owned(); + unsafe { berd_siri_tts_free_string(error) }; + message +} + +#[cfg(target_os = "macos")] +fn take_string(value: *mut c_char) -> Option { + if value.is_null() { + return None; + } + // SAFETY: Bridge strings are malloc-owned and paired with this free function. + let result = unsafe { CStr::from_ptr(value) } + .to_string_lossy() + .into_owned(); + unsafe { berd_siri_tts_free_string(value) }; + Some(result) +} + +#[cfg(test)] +mod tests { + #[cfg(target_os = "macos")] + use super::{ + berd_siri_tts_test_closed_pcm_gate_ignores_late_callback, receive_pcm, + receive_pcm_until_complete, CallbackContext, SiriTts, MAX_PENDING_PCM_FRAMES, + }; + use super::{ + download_voice_with, parse_catalog_json, parse_languages_json, + validate_download_wait_timeout, SiriDownloadAvailabilityWait, SiriVoice, SiriVoiceCatalog, + SiriVoiceDownloadError, SiriVoiceIdentity, MAX_SIRI_DOWNLOAD_WAIT_TIMEOUT, + MIN_SIRI_DOWNLOAD_WAIT_TIMEOUT, + }; + #[cfg(target_os = "macos")] + use crate::{TtsBackend, TtsOutcome, TtsSynthesisEvent}; + use std::cell::Cell; + #[cfg(target_os = "macos")] + use std::collections::VecDeque; + #[cfg(target_os = "macos")] + use std::sync::atomic::{AtomicBool, Ordering}; + #[cfg(target_os = "macos")] + use std::sync::{mpsc, Arc, Mutex}; + use std::time::Duration; + #[cfg(target_os = "macos")] + use std::time::Instant; + + #[test] + fn catalog_decode_normalizes_identity_and_rejects_malformed_entries() { + let catalog = parse_catalog_json( + r#"[{"name":"Aaron","language":"en_US","sizeBytes":42,"installed":true}]"#, + ) + .expect("valid catalog"); + assert_eq!(catalog.len(), 1); + assert_eq!(catalog[0].name, "Aaron"); + assert_eq!(catalog[0].language, "en-US"); + assert_eq!(catalog[0].size_bytes, 42); + assert!(catalog[0].installed); + + assert!(parse_catalog_json("not json").is_err()); + assert!(parse_catalog_json( + r#"[{"name":"","language":"en-US","sizeBytes":42,"installed":true}]"#, + ) + .is_err()); + assert!(parse_catalog_json( + r#"[{"name":"Aaron","language":"en--US","sizeBytes":42,"installed":true}]"#, + ) + .is_err()); + } + + #[test] + fn identity_preserves_exact_name_and_normalizes_bcp47_language() { + let identity = SiriVoiceIdentity::new("Aaron", "ZH_hans_cn").expect("identity"); + assert_eq!(identity.name, "Aaron"); + assert_eq!(identity.language, "zh-Hans-CN"); + assert_eq!( + SiriVoiceIdentity::new("Aaron", "en_US_u_CA_gregory") + .unwrap() + .language, + "en-US-u-ca-gregory" + ); + + let voices = parse_catalog_json( + r#"[{"name":"Aaron","language":"zh-Hans-CN","sizeBytes":42,"installed":true}]"#, + ) + .expect("catalog"); + assert!(voices[0].matches(&identity)); + assert!(!voices[0].matches( + &SiriVoiceIdentity::new("aaron", "zh-Hans-CN").expect("case-sensitive identity") + )); + } + + #[test] + fn catalog_decode_preserves_installed_status_for_exact_voices() { + let voices = parse_catalog_json( + r#"[ + {"name":"Aaron","language":"en-US","sizeBytes":42,"installed":true}, + {"name":"Quinn","language":"en_US","sizeBytes":84,"installed":false} + ]"#, + ) + .expect("catalog"); + assert!(voices[0].installed); + assert!(!voices[1].installed); + assert!(voices[0].matches(&SiriVoiceIdentity::new("Aaron", "en_US").unwrap())); + assert!(voices[1].matches(&SiriVoiceIdentity::new("Quinn", "en-US").unwrap())); + } + + #[test] + fn languages_decode_uses_the_same_normalization_and_deduplicates() { + assert_eq!( + parse_languages_json(r#"["en_US","en-US","zh_hans_cn"]"#).expect("languages"), + ["en-US", "zh-Hans-CN"] + ); + assert!(parse_languages_json(r#"["en--US"]"#).is_err()); + } + + #[test] + fn download_wait_timeout_is_explicitly_bounded() { + assert!(validate_download_wait_timeout(MIN_SIRI_DOWNLOAD_WAIT_TIMEOUT).is_ok()); + assert!(validate_download_wait_timeout(MAX_SIRI_DOWNLOAD_WAIT_TIMEOUT).is_ok()); + assert!(validate_download_wait_timeout(Duration::ZERO).is_err()); + assert!(validate_download_wait_timeout( + MAX_SIRI_DOWNLOAD_WAIT_TIMEOUT + Duration::from_secs(1) + ) + .is_err()); + } + + #[test] + fn download_preflights_exact_catalog_identity_before_native_mutation() { + let requested = SiriVoiceIdentity::new("Aaron", "en_US").unwrap(); + let wait = SiriDownloadAvailabilityWait::default(); + let download_calls = Cell::new(0); + let missing = download_voice_with( + &requested, + wait, + |_| { + Ok(SiriVoiceCatalog { + available_languages: vec!["en-US".into()], + voices: vec![SiriVoice { + name: "aaron".into(), + language: "en-US".into(), + size_bytes: 42, + installed: false, + }], + }) + }, + |_, _| { + download_calls.set(download_calls.get() + 1); + Ok(()) + }, + ) + .unwrap_err(); + assert_eq!(missing, SiriVoiceDownloadError::NotFound(requested.clone())); + assert_eq!(download_calls.get(), 0); + + let available = |installed| SiriVoiceCatalog { + available_languages: vec!["en-US".into()], + voices: vec![SiriVoice { + name: "Aaron".into(), + language: "en-US".into(), + size_bytes: 42, + installed, + }], + }; + assert_eq!( + download_voice_with( + &requested, + wait, + |_| Ok(available(true)), + |_, _| { + download_calls.set(download_calls.get() + 1); + Ok(()) + }, + ) + .unwrap(), + requested + ); + assert_eq!(download_calls.get(), 0); + + download_voice_with( + &requested, + wait, + |_| Ok(available(false)), + |identity, duration| { + assert_eq!(identity, &requested); + assert_eq!(duration.as_secs(), wait.seconds()); + download_calls.set(download_calls.get() + 1); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(download_calls.get(), 1); + } + + #[test] + #[cfg(target_os = "macos")] + fn pcm_receive_loop_resets_progress_deadline_and_cancels_a_stall() { + let callback_cancelled = AtomicBool::new(false); + let queue_overflowed = AtomicBool::new(false); + let pending_frames = Arc::new(Mutex::new(VecDeque::new())); + let (sender, receiver) = mpsc::sync_channel(2); + let producer_frames = Arc::clone(&pending_frames); + let producer = std::thread::spawn(move || { + for sample in [0.1, 0.2] { + std::thread::sleep(Duration::from_millis(5)); + producer_frames.lock().unwrap().push_back(sample); + sender.send(()).unwrap(); + } + }); + let mut samples = Vec::new(); + let mut idle_polls = 0; + receive_pcm_until_complete( + receiver, + &callback_cancelled, + &queue_overflowed, + &pending_frames, + Duration::from_millis(2), + Duration::from_secs(1), + &mut |event| { + match event { + TtsSynthesisEvent::Frames(frames) => samples.extend_from_slice(frames), + TtsSynthesisEvent::Poll => idle_polls += 1, + } + Ok(()) + }, + ) + .unwrap(); + producer.join().unwrap(); + assert_eq!(samples, [0.1, 0.2]); + assert!(idle_polls > 0); + assert!(!callback_cancelled.load(Ordering::SeqCst)); + + let (_sender, receiver) = mpsc::sync_channel(1); + let queue_overflowed = AtomicBool::new(false); + let pending_frames = Mutex::new(VecDeque::new()); + let error = receive_pcm_until_complete( + receiver, + &callback_cancelled, + &queue_overflowed, + &pending_frames, + Duration::from_millis(1), + Duration::from_millis(5), + &mut |_| Ok(()), + ) + .unwrap_err(); + assert_eq!(error, "Siri synthesis stopped making progress"); + assert!(callback_cancelled.load(Ordering::SeqCst)); + } + + #[test] + #[cfg(target_os = "macos")] + fn native_pcm_callback_coalesces_tiny_frames_without_blocking_on_playback() { + let active = AtomicBool::new(true); + let callback_cancelled = AtomicBool::new(false); + let queue_overflowed = AtomicBool::new(false); + let pending_frames = Mutex::new(VecDeque::new()); + let (sender, receiver) = mpsc::sync_channel(1); + let (finished_tx, finished_rx) = mpsc::channel(); + std::thread::scope(|scope| { + let worker = scope.spawn(|| { + let mut context = CallbackContext { + active: &active, + callback_cancelled: &callback_cancelled, + queue_overflowed: &queue_overflowed, + frames: &pending_frames, + notification: sender, + }; + let sample = 0.25_f32; + for _ in 0..10_000 { + // SAFETY: The sample slice and callback context remain live for this call. + assert!(unsafe { + receive_pcm( + &sample, + 1, + (&mut context as *mut CallbackContext<'_>).cast(), + ) + }); + } + finished_tx.send(()).unwrap(); + }); + let completed_without_playback = + finished_rx.recv_timeout(Duration::from_millis(50)).is_ok(); + if !completed_without_playback { + drop(receiver); + } + worker.join().unwrap(); + assert!( + completed_without_playback, + "the native Siri callback must copy PCM without waiting for playback credit" + ); + }); + assert_eq!(pending_frames.lock().unwrap().len(), 10_000); + assert!(!queue_overflowed.load(Ordering::SeqCst)); + assert!(!callback_cancelled.load(Ordering::SeqCst)); + } + + #[test] + #[cfg(target_os = "macos")] + fn native_pcm_callback_fails_instead_of_exceeding_its_memory_bound() { + let active = AtomicBool::new(true); + let callback_cancelled = AtomicBool::new(false); + let queue_overflowed = AtomicBool::new(false); + let pending_frames = Mutex::new(VecDeque::from(vec![0.0; MAX_PENDING_PCM_FRAMES])); + let (sender, receiver) = mpsc::sync_channel(1); + let mut context = CallbackContext { + active: &active, + callback_cancelled: &callback_cancelled, + queue_overflowed: &queue_overflowed, + frames: &pending_frames, + notification: sender, + }; + let sample = 0.25_f32; + // SAFETY: The sample and callback context remain live for this call. + assert!(!unsafe { + receive_pcm( + &sample, + 1, + (&mut context as *mut CallbackContext<'_>).cast(), + ) + }); + let error = receive_pcm_until_complete( + receiver, + &callback_cancelled, + &queue_overflowed, + &pending_frames, + Duration::from_millis(1), + Duration::from_secs(1), + &mut |_| Ok(()), + ) + .unwrap_err(); + assert_eq!(error, "Siri synthesis exceeded its bounded PCM queue"); + assert_eq!(pending_frames.lock().unwrap().len(), MAX_PENDING_PCM_FRAMES); + assert!(callback_cancelled.load(Ordering::SeqCst)); + } + + #[test] + #[cfg(target_os = "macos")] + fn native_completion_closes_the_borrowed_pcm_callback_context() { + // SAFETY: This native regression owns its stack canary for the complete + // synchronous call and performs no synthesis, device, or network work. + assert!(unsafe { berd_siri_tts_test_closed_pcm_gate_ignores_late_callback() }); + } + + #[test] + #[cfg(target_os = "macos")] + fn exact_uninstalled_voice_is_rejected_without_synthesis() { + let error = SiriTts::new("en-US", "__berd_voice_does_not_exist__", 1.0).unwrap_err(); + assert!(error.contains("not installed") || error.contains("validating Siri voice")); + } + + #[test] + #[cfg(target_os = "macos")] + #[ignore = "uses the private macOS Siri voice catalog"] + fn native_catalog_returns_normalized_public_identities() { + let catalog = super::load_voice_catalog(None).expect("load Siri catalog"); + assert!(!catalog.available_languages.is_empty()); + for voice in &catalog.voices { + assert_eq!( + SiriVoiceIdentity::new(voice.name.clone(), &voice.language).unwrap(), + voice.identity() + ); + if voice.installed { + super::validate_installed_voice(&voice.identity()) + .expect("catalog-installed voice validates exactly"); + } + } + } + + #[test] + #[cfg(target_os = "macos")] + #[ignore = "requires BERD_SIRI_TEST_VOICE and invokes private sirittsd synthesis"] + fn installed_voice_synthesizes_normalized_pcm_without_an_output_device() { + let voice = std::env::var("BERD_SIRI_TEST_VOICE").unwrap(); + let language = std::env::var("BERD_SIRI_TEST_LANGUAGE").unwrap_or_else(|_| "en-US".into()); + let backend = SiriTts::new(&language, &voice, 1.0).unwrap(); + let mut frames = Vec::new(); + let outcome = backend + .synthesize( + "This is an in-memory Siri synthesis test.", + &AtomicBool::new(true), + &mut |chunk| { + frames.extend_from_slice(chunk); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(outcome, TtsOutcome::Completed); + assert_eq!(backend.pcm_spec().sample_rate, 48_000); + assert!(!frames.is_empty()); + assert!(frames.iter().all(|sample| sample.is_finite())); + } + + #[test] + #[cfg(target_os = "macos")] + #[ignore = "requires BERD_SIRI_TEST_VOICE and invokes private sirittsd synthesis"] + fn cancellation_during_pcm_delivery_returns_promptly() { + let voice = std::env::var("BERD_SIRI_TEST_VOICE").unwrap(); + let language = std::env::var("BERD_SIRI_TEST_LANGUAGE").unwrap_or_else(|_| "en-US".into()); + let backend = SiriTts::new(&language, &voice, 1.0).unwrap(); + let active = AtomicBool::new(true); + let started = Instant::now(); + let outcome = backend + .synthesize( + "This deliberately long sentence keeps Siri synthesis active long enough to test cancellation while decoded audio is crossing the native boundary and must return without leaving the worker or bounded channel stuck.", + &active, + &mut |chunk| { + assert!(!chunk.is_empty()); + active.store(false, Ordering::SeqCst); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(outcome, TtsOutcome::Cancelled); + assert!(started.elapsed() < Duration::from_secs(2)); + } +} diff --git a/src-tauri/crates/berd-voice/src/synthesis.rs b/src-tauri/crates/berd-voice/src/synthesis.rs new file mode 100644 index 000000000..fbe5b9e9e --- /dev/null +++ b/src-tauri/crates/berd-voice/src/synthesis.rs @@ -0,0 +1,283 @@ +use std::io::{Seek, Write}; +use std::sync::atomic::AtomicBool; + +use crate::{TtsBackend, TtsOutcome, TtsSynthesisEvent}; + +const MAX_SYNTHESIS_SECONDS: u64 = 10 * 60; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WavSynthesis { + pub sample_rate: u32, + pub frames: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WavSynthesisErrorKind { + Backend, + Cancelled, + Empty, + InvalidPcm, + TooLong, + Output, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct WavSynthesisError { + pub kind: WavSynthesisErrorKind, + pub detail: String, +} + +impl WavSynthesisError { + fn new(kind: WavSynthesisErrorKind, detail: impl Into) -> Self { + Self { + kind, + detail: detail.into(), + } + } +} + +pub fn synthesize_pcm16_wav( + backend: &dyn TtsBackend, + text: &str, + output: impl Write + Seek, +) -> Result { + let spec = backend.pcm_spec(); + if spec.sample_rate == 0 || !spec.playback_rate.is_finite() || spec.playback_rate != 1.0 { + return Err(WavSynthesisError::new( + WavSynthesisErrorKind::InvalidPcm, + "WAV synthesis requires a nonzero sample rate and playback rate 1.0", + )); + } + let wav_spec = hound::WavSpec { + channels: 1, + sample_rate: spec.sample_rate, + bits_per_sample: 16, + sample_format: hound::SampleFormat::Int, + }; + let mut writer = hound::WavWriter::new(output, wav_spec).map_err(|error| { + WavSynthesisError::new(WavSynthesisErrorKind::Output, error.to_string()) + })?; + let max_frames = u64::from(spec.sample_rate) + .checked_mul(MAX_SYNTHESIS_SECONDS) + .expect("u32 sample rate times ten minutes fits u64"); + let mut frames = 0_u64; + let mut callback_failure = None; + let active = AtomicBool::new(true); + let outcome = backend.synthesize_with_poll(text, &active, &mut |event| { + let TtsSynthesisEvent::Frames(samples) = event else { + return Ok(()); + }; + if samples.is_empty() { + return Ok(()); + } + let incoming = samples.len() as u64; + if frames.saturating_add(incoming) > max_frames { + callback_failure = Some(WavSynthesisError::new( + WavSynthesisErrorKind::TooLong, + "synthesis exceeds ten minutes of source PCM", + )); + return Err("synthesis output is too long".into()); + } + for &sample in samples { + if !sample.is_finite() || !(-1.0..=1.0).contains(&sample) { + callback_failure = Some(WavSynthesisError::new( + WavSynthesisErrorKind::InvalidPcm, + "synthesis produced non-finite or out-of-unit PCM", + )); + return Err("synthesis produced invalid PCM".into()); + } + let quantized = if sample < 0.0 { + (sample * 32_768.0).round() as i16 + } else { + (sample * 32_767.0).round() as i16 + }; + if let Err(error) = writer.write_sample(quantized) { + callback_failure = Some(WavSynthesisError::new( + WavSynthesisErrorKind::Output, + error.to_string(), + )); + return Err("could not write WAV output".into()); + } + } + frames += incoming; + Ok(()) + }); + if let Some(error) = callback_failure { + return Err(error); + } + match outcome { + Err(error) => { + return Err(WavSynthesisError::new( + WavSynthesisErrorKind::Backend, + error, + )) + } + Ok(TtsOutcome::Cancelled) => { + return Err(WavSynthesisError::new( + WavSynthesisErrorKind::Cancelled, + "synthesis was cancelled", + )) + } + Ok(TtsOutcome::Completed) if frames == 0 => { + return Err(WavSynthesisError::new( + WavSynthesisErrorKind::Empty, + "synthesis completed without PCM", + )) + } + Ok(TtsOutcome::Completed) => {} + } + writer.finalize().map_err(|error| { + WavSynthesisError::new(WavSynthesisErrorKind::Output, error.to_string()) + })?; + Ok(WavSynthesis { + sample_rate: spec.sample_rate, + frames, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{TtsOutcome, TtsPcmSpec, TtsSynthesisEvent}; + use std::io::Cursor; + use std::sync::atomic::AtomicBool; + + struct FakeTts { + spec: TtsPcmSpec, + events: Vec>, + outcome: Result, + } + + impl TtsBackend for FakeTts { + fn pcm_spec(&self) -> TtsPcmSpec { + self.spec + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + _on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + unreachable!("WAV synthesis uses lifecycle polling") + } + + fn synthesize_with_poll( + &self, + _text: &str, + _active: &AtomicBool, + on_event: &mut dyn FnMut(TtsSynthesisEvent<'_>) -> Result<(), String>, + ) -> Result { + on_event(TtsSynthesisEvent::Poll)?; + for frames in &self.events { + on_event(TtsSynthesisEvent::Frames(frames))?; + } + self.outcome.clone() + } + } + + fn fake(events: Vec>) -> FakeTts { + FakeTts { + spec: TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + events, + outcome: Ok(TtsOutcome::Completed), + } + } + + fn failure( + spec: TtsPcmSpec, + events: Vec>, + outcome: Result, + ) -> WavSynthesisErrorKind { + synthesize_pcm16_wav( + &FakeTts { + spec, + events, + outcome, + }, + "hello", + Cursor::new(Vec::new()), + ) + .unwrap_err() + .kind + } + + #[test] + fn writes_exact_mono_pcm16_wav_across_callbacks_and_ignores_poll() { + let mut bytes = Cursor::new(Vec::new()); + let result = synthesize_pcm16_wav( + &fake(vec![vec![-1.0, -0.5], vec![], vec![0.0, 0.5, 1.0]]), + "hello", + &mut bytes, + ) + .unwrap(); + assert_eq!(result.sample_rate, 24_000); + assert_eq!(result.frames, 5); + let bytes = bytes.into_inner(); + assert_eq!(&bytes[0..4], b"RIFF"); + assert_eq!(&bytes[8..12], b"WAVE"); + assert_eq!(u16::from_le_bytes(bytes[22..24].try_into().unwrap()), 1); + assert_eq!( + u32::from_le_bytes(bytes[24..28].try_into().unwrap()), + 24_000 + ); + assert_eq!(u16::from_le_bytes(bytes[34..36].try_into().unwrap()), 16); + assert_eq!(u32::from_le_bytes(bytes[40..44].try_into().unwrap()), 10); + let samples = bytes[44..] + .chunks_exact(2) + .map(|sample| i16::from_le_bytes(sample.try_into().unwrap())) + .collect::>(); + assert_eq!(samples, [-32768, -16384, 0, 16384, 32767]); + } + + #[test] + fn rejects_invalid_empty_cancelled_failed_and_oversized_output() { + let spec = TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }; + for invalid in [f32::NAN, f32::INFINITY, -1.01, 1.01] { + assert_eq!( + failure(spec, vec![vec![invalid]], Ok(TtsOutcome::Completed)), + WavSynthesisErrorKind::InvalidPcm + ); + } + assert_eq!( + failure(spec, vec![], Ok(TtsOutcome::Completed)), + WavSynthesisErrorKind::Empty + ); + assert_eq!( + failure(spec, vec![vec![0.1]], Ok(TtsOutcome::Cancelled)), + WavSynthesisErrorKind::Cancelled + ); + assert_eq!( + failure(spec, vec![vec![0.1]], Err("provider failed".into())), + WavSynthesisErrorKind::Backend + ); + assert_eq!( + failure( + TtsPcmSpec { + sample_rate: 1, + playback_rate: 1.0, + }, + vec![vec![0.1; 601]], + Ok(TtsOutcome::Completed), + ), + WavSynthesisErrorKind::TooLong + ); + assert_eq!( + failure( + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 2.0, + }, + vec![vec![0.1]], + Ok(TtsOutcome::Completed), + ), + WavSynthesisErrorKind::InvalidPcm + ); + } +} diff --git a/src-tauri/crates/berd-voice/src/tts.rs b/src-tauri/crates/berd-voice/src/tts.rs new file mode 100644 index 000000000..5c56d8868 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/tts.rs @@ -0,0 +1,265 @@ +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use crate::openai::{stream_openai_pcm, OpenAiPcmOutcome, OpenAiSpeechConfig}; +use crate::{load_pocket_voice_style, load_text_to_speech, PocketTts, VoiceStyle, SAMPLE_RATE}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TtsPcmSpec { + pub sample_rate: u32, + pub playback_rate: f32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TtsOutcome { + Completed, + Cancelled, +} + +pub enum TtsSynthesisEvent<'a> { + Frames(&'a [f32]), + /// A lifecycle polling opportunity while synthesis is blocked waiting for + /// more PCM or a terminal provider result. + Poll, +} + +/// A backend-neutral source of normalized mono, unit-scale Float32 PCM. +/// +/// Turn admission, output-device ownership, buffering, playback, and delivery +/// events remain the session host's responsibility. +pub trait TtsBackend: Send + Sync { + fn pcm_spec(&self) -> TtsPcmSpec; + + fn synthesize( + &self, + text: &str, + active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result; + + /// Synthesizes while allowing backends with blocking waits to yield host + /// lifecycle polling. Backends without such waits keep the simple source + /// contract and use the default implementation. + fn synthesize_with_poll( + &self, + text: &str, + active: &AtomicBool, + on_event: &mut dyn FnMut(TtsSynthesisEvent<'_>) -> Result<(), String>, + ) -> Result { + self.synthesize(text, active, &mut |frames| { + on_event(TtsSynthesisEvent::Frames(frames)) + }) + } +} + +pub struct OpenAiTts { + client: reqwest::Client, + config: OpenAiSpeechConfig, +} + +pub struct PocketTtsBackend { + engine: PocketTts, + style: VoiceStyle, + playback_rate: f32, +} + +impl PocketTtsBackend { + pub fn new(model_dir: &Path, voice_id: &str, playback_rate: f32) -> Result { + if !playback_rate.is_finite() || !(0.75..=2.0).contains(&playback_rate) { + return Err("Pocket rate must be between 0.75 and 2.0".into()); + } + let model_dir_str = model_dir.to_str().ok_or_else(|| { + format!( + "Pocket model path is not valid UTF-8: {}", + model_dir.display() + ) + })?; + let style = load_pocket_voice_style(model_dir, voice_id)?; + let engine = load_text_to_speech(model_dir_str)?; + Ok(Self { + engine, + style, + playback_rate, + }) + } +} + +impl TtsBackend for PocketTtsBackend { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: SAMPLE_RATE, + playback_rate: self.playback_rate, + } + } + + fn synthesize( + &self, + text: &str, + active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + if !active.load(Ordering::SeqCst) { + return Ok(TtsOutcome::Cancelled); + } + let mut callback_error = None; + let completed = + self.engine + .synth_chunk_streaming(text, &self.style, 12, &mut |frames| { + if !active.load(Ordering::SeqCst) { + return false; + } + if frames.is_empty() { + return true; + } + match on_frames(&frames) { + Ok(()) => true, + Err(error) => { + callback_error = Some(error); + false + } + } + })?; + if let Some(error) = callback_error { + return Err(error); + } + Ok(if completed && active.load(Ordering::SeqCst) { + TtsOutcome::Completed + } else { + TtsOutcome::Cancelled + }) + } +} + +impl OpenAiTts { + pub fn new(config: OpenAiSpeechConfig) -> Result { + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(15)) + .read_timeout(Duration::from_secs(120)) + .build() + .map_err(|error| error.to_string())?; + Ok(Self { client, config }) + } +} + +impl TtsBackend for OpenAiTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + } + } + + fn synthesize( + &self, + text: &str, + active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + if !active.load(Ordering::SeqCst) { + return Ok(TtsOutcome::Cancelled); + } + let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; + match runtime.block_on(stream_openai_pcm( + &self.client, + &self.config, + text, + active, + on_frames, + ))? { + OpenAiPcmOutcome::Completed => Ok(TtsOutcome::Completed), + OpenAiPcmOutcome::Cancelled => Ok(TtsOutcome::Cancelled), + } + } +} + +#[cfg(test)] +mod tests { + use super::{PocketTtsBackend, TtsBackend, TtsOutcome, TtsPcmSpec}; + use std::path::Path; + use std::sync::atomic::{AtomicBool, Ordering}; + + struct FakeTts; + + impl TtsBackend for FakeTts { + fn pcm_spec(&self) -> TtsPcmSpec { + TtsPcmSpec { + sample_rate: 16_000, + playback_rate: 1.25, + } + } + + fn synthesize( + &self, + _text: &str, + _active: &AtomicBool, + on_frames: &mut dyn FnMut(&[f32]) -> Result<(), String>, + ) -> Result { + on_frames(&[0.0, 0.5])?; + Ok(TtsOutcome::Completed) + } + } + + #[test] + fn trait_keeps_engine_pcm_separate_from_output_policy() { + let backend: &dyn TtsBackend = &FakeTts; + let mut received = Vec::new(); + let outcome = backend + .synthesize("hello", &AtomicBool::new(true), &mut |frames| { + received.extend_from_slice(frames); + Ok(()) + }) + .unwrap(); + assert_eq!(backend.pcm_spec().sample_rate, 16_000); + assert_eq!(outcome, TtsOutcome::Completed); + assert_eq!(received, [0.0, 0.5]); + } + + #[test] + #[ignore = "requires BERD_POCKET_TEST_MODEL_DIR with a complete Pocket bundle"] + fn pocket_backend_synthesizes_in_memory_and_cancels() { + let model_dir = std::env::var("BERD_POCKET_TEST_MODEL_DIR").unwrap(); + let voice = std::env::var("BERD_POCKET_TEST_VOICE").unwrap_or_else(|_| "george".into()); + let backend = PocketTtsBackend::new(Path::new(&model_dir), &voice, 1.0).unwrap(); + let active = AtomicBool::new(true); + let mut frames = Vec::new(); + assert_eq!( + backend + .synthesize("Pocket synthesis works.", &active, &mut |chunk| { + frames.extend_from_slice(chunk); + Ok(()) + }) + .unwrap(), + TtsOutcome::Completed + ); + assert!(!frames.is_empty()); + assert!(frames.iter().all(|sample| sample.is_finite())); + + let active = AtomicBool::new(true); + let mut received = false; + assert_eq!( + backend + .synthesize( + "This longer sentence is cancelled during local inference.", + &active, + &mut |chunk| { + received |= !chunk.is_empty(); + active.store(false, Ordering::SeqCst); + Ok(()) + }, + ) + .unwrap(), + TtsOutcome::Cancelled + ); + assert!(received); + + let error = backend + .synthesize( + "Pocket output errors are preserved.", + &AtomicBool::new(true), + &mut |_chunk| Err("fake output failed".into()), + ) + .unwrap_err(); + assert_eq!(error, "fake output failed"); + } +} diff --git a/src-tauri/crates/berd-voice/swift/BerdMacSpeechBridge/Package.swift b/src-tauri/crates/berd-voice/swift/BerdMacSpeechBridge/Package.swift new file mode 100644 index 000000000..50e3b047c --- /dev/null +++ b/src-tauri/crates/berd-voice/swift/BerdMacSpeechBridge/Package.swift @@ -0,0 +1,23 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "BerdMacSpeechBridge", + platforms: [.macOS(.v14)], + products: [ + .library( + name: "BerdMacSpeechBridge", + type: .static, + targets: ["BerdMacSpeechBridge"] + ) + ], + targets: [ + .target( + name: "BerdMacSpeechBridge", + linkerSettings: [ + .linkedFramework("AVFoundation"), + .linkedFramework("Speech"), + ] + ) + ] +) diff --git a/src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdMacSpeechBridge.swift b/src-tauri/crates/berd-voice/swift/BerdMacSpeechBridge/Sources/BerdMacSpeechBridge/BerdMacSpeechBridge.swift similarity index 100% rename from src-tauri/swift/BerdAirPodsBridge/Sources/BerdAirPodsBridge/BerdMacSpeechBridge.swift rename to src-tauri/crates/berd-voice/swift/BerdMacSpeechBridge/Sources/BerdMacSpeechBridge/BerdMacSpeechBridge.swift diff --git a/src-tauri/crates/berd-voice/tests/management_cli.rs b/src-tauri/crates/berd-voice/tests/management_cli.rs new file mode 100644 index 000000000..180d2718d --- /dev/null +++ b/src-tauri/crates/berd-voice/tests/management_cli.rs @@ -0,0 +1,267 @@ +use std::process::Command; + +use serde_json::Value; + +fn berd_voice(args: &[&str]) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_berd-voice")) + .args(args) + .output() + .expect("run berd-voice") +} + +#[test] +fn management_usage_errors_are_exit_two_and_do_not_emit_json() { + for args in [ + vec!["voices", "download", "--voice", "Aaron"], + vec![ + "voices", + "download", + "--voice", + "Aaron", + "--language", + "en-US", + "--availability-wait-seconds", + "0", + ], + vec!["models", "macos", "status", "extra"], + vec!["models", "pocket", "status"], + vec!["models", "parakeet", "install", "--store-root", "relative"], + vec!["models", "pocket", "status", "--store-root", "/tmp/./store"], + ] { + let output = berd_voice(&args); + assert_eq!(output.status.code(), Some(2), "{args:?}"); + assert!(output.stdout.is_empty(), "{args:?}"); + assert!( + String::from_utf8(output.stderr) + .expect("UTF-8 stderr") + .contains("usage:"), + "{args:?}" + ); + } +} + +#[test] +fn local_model_management_is_process_stable_without_network_or_root_creation() { + let temporary = tempfile::tempdir().expect("temporary directory"); + let missing_store = temporary.path().join("missing-store"); + let missing_store_arg = missing_store.to_str().expect("UTF-8 path"); + + for (engine, model_id) in [ + ("pocket", "native-voice-v2"), + ("parakeet", "parakeet-tdt-ctc-110m-en-int8"), + ] { + let output = berd_voice(&[ + "models", + engine, + "status", + "--store-root", + missing_store_arg, + ]); + assert!( + output.status.success(), + "{engine}: {}", + String::from_utf8_lossy(&output.stderr) + ); + let lines = String::from_utf8(output.stdout).expect("UTF-8 stdout"); + let lines = lines.lines().collect::>(); + assert_eq!(lines.len(), 1, "{engine}"); + let value: Value = serde_json::from_str(lines[0]).expect("JSON result"); + assert_eq!(value["schemaVersion"], 1); + assert_eq!(value["operation"], format!("models.{engine}.status")); + assert_eq!(value["event"], "result"); + assert_eq!(value["result"]["modelId"], model_id); + assert_eq!(value["result"]["state"], "missing"); + assert_eq!(value["result"]["ready"], false); + assert!(value["result"]["verifiedBytes"].is_null()); + assert!(value["result"]["totalDownloadBytes"].as_u64().unwrap() > 0); + assert!(output.stderr.is_empty()); + assert!( + !missing_store.exists(), + "status created {missing_store_arg}" + ); + } + + let output = berd_voice(&["models", "pocket", "voices"]); + assert!(output.status.success()); + let lines = String::from_utf8(output.stdout).expect("UTF-8 stdout"); + let lines = lines.lines().collect::>(); + assert_eq!(lines.len(), 1); + let value: Value = serde_json::from_str(lines[0]).expect("JSON result"); + assert_eq!(value["operation"], "models.pocket.voices"); + assert_eq!(value["result"]["voiceLicenseId"], "CC-BY-4.0"); + let voices = value["result"]["voices"].as_array().unwrap(); + assert_eq!(voices.len(), 12); + assert!(voices.iter().all(|voice| { + voice.as_object().is_some_and(|voice| { + voice.len() == 2 && voice.contains_key("id") && voice.contains_key("name") + }) + })); + + for (engine, relative_file) in [ + ("pocket", "native-voice-v2/bundle.json"), + ("parakeet", "native-voice-v2/stt/model.int8.onnx"), + ] { + let invalid_store = temporary.path().join(format!("invalid-{engine}")); + let invalid_file = invalid_store.join(relative_file); + std::fs::create_dir_all(invalid_file.parent().unwrap()).expect("create invalid bundle"); + std::fs::write(invalid_file, b"invalid").expect("write invalid bundle"); + let output = berd_voice(&[ + "models", + engine, + "status", + "--store-root", + invalid_store.to_str().expect("UTF-8 path"), + ]); + assert!(output.status.success()); + let value: Value = serde_json::from_slice(&output.stdout).expect("JSON result"); + assert_eq!(value["result"]["state"], "invalid"); + assert_eq!(value["result"]["ready"], false); + assert!(value["result"]["verifiedBytes"].is_null()); + } + + let blocked_store = temporary.path().join("blocked-store"); + std::fs::write(&blocked_store, b"not a directory").expect("write blocked store"); + for engine in ["pocket", "parakeet"] { + let output = berd_voice(&[ + "models", + engine, + "install", + "--store-root", + blocked_store.to_str().expect("UTF-8 path"), + ]); + assert_eq!(output.status.code(), Some(1)); + let lines = String::from_utf8(output.stdout).expect("UTF-8 stdout"); + let lines = lines.lines().collect::>(); + assert_eq!(lines.len(), 1, "{engine}"); + let value: Value = serde_json::from_str(lines[0]).expect("JSON error"); + assert_eq!(value["operation"], format!("models.{engine}.install")); + assert_eq!(value["event"], "error"); + assert_eq!(value["error"]["code"], "io_failed"); + assert!(!output.stderr.is_empty()); + } +} + +#[cfg(not(target_os = "macos"))] +#[test] +fn unsupported_platform_management_contract_is_process_stable() { + for (args, exit, operation, event, supported_or_code) in [ + (vec!["voices", "list"], 0, "voices.list", "result", "false"), + ( + vec!["models", "macos", "status"], + 0, + "models.macos.status", + "result", + "false", + ), + ( + vec![ + "voices", + "download", + "--voice", + "Aaron", + "--language", + "en-US", + ], + 1, + "voices.download", + "error", + "unsupported", + ), + ( + vec!["models", "macos", "install"], + 1, + "models.macos.install", + "error", + "unsupported", + ), + ] { + let output = berd_voice(&args); + assert_eq!(output.status.code(), Some(exit), "{args:?}"); + let stdout = String::from_utf8(output.stdout).expect("UTF-8 stdout"); + let lines = stdout.lines().collect::>(); + assert_eq!(lines.len(), 1, "{args:?}"); + let value: Value = serde_json::from_str(lines[0]).expect("JSON terminal"); + assert_eq!(value["schemaVersion"], 1); + assert_eq!(value["operation"], operation); + assert_eq!(value["event"], event); + if event == "result" { + assert_eq!(value["result"]["supported"], false); + assert!(output.stderr.is_empty()); + } else { + assert_eq!(value["error"]["code"], supported_or_code); + assert!(!output.stderr.is_empty()); + } + } +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "safe opt-in probe of the native Siri catalog; no download, synthesis, or playback"] +fn native_voice_list_emits_one_terminal_json_line() { + let output = berd_voice(&["voices", "list", "--language", "en_US"]); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let lines = String::from_utf8(output.stdout).expect("UTF-8 stdout"); + let lines = lines.lines().collect::>(); + assert_eq!(lines.len(), 1); + let value: Value = serde_json::from_str(lines[0]).expect("JSON result"); + assert_eq!(value["schemaVersion"], 1); + assert_eq!(value["operation"], "voices.list"); + assert_eq!(value["event"], "result"); + assert_eq!(value["result"]["supported"], true); + assert!(output.stderr.is_empty()); + for voice in value["result"]["voices"].as_array().expect("voice array") { + assert_eq!(voice["language"], "en-US"); + } +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "safe opt-in exact-catalog miss; proves failure before download mutation"] +fn native_missing_voice_emits_one_not_found_terminal() { + let output = berd_voice(&[ + "voices", + "download", + "--voice", + "__berd_voice_missing__", + "--language", + "en-US", + "--availability-wait-seconds", + "1", + ]); + assert_eq!(output.status.code(), Some(1)); + let lines = String::from_utf8(output.stdout).expect("UTF-8 stdout"); + let lines = lines.lines().collect::>(); + assert_eq!(lines.len(), 1); + let value: Value = serde_json::from_str(lines[0]).expect("JSON error"); + assert_eq!(value["schemaVersion"], 1); + assert_eq!(value["operation"], "voices.download"); + assert_eq!(value["event"], "error"); + assert_eq!(value["error"]["code"], "voice_not_found"); + assert!(!output.stderr.is_empty()); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "safe opt-in probe of native SpeechTranscriber status; no installation or audio"] +fn native_macos_model_status_emits_one_terminal_json_line() { + let output = berd_voice(&["models", "macos", "status"]); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let lines = String::from_utf8(output.stdout).expect("UTF-8 stdout"); + let lines = lines.lines().collect::>(); + assert_eq!(lines.len(), 1); + let value: Value = serde_json::from_str(lines[0]).expect("JSON result"); + assert_eq!(value["schemaVersion"], 1); + assert_eq!(value["operation"], "models.macos.status"); + assert_eq!(value["event"], "result"); + assert!(value["result"]["supported"].is_boolean()); + assert!(value["result"]["ready"].is_boolean()); + assert!(output.stderr.is_empty()); +} diff --git a/src-tauri/crates/berd-voice/tests/session_protocol.rs b/src-tauri/crates/berd-voice/tests/session_protocol.rs new file mode 100644 index 000000000..f029ff6a5 --- /dev/null +++ b/src-tauri/crates/berd-voice/tests/session_protocol.rs @@ -0,0 +1,515 @@ +use std::fs::File; +use std::io::{BufRead, BufReader, Read, Write}; +use std::os::fd::{AsRawFd, FromRawFd}; +use std::os::unix::net::UnixStream; +use std::os::unix::process::CommandExt; +use std::process::{Child, Command, Stdio}; +use std::sync::{mpsc, Arc, Mutex}; +use std::time::Duration; + +use serde_json::{json, Value}; + +struct ChildGuard(Option); + +impl Drop for ChildGuard { + fn drop(&mut self) { + if let Some(child) = self.0.as_mut() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +fn write_session_json(writer: &mut impl Write, value: &Value) { + let payload = serde_json::to_vec(value).unwrap(); + writer.write_all(b"BV").unwrap(); + writer.write_all(&[2, 1]).unwrap(); + writer + .write_all(&(payload.len() as u32).to_le_bytes()) + .unwrap(); + writer.write_all(&payload).unwrap(); +} + +fn session_command() -> (Command, File, UnixStream) { + let (pcm, host) = UnixStream::pair().unwrap(); + let source_fd = unsafe { libc::fcntl(pcm.as_raw_fd(), libc::F_DUPFD_CLOEXEC, 64) }; + assert!(source_fd >= 64); + let inherited = unsafe { File::from_raw_fd(source_fd) }; + let mut command = Command::new(env!("CARGO_BIN_EXE_berd-voice")); + command.args(["session", "--pcm-output-fd", "9"]); + unsafe { + command.pre_exec(move || { + if libc::dup2(source_fd, 9) < 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::fcntl(9, libc::F_SETFD, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + (command, inherited, host) +} + +fn spawn_audio_host( + mut reader: UnixStream, + stdin: Arc>, +) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let mut current = None::<(u64, u64, u64)>; + loop { + let mut header = [0_u8; 8]; + match reader.read_exact(&mut header) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => return, + Err(error) => panic!("audio pipe read failed: {error}"), + } + assert_eq!(&header[..2], b"BA"); + assert_eq!(header[2], 2); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + let mut payload = vec![0_u8; length]; + reader.read_exact(&mut payload).unwrap(); + let message = match header[3] { + 1 => { + assert_eq!(payload.len(), 16); + let speech_id = u64::from_le_bytes(payload[..8].try_into().unwrap()); + current = Some((speech_id, 0, 0)); + json!({"type":"audio_begin_accepted","speech_id":speech_id}) + } + 2 => { + assert!(payload.len() >= 20); + let speech_id = u64::from_le_bytes(payload[..8].try_into().unwrap()); + let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); + let frames = u64::try_from((payload.len() - 16) / 4).unwrap(); + let state = current.as_mut().expect("chunk follows begin"); + assert_eq!(state.0, speech_id); + assert_eq!(sequence, state.1 + 1); + state.1 = sequence; + state.2 += frames; + let mut writer = stdin.lock().unwrap(); + write_session_json( + &mut *writer, + &json!({"type":"audio_chunk_accepted","speech_id":speech_id,"sequence":sequence}), + ); + write_session_json( + &mut *writer, + &json!({"type":"audio_played","speech_id":speech_id,"played_frames":state.2}), + ); + writer.flush().unwrap(); + continue; + } + 3 => { + assert_eq!(payload.len(), 24); + let speech_id = u64::from_le_bytes(payload[..8].try_into().unwrap()); + let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); + let frames = u64::from_le_bytes(payload[16..24].try_into().unwrap()); + assert_eq!(current, Some((speech_id, sequence, frames))); + current = None; + json!({"type":"audio_drained","speech_id":speech_id,"sequence":sequence,"played_frames":frames}) + } + 4 => { + assert_eq!(payload.len(), 8); + let speech_id = u64::from_le_bytes(payload.try_into().unwrap()); + let played_frames = current + .take() + .filter(|state| state.0 == speech_id) + .map_or(0, |state| state.2); + json!({"type":"audio_cancelled","speech_id":speech_id,"played_frames":played_frames}) + } + kind => panic!("unknown audio record kind {kind}"), + }; + let mut writer = stdin.lock().unwrap(); + write_session_json(&mut *writer, &message); + writer.flush().unwrap(); + } + }) +} + +#[test] +fn session_rejects_a_read_only_pcm_descriptor_before_hello() { + let mut descriptors = [-1; 2]; + assert_eq!(unsafe { libc::pipe(descriptors.as_mut_ptr()) }, 0); + let read = unsafe { File::from_raw_fd(descriptors[0]) }; + let write_guard = unsafe { File::from_raw_fd(descriptors[1]) }; + let source_fd = unsafe { libc::fcntl(read.as_raw_fd(), libc::F_DUPFD_CLOEXEC, 64) }; + assert!(source_fd >= 64); + let read_guard = unsafe { File::from_raw_fd(source_fd) }; + let mut command = Command::new(env!("CARGO_BIN_EXE_berd-voice")); + command + .args(["session", "--pcm-output-fd", "9", "--tts-backend", "openai"]) + .env("OPENAI_API_KEY", "test-key-not-used") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + unsafe { + command.pre_exec(move || { + if libc::dup2(source_fd, 9) < 0 || libc::fcntl(9, libc::F_SETFD, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let child = command.spawn().unwrap(); + drop(read_guard); + drop(write_guard); + let output = child.wait_with_output().unwrap(); + assert_eq!(output.status.code(), Some(1)); + assert!(output.stdout.is_empty()); + assert!(String::from_utf8_lossy(&output.stderr) + .contains("PCM output file descriptor is not writable")); +} + +#[test] +fn framed_hello_reports_input_initialization_failure_before_ready() { + let missing = std::env::temp_dir().join(format!( + "berd-voice-missing-parakeet-{}", + std::process::id() + )); + assert!(!missing.exists(), "test path must remain absent"); + let (mut command, _pcm, _host) = session_command(); + let mut child = command + .args([ + "--tts-backend", + "openai", + "--stt-backend", + "parakeet", + "--stt-model-dir", + missing.to_str().unwrap(), + ]) + .env("OPENAI_API_KEY", "test-key-not-used-before-synthesis") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + write_session_json( + &mut stdin, + &json!({"type":"hello","id":1,"input_during_tts":"allow_barge_in"}), + ); + stdin.flush().unwrap(); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + drop(stdin); + assert!(child.wait().unwrap().success()); + let message: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(message["type"], "fatal"); + assert!(!message["message"].as_str().unwrap().is_empty()); +} + +#[test] +#[ignore = "requires an installed Siri voice and current-locale macOS SpeechTranscriber model"] +fn siri_session_reaches_ready_without_openai_credentials() { + let voice = std::env::var("BERD_SIRI_TEST_VOICE").unwrap(); + let language = std::env::var("BERD_SIRI_TEST_LANGUAGE").unwrap_or_else(|_| "en-US".into()); + let (mut command, _pcm, _host) = session_command(); + let mut child = command + .args([ + "--tts-backend", + "siri", + "--voice", + &voice, + "--language", + &language, + ]) + .env_remove("OPENAI_API_KEY") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + let mut receive = || { + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + serde_json::from_str::(&line).unwrap() + }; + write_session_json( + &mut stdin, + &json!({"type":"hello","id":1,"input_during_tts":"allow_barge_in"}), + ); + stdin.flush().unwrap(); + let ready = receive(); + assert_eq!(ready["type"], "ready"); + assert_eq!(ready["protocol"], 2); + assert_eq!(ready["session"]["tts"]["backend"], "siri"); + assert_eq!(ready["session"]["tts"]["voice"], voice); + assert_eq!(ready["session"]["tts"]["language"], language); + assert_eq!(ready["session"]["tts"]["rate"], 1.0); + assert_eq!( + ready["session"]["input_during_tts"], + json!({"revision":1,"policy":"allow_barge_in"}) + ); + write_session_json( + &mut stdin, + &json!({ + "type":"set_input_during_tts", + "id":20, + "expected_revision":1, + "policy":"suppress_input" + }), + ); + stdin.flush().unwrap(); + assert_eq!( + receive(), + json!({ + "type":"input_during_tts_result", + "id":20, + "outcome":"applied", + "snapshot":{"revision":2,"policy":"suppress_input"} + }) + ); + write_session_json( + &mut stdin, + &json!({ + "type":"set_input_during_tts", + "id":21, + "expected_revision":1, + "policy":"allow_barge_in" + }), + ); + stdin.flush().unwrap(); + assert_eq!( + receive(), + json!({ + "type":"input_during_tts_result", + "id":21, + "outcome":"rejected", + "snapshot":{"revision":2,"policy":"suppress_input"} + }) + ); + write_session_json( + &mut stdin, + &json!({ + "type":"set_tts_settings", + "id":2, + "expected_revision":1, + "settings":{ + "backend":"siri", + "voice":voice, + "language":language, + "rate":2.0 + } + }), + ); + stdin.flush().unwrap(); + let applied = receive(); + assert_eq!(applied["type"], "tts_settings_result"); + assert_eq!(applied["id"], 2); + assert_eq!(applied["outcome"], "applied"); + assert_eq!(applied["snapshot"]["revision"], 2); + assert_eq!(applied["snapshot"]["rate"], 2.0); + assert!(applied.get("message").is_none()); + write_session_json(&mut stdin, &json!({"type":"shutdown"})); + drop(stdin); + assert!(child.wait().unwrap().success()); +} + +#[test] +#[ignore = "requires a Pocket bundle and current-locale macOS SpeechTranscriber model"] +fn pocket_session_reaches_ready_without_openai_credentials() { + let model_dir = std::env::var("BERD_POCKET_TEST_MODEL_DIR").unwrap(); + let voice = std::env::var("BERD_POCKET_TEST_VOICE").unwrap_or_else(|_| "george".into()); + let (mut command, _pcm, _host) = session_command(); + let mut child = command + .args([ + "--tts-backend", + "pocket", + "--model-dir", + &model_dir, + "--voice", + &voice, + ]) + .env_remove("OPENAI_API_KEY") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + write_session_json( + &mut stdin, + &json!({"type":"hello","id":1,"input_during_tts":"allow_barge_in"}), + ); + write_session_json(&mut stdin, &json!({"type":"shutdown"})); + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let messages: Vec = String::from_utf8(output.stdout) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0]["type"], "ready"); + assert_eq!(messages[0]["session"]["tts"]["backend"], "pocket"); + assert_eq!(messages[0]["session"]["tts"]["voice"], voice); + assert_eq!(messages[0]["session"]["tts"]["rate"], 1.0); +} + +#[test] +#[ignore = "requires installed Siri voice and current-locale macOS SpeechTranscriber model"] +fn explicit_macos_stt_session_reaches_ready_without_audio() { + let voice = std::env::var("BERD_SIRI_TEST_VOICE").unwrap(); + let language = std::env::var("BERD_SIRI_TEST_LANGUAGE").unwrap_or_else(|_| "en-US".into()); + let (mut command, _pcm, _host) = session_command(); + let mut child = command + .args([ + "--tts-backend", + "siri", + "--voice", + &voice, + "--language", + &language, + "--stt-backend", + "macos", + ]) + .env_remove("OPENAI_API_KEY") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + write_session_json( + &mut stdin, + &json!({"type":"hello","id":1,"input_during_tts":"allow_barge_in"}), + ); + write_session_json(&mut stdin, &json!({"type":"shutdown"})); + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let messages: Vec = String::from_utf8(output.stdout) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0]["type"], "ready"); + assert_eq!(messages[0]["session"]["tts"]["backend"], "siri"); +} + +#[test] +#[ignore = "requires Siri voice and current-locale macOS SpeechTranscriber model"] +fn siri_remote_output_supports_consecutive_turns_and_cancellation() { + let voice = std::env::var("BERD_SIRI_TEST_VOICE").unwrap(); + let language = std::env::var("BERD_SIRI_TEST_LANGUAGE").unwrap_or_else(|_| "en-US".into()); + let (mut command, pcm_guard, host) = session_command(); + let child = command + .args([ + "--tts-backend", + "siri", + "--voice", + &voice, + "--language", + &language, + "--rate", + "1.5", + ]) + .env_remove("OPENAI_API_KEY") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + drop(pcm_guard); + let mut child = ChildGuard(Some(child)); + let child_process = child.0.as_mut().unwrap(); + let stdin = Arc::new(Mutex::new(child_process.stdin.take().unwrap())); + let audio_host = spawn_audio_host(host, Arc::clone(&stdin)); + let stdout = child_process.stdout.take().unwrap(); + let (sender, receiver) = mpsc::channel(); + let reader = std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + if sender.send(line).is_err() { + break; + } + } + }); + let send = |message: Value| { + let mut stdin = stdin.lock().unwrap(); + write_session_json(&mut *stdin, &message); + stdin.flush().unwrap(); + }; + let receive = || -> Value { + let line = receiver + .recv_timeout(Duration::from_secs(10)) + .expect("child did not emit a bounded response") + .unwrap(); + serde_json::from_str(&line).unwrap() + }; + + send(json!({"type":"hello","id":1,"input_during_tts":"allow_barge_in"})); + assert_eq!(receive()["type"], "ready"); + + send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":null, + "text":"First completed turn." + })); + let first = receive(); + assert_eq!(first["type"], "admitted"); + let first_speech_id = first["speech_id"].as_u64().unwrap(); + send(json!({"type":"output_ready","id":2,"speech_id":first_speech_id})); + assert_eq!(receive()["type"], "output_ready_result"); + assert_eq!(receive()["type"], "speech_started"); + let first_terminal = receive(); + assert_eq!( + first_terminal["type"], "speech_completed", + "unexpected first terminal: {first_terminal}" + ); + + send(json!({ + "type":"prepare_speak", + "id":3, + "acknowledgement":null, + "text":"This deliberately long Siri phrase keeps queued output active until playback completes and the session must promptly return to idle." + })); + let admitted = receive(); + assert_eq!(admitted["type"], "admitted"); + let speech_id = admitted["speech_id"].as_u64().unwrap(); + send(json!({"type":"output_ready","id":3,"speech_id":speech_id})); + assert_eq!(receive()["type"], "output_ready_result"); + assert_eq!(receive()["type"], "speech_started"); + assert_eq!(receive()["type"], "speech_completed"); + + send(json!({ + "type":"prepare_speak", + "id":4, + "acknowledgement":null, + "text":"This second deliberately long Siri phrase stays active until targeted cancellation interrupts playback and the session must promptly return to idle." + })); + let interruptible = receive(); + assert_eq!(interruptible["type"], "admitted"); + let speech_id = interruptible["speech_id"].as_u64().unwrap(); + send(json!({"type":"output_ready","id":4,"speech_id":speech_id})); + assert_eq!(receive()["type"], "output_ready_result"); + assert_eq!(receive()["type"], "speech_started"); + send(json!({"type":"cancel","id":4})); + assert_eq!(receive()["type"], "cancel_result"); + assert_eq!(receive()["type"], "speech_interrupted"); + + send(json!({"type":"prepare_speak","id":5,"acknowledgement":null,"text":"next"})); + let next = receive(); + assert_eq!(next["type"], "admitted"); + send(json!({"type":"cancel","id":5})); + assert_eq!(receive()["type"], "cancel_result"); + assert_eq!(receive()["type"], "speech_interrupted"); + send(json!({"type":"shutdown"})); + drop(stdin); + let mut child_process = child.0.take().unwrap(); + assert!(child_process.wait().unwrap().success()); + reader.join().unwrap(); + audio_host.join().unwrap(); +} diff --git a/src-tauri/crates/berd-voice/tests/synthesis_cli.rs b/src-tauri/crates/berd-voice/tests/synthesis_cli.rs new file mode 100644 index 000000000..c3ce89059 --- /dev/null +++ b/src-tauri/crates/berd-voice/tests/synthesis_cli.rs @@ -0,0 +1,131 @@ +use std::process::Command; + +use serde_json::Value; + +fn berd_voice(args: &[&str]) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_berd-voice")) + .args(args) + .env_remove("OPENAI_API_KEY") + .env_remove("OPENAI_BASE_URL") + .env_remove("OPENAI_TTS_MODEL") + .env_remove("OPENAI_TTS_VOICE") + .output() + .expect("run berd-voice") +} + +fn terminal(output: &std::process::Output) -> Value { + let stdout = String::from_utf8(output.stdout.clone()).expect("UTF-8 stdout"); + let lines = stdout.lines().collect::>(); + assert_eq!(lines.len(), 1, "{stdout:?}"); + serde_json::from_str(lines[0]).expect("JSON terminal") +} + +#[test] +fn synthesis_usage_errors_are_exit_two_and_emit_no_json() { + for args in [ + vec![ + "synthesize", + "--tts-backend", + "openai", + "--model", + "gpt-test", + "--voice", + "marin", + "--text", + "hello", + "--output", + "voice.wav", + ], + vec![ + "synthesize", + "--tts-backend", + "pocket", + "--model-dir", + "/models/pocket", + "--voice", + "mary", + "--rate", + "2", + "--text", + "hello", + "--output", + "voice.wav", + ], + vec![ + "synthesize", + "--tts-backend", + "siri", + "--voice", + "Aaron", + "--language", + "en-US", + "--text", + "hello", + "--output", + "-", + ], + ] { + let output = berd_voice(&args); + assert_eq!(output.status.code(), Some(2), "{args:?}"); + assert!(output.stdout.is_empty(), "{args:?}"); + assert!(String::from_utf8_lossy(&output.stderr).contains("usage:")); + } +} + +#[test] +fn existing_output_fails_before_openai_credentials_or_any_request() { + let directory = tempfile::tempdir().expect("temporary directory"); + let path = directory.path().join("owned.wav"); + std::fs::write(&path, b"owned").expect("write target"); + let output = berd_voice(&[ + "synthesize", + "--tts-backend", + "openai", + "--model", + "gpt-test", + "--voice", + "marin", + "--allow-paid-openai", + "--text", + "private prompt", + "--output", + path.to_str().expect("UTF-8 path"), + ]); + assert_eq!(output.status.code(), Some(1)); + let value = terminal(&output); + assert_eq!(value["schemaVersion"], 1); + assert_eq!(value["operation"], "synthesize"); + assert_eq!(value["event"], "error"); + assert_eq!(value["error"]["code"], "output_unavailable"); + assert_eq!(std::fs::read(path).unwrap(), b"owned"); + assert!(!String::from_utf8_lossy(&output.stdout).contains("private prompt")); +} + +#[test] +fn missing_openai_key_is_one_sanitized_terminal_and_leaves_no_artifact() { + let directory = tempfile::tempdir().expect("temporary directory"); + let path = directory.path().join("voice.wav"); + let output = berd_voice(&[ + "synthesize", + "--tts-backend", + "openai", + "--model", + "gpt-test", + "--voice", + "marin", + "--allow-paid-openai", + "--text", + "private prompt", + "--output", + path.to_str().expect("UTF-8 path"), + ]); + assert_eq!(output.status.code(), Some(1)); + let value = terminal(&output); + assert_eq!(value["error"]["code"], "backend_unavailable"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!stdout.contains("private prompt")); + assert!(!stdout.contains(path.to_str().unwrap())); + assert!(!path.exists()); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + assert!(String::from_utf8_lossy(&output.stderr).contains("OPENAI_API_KEY is required")); +} diff --git a/src-tauri/src/commands/mac_speech.rs b/src-tauri/src/commands/mac_speech.rs index 243e0a5ec..e50de3988 100644 --- a/src-tauri/src/commands/mac_speech.rs +++ b/src-tauri/src/commands/mac_speech.rs @@ -10,7 +10,6 @@ use tauri::Emitter; #[cfg(target_os = "macos")] const STATUS_EVENT: &str = "mac-speech:status"; static STATUS_REVISION: AtomicU64 = AtomicU64::new(0); -pub const RECOGNITION_FINISH_TIMEOUT_SECONDS: u64 = 5; #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -26,234 +25,6 @@ pub struct MacSpeechStatus { pub revision: u64, } -#[cfg(target_os = "macos")] -mod bridge { - use std::{ - ffi::{c_char, c_void, CStr}, - ptr, - }; - - use serde::Deserialize; - use tokio::sync::mpsc; - - #[derive(Debug, Deserialize)] - #[serde(rename_all = "camelCase")] - struct BridgeStatus { - supported: bool, - locale: Option, - locale_supported: bool, - model_status: String, - ready: bool, - } - - #[derive(Debug)] - pub enum RecognitionEvent { - Final(String), - Finished, - Failed(String), - } - - unsafe extern "C" { - fn berd_macos_stt_is_supported() -> bool; - fn berd_macos_stt_status_json( - locale: *const c_char, - error_out: *mut *mut c_char, - ) -> *mut c_char; - fn berd_macos_stt_install_model( - locale: *const c_char, - progress: Option, - context: *mut c_void, - error_out: *mut *mut c_char, - ) -> bool; - fn berd_macos_stt_create( - locale: *const c_char, - event: Option, - context: *mut c_void, - error_out: *mut *mut c_char, - ) -> *mut c_void; - fn berd_macos_stt_push( - handle: *mut c_void, - samples: *const f32, - count: isize, - sample_rate: f64, - error_out: *mut *mut c_char, - ) -> bool; - fn berd_macos_stt_finish( - handle: *mut c_void, - timeout_seconds: f64, - error_out: *mut *mut c_char, - ) -> bool; - fn berd_macos_stt_cancel(handle: *mut c_void); - fn berd_macos_stt_release(handle: *mut c_void); - fn berd_macos_stt_free_string(value: *mut c_char); - } - - fn take_string(value: *mut c_char) -> Option { - if value.is_null() { - return None; - } - let result = unsafe { CStr::from_ptr(value) } - .to_string_lossy() - .into_owned(); - unsafe { berd_macos_stt_free_string(value) }; - Some(result) - } - - fn take_error(value: *mut c_char, fallback: &str) -> String { - take_string(value).unwrap_or_else(|| fallback.to_string()) - } - - pub fn supported() -> bool { - unsafe { berd_macos_stt_is_supported() } - } - - pub fn status() -> Result { - if !supported() { - return Ok(super::unsupported_status()); - } - let mut error = ptr::null_mut(); - let json = unsafe { berd_macos_stt_status_json(ptr::null(), &mut error) }; - let json = take_string(json).ok_or_else(|| { - take_error(error, "Could not read the macOS speech recognition status.") - })?; - let status: BridgeStatus = serde_json::from_str(&json) - .map_err(|error| format!("decode macOS speech status: {error}"))?; - Ok(super::MacSpeechStatus { - supported: status.supported, - unavailable_reason: (!status.supported) - .then(|| "Apple speech recognition is unavailable.".to_string()), - locale: status.locale.unwrap_or_default(), - locale_supported: status.locale_supported, - model_installed: status.ready, - installing: status.model_status == "downloading", - progress: None, - error: None, - revision: super::STATUS_REVISION.load(std::sync::atomic::Ordering::Acquire), - }) - } - - pub fn install( - progress: unsafe extern "C" fn(f64, *mut c_void), - context: *mut c_void, - ) -> Result<(), String> { - let mut error = ptr::null_mut(); - let installed = unsafe { - berd_macos_stt_install_model(ptr::null(), Some(progress), context, &mut error) - }; - if installed { - Ok(()) - } else { - Err(take_error( - error, - "Could not install the macOS speech recognition model.", - )) - } - } - - struct RecognitionContext { - events: mpsc::UnboundedSender, - } - - unsafe extern "C" fn recognition_event(code: i32, text: *const c_char, context: *mut c_void) { - if context.is_null() { - return; - } - let context = unsafe { &*(context.cast::()) }; - let text = (!text.is_null()) - .then(|| unsafe { CStr::from_ptr(text).to_string_lossy().into_owned() }); - let event = match code { - 1 => RecognitionEvent::Final(text.unwrap_or_default()), - 2 => RecognitionEvent::Finished, - 3 => RecognitionEvent::Failed( - text.unwrap_or_else(|| "macOS speech recognition failed.".to_string()), - ), - _ => return, - }; - let _ = context.events.send(event); - } - - pub struct RecognitionSession { - handle: *mut c_void, - context: *mut RecognitionContext, - } - - impl RecognitionSession { - pub fn new(events: mpsc::UnboundedSender) -> Result { - let context = Box::into_raw(Box::new(RecognitionContext { events })); - let mut error = ptr::null_mut(); - let handle = unsafe { - berd_macos_stt_create( - ptr::null(), - Some(recognition_event), - context.cast(), - &mut error, - ) - }; - if handle.is_null() { - unsafe { drop(Box::from_raw(context)) }; - return Err(take_error( - error, - "Could not start macOS speech recognition.", - )); - } - Ok(Self { handle, context }) - } - - pub fn push(&mut self, samples: &[f32]) -> Result<(), String> { - let mut error = ptr::null_mut(); - let pushed = unsafe { - berd_macos_stt_push( - self.handle, - samples.as_ptr(), - samples.len() as isize, - 48_000.0, - &mut error, - ) - }; - if pushed { - Ok(()) - } else { - Err(take_error( - error, - "Could not send audio to macOS speech recognition.", - )) - } - } - - pub fn finish(&mut self) -> Result<(), String> { - let mut error = ptr::null_mut(); - let finished = unsafe { - berd_macos_stt_finish( - self.handle, - super::RECOGNITION_FINISH_TIMEOUT_SECONDS as f64, - &mut error, - ) - }; - if finished { - Ok(()) - } else { - Err(take_error( - error, - "macOS speech recognition did not finish.", - )) - } - } - - pub fn cancel(&mut self) { - unsafe { berd_macos_stt_cancel(self.handle) }; - } - } - - impl Drop for RecognitionSession { - fn drop(&mut self) { - unsafe { - berd_macos_stt_release(self.handle); - drop(Box::from_raw(self.context)); - } - } - } -} - fn unsupported_status() -> MacSpeechStatus { MacSpeechStatus { supported: false, @@ -271,7 +42,22 @@ fn unsupported_status() -> MacSpeechStatus { pub fn status() -> Result { #[cfg(target_os = "macos")] { - bridge::status() + if !berd_voice::mac_speech::mac_speech_is_supported() { + return Ok(unsupported_status()); + } + let status = berd_voice::mac_speech::mac_speech_status()?; + Ok(MacSpeechStatus { + supported: status.supported, + unavailable_reason: (!status.supported) + .then(|| "Apple speech recognition is unavailable.".to_string()), + locale: status.locale.unwrap_or_default(), + locale_supported: status.locale_supported, + model_installed: status.ready, + installing: status.model_status == "downloading", + progress: None, + error: None, + revision: STATUS_REVISION.load(Ordering::Acquire), + }) } #[cfg(not(target_os = "macos"))] { @@ -314,36 +100,18 @@ pub async fn get_mac_speech_status() -> Result { pub async fn install_mac_speech_model(app: AppHandle) -> Result { #[cfg(target_os = "macos")] { - use std::ffi::c_void; - - struct ProgressContext { - app: AppHandle, - } - - unsafe extern "C" fn progress(value: f64, context: *mut c_void) { - if context.is_null() { - return; - } - let context = unsafe { &*(context.cast::()) }; - let mut next = status().unwrap_or_else(|error| MacSpeechStatus { - error: Some(error), - ..unsupported_status() - }); - next.installing = true; - next.progress = Some(value); - next.revision = STATUS_REVISION.fetch_add(1, Ordering::AcqRel) + 1; - let _ = context.app.emit(STATUS_EVENT, next); - } - - let context = Box::into_raw(Box::new(ProgressContext { app: app.clone() })); - // Raw pointers are intentionally not `Send`; move the address across - // the blocking-task boundary and reconstruct it only on that thread. - let context_address = context as usize; + let progress_app = app.clone(); let result = match tauri::async_runtime::spawn_blocking(move || { - let context = context_address as *mut ProgressContext; - let result = bridge::install(progress, context.cast()); - unsafe { drop(Box::from_raw(context)) }; - result + berd_voice::mac_speech::install_mac_speech_model(move |value| { + let mut next = status().unwrap_or_else(|error| MacSpeechStatus { + error: Some(error), + ..unsupported_status() + }); + next.installing = true; + next.progress = Some(value); + next.revision = STATUS_REVISION.fetch_add(1, Ordering::AcqRel) + 1; + let _ = progress_app.emit(STATUS_EVENT, next); + }) }) .await { @@ -371,9 +139,6 @@ pub async fn install_mac_speech_model(app: AppHandle) -> Result(input_muted: &Arc, mute_epoch: &Arc, on_change: F) -> bool +pub fn start(on_change: F) -> bool where F: Fn(bool) + Send + Sync + 'static, { - clear(input_muted); - #[cfg(target_os = "macos")] - return match macos::install(Arc::clone(input_muted), Arc::clone(mute_epoch), on_change) { + return match macos::install(on_change) { Ok(()) => true, Err(error) => { log::info!("AirPods input mute listener is unavailable: {error}"); @@ -20,61 +13,33 @@ where #[cfg(not(target_os = "macos"))] { - let _ = (mute_epoch, on_change); + let _ = on_change; false } } -pub fn stop(input_muted: &Arc) { - clear(input_muted); - +pub fn stop() { #[cfg(target_os = "macos")] if let Err(error) = macos::uninstall() { log::info!("Could not stop the AirPods input mute listener: {error}"); } } -pub fn set_muted( - input_muted: &AtomicBool, - mute_epoch: &AtomicU64, - muted: bool, -) -> Result<(), String> { +pub fn set_muted(muted: bool) -> Result<(), String> { #[cfg(target_os = "macos")] { - macos::set_muted(muted)?; - apply_change(input_muted, mute_epoch, muted, &|_| {}); - Ok(()) + macos::set_muted(muted) } #[cfg(not(target_os = "macos"))] { - let _ = (input_muted, mute_epoch, muted); + let _ = muted; Err("native microphone mute is only available on macOS".to_string()) } } -fn clear(input_muted: &AtomicBool) { - input_muted.store(false, Ordering::Release); -} - -#[cfg(any(target_os = "macos", test))] -fn apply_change( - input_muted: &AtomicBool, - mute_epoch: &AtomicU64, - muted: bool, - on_change: &dyn Fn(bool), -) { - if input_muted.swap(muted, Ordering::AcqRel) != muted { - if muted { - mute_epoch.fetch_add(1, Ordering::AcqRel); - } - on_change(muted); - } -} - #[cfg(target_os = "macos")] mod macos { - use super::*; use block2::RcBlock; use objc2::runtime::Bool; use objc2_avf_audio::AVAudioApplication; @@ -84,11 +49,7 @@ mod macos { fn berd_airpods_capture_stop(); } - pub fn install( - input_muted: Arc, - mute_epoch: Arc, - on_change: F, - ) -> Result<(), String> + pub fn install(on_change: F) -> Result<(), String> where F: Fn(bool) + Send + Sync + 'static, { @@ -97,10 +58,8 @@ mod macos { let application = unsafe { AVAudioApplication::sharedInstance() }; let handler = RcBlock::new(move |muted: Bool| { let muted = muted.as_bool(); - apply_change(&input_muted, &mute_epoch, muted, &|muted| { - log::info!("AirPods input mute changed muted={muted}"); - on_change(muted); - }); + log::info!("AirPods input mute changed muted={muted}"); + on_change(muted); Bool::YES }); // SAFETY: The block has the generated AVFAudio signature. The API @@ -144,27 +103,3 @@ mod macos { .map_err(|error| error.localizedDescription().to_string()) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn lifecycle_boundary_clears_mute() { - let input_muted = Arc::new(AtomicBool::new(true)); - clear(&input_muted); - assert!(!input_muted.load(Ordering::Acquire)); - } - - #[test] - fn mute_edge_advances_epoch_even_after_immediate_unmute() { - let input_muted = AtomicBool::new(false); - let mute_epoch = AtomicU64::new(0); - - apply_change(&input_muted, &mute_epoch, true, &|_| {}); - apply_change(&input_muted, &mute_epoch, false, &|_| {}); - - assert!(!input_muted.load(Ordering::Acquire)); - assert_eq!(mute_epoch.load(Ordering::Acquire), 1); - } -} diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 042ccc754..deb36a6e8 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -1,52 +1,63 @@ //! Native speech recognition for Desktop voice conversations. +#[cfg(any(test, target_os = "macos"))] +use std::time::Instant; use std::{ collections::{HashMap, VecDeque}, - path::PathBuf, sync::{ - atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}, - mpsc::{self, Receiver, SyncSender, TrySendError}, + atomic::{AtomicBool, Ordering}, Arc, Mutex, }, - thread, time::Duration, }; -#[cfg(any(test, target_os = "macos"))] -use std::collections::BTreeMap; - -#[cfg(target_os = "macos")] -use std::time::Instant; - use serde::{Deserialize, Serialize}; use tauri::{AppHandle, Emitter, Manager, State, WebviewWindow}; -use tokio::sync::mpsc as tokio_mpsc; +use tokio::sync::Notify; use super::mac_speech; use super::{ - native_input_mute, pocket_voice::parakeet_model_dir, voice_capture::VoiceCaptureState, + native_input_mute, + pocket_voice::{parakeet_model_dir, parakeet_model_for_loading}, + voice_capture::VoiceCaptureState, }; pub(crate) const EVENT_NAME: &str = "voice-conversation:event"; -const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; -const AUDIO_QUEUE_DEPTH: usize = 50; const MAX_PENDING_TRANSCRIPTS: usize = 64; const MAX_TRANSCRIPT_DELIVERY_ATTEMPTS: u8 = 3; -const MAX_SPEECH_SAMPLES: usize = 16_000 * 30; -const VAD_FRAME_SAMPLES: usize = 256; const VAD_THRESHOLD: f32 = 0.5; -// Keep ordinary pauses between words inside one offline recognition request. -// At 16 kHz with 256-sample frames this is 1.2 seconds. -const SILENCE_FLUSH_FRAMES: usize = 75; -const FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS: u64 = 5; -const FINAL_TRANSCRIPT_DELIVERY_TIMEOUT: Duration = - Duration::from_secs(FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS); -const OPENAI_NETWORK_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); -const OPENAI_FINAL_WRITE_TIMEOUT: Duration = Duration::from_secs(1); -const OPENAI_STARTUP_TIMEOUT: Duration = Duration::from_secs(12); -const OPENAI_PRE_ROLL_CHUNKS: usize = 15; -const STT_WORKER_SHUTDOWN_TIMEOUT_SECONDS: u64 = - mac_speech::RECOGNITION_FINISH_TIMEOUT_SECONDS + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS + 1; +const INPUT_STARTUP_TIMEOUT: Duration = Duration::from_secs(60); + +#[cfg(any(test, target_os = "macos"))] +pub(crate) fn output_latency_grace_elapsed( + playback_drained: bool, + guard_active: bool, + playback_drained_at: &mut Option, + output_latency_grace: Duration, + now: Instant, +) -> bool { + if !guard_active || !playback_drained { + *playback_drained_at = None; + return false; + } + let drained_at = *playback_drained_at.get_or_insert(now); + now.saturating_duration_since(drained_at) >= output_latency_grace +} + +#[cfg(any(test, target_os = "macos"))] +pub(crate) fn output_latency_grace_remaining( + guard_active: bool, + playback_drained_at: Option, + output_latency_grace: Duration, + now: Instant, +) -> Duration { + if !guard_active { + return Duration::ZERO; + } + playback_drained_at.map_or(output_latency_grace, |drained_at| { + output_latency_grace.saturating_sub(now.saturating_duration_since(drained_at)) + }) +} #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] #[serde(rename_all = "kebab-case")] @@ -56,26 +67,6 @@ pub enum VoiceInputBackend { Openai, } -fn active_vad_threshold_for_speech( - assistant_speaking: &AtomicBool, - assistant_vad_threshold: &AtomicU32, - speech_vad_threshold: f32, -) -> f32 { - if assistant_speaking.load(Ordering::Acquire) { - f32::from_bits(assistant_vad_threshold.load(Ordering::Acquire)) - } else { - speech_vad_threshold - } -} - -#[cfg(test)] -fn active_vad_threshold( - assistant_speaking: &AtomicBool, - assistant_vad_threshold: &AtomicU32, -) -> f32 { - active_vad_threshold_for_speech(assistant_speaking, assistant_vad_threshold, VAD_THRESHOLD) -} - #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct MicrophoneMuteRequest { @@ -96,6 +87,47 @@ pub struct AssistantSpeakingRequest { renderer_epoch: u64, } +#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct VoiceTranscriptReference { + lifecycle_id: String, + id: String, + revision: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrepareAssistantSpeechRequest { + session_id: String, + expected_revision: u64, + text: String, + acknowledgement: Option, + renderer_id: String, + renderer_epoch: u64, +} + +#[derive(Clone, Debug, Serialize)] +#[serde( + tag = "outcome", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +pub enum PrepareAssistantSpeechOutcome { + Pending, + NotAdmitted, + Admitted { speech_id: u64 }, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CancelAssistantSpeechRequest { + session_id: String, + expected_revision: u64, + speech_id: u64, + renderer_id: String, + renderer_epoch: u64, +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub enum InterruptionSensitivity { @@ -105,7 +137,6 @@ pub enum InterruptionSensitivity { } impl InterruptionSensitivity { - #[cfg(any(test, target_os = "macos"))] pub(crate) fn vad_threshold(self) -> f32 { match self { Self::Less => 0.8, @@ -205,12 +236,267 @@ struct Runtime { lifecycle_id: Option, revision: u64, owner: Option, - pipeline: Option, + pipeline: Option, controls_ready: bool, controls_suppressed: bool, controls_visibility_generation: u64, controls_window_revision: Option, native_microphone_mute_control: bool, + admission: Option>, + voice_input_quarantined: bool, +} + +#[derive(Debug)] +struct ActiveAdmission { + speech_id: u64, + playback_active: Option>, +} + +#[derive(Debug, Default)] +struct BerdAdmissionInner { + core: berd_voice::session::SessionCore, + next_token: u64, + tokens: HashMap, + active: Option, + closed: bool, +} + +#[derive(Debug, Default)] +struct BerdAdmissionCoordinator { + inner: Mutex, + changed: Notify, +} + +impl BerdAdmissionCoordinator { + fn add_final(&self, reference: VoiceTranscriptReference, text: String) -> Result { + let mut inner = self + .inner + .lock() + .map_err(|_| "voice admission lock was poisoned".to_string())?; + if inner.closed { + return Err("The voice conversation is no longer running.".to_string()); + } + let token = inner.next_token.saturating_add(1); + inner.core.add_final(token, text)?; + inner.next_token = token; + inner.tokens.insert(reference, token); + Self::interrupt_locked(&mut inner); + drop(inner); + self.changed.notify_waiters(); + Ok(token) + } + + fn confirm(&self, reference: &VoiceTranscriptReference) -> Result { + let mut inner = self + .inner + .lock() + .map_err(|_| "voice admission lock was poisoned".to_string())?; + let confirmed = inner + .tokens + .get(reference) + .copied() + .is_some_and(|token| inner.core.confirm_exact(token)); + drop(inner); + if confirmed { + self.changed.notify_waiters(); + } + Ok(confirmed) + } + + fn discard(&self, reference: &VoiceTranscriptReference) -> Result { + let mut inner = self + .inner + .lock() + .map_err(|_| "voice admission lock was poisoned".to_string())?; + let discarded = inner + .tokens + .get(reference) + .copied() + .is_some_and(|token| inner.core.discard_final(token)); + if discarded { + inner.tokens.remove(reference); + } + drop(inner); + if discarded { + self.changed.notify_waiters(); + } + Ok(discarded) + } + + fn set_user_speaking(&self, active: bool) { + if let Ok(mut inner) = self.inner.lock() { + if inner.core.set_user_speaking(active) { + Self::interrupt_locked(&mut inner); + } + } + self.changed.notify_waiters(); + } + + fn set_recognition_pending(&self, active: bool) { + if let Ok(mut inner) = self.inner.lock() { + if inner.core.set_recognition_pending(active) { + Self::interrupt_locked(&mut inner); + } + } + self.changed.notify_waiters(); + } + + async fn prepare( + &self, + text: String, + acknowledgement: Option, + ) -> Result { + loop { + let notified = self.changed.notified(); + let outcome = { + let mut inner = self + .inner + .lock() + .map_err(|_| "voice admission lock was poisoned".to_string())?; + if inner.closed { + return Err("The voice conversation is no longer running.".to_string()); + } + let acknowledgement = match acknowledgement.as_ref() { + Some(reference) => { + let Some(token) = inner.tokens.get(reference).copied() else { + return Ok(PrepareAssistantSpeechOutcome::Pending); + }; + Some(token) + } + None => None, + }; + match inner.core.prepare_after_host_confirmation( + berd_voice::session::PrepareRequest { + id: 0, + acknowledgement, + text: text.clone(), + }, + ) { + berd_voice::session::PrepareOutcome::Hold => None, + berd_voice::session::PrepareOutcome::Pending(_) => { + Some(PrepareAssistantSpeechOutcome::Pending) + } + berd_voice::session::PrepareOutcome::NotAdmitted(_) => { + Some(PrepareAssistantSpeechOutcome::NotAdmitted) + } + berd_voice::session::PrepareOutcome::Admitted { speech_id, .. } => { + inner.active = Some(ActiveAdmission { + speech_id, + playback_active: None, + }); + Some(PrepareAssistantSpeechOutcome::Admitted { speech_id }) + } + } + }; + if let Some(outcome) = outcome { + return Ok(outcome); + } + notified.await; + } + } + + #[cfg(any(test, target_os = "macos"))] + fn claim( + self: &Arc, + speech_id: u64, + playback_active: Arc, + ) -> Result, String> { + let mut inner = self + .inner + .lock() + .map_err(|_| "voice admission lock was poisoned".to_string())?; + let owns_reservation = inner + .active + .as_ref() + .is_some_and(|active| active.speech_id == speech_id); + if inner.closed || !owns_reservation || !inner.core.mark_started(speech_id) { + return Ok(None); + } + if let Some(active) = inner.active.as_mut() { + active.playback_active = Some(playback_active); + } + Ok(Some(AdmissionPlaybackGuard { + coordinator: Arc::clone(self), + speech_id, + })) + } + + fn cancel(&self, speech_id: u64) -> Result { + let mut inner = self + .inner + .lock() + .map_err(|_| "voice admission lock was poisoned".to_string())?; + let Some(active) = inner.active.as_ref() else { + return Ok(false); + }; + if active.speech_id != speech_id { + return Ok(false); + } + if let Some(playback_active) = active.playback_active.as_ref() { + playback_active.store(false, Ordering::SeqCst); + } else { + inner.core.finish(speech_id); + inner.active = None; + } + drop(inner); + self.changed.notify_waiters(); + Ok(true) + } + + #[cfg(any(test, target_os = "macos"))] + fn finish(&self, speech_id: u64) { + if let Ok(mut inner) = self.inner.lock() { + if inner + .active + .as_ref() + .is_some_and(|active| active.speech_id == speech_id) + { + inner.core.finish(speech_id); + inner.active = None; + } + } + self.changed.notify_waiters(); + } + + fn close(&self) { + if let Ok(mut inner) = self.inner.lock() { + inner.closed = true; + if let Some(active) = inner.active.take() { + if let Some(playback_active) = active.playback_active { + playback_active.store(false, Ordering::SeqCst); + } + inner.core.finish(active.speech_id); + } + } + self.changed.notify_waiters(); + } + + fn interrupt_locked(inner: &mut BerdAdmissionInner) { + let Some(active) = inner.active.as_ref() else { + return; + }; + if let Some(playback_active) = active.playback_active.as_ref() { + playback_active.store(false, Ordering::SeqCst); + } else { + let speech_id = active.speech_id; + inner.core.finish(speech_id); + inner.active = None; + } + } +} + +#[must_use = "voice admission remains active until the backend terminal path drops this guard"] +#[cfg(any(test, target_os = "macos"))] +pub(crate) struct AdmissionPlaybackGuard { + coordinator: Arc, + speech_id: u64, +} + +#[cfg(any(test, target_os = "macos"))] +impl Drop for AdmissionPlaybackGuard { + fn drop(&mut self) { + self.coordinator.finish(self.speech_id); + } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -242,7 +528,7 @@ struct VoiceStartBlock { type StopSnapshot = ( Option, u64, - Option, + Option, Option<(RuntimeOwner, String)>, ); @@ -252,6 +538,7 @@ struct StopCompletion { next_revision: u64, owner: RuntimeOwner, owner_id: String, + shutdown_error: Option, } #[derive(Clone, Default)] @@ -260,63 +547,156 @@ pub struct NativeVoiceState { stop_serial: Arc>, start_blocks: Arc>>>, pending: Arc>>, - capture_suppressions: Arc, microphone_muted: Arc, - input_muted: Arc, - input_mute_epoch: Arc, - assistant_speaking: Arc, - assistant_vad_threshold: Arc, - #[cfg(any(test, target_os = "macos"))] - assistant_speech_generation: Arc, - #[cfg(any(test, target_os = "macos"))] - assistant_speech_lifetimes: Arc>>, -} - -#[must_use = "capture suppression ends when the guard is dropped"] -pub struct CaptureSuppressionGuard { - capture_suppressions: Arc, -} - -impl Drop for CaptureSuppressionGuard { - fn drop(&mut self) { - let previous = self.capture_suppressions.fetch_sub(1, Ordering::SeqCst); - debug_assert!(previous > 0, "capture suppression guard underflow"); - log::info!( - "[voice-echo-guard] capture resumed suppression_count={}", - previous.saturating_sub(1) - ); - } + input_controls: berd_voice::input::VoiceInputControls, } #[must_use = "assistant speech policy ends when the guard is dropped"] -#[cfg(any(test, target_os = "macos"))] pub(crate) struct AssistantSpeechGuard { - _capture_suppression: Option, - assistant_speaking: Arc, - assistant_vad_threshold: Arc, - assistant_speech_lifetimes: Arc>>, - generation: u64, + _activity: Option, } -#[cfg(any(test, target_os = "macos"))] -impl Drop for AssistantSpeechGuard { - fn drop(&mut self) { - let mut lifetimes = self - .assistant_speech_lifetimes +impl NativeVoiceState { + fn ensure_voice_input_not_quarantined(&self) -> Result<(), String> { + let runtime = self + .runtime .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - lifetimes.remove(&self.generation); - if let Some((_, threshold)) = lifetimes.last_key_value() { - self.assistant_vad_threshold - .store(*threshold, Ordering::Release); - self.assistant_speaking.store(true, Ordering::Release); + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.voice_input_quarantined { + Err("Voice recognition did not stop safely. Restart Berd before starting another voice conversation.".to_string()) } else { - self.assistant_speaking.store(false, Ordering::Release); + Ok(()) } } -} -impl NativeVoiceState { + fn record_voice_input_finish( + &self, + result: Result<(), berd_voice::input::VoiceInputFinishError>, + ) -> Option { + let error = result.err()?; + log::error!("Native voice recognizer shutdown failed: {error}"); + if error.is_quarantined() { + if let Ok(mut runtime) = self.runtime.lock() { + runtime.voice_input_quarantined = true; + } + Some("Voice recognition did not stop safely. Restart Berd before starting another voice conversation.".to_string()) + } else { + Some(error.to_string()) + } + } + + async fn finish_uninstalled_pipeline( + &self, + pipeline: berd_voice::input::VoiceInputRuntime, + startup_error: String, + ) -> String { + self.record_voice_input_finish(shutdown_pipeline(pipeline).await) + .unwrap_or(startup_error) + } + + fn admission_target( + &self, + caller_window_label: &str, + session_id: &str, + expected_revision: u64, + ) -> Result>, String> { + let runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(None); + } + if runtime + .owner + .as_ref() + .map(|owner| owner.window_label.as_str()) + != Some(caller_window_label) + { + return Err("Only the voice conversation owner can prepare assistant speech.".into()); + } + Ok(runtime.admission.clone()) + } + + #[cfg(target_os = "macos")] + pub(crate) fn claim_assistant_speech( + &self, + session_id: &str, + expected_revision: u64, + speech_id: u64, + playback_active: Arc, + ) -> Result, String> { + let admission = { + let runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(None); + } + runtime.admission.clone() + }; + admission + .ok_or_else(|| "Voice admission is not available.".to_string())? + .claim(speech_id, playback_active) + } + + fn acknowledge_transcript( + &self, + session_id: &str, + id: &str, + revision: u64, + ) -> Result<(), String> { + let admission = self.runtime.lock().ok().and_then(|runtime| { + (runtime.session_id.as_deref() == Some(session_id) && runtime.revision == revision) + .then(|| runtime.admission.clone()) + .flatten() + }); + let mut pending = self + .pending + .lock() + .map_err(|_| "native transcript queue lock was poisoned".to_string())?; + if let Some(index) = pending.iter().position(|item| { + item.session_id == session_id && item.id == id && item.revision == revision + }) { + let transcript = &pending[index]; + if let Some(admission) = admission { + let reference = VoiceTranscriptReference { + lifecycle_id: transcript.lifecycle_id.clone(), + id: transcript.id.clone(), + revision, + }; + if !admission.confirm(&reference)? { + return Err("The voice transcript is not tracked by admission.".to_string()); + } + } + pending.remove(index); + } + Ok(()) + } + + fn reject_transcript( + &self, + session_id: &str, + id: &str, + revision: u64, + ) -> Result { + let admission = self.runtime.lock().ok().and_then(|runtime| { + (runtime.session_id.as_deref() == Some(session_id) && runtime.revision == revision) + .then(|| runtime.admission.clone()) + .flatten() + }); + let mut pending = self + .pending + .lock() + .map_err(|_| "native transcript queue lock was poisoned".to_string())?; + reject_pending_transcript(&mut pending, session_id, id, revision, admission.as_deref()) + } + fn block_starts( &self, session_id: String, @@ -389,51 +769,22 @@ impl NativeVoiceState { .is_ok_and(|blocks| blocks.contains_key(session_id)) } - pub fn suppress_capture(&self) -> CaptureSuppressionGuard { - let previous = self.capture_suppressions.fetch_add(1, Ordering::SeqCst); - log::info!( - "[voice-echo-guard] capture suppressed suppression_count={}", - previous + 1 - ); - CaptureSuppressionGuard { - capture_suppressions: Arc::clone(&self.capture_suppressions), - } - } - - #[cfg(any(test, target_os = "macos"))] pub(crate) fn begin_assistant_speech( &self, sensitivity: InterruptionSensitivity, - suppress_capture: bool, + input_during_tts: berd_voice::input::InputDuringTtsPolicy, ) -> AssistantSpeechGuard { - let mut lifetimes = self - .assistant_speech_lifetimes - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let generation = self - .assistant_speech_generation - .fetch_add(1, Ordering::AcqRel) - .wrapping_add(1); - let threshold = sensitivity.vad_threshold().to_bits(); - lifetimes.insert(generation, threshold); - self.assistant_vad_threshold - .store(threshold, Ordering::Release); - self.assistant_speaking.store(true, Ordering::Release); + let activity = self + .input_controls + .begin_assistant_activity(sensitivity.vad_threshold(), input_during_tts) + .ok(); AssistantSpeechGuard { - _capture_suppression: suppress_capture.then(|| self.suppress_capture()), - assistant_speaking: Arc::clone(&self.assistant_speaking), - assistant_vad_threshold: Arc::clone(&self.assistant_vad_threshold), - assistant_speech_lifetimes: Arc::clone(&self.assistant_speech_lifetimes), - generation, + _activity: activity, } } - fn capture_is_suppressed(&self) -> bool { - self.capture_suppressions.load(Ordering::SeqCst) > 0 - } - pub fn microphone_is_muted(&self) -> bool { - self.microphone_muted.load(Ordering::SeqCst) || self.input_muted.load(Ordering::Acquire) + self.microphone_muted.load(Ordering::SeqCst) || self.input_controls.is_host_muted() } pub fn active_session_target(&self) -> Option<(String, String)> { @@ -604,17 +955,14 @@ impl NativeVoiceState { } let native_microphone_mute_control = runtime.native_microphone_mute_control; if native_microphone_mute_control { - native_input_mute::set_muted(&self.input_muted, &self.input_mute_epoch, muted)?; + native_input_mute::set_muted(muted)?; } // Native input mute is authoritative when installed so a hardware // unmute cannot be masked by a stale renderer fallback latch. let software_muted = software_microphone_mute(native_microphone_mute_control, muted); - let previous_software_muted = - self.microphone_muted.swap(software_muted, Ordering::SeqCst); - if !native_microphone_mute_control && previous_software_muted != software_muted { - self.input_muted.store(software_muted, Ordering::Release); - self.input_mute_epoch.fetch_add(1, Ordering::AcqRel); - } + self.microphone_muted + .store(software_muted, Ordering::SeqCst); + self.input_controls.set_host_muted(muted); owner_window_label }; Ok(Some(owner_window_label)) @@ -758,310 +1106,11 @@ impl NativeVoiceState { } } -enum SttMessage { - Speaking(bool), - Final { - text: String, - delivered: Option>, - }, - Failed(String), -} - -struct SttPipeline { - audio_tx: SyncSender, - audio_seen: AtomicBool, - shutdown: Arc, - discard_on_shutdown: Arc, - input_muted: Arc, - input_mute_epoch: Arc, - shutdown_mute_epoch: Arc, - thread: Option>, -} - -struct AudioBatch { - bytes: Vec, - mute_epoch: u64, +async fn shutdown_pipeline( + pipeline: berd_voice::input::VoiceInputRuntime, +) -> Result<(), berd_voice::input::VoiceInputFinishError> { + pipeline.finish().await } - -type OpenAiSttPipelineStart = ( - SttPipeline, - tokio_mpsc::Receiver, - mpsc::Receiver>, -); - -impl SttPipeline { - fn new_parakeet( - model_dir: PathBuf, - input_muted: Arc, - input_mute_epoch: Arc, - assistant_speaking: Arc, - assistant_vad_threshold: Arc, - speech_vad_threshold: f32, - ) -> Result<(Self, tokio_mpsc::Receiver), String> { - let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); - let (event_tx, event_rx) = tokio_mpsc::channel(64); - let shutdown = Arc::new(AtomicBool::new(false)); - let discard_on_shutdown = Arc::new(AtomicBool::new(false)); - let shutdown_mute_epoch = Arc::new(AtomicU64::new(0)); - let worker_shutdown = Arc::clone(&shutdown); - let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); - let worker_input_muted = Arc::clone(&input_muted); - let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); - let worker_shutdown_mute_epoch = Arc::clone(&shutdown_mute_epoch); - let thread = thread::Builder::new() - .name("berd-native-stt".into()) - .spawn(move || { - stt_worker( - model_dir, - audio_rx, - event_tx, - worker_shutdown, - worker_discard_on_shutdown, - worker_input_muted, - worker_input_mute_epoch, - worker_shutdown_mute_epoch, - assistant_speaking, - assistant_vad_threshold, - speech_vad_threshold, - ) - }) - .map_err(|error| format!("start native transcription: {error}"))?; - Ok(( - Self { - audio_tx, - audio_seen: AtomicBool::new(false), - shutdown, - discard_on_shutdown, - input_muted, - input_mute_epoch, - shutdown_mute_epoch, - thread: Some(thread), - }, - event_rx, - )) - } - - fn new_openai( - api_key: String, - input_muted: Arc, - input_mute_epoch: Arc, - assistant_speaking: Arc, - assistant_vad_threshold: Arc, - speech_vad_threshold: f32, - ) -> Result { - let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); - let (event_tx, event_rx) = tokio_mpsc::channel(64); - let (startup_tx, startup_rx) = mpsc::sync_channel(0); - let endpoint = super::openai_audio::realtime_endpoint()?; - let model = super::openai_audio::transcription_model(); - let shutdown = Arc::new(AtomicBool::new(false)); - let discard_on_shutdown = Arc::new(AtomicBool::new(false)); - let shutdown_mute_epoch = Arc::new(AtomicU64::new(0)); - let worker_shutdown = Arc::clone(&shutdown); - let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); - let worker_input_muted = Arc::clone(&input_muted); - let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); - let worker_shutdown_mute_epoch = Arc::clone(&shutdown_mute_epoch); - let thread = thread::Builder::new() - .name("berd-openai-stt".into()) - .spawn(move || { - openai_stt_worker( - api_key, - endpoint, - model, - audio_rx, - event_tx, - worker_shutdown, - worker_discard_on_shutdown, - worker_input_muted, - worker_input_mute_epoch, - worker_shutdown_mute_epoch, - assistant_speaking, - assistant_vad_threshold, - speech_vad_threshold, - startup_tx, - ) - }) - .map_err(|error| format!("start OpenAI speech recognition: {error}"))?; - Ok(( - Self { - audio_tx, - audio_seen: AtomicBool::new(false), - shutdown, - discard_on_shutdown, - input_muted, - input_mute_epoch, - shutdown_mute_epoch, - thread: Some(thread), - }, - event_rx, - startup_rx, - )) - } - - fn new_macos( - input_muted: Arc, - input_mute_epoch: Arc, - assistant_speaking: Arc, - assistant_vad_threshold: Arc, - speech_vad_threshold: f32, - ) -> Result<(Self, tokio_mpsc::Receiver), String> { - #[cfg(not(target_os = "macos"))] - { - let _ = ( - input_muted, - input_mute_epoch, - assistant_speaking, - assistant_vad_threshold, - speech_vad_threshold, - ); - Err("macOS speech recognition requires macOS 26 or later.".to_string()) - } - #[cfg(target_os = "macos")] - { - let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); - let (event_tx, event_rx) = tokio_mpsc::channel(64); - let shutdown = Arc::new(AtomicBool::new(false)); - let discard_on_shutdown = Arc::new(AtomicBool::new(false)); - let shutdown_mute_epoch = Arc::new(AtomicU64::new(0)); - let worker_shutdown = Arc::clone(&shutdown); - let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); - let worker_input_muted = Arc::clone(&input_muted); - let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); - let worker_shutdown_mute_epoch = Arc::clone(&shutdown_mute_epoch); - let thread = thread::Builder::new() - .name("berd-macos-stt".into()) - .spawn(move || { - macos_stt_worker( - audio_rx, - event_tx, - worker_shutdown, - worker_discard_on_shutdown, - worker_input_muted, - worker_input_mute_epoch, - worker_shutdown_mute_epoch, - assistant_speaking, - assistant_vad_threshold, - speech_vad_threshold, - ) - }) - .map_err(|error| format!("start macOS speech recognition: {error}"))?; - Ok(( - Self { - audio_tx, - audio_seen: AtomicBool::new(false), - shutdown, - discard_on_shutdown, - input_muted, - input_mute_epoch, - shutdown_mute_epoch, - thread: Some(thread), - }, - event_rx, - )) - } - } - - fn push(&self, bytes: Vec) -> Result<(), String> { - if bytes.len() > MAX_AUDIO_BATCH_BYTES { - return Err(format!( - "audio batch is {} bytes; maximum is {MAX_AUDIO_BATCH_BYTES}", - bytes.len() - )); - } - if !bytes.len().is_multiple_of(4) { - return Err("audio batch must contain complete f32 samples".to_string()); - } - if self.input_muted.load(Ordering::Acquire) { - return Ok(()); - } - let mute_epoch = self.input_mute_epoch.load(Ordering::Acquire); - if self.input_muted.load(Ordering::Acquire) - || mute_epoch != self.input_mute_epoch.load(Ordering::Acquire) - { - return Ok(()); - } - if !self.audio_seen.swap(true, Ordering::AcqRel) { - log::info!( - "Native STT received its first audio batch ({} bytes)", - bytes.len() - ); - } - match self.audio_tx.try_send(AudioBatch { bytes, mute_epoch }) { - Ok(()) => Ok(()), - Err(TrySendError::Full(_)) => Err( - "Native voice audio overrun: transcription could not keep up with microphone input." - .to_string(), - ), - Err(TrySendError::Disconnected(_)) => { - Err("Native voice transcription is no longer running.".to_string()) - } - } - } - - fn begin_shutdown(&mut self) -> Option> { - self.signal_shutdown(); - self.thread.take() - } - - fn signal_shutdown(&self) { - if self.shutdown.load(Ordering::Acquire) { - return; - } - self.shutdown_mute_epoch.store( - self.input_mute_epoch.load(Ordering::Acquire), - Ordering::Release, - ); - self.latch_muted_shutdown(); - self.shutdown.store(true, Ordering::Release); - } - - fn latch_muted_shutdown(&self) { - if self.input_muted.load(Ordering::Acquire) { - self.discard_on_shutdown.store(true, Ordering::Release); - } - } -} - -impl Drop for SttPipeline { - fn drop(&mut self) { - if let Some(worker) = self.begin_shutdown() { - let _ = thread::Builder::new() - .name("berd-native-stt-reaper".into()) - .spawn(move || { - let _ = worker.join(); - }); - } - } -} - -#[cfg(not(test))] -const STT_WORKER_SHUTDOWN_TIMEOUT: Duration = - Duration::from_secs(STT_WORKER_SHUTDOWN_TIMEOUT_SECONDS); -#[cfg(test)] -const STT_WORKER_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100); - -async fn shutdown_pipeline(mut pipeline: SttPipeline) { - let worker = pipeline.begin_shutdown(); - drop(pipeline); - if let Some(worker) = worker { - let deadline = tokio::time::Instant::now() + STT_WORKER_SHUTDOWN_TIMEOUT; - while !worker.is_finished() { - if tokio::time::Instant::now() >= deadline { - log::error!( - "Native voice recognizer did not stop within {:?}; detaching it", - STT_WORKER_SHUTDOWN_TIMEOUT - ); - return; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - if worker.join().is_err() { - log::error!("Native voice recognizer worker panicked during shutdown"); - } - } -} - async fn status_with_availability( state: &NativeVoiceState, parakeet_available: bool, @@ -1071,14 +1120,20 @@ where F: FnMut() -> Fut, Fut: std::future::Future, { - let (session_active, revision_before_availability) = { + let (session_active, revision_before_availability, quarantined_before_availability) = { let runtime = state .runtime .lock() .unwrap_or_else(|error| error.into_inner()); - (runtime.session_id.is_some(), runtime.revision) + ( + runtime.session_id.is_some(), + runtime.revision, + runtime.voice_input_quarantined, + ) }; - let macos_available = if needs_macos_status(session_active, parakeet_available) { + let macos_available = if !quarantined_before_availability + && needs_macos_status(session_active, parakeet_available) + { macos_status().await } else { false @@ -1095,9 +1150,11 @@ where .as_ref() .map(|owner| owner.window_label.clone()), runtime.revision, + runtime.voice_input_quarantined, ) }; - let macos_available = if snapshot.2 != revision_before_availability + let macos_available = if !snapshot.3 + && snapshot.2 != revision_before_availability && needs_macos_status(snapshot.0.is_some(), parakeet_available) { let available = macos_status().await; @@ -1113,22 +1170,27 @@ where .as_ref() .map(|owner| owner.window_label.clone()), runtime.revision, + runtime.voice_input_quarantined, ) }; available } else { macos_available }; - let (session_id, owner_window_label, revision) = snapshot; - let (available, unavailable_reason) = - if session_id.is_some() || parakeet_available || macos_available { - (true, None) - } else { - ( + let (session_id, owner_window_label, revision, voice_input_quarantined) = snapshot; + let (available, unavailable_reason) = if voice_input_quarantined { + ( false, - Some("Download speech recognition before starting a call.".to_string()), + Some("Voice recognition did not stop safely. Restart Berd before starting another voice conversation.".to_string()), ) - }; + } else if session_id.is_some() || parakeet_available || macos_available { + (true, None) + } else { + ( + false, + Some("Download speech recognition before starting a call.".to_string()), + ) + }; NativeVoiceStatus { available, unavailable_reason, @@ -1206,6 +1268,58 @@ pub fn release_native_voice_conversation_start_block( state.release_start_block(&session_id, &token) } +#[tauri::command] +pub async fn prepare_native_voice_assistant_speech( + state: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, + webview_window: WebviewWindow, + request: PrepareAssistantSpeechRequest, +) -> Result { + let admission = capture.with_active_renderer( + webview_window.label(), + &request.renderer_id, + request.renderer_epoch, + || { + state.admission_target( + webview_window.label(), + &request.session_id, + request.expected_revision, + ) + }, + )?; + let Some(admission) = admission else { + return Ok(PrepareAssistantSpeechOutcome::NotAdmitted); + }; + admission + .prepare(request.text, request.acknowledgement) + .await +} + +#[tauri::command] +pub fn cancel_native_voice_assistant_speech( + state: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, + webview_window: WebviewWindow, + request: CancelAssistantSpeechRequest, +) -> Result { + let admission = capture.with_active_renderer( + webview_window.label(), + &request.renderer_id, + request.renderer_epoch, + || { + state.admission_target( + webview_window.label(), + &request.session_id, + request.expected_revision, + ) + }, + )?; + match admission { + Some(admission) => admission.cancel(request.speech_id), + None => Ok(false), + } +} + #[tauri::command] pub fn drain_native_voice_conversation_transcripts( state: State<'_, NativeVoiceState>, @@ -1228,14 +1342,7 @@ pub fn acknowledge_native_voice_conversation_transcript( id: String, revision: u64, ) -> Result<(), String> { - state - .pending - .lock() - .map_err(|_| "native transcript queue lock was poisoned".to_string())? - .retain(|item| { - !(item.session_id == session_id && item.id == id && item.revision == revision) - }); - Ok(()) + state.acknowledge_transcript(&session_id, &id, revision) } #[tauri::command] @@ -1245,16 +1352,7 @@ pub fn reject_native_voice_conversation_transcript( id: String, revision: u64, ) -> Result { - let mut pending = state - .pending - .lock() - .map_err(|_| "native transcript queue lock was poisoned".to_string())?; - Ok(reject_pending_transcript( - &mut pending, - &session_id, - &id, - revision, - )) + state.reject_transcript(&session_id, &id, revision) } fn reject_pending_transcript( @@ -1262,23 +1360,35 @@ fn reject_pending_transcript( session_id: &str, id: &str, revision: u64, -) -> TranscriptRejection { + admission: Option<&BerdAdmissionCoordinator>, +) -> Result { let Some(index) = pending.iter().position(|item| { item.session_id == session_id && item.id == id && item.revision == revision }) else { - return TranscriptRejection { + return Ok(TranscriptRejection { attempts: MAX_TRANSCRIPT_DELIVERY_ATTEMPTS, terminal: true, - }; + }); }; let attempts = pending[index].delivery_attempts.saturating_add(1); let terminal = attempts >= MAX_TRANSCRIPT_DELIVERY_ATTEMPTS; if terminal { + if let Some(admission) = admission { + let transcript = &pending[index]; + let reference = VoiceTranscriptReference { + lifecycle_id: transcript.lifecycle_id.clone(), + id: transcript.id.clone(), + revision, + }; + if !admission.discard(&reference)? { + return Err("The voice transcript is not tracked by admission.".to_string()); + } + } pending.remove(index); } else { pending[index].delivery_attempts = attempts; } - TranscriptRejection { attempts, terminal } + Ok(TranscriptRejection { attempts, terminal }) } #[tauri::command] @@ -1305,6 +1415,7 @@ pub async fn start_native_voice_conversation( { return Err("An OpenAI Realtime voice conversation is already active.".to_string()); } + state.ensure_voice_input_not_quarantined()?; if input_backend == VoiceInputBackend::Macos && !mac_speech::status_async().await?.model_installed { @@ -1312,7 +1423,12 @@ pub async fn start_native_voice_conversation( "Download the macOS speech recognition model before starting a call.".to_string(), ); } - if input_backend == VoiceInputBackend::Openai { + let openai_api_key = if input_backend == VoiceInputBackend::Openai { + Some(super::openai_audio::stt_api_key()?) + } else { + None + }; + if openai_api_key.is_some() { let deadline = tokio::time::Instant::now() + Duration::from_secs(1); while !webview_window.is_focused().unwrap_or(false) && tokio::time::Instant::now() < deadline @@ -1335,71 +1451,61 @@ pub async fn start_native_voice_conversation( ) }) .await?; - let openai_api_key = if input_backend == VoiceInputBackend::Openai { - Some(super::openai_audio::stt_api_key()?) - } else { - None - }; let mut microphone_claimed = capture.claim_microphone( window_label.clone(), renderer_id.clone(), renderer_epoch, owner_id.clone(), )?; - let speech_vad_threshold = VAD_THRESHOLD; - let pipeline = match input_backend { - VoiceInputBackend::Parakeet => parakeet_model_dir(&app).and_then(|model_dir| { - SttPipeline::new_parakeet( - model_dir, - Arc::clone(&state.input_muted), - Arc::clone(&state.input_mute_epoch), - Arc::clone(&state.assistant_speaking), - Arc::clone(&state.assistant_vad_threshold), - speech_vad_threshold, - ) - }), - VoiceInputBackend::Macos => SttPipeline::new_macos( - Arc::clone(&state.input_muted), - Arc::clone(&state.input_mute_epoch), - Arc::clone(&state.assistant_speaking), - Arc::clone(&state.assistant_vad_threshold), - speech_vad_threshold, - ), - VoiceInputBackend::Openai => { - let startup = SttPipeline::new_openai( - openai_api_key.expect("OpenAI key resolved for OpenAI input"), - Arc::clone(&state.input_muted), - Arc::clone(&state.input_mute_epoch), - Arc::clone(&state.assistant_speaking), - Arc::clone(&state.assistant_vad_threshold), - speech_vad_threshold, - ); - match startup { - Err(error) => Err(error), - Ok((pipeline, events, startup)) => { - let startup_result = tokio::task::spawn_blocking(move || { - startup.recv_timeout(OPENAI_STARTUP_TIMEOUT) - }) - .await; - match startup_result { - Ok(Ok(Ok(()))) => Ok((pipeline, events)), - Ok(Ok(Err(error))) => { - drop(pipeline); - Err(error) - } - Ok(Err(error)) => { - drop(pipeline); - Err(format!("OpenAI transcription did not start: {error}")) - } - Err(error) => { - drop(pipeline); - Err(format!("wait for OpenAI transcription startup: {error}")) - } - } + let mut parakeet_assets = None; + let engine = match input_backend { + VoiceInputBackend::Parakeet => { + parakeet_model_for_loading(&app).map(|(model_dir, assets)| { + parakeet_assets = Some(assets); + berd_voice::input::VoiceInputEngineConfig::Parakeet { model_dir } + }) + } + VoiceInputBackend::Macos => { + #[cfg(target_os = "macos")] + { + Ok(berd_voice::input::VoiceInputEngineConfig::MacSpeech) + } + #[cfg(not(target_os = "macos"))] + { + if microphone_claimed { + capture.release_microphone( + &window_label, + &renderer_id, + renderer_epoch, + &owner_id, + ); } + return Err("macOS speech recognition requires macOS 26 or later.".to_string()); + } + } + VoiceInputBackend::Openai => super::openai_audio::realtime_endpoint().map(|endpoint| { + berd_voice::input::VoiceInputEngineConfig::OpenAi { + endpoint, + api_key: openai_api_key.expect("OpenAI key resolved for OpenAI input"), + model: super::openai_audio::transcription_model(), + } + }), + }; + let engine = match engine { + Ok(engine) => engine, + Err(error) => { + if microphone_claimed { + capture.release_microphone(&window_label, &renderer_id, renderer_epoch, &owner_id); } + return Err(error); } }; + let pipeline = + berd_voice::input::VoiceInputRuntime::start(berd_voice::input::VoiceInputConfig { + engine, + speech_vad_threshold: VAD_THRESHOLD, + controls: state.input_controls.clone(), + }); let (pipeline, mut events) = match pipeline { Ok(result) => result, Err(error) => { @@ -1409,6 +1515,22 @@ pub async fn start_native_voice_conversation( return Err(error); } }; + let readiness = match tokio::time::timeout(INPUT_STARTUP_TIMEOUT, events.recv()).await { + Ok(Some(berd_voice::input::VoiceInputEvent::Ready)) => Ok(()), + Ok(Some(berd_voice::input::VoiceInputEvent::Failed(error))) => Err(error), + Ok(Some(_)) => Err("Voice input emitted activity before it was ready.".to_string()), + Ok(None) => Err("Voice input stopped before it was ready.".to_string()), + Err(_) => Err("Voice input did not become ready within 60 seconds.".to_string()), + }; + if let Err(error) = readiness { + let error = state.finish_uninstalled_pipeline(pipeline, error).await; + if microphone_claimed { + capture.release_microphone(&window_label, &renderer_id, renderer_epoch, &owner_id); + } + return Err(error); + } + drop(parakeet_assets); + let input_controls = pipeline.controls(); if let Err(error) = validate_voice_target_session( capture.inner(), &window_sessions, @@ -1418,6 +1540,7 @@ pub async fn start_native_voice_conversation( &session_id, Some(foreground_generation), ) { + let error = state.finish_uninstalled_pipeline(pipeline, error).await; drop(lifecycle_guard); if microphone_claimed { capture.release_microphone(&window_label, &renderer_id, renderer_epoch, &owner_id); @@ -1434,6 +1557,7 @@ pub async fn start_native_voice_conversation( ) { Ok(()) => {} Err(error) => { + let error = state.finish_uninstalled_pipeline(pipeline, error).await; drop(lifecycle_guard); if microphone_claimed { capture.release_microphone(&window_label, &renderer_id, renderer_epoch, &owner_id); @@ -1441,6 +1565,7 @@ pub async fn start_native_voice_conversation( return Err(error); } } + let mut pipeline = Some(pipeline); let install_result = (|| -> Result<(u64, String), String> { let start_blocks = state .start_blocks @@ -1453,6 +1578,9 @@ pub async fn start_native_voice_conversation( .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.voice_input_quarantined { + return Err("Voice recognition did not stop safely. Restart Berd before starting another voice conversation.".to_string()); + } if runtime.session_id.is_some() { return Err("A native voice conversation is already active.".to_string()); } @@ -1462,7 +1590,8 @@ pub async fn start_native_voice_conversation( runtime.owner = Some(RuntimeOwner { window_label: window_label.clone(), }); - runtime.pipeline = Some(pipeline); + runtime.pipeline = pipeline.take(); + runtime.admission = Some(Arc::new(BerdAdmissionCoordinator::default())); runtime.controls_ready = false; // Voice always starts from its owning session, where the in-session // controls are already available. The owner renderer reveals the @@ -1474,16 +1603,17 @@ pub async fn start_native_voice_conversation( let mute_app = app.clone(); let mute_window = webview_window.clone(); let mute_session_id = session_id.clone(); - runtime.native_microphone_mute_control = - native_input_mute::start(&state.input_muted, &state.input_mute_epoch, move |muted| { - let event = NativeVoiceEvent::MicrophoneMute { - session_id: mute_session_id.clone(), - muted, - revision: runtime_revision, - }; - let _ = mute_window.emit(EVENT_NAME, event.clone()); - super::voice_buddy::emit(&mute_app, event); - }); + let native_input_controls = input_controls.clone(); + runtime.native_microphone_mute_control = native_input_mute::start(move |muted| { + native_input_controls.set_host_muted(muted); + let event = NativeVoiceEvent::MicrophoneMute { + session_id: mute_session_id.clone(), + muted, + revision: runtime_revision, + }; + let _ = mute_window.emit(EVENT_NAME, event.clone()); + super::voice_buddy::emit(&mute_app, event); + }); Ok(( runtime.revision, runtime.lifecycle_id.clone().unwrap_or_default(), @@ -1492,6 +1622,10 @@ pub async fn start_native_voice_conversation( let (revision, lifecycle_id) = match install_result { Ok(lifecycle) => lifecycle, Err(error) => { + let error = match pipeline.take() { + Some(pipeline) => state.finish_uninstalled_pipeline(pipeline, error).await, + None => error, + }; drop(lifecycle_guard); if microphone_claimed { capture.release_microphone(&window_label, &renderer_id, renderer_epoch, &owner_id); @@ -1550,19 +1684,29 @@ pub async fn start_native_voice_conversation( let event_window = webview_window.clone(); let runtime = Arc::clone(&state.runtime); let pending = Arc::clone(&state.pending); + let admission = state + .runtime + .lock() + .ok() + .and_then(|runtime| runtime.admission.clone()) + .ok_or_else(|| "Voice admission was not initialized.".to_string())?; let event_state = state.inner().clone(); - let input_muted = Arc::clone(&state.input_muted); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { let active = runtime.lock().ok().is_some_and(|current| { current.session_id.as_deref() == Some(session_id.as_str()) && current.revision == revision + && !current.voice_input_quarantined }); if !active { break; } match event { - SttMessage::Speaking(speaking) => { + berd_voice::input::VoiceInputEvent::Ready => { + log::warn!("Voice input emitted duplicate readiness"); + } + berd_voice::input::VoiceInputEvent::SpeakingChanged(speaking) => { + admission.set_user_speaking(speaking); let event = NativeVoiceEvent::Activity { session_id: session_id.clone(), activity: if speaking { @@ -1575,7 +1719,15 @@ pub async fn start_native_voice_conversation( let _ = event_window.emit(EVENT_NAME, event.clone()); super::voice_buddy::emit(&event_app, event); } - SttMessage::Final { text, delivered } => { + berd_voice::input::VoiceInputEvent::RecognitionPendingChanged(pending) => { + admission.set_recognition_pending(pending); + // The runtime owns recognition-pending sequencing. Berd's + // renderer does not project that state yet. + } + berd_voice::input::VoiceInputEvent::FinalTranscript { + text, + storage_receipt, + } => { let transcript = PendingTranscript { session_id: session_id.clone(), lifecycle_id: lifecycle_id.clone(), @@ -1584,21 +1736,20 @@ pub async fn start_native_voice_conversation( revision, delivery_attempts: 0, }; - let Ok((accepted, evicted)) = enqueue_transcript_if_active( + let Ok(disposition) = store_final_if_active( &runtime, &pending, + &admission, &session_id, revision, transcript.clone(), + || storage_receipt.stored(), ) else { break; }; - if !accepted { - if let Some(delivered) = delivered { - let _ = delivered.send(()); - } + let StoredFinal::Stored { evicted } = disposition else { break; - } + }; if evicted.is_some() { let _ = event_window.emit( EVENT_NAME, @@ -1621,11 +1772,8 @@ pub async fn start_native_voice_conversation( delivery_attempts: transcript.delivery_attempts, }, ); - if let Some(delivered) = delivered { - let _ = delivered.send(()); - } } - SttMessage::Failed(message) => { + berd_voice::input::VoiceInputEvent::Failed(message) => { let _stop_guard = event_state.stop_serial.lock().await; let pipeline = { let Ok(mut current) = runtime.lock() else { @@ -1636,24 +1784,31 @@ pub async fn start_native_voice_conversation( { break; } - native_input_mute::stop(&input_muted); + native_input_mute::stop(); + event_state.input_controls.set_host_muted(false); current.native_microphone_mute_control = false; + if let Some(admission) = current.admission.take() { + admission.close(); + } current.session_id = None; current.lifecycle_id = None; current.owner = None; current.revision = current.revision.wrapping_add(1); current.pipeline.take() }; - if let Some(pipeline) = pipeline { - shutdown_pipeline(pipeline).await; - } + let shutdown_error = match pipeline { + Some(pipeline) => { + event_state.record_voice_input_finish(shutdown_pipeline(pipeline).await) + } + None => None, + }; event_state.microphone_muted.store(false, Ordering::SeqCst); event_app .state::() .release_owner(&window_label, &owner_id); let terminal_event = NativeVoiceEvent::Error { session_id: Some(session_id.clone()), - message, + message: shutdown_error.unwrap_or(message), revision: revision.wrapping_add(1), terminal: true, }; @@ -2001,11 +2156,12 @@ impl NativeVoiceState { next_revision, owner, owner_id, + shutdown_error, }) = completion else { return Ok(false); }; - if let Some(failure_message) = failure_message { + if let Some(failure_message) = shutdown_error.as_deref().or(failure_message) { let failure_event = NativeVoiceEvent::Error { session_id: Some(session_id.clone()), message: failure_message.to_string(), @@ -2031,15 +2187,6 @@ impl NativeVoiceState { Ok(true) } - #[cfg(test)] - async fn stop_lifecycle( - &self, - expected_lifecycle: Option<(&str, u64)>, - ) -> Result, String> { - let _stop_guard = self.stop_serial.lock().await; - self.stop_lifecycle_locked(expected_lifecycle).await - } - async fn stop_lifecycle_locked( &self, expected_lifecycle: Option<(&str, u64)>, @@ -2051,11 +2198,12 @@ impl NativeVoiceState { }; // Keep the lifecycle current through the bounded shutdown window so a // cooperative worker can flush its final utterance durably. A worker - // that misses the deadline is detached; its revision-bound late events - // are discarded rather than leaking into a replacement lifecycle. - if let Some(pipeline) = pipeline { - shutdown_pipeline(pipeline).await; - } + // that misses the deadline is quarantined; its revision-bound late + // events are discarded and this process cannot start a replacement. + let shutdown_error = match pipeline { + Some(pipeline) => self.record_voice_input_finish(shutdown_pipeline(pipeline).await), + None => None, + }; let (stopped, next_revision) = { let mut runtime = self .runtime @@ -2063,8 +2211,12 @@ impl NativeVoiceState { .map_err(|_| "native voice state lock was poisoned".to_string())?; let stopped = runtime.revision == revision && runtime.session_id == session_id; if stopped { - native_input_mute::stop(&self.input_muted); + native_input_mute::stop(); + self.input_controls.set_host_muted(false); runtime.native_microphone_mute_control = false; + if let Some(admission) = runtime.admission.take() { + admission.close(); + } runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -2084,6 +2236,7 @@ impl NativeVoiceState { next_revision, owner, owner_id, + shutdown_error, })) } @@ -2105,17 +2258,22 @@ impl NativeVoiceState { runtime.owner.clone(), ) }; - if let Some(pipeline) = pipeline { - shutdown_pipeline(pipeline).await; - } + let shutdown_error = match pipeline { + Some(pipeline) => self.record_voice_input_finish(shutdown_pipeline(pipeline).await), + None => None, + }; let next_revision = { let mut runtime = self .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.revision == revision && runtime.session_id == session_id { - native_input_mute::stop(&self.input_muted); + native_input_mute::stop(); + self.input_controls.set_host_muted(false); runtime.native_microphone_mute_control = false; + if let Some(admission) = runtime.admission.take() { + admission.close(); + } runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -2126,6 +2284,18 @@ impl NativeVoiceState { self.microphone_muted.store(false, Ordering::SeqCst); if let (Some(owner), Some(session_id)) = (owner, session_id) { capture.release_owner(&owner.window_label, &native_owner_id(&session_id)); + if let Some(message) = shutdown_error { + let failure_event = NativeVoiceEvent::Error { + session_id: Some(session_id.clone()), + message, + revision: next_revision, + terminal: true, + }; + if let Some(window) = app.get_webview_window(&owner.window_label) { + let _ = window.emit(EVENT_NAME, failure_event.clone()); + } + super::voice_buddy::emit(app, failure_event); + } let shutdown_event = NativeVoiceEvent::CleanShutdown { session_id, revision: next_revision, @@ -2262,9 +2432,12 @@ impl NativeVoiceState { return; }; if let Some(pipeline) = runtime.pipeline.as_ref() { - pipeline.latch_muted_shutdown(); + if pipeline.controls().is_muted() { + pipeline.cancel(); + } } - native_input_mute::stop(&self.input_muted); + native_input_mute::stop(); + self.input_controls.set_host_muted(false); runtime.native_microphone_mute_control = false; ( runtime.session_id.clone(), @@ -2276,6 +2449,9 @@ impl NativeVoiceState { self.microphone_muted.store(false, Ordering::SeqCst); if let Ok(mut runtime) = self.runtime.lock() { if runtime.revision == revision && runtime.session_id == session_id { + if let Some(admission) = runtime.admission.take() { + admission.close(); + } runtime.session_id = None; runtime.lifecycle_id = None; runtime.owner = None; @@ -2356,15 +2532,30 @@ fn push_audio_for_window( { return Err("Only the owning window may send native voice audio.".to_string()); } - if state.capture_is_suppressed() || state.microphone_is_muted() { - return Ok(()); - } if let Some(pipeline) = runtime.pipeline.as_ref() { - pipeline.push(bytes)?; + if pipeline.controls().is_muted() { + return Ok(()); + } + pipeline.try_push_frame(decode_voice_input_frame(&bytes)?)?; } Ok(()) } +fn decode_voice_input_frame(bytes: &[u8]) -> Result { + if bytes.len() != berd_voice::input::INPUT_FRAME_SAMPLES * size_of::() { + return Err(format!( + "native voice audio must contain exactly {} mono f32 samples", + berd_voice::input::INPUT_FRAME_SAMPLES + )); + } + let samples = bytes + .chunks_exact(size_of::()) + .map(|sample| f32::from_le_bytes(sample.try_into().expect("four-byte chunk"))) + .collect::>(); + berd_voice::input::VoiceInputFrame::try_from_samples(&samples) +} + +#[cfg(test)] fn enqueue_pending_transcript( queue: &mut VecDeque, transcript: PendingTranscript, @@ -2376,1157 +2567,559 @@ fn enqueue_pending_transcript( evicted } -fn enqueue_transcript_if_active( +#[derive(Debug)] +enum StoredFinal { + Stored { evicted: Option }, + Inactive, +} + +fn store_final_if_active( runtime: &Mutex, pending: &Mutex>, + admission: &BerdAdmissionCoordinator, expected_session_id: &str, expected_revision: u64, transcript: PendingTranscript, -) -> Result<(bool, Option), String> { + mark_stored: impl FnOnce(), +) -> Result { let runtime = runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; if runtime.session_id.as_deref() != Some(expected_session_id) || runtime.revision != expected_revision + || runtime.voice_input_quarantined { - return Ok((false, None)); + return Ok(StoredFinal::Inactive); } let mut pending = pending .lock() .map_err(|_| "pending transcript lock was poisoned".to_string())?; - let evicted = enqueue_pending_transcript(&mut pending, transcript); - Ok((true, evicted)) -} - -#[cfg(target_os = "macos")] -fn forward_macos_events( - events: &mut tokio_mpsc::UnboundedReceiver, - output: &tokio_mpsc::Sender, - delivery_deadline: Option, -) -> Result<(), ()> { - while let Ok(event) = events.try_recv() { - match event { - mac_speech::RecognitionEvent::Final(text) => { - let text = text.trim().to_string(); - if text.is_empty() { - continue; - } - let delivered = delivery_deadline.map(|_| { - let (sender, receiver) = mpsc::sync_channel(0); - (sender, receiver) - }); - let sender = delivered.as_ref().map(|(sender, _)| sender.clone()); - if output - .blocking_send(SttMessage::Final { - text, - delivered: sender, - }) - .is_err() - { - return Err(()); - } - if let (Some(deadline), Some((_, receiver))) = (delivery_deadline, delivered) { - let remaining = deadline.saturating_duration_since(Instant::now()); - if !remaining.is_zero() { - let _ = receiver.recv_timeout(remaining); - } - } - } - mac_speech::RecognitionEvent::Finished => { - if delivery_deadline.is_none() { - let _ = output.blocking_send(SttMessage::Failed( - "macOS speech recognition stopped unexpectedly.".to_string(), - )); - return Err(()); - } - } - mac_speech::RecognitionEvent::Failed(message) => { - if delivery_deadline.is_none() { - let _ = output.blocking_send(SttMessage::Failed(message)); - } else { - log::error!("macOS speech recognition failed while finishing: {message}"); - } - return Err(()); + let reference = VoiceTranscriptReference { + lifecycle_id: transcript.lifecycle_id.clone(), + id: transcript.id.clone(), + revision: transcript.revision, + }; + let evicted = (pending.len() >= MAX_PENDING_TRANSCRIPTS) + .then(|| pending.front().cloned()) + .flatten(); + admission.add_final(reference, transcript.text.clone())?; + if let Some(evicted) = evicted.as_ref() { + if evicted.session_id == transcript.session_id + && evicted.lifecycle_id == transcript.lifecycle_id + && evicted.revision == transcript.revision + { + let evicted_reference = VoiceTranscriptReference { + lifecycle_id: evicted.lifecycle_id.clone(), + id: evicted.id.clone(), + revision: evicted.revision, + }; + if !admission.discard(&evicted_reference)? { + return Err("The evicted voice transcript is not tracked by admission.".to_string()); } } + pending.pop_front(); } - Ok(()) + pending.push_back(transcript); + drop(pending); + drop(runtime); + // This acknowledgement is the durability boundary. UI delivery below is + // best effort and must not delay or decide whether the engine may finish. + mark_stored(); + Ok(StoredFinal::Stored { evicted }) } -#[cfg(target_os = "macos")] -fn new_macos_recognition_session() -> Result< - ( - mac_speech::RecognitionSession, - tokio_mpsc::UnboundedReceiver, - ), - String, -> { - let (events_tx, events_rx) = tokio_mpsc::unbounded_channel(); - mac_speech::RecognitionSession::new(events_tx).map(|session| (session, events_rx)) -} +#[cfg(test)] +mod tests { + use super::*; + use std::{thread, time::Instant}; -#[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -fn macos_stt_worker( - audio_rx: Receiver, - event_tx: tokio_mpsc::Sender, - shutdown: Arc, - discard_on_shutdown: Arc, - input_muted: Arc, - input_mute_epoch: Arc, - shutdown_mute_epoch: Arc, - assistant_speaking: Arc, - assistant_vad_threshold: Arc, - speech_vad_threshold: f32, -) { - use rubato::{Fft, FixedSync, Resampler}; - - let (mut session, mut recognition_events) = match new_macos_recognition_session() { - Ok(session) => session, - Err(error) => { - let _ = event_tx.blocking_send(SttMessage::Failed(error)); - return; - } - }; - let mut resampler = match Fft::::new(48_000, 16_000, 1024, 2, 1, FixedSync::Input) { - Ok(resampler) => resampler, - Err(error) => { - let _ = event_tx.blocking_send(SttMessage::Failed(format!( - "Could not initialize native audio resampling: {error}" - ))); - return; - } - }; - let chunk_in = resampler.input_frames_next(); - let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); - let mut input_48k = Vec::new(); - let mut leftover_16k = Vec::new(); - let mut silence_frames = 0_usize; - let mut in_speech = false; - let mut observed_mute_epoch = input_mute_epoch.load(Ordering::Acquire); - - loop { - if forward_macos_events(&mut recognition_events, &event_tx, None).is_err() { - return; + fn pending_transcript( + session_id: &str, + lifecycle_id: &str, + id: &str, + revision: u64, + ) -> PendingTranscript { + PendingTranscript { + session_id: session_id.to_string(), + lifecycle_id: lifecycle_id.to_string(), + id: id.to_string(), + text: format!("text-{id}"), + revision, + delivery_attempts: 0, } - let batch = match audio_rx.recv_timeout(Duration::from_millis(50)) { - Ok(batch) => Some(batch), - Err(mpsc::RecvTimeoutError::Timeout) => None, - Err(mpsc::RecvTimeoutError::Disconnected) => break, + } + + #[test] + fn final_that_linearizes_before_stop_is_stored_and_acknowledged_once() { + let runtime = Arc::new(Mutex::new(Runtime { + session_id: Some("session-a".into()), + lifecycle_id: Some("lifecycle-a".into()), + revision: 4, + ..Runtime::default() + })); + let pending = Arc::new(Mutex::new(VecDeque::new())); + let admission = Arc::new(BerdAdmissionCoordinator::default()); + let acknowledgements = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let pending_gate = pending.lock().expect("hold pending queue"); + + let final_thread = { + let runtime = Arc::clone(&runtime); + let pending = Arc::clone(&pending); + let pending_at_ack = Arc::clone(&pending); + let admission = Arc::clone(&admission); + let acknowledgements = Arc::clone(&acknowledgements); + thread::spawn(move || { + store_final_if_active( + &runtime, + &pending, + &admission, + "session-a", + 4, + pending_transcript("session-a", "lifecycle-a", "final", 4), + || { + assert_eq!( + pending_at_ack.lock().expect("inspect stored final")[0].id, + "final" + ); + acknowledgements.fetch_add(1, Ordering::SeqCst); + }, + ) + .expect("store final") + }) }; - let (shutting_down, current_mute_epoch) = - sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); - if current_mute_epoch != observed_mute_epoch { - observed_mute_epoch = current_mute_epoch; - input_48k.clear(); - leftover_16k.clear(); - silence_frames = 0; - if std::mem::take(&mut in_speech) { - let _ = event_tx.blocking_send(SttMessage::Speaking(false)); - } - // A mute boundary must discard Apple's partial hypothesis just as - // the Parakeet worker discards its buffered utterance. - session.cancel(); - if shutting_down { - break; - } - match new_macos_recognition_session() { - Ok((next_session, next_events)) => { - session = next_session; - recognition_events = next_events; - } - Err(error) => { - let _ = event_tx.blocking_send(SttMessage::Failed(error)); - return; - } - } - } - if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || batch.is_none()) { - break; - } - if !shutting_down && input_muted.load(Ordering::Acquire) { - continue; - } - let Some(batch) = batch else { - continue; + let deadline = Instant::now() + Duration::from_secs(1); + while runtime.try_lock().is_ok() && Instant::now() < deadline { + thread::yield_now(); + } + assert!(runtime.try_lock().is_err(), "final holds lifecycle lock"); + let close_thread = { + let runtime = Arc::clone(&runtime); + thread::spawn(move || { + let mut runtime = runtime.lock().expect("close lifecycle"); + runtime.session_id = None; + runtime.lifecycle_id = None; + runtime.revision += 1; + }) }; - if batch.mute_epoch != observed_mute_epoch { - continue; - } + drop(pending_gate); - let samples: Vec = batch - .bytes - .chunks_exact(4) - .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])) - .collect(); - // Feed Apple continuously. Earshot below controls activity only and - // never holds audio until its silence boundary. - if let Err(error) = session.push(&samples) { - let _ = event_tx.blocking_send(SttMessage::Failed(error)); - return; - } - - input_48k.extend_from_slice(&samples); - while input_48k.len() >= chunk_in { - let chunk: Vec = input_48k.drain(..chunk_in).collect(); - leftover_16k.extend_from_slice(&resample(&mut resampler, &chunk)); - while leftover_16k.len() >= VAD_FRAME_SAMPLES { - let frame: Vec = leftover_16k.drain(..VAD_FRAME_SAMPLES).collect(); - let clamped: Vec = - frame.iter().map(|sample| sample.clamp(-1.0, 1.0)).collect(); - let vad_threshold = active_vad_threshold_for_speech( - &assistant_speaking, - &assistant_vad_threshold, - speech_vad_threshold, - ); - if vad.predict_f32(&clamped) > vad_threshold { - silence_frames = 0; - if !in_speech { - in_speech = true; - let _ = event_tx.blocking_send(SttMessage::Speaking(true)); - } - } else if in_speech { - silence_frames += 1; - if silence_frames >= SILENCE_FLUSH_FRAMES { - silence_frames = 0; - in_speech = false; - let _ = event_tx.blocking_send(SttMessage::Speaking(false)); - } - } - } - } - if forward_macos_events(&mut recognition_events, &event_tx, None).is_err() { - return; - } - } - - if in_speech { - let _ = event_tx.blocking_send(SttMessage::Speaking(false)); - } - if discard_on_shutdown.load(Ordering::Acquire) { - session.cancel(); - return; - } - if let Err(error) = session.finish() { - log::error!("Could not finish macOS speech recognition: {error}"); - return; + assert!(matches!( + final_thread.join().expect("join final"), + StoredFinal::Stored { .. } + )); + close_thread.join().expect("join close"); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 1); + assert_eq!(pending.lock().expect("pending queue").len(), 1); + let admission = admission.inner.lock().expect("admission state"); + assert_eq!(admission.next_token, 1); + assert_eq!(admission.tokens.len(), 1); + assert_eq!(admission.core.utterances_after(0).len(), 1); } - let delivery_deadline = Instant::now() + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT; - let _ = forward_macos_events(&mut recognition_events, &event_tx, Some(delivery_deadline)); -} - -#[derive(Debug, PartialEq, Eq)] -struct OpenAiCommittedTurn { - item_id: String, - mute_epoch: u64, -} -fn block_on_openai_operation( - runtime: &tokio::runtime::Runtime, - shutdown: &AtomicBool, - future: F, - action: &str, -) -> Result, String> -where - F: std::future::Future>, - E: std::fmt::Display, -{ - runtime.block_on(async { - let wait_for_shutdown = async { - while !shutdown.load(Ordering::Acquire) { - tokio::time::sleep(Duration::from_millis(20)).await; - } + #[test] + fn stop_that_linearizes_before_final_drops_without_acknowledging() { + let runtime = Arc::new(Mutex::new(Runtime { + session_id: Some("session-a".into()), + lifecycle_id: Some("lifecycle-a".into()), + revision: 4, + ..Runtime::default() + })); + let pending = Arc::new(Mutex::new(VecDeque::new())); + let admission = Arc::new(BerdAdmissionCoordinator::default()); + let acknowledgements = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut close = runtime.lock().expect("hold lifecycle for stop"); + let final_thread = { + let runtime = Arc::clone(&runtime); + let pending = Arc::clone(&pending); + let admission = Arc::clone(&admission); + let acknowledgements = Arc::clone(&acknowledgements); + thread::spawn(move || { + store_final_if_active( + &runtime, + &pending, + &admission, + "session-a", + 4, + pending_transcript("session-a", "lifecycle-a", "late", 4), + || { + acknowledgements.fetch_add(1, Ordering::SeqCst); + }, + ) + .expect("dispose final") + }) }; - tokio::select! { - result = tokio::time::timeout(OPENAI_NETWORK_OPERATION_TIMEOUT, future) => { - match result { - Ok(Ok(value)) => Ok(Some(value)), - Ok(Err(error)) => Err(format!("{action}: {error}")), - Err(_) => Err(format!("{action}: operation timed out")), - } - } - () = wait_for_shutdown => Ok(None), - } - }) -} + close.session_id = None; + close.lifecycle_id = None; + close.revision = 5; + drop(close); -async fn wait_for_openai_transcription_ready(socket: &mut S) -> Result<(), String> -where - S: futures_util::Stream> + Unpin, - E: std::fmt::Display, -{ - use futures_util::StreamExt; - use tokio_tungstenite::tungstenite::Message; - - loop { - match socket.next().await { - Some(Ok(Message::Text(text))) => { - let Ok(value) = serde_json::from_str::(&text) else { - continue; - }; - match value.get("type").and_then(|value| value.as_str()) { - Some("session.updated") => return Ok(()), - Some("error") => { - return Err(value - .pointer("/error/message") - .and_then(|value| value.as_str()) - .unwrap_or("OpenAI rejected the transcription session configuration.") - .to_string()); - } - _ => {} - } - } - Some(Ok(Message::Close(_))) | None => { - return Err( - "OpenAI realtime transcription disconnected before it was ready.".to_string(), - ); - } - Some(Ok(_)) => {} - Some(Err(error)) => { - return Err(format!( - "OpenAI realtime transcription failed before it was ready: {error}" - )); - } - } + assert!(matches!( + final_thread.join().expect("join final"), + StoredFinal::Inactive + )); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 0); + assert!(pending.lock().expect("pending queue").is_empty()); + let admission = admission.inner.lock().expect("admission state"); + assert_eq!(admission.next_token, 0); + assert!(admission.tokens.is_empty()); + assert!(admission.core.utterances_after(0).is_empty()); } -} -fn push_openai_pre_roll(pre_roll: &mut VecDeque>, pcm: Vec) { - pre_roll.push_back(pcm); - while pre_roll.len() > OPENAI_PRE_ROLL_CHUNKS { - pre_roll.pop_front(); - } -} + #[test] + fn old_revision_final_cannot_reach_replacement_but_current_final_can() { + let runtime = Mutex::new(Runtime { + session_id: Some("session-b".into()), + lifecycle_id: Some("lifecycle-b".into()), + revision: 5, + ..Runtime::default() + }); + let pending = Mutex::new(VecDeque::new()); + let admission = BerdAdmissionCoordinator::default(); + let acknowledgements = std::sync::atomic::AtomicUsize::new(0); -fn openai_turn_reached_limit(samples_16k: usize) -> bool { - samples_16k >= MAX_SPEECH_SAMPLES -} + assert!(matches!( + store_final_if_active( + &runtime, + &pending, + &admission, + "session-a", + 4, + pending_transcript("session-a", "lifecycle-a", "old", 4), + || { + acknowledgements.fetch_add(1, Ordering::SeqCst); + }, + ) + .expect("reject old final"), + StoredFinal::Inactive + )); + assert!(matches!( + store_final_if_active( + &runtime, + &pending, + &admission, + "session-b", + 5, + pending_transcript("session-b", "lifecycle-b", "new", 5), + || { + acknowledgements.fetch_add(1, Ordering::SeqCst); + }, + ) + .expect("store replacement final"), + StoredFinal::Stored { .. } + )); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 1); + let pending = pending.lock().expect("pending queue"); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].id, "new"); + let admission = admission.inner.lock().expect("admission state"); + assert_eq!(admission.next_token, 1); + assert_eq!(admission.tokens.len(), 1); + assert_eq!(admission.core.utterances_after(0).len(), 1); + } -fn record_openai_transcription_event( - value: &serde_json::Value, - current_mute_epoch: u64, - pending_commit_epochs: &mut VecDeque, - committed: &mut VecDeque, - completed: &mut HashMap, -) -> Result, String> { - let mut newly_committed = None; - match value.get("type").and_then(|value| value.as_str()) { - Some("input_audio_buffer.committed") => { - if let Some(item_id) = value.get("item_id").and_then(|value| value.as_str()) { - let turn = OpenAiCommittedTurn { - item_id: item_id.to_string(), - mute_epoch: pending_commit_epochs - .pop_front() - .unwrap_or(current_mute_epoch), - }; - if turn.mute_epoch == current_mute_epoch { - committed.push_back(OpenAiCommittedTurn { - item_id: turn.item_id.clone(), - mute_epoch: turn.mute_epoch, - }); - } - newly_committed = Some(turn); - } - } - Some("conversation.item.input_audio_transcription.completed") => { - if let (Some(item_id), Some(transcript)) = ( - value.get("item_id").and_then(|value| value.as_str()), - value.get("transcript").and_then(|value| value.as_str()), - ) { - if committed - .iter() - .any(|turn| turn.item_id == item_id && turn.mute_epoch == current_mute_epoch) - { - completed.insert(item_id.to_string(), transcript.trim().to_string()); - } - } - } - Some("conversation.item.input_audio_transcription.failed") => { - if value - .get("item_id") - .and_then(|value| value.as_str()) - .is_some_and(|item_id| { - !committed.iter().any(|turn| { - turn.item_id == item_id && turn.mute_epoch == current_mute_epoch - }) + #[test] + fn current_final_evicts_superseded_lifecycle_recovery_without_current_admission() { + let runtime = Mutex::new(Runtime { + session_id: Some("session-new".into()), + lifecycle_id: Some("lifecycle-new".into()), + revision: 5, + ..Runtime::default() + }); + let pending = Mutex::new( + (0..MAX_PENDING_TRANSCRIPTS) + .map(|index| { + pending_transcript("session-old", "lifecycle-old", &format!("old-{index}"), 4) }) - { - return Ok(None); - } - return Err(value - .pointer("/error/message") - .and_then(|value| value.as_str()) - .unwrap_or("OpenAI realtime transcription failed.") - .to_string()); - } - Some("error") => { - return Err(value - .pointer("/error/message") - .and_then(|value| value.as_str()) - .unwrap_or("OpenAI realtime transcription failed.") - .to_string()); - } - _ => {} - } - Ok(newly_committed) -} + .collect(), + ); + let admission = BerdAdmissionCoordinator::default(); -fn deliver_completed_openai_turns( - committed: &mut VecDeque, - completed: &mut HashMap, - event_tx: &tokio_mpsc::Sender, - current_mute_epoch: u64, - final_item_id: Option<&str>, - final_delivery: &mut Option>, -) { - while committed - .front() - .is_some_and(|turn| completed.contains_key(&turn.item_id)) - { - let turn = committed.pop_front().expect("checked front"); - let text = completed.remove(&turn.item_id).unwrap_or_default(); - if turn.mute_epoch != current_mute_epoch { - continue; - } - let delivered = (Some(turn.item_id.as_str()) == final_item_id) - .then(|| final_delivery.take()) - .flatten(); - deliver_recognition_result(text, event_tx, delivered); - } -} + let result = store_final_if_active( + &runtime, + &pending, + &admission, + "session-new", + 5, + pending_transcript("session-new", "lifecycle-new", "new", 5), + || {}, + ) + .expect("store current final while evicting stale recovery"); -#[allow(clippy::too_many_arguments)] // Worker boundary keeps channel and mute lifecycle inputs explicit. -fn openai_stt_worker( - key: String, - endpoint: String, - model: String, - audio_rx: Receiver, - event_tx: tokio_mpsc::Sender, - shutdown: Arc, - discard_on_shutdown: Arc, - input_muted: Arc, - input_mute_epoch: Arc, - shutdown_mute_epoch: Arc, - assistant_speaking: Arc, - assistant_vad_threshold: Arc, - speech_vad_threshold: f32, - startup_tx: SyncSender>, -) { - use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; - use futures_util::{SinkExt, StreamExt}; - use rubato::{Fft, FixedSync, Resampler}; - use tokio_tungstenite::tungstenite::{client::IntoClientRequest, Message}; - - macro_rules! fail_openai_startup { - ($error:expr) => {{ - let error = $error; - let _ = startup_tx.send(Err(error.clone())); - let _ = event_tx.blocking_send(SttMessage::Failed(error)); - return; - }}; + assert!(matches!(result, StoredFinal::Stored { evicted: Some(_) })); + let pending = pending.lock().unwrap(); + assert_eq!(pending.len(), MAX_PENDING_TRANSCRIPTS); + assert_eq!(pending.front().unwrap().id, "old-1"); + assert_eq!(pending.back().unwrap().id, "new"); } - let runtime = match tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - { - Ok(runtime) => runtime, - Err(error) => { - fail_openai_startup!(format!( - "Could not initialize OpenAI realtime transcription: {error}" - )); - } - }; - if let Err(existing) = rustls::crypto::aws_lc_rs::default_provider().install_default() { - // Another dependency may have installed the same process-wide provider first. - drop(existing); - } - let mut request = match endpoint.into_client_request() { - Ok(request) => request, - Err(error) => { - fail_openai_startup!(format!("prepare OpenAI realtime connection: {error}")); - } - }; - let authorization = match format!("Bearer {key}").parse() { - Ok(authorization) => authorization, - Err(_) => { - fail_openai_startup!("OpenAI API key is not a valid header value".to_string()); - } - }; - request.headers_mut().insert("Authorization", authorization); - let connection = block_on_openai_operation( - &runtime, - &shutdown, - tokio_tungstenite::connect_async(request), - "connect to OpenAI realtime transcription", - ); - let mut socket = match connection { - Ok(Some((socket, _))) => socket, - Ok(None) => return, - Err(error) => { - fail_openai_startup!(error); + #[tokio::test] + async fn nonquiescent_finish_blocks_restart_and_projects_unavailable() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("install old lifecycle"); + runtime.session_id = Some("session-a".into()); + runtime.lifecycle_id = Some("lifecycle-a".into()); + runtime.revision = 4; } - }; + let message = state + .record_voice_input_finish(Err(berd_voice::input::VoiceInputFinishError::Quarantined { + timeout: Duration::from_millis(20), + })) + .expect("quarantine is terminal"); + assert!(message.contains("Restart Berd")); + assert!(state.ensure_voice_input_not_quarantined().is_err()); - macro_rules! send_openai_startup_message { - ($message:expr, $action:literal) => { - match block_on_openai_operation(&runtime, &shutdown, socket.send($message), $action) { - Ok(Some(())) => {} - Ok(None) => return, - Err(error) => { - fail_openai_startup!(error); - } - } - }; + let admission = BerdAdmissionCoordinator::default(); + let acknowledgements = std::sync::atomic::AtomicUsize::new(0); + assert!(matches!( + store_final_if_active( + &state.runtime, + &state.pending, + &admission, + "session-a", + 4, + pending_transcript("session-a", "lifecycle-a", "too-late", 4), + || { + acknowledgements.fetch_add(1, Ordering::SeqCst); + }, + ) + .expect("quarantine rejects late final"), + StoredFinal::Inactive + )); + assert_eq!(acknowledgements.load(Ordering::SeqCst), 0); + assert!(state.pending.lock().expect("pending queue").is_empty()); + + let status = status_with_availability(&state, true, || async { true }).await; + assert!(!status.available); + assert!(status + .unavailable_reason + .as_deref() + .is_some_and(|reason| reason.contains("Restart Berd"))); } - let session_update = serde_json::json!({ - "type": "session.update", - "session": { - "type": "transcription", - "audio": { "input": { - "format": { "type": "audio/pcm", "rate": 24000 }, - "transcription": { "model": model, "delay": "low" }, - "turn_detection": null - }} - } - }); - send_openai_startup_message!( - Message::Text(session_update.to_string().into()), - "configure OpenAI realtime transcription" - ); - let mut resampler = match Fft::::new(48_000, 24_000, 960, 2, 1, FixedSync::Input) { - Ok(resampler) => resampler, - Err(error) => { - fail_openai_startup!(format!( - "Could not initialize OpenAI audio resampling: {error}" - )); - } - }; - let readiness = block_on_openai_operation( - &runtime, - &shutdown, - wait_for_openai_transcription_ready(&mut socket), - "wait for OpenAI realtime transcription readiness", - ); - match readiness { - Ok(Some(())) => {} - Ok(None) => return, - Err(error) => fail_openai_startup!(error), - } - if startup_tx.send(Ok(())).is_err() { - return; - } - let chunk_in = resampler.input_frames_next(); - let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); - let mut input_48k = Vec::new(); - let mut vad_16k = Vec::new(); - let mut silence_frames = 0_usize; - let mut in_speech = false; - let mut turn_has_audio = false; - let mut turn_samples_16k = 0_usize; - let mut pre_roll = VecDeque::>::new(); - let mut observed_mute_epoch = input_mute_epoch.load(Ordering::Acquire); - let mut pending_commit_epochs = VecDeque::::new(); - let mut committed_turns = VecDeque::::new(); - let mut completed_turns = HashMap::::new(); - - macro_rules! send_openai_stream_message { - ($message:expr, $action:literal, $on_shutdown:block) => { - match block_on_openai_operation(&runtime, &shutdown, socket.send($message), $action) { - Ok(Some(())) => {} - Ok(None) => $on_shutdown - Err(error) => { - let _ = event_tx.blocking_send(SttMessage::Failed(error)); - return; - } - } - }; + #[test] + fn quiescent_completion_and_worker_panic_do_not_poison_restart() { + let completed = NativeVoiceState::default(); + assert_eq!(completed.record_voice_input_finish(Ok(())), None); + completed + .ensure_voice_input_not_quarantined() + .expect("joined worker remains restartable"); + + let panicked = NativeVoiceState::default(); + assert_eq!( + panicked.record_voice_input_finish(Err( + berd_voice::input::VoiceInputFinishError::WorkerPanicked, + )), + Some("voice input runtime worker panicked".to_string()) + ); + panicked + .ensure_voice_input_not_quarantined() + .expect("joined panic is quiescent and restartable"); } - 'worker: loop { - let (shutting_down, current_mute_epoch) = - sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); - if shutting_down { - break; - } - if current_mute_epoch != observed_mute_epoch { - observed_mute_epoch = current_mute_epoch; - input_48k.clear(); - vad_16k.clear(); - silence_frames = 0; - turn_has_audio = false; - turn_samples_16k = 0; - pre_roll.clear(); - committed_turns.clear(); - completed_turns.clear(); - if std::mem::take(&mut in_speech) { - let _ = event_tx.blocking_send(SttMessage::Speaking(false)); - } - send_openai_stream_message!( - Message::Text( - serde_json::json!({"type":"input_audio_buffer.clear"}) - .to_string() - .into() - ), - "clear muted OpenAI transcription audio", - { break 'worker } - ); - } - while let Some(event) = runtime.block_on(async { - tokio::time::timeout(Duration::from_millis(1), socket.next()) - .await - .ok() - .flatten() - }) { - let (shutting_down, current_mute_epoch) = - sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); - if shutting_down { - break 'worker; - } - if current_mute_epoch != observed_mute_epoch { - continue 'worker; - } - let message = match event { - Ok(Message::Text(text)) => text, - Ok(Message::Close(_)) => { - let _ = event_tx.blocking_send(SttMessage::Failed( - "OpenAI realtime transcription disconnected.".to_string(), - )); - return; - } - Ok(_) => continue, - Err(error) => { - let _ = event_tx.blocking_send(SttMessage::Failed(format!( - "OpenAI realtime transcription failed: {error}" - ))); - return; - } - }; - let Ok(value) = serde_json::from_str::(&message) else { - continue; - }; - let recorded_turn = match record_openai_transcription_event( - &value, - observed_mute_epoch, - &mut pending_commit_epochs, - &mut committed_turns, - &mut completed_turns, - ) { - Ok(turn) => turn, - Err(error) => { - let _ = event_tx.blocking_send(SttMessage::Failed(error)); - return; - } - }; - if let Some(turn) = recorded_turn.filter(|turn| turn.mute_epoch != observed_mute_epoch) - { - send_openai_stream_message!( - Message::Text( - serde_json::json!({ - "type": "conversation.item.delete", - "item_id": turn.item_id, - }) - .to_string() - .into() - ), - "discard muted OpenAI transcription turn", - { break 'worker } - ); - } - deliver_completed_openai_turns( - &mut committed_turns, - &mut completed_turns, - &event_tx, - input_mute_epoch.load(Ordering::Acquire), - None, - &mut None, - ); + fn transcript_reference(id: &str) -> VoiceTranscriptReference { + VoiceTranscriptReference { + lifecycle_id: "lifecycle-1".to_string(), + id: id.to_string(), + revision: 4, } + } - let batch = match audio_rx.recv_timeout(Duration::from_millis(20)) { - Ok(batch) => Some(batch), - Err(mpsc::RecvTimeoutError::Timeout) => None, - Err(mpsc::RecvTimeoutError::Disconnected) => break, + #[tokio::test] + async fn admission_holds_without_confirming_then_observes_a_final() { + let admission = Arc::new(BerdAdmissionCoordinator::default()); + let first = transcript_reference("first"); + admission.add_final(first.clone(), "one".into()).unwrap(); + admission.set_user_speaking(true); + + let waiting = { + let admission = Arc::clone(&admission); + let first = first.clone(); + tokio::spawn(async move { admission.prepare("reply".into(), Some(first)).await }) }; - let (shutting_down, current_mute_epoch) = - sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); - // Stop consuming queued microphone batches as soon as shutdown begins. - // The bounded finalization below commits only audio already accepted by - // this worker, so network backpressure cannot extend the owner timeout. - if shutting_down { - break; - } - if current_mute_epoch != observed_mute_epoch { - continue 'worker; - } - if input_muted.load(Ordering::Acquire) { - continue; - } - let Some(batch) = batch else { continue }; - if batch.mute_epoch != observed_mute_epoch { - continue; - } - input_48k.extend( - batch - .bytes - .chunks_exact(4) - .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])), + tokio::task::yield_now().await; + assert_eq!( + admission.inner.lock().unwrap().core.confirmed_token(), + 0, + "held preparation must not apply its causal acknowledgement" ); - while input_48k.len() >= chunk_in { - let chunk: Vec = input_48k.drain(..chunk_in).collect(); - let pcm_24k = resample(&mut resampler, &chunk); - let pcm_bytes: Vec = pcm_24k - .iter() - .flat_map(|sample| { - ((sample.clamp(-1.0, 1.0) * i16::MAX as f32).round() as i16).to_le_bytes() - }) - .collect(); - // Earshot requires 16 kHz; use every third 48 kHz source sample for activity only. - vad_16k.extend(chunk.iter().step_by(3).copied()); - let mut speech_started = false; - let mut should_commit = false; - while vad_16k.len() >= VAD_FRAME_SAMPLES { - let frame: Vec = vad_16k.drain(..VAD_FRAME_SAMPLES).collect(); - let threshold = active_vad_threshold_for_speech( - &assistant_speaking, - &assistant_vad_threshold, - speech_vad_threshold, - ); - if vad.predict_f32(&frame) > threshold { - silence_frames = 0; - if !in_speech { - in_speech = true; - speech_started = true; - let _ = event_tx.blocking_send(SttMessage::Speaking(true)); - } - } else if in_speech { - silence_frames += 1; - if silence_frames >= SILENCE_FLUSH_FRAMES { - should_commit = true; - silence_frames = 0; - in_speech = false; - } - } - } - if speech_started { - while let Some(pre_roll_bytes) = pre_roll.pop_front() { - send_openai_stream_message!( - Message::Text( - serde_json::json!({ - "type": "input_audio_buffer.append", - "audio": BASE64.encode(pre_roll_bytes) - }) - .to_string() - .into() - ), - "stream OpenAI transcription pre-roll", - { break 'worker } - ); - } - } + admission + .add_final(transcript_reference("second"), "two".into()) + .unwrap(); + admission.set_user_speaking(false); + assert!(matches!( + waiting.await.unwrap().unwrap(), + PrepareAssistantSpeechOutcome::Pending + )); + } - if speech_started || in_speech || should_commit { - send_openai_stream_message!( - Message::Text( - serde_json::json!({ - "type": "input_audio_buffer.append", - "audio": BASE64.encode(pcm_bytes) - }) - .to_string() - .into() - ), - "stream audio to OpenAI realtime transcription", - { break 'worker } - ); - turn_has_audio = true; - turn_samples_16k = turn_samples_16k.saturating_add(chunk.len() / 3); - } else { - push_openai_pre_roll(&mut pre_roll, pcm_bytes); - } + #[tokio::test] + async fn stale_exact_causal_reference_does_not_inherit_global_confirmation() { + let admission = BerdAdmissionCoordinator::default(); + let first = transcript_reference("first"); + let second = transcript_reference("second"); + admission.add_final(first.clone(), "one".into()).unwrap(); + admission.add_final(second.clone(), "two".into()).unwrap(); + assert!(admission.confirm(&second).unwrap()); - if should_commit || openai_turn_reached_limit(turn_samples_16k) { - send_openai_stream_message!( - Message::Text( - serde_json::json!({"type":"input_audio_buffer.commit"}) - .to_string() - .into() - ), - "commit OpenAI transcription turn", - { break 'worker } - ); - pending_commit_epochs.push_back(observed_mute_epoch); - turn_has_audio = false; - turn_samples_16k = 0; - silence_frames = 0; - if std::mem::take(&mut in_speech) || should_commit { - let _ = event_tx.blocking_send(SttMessage::Speaking(false)); - } - } - } - } - if discard_on_shutdown.load(Ordering::Acquire) { - return; - } - let mut final_item_id = None::; - let mut final_delivery = None::>; - let mut final_delivered = None::>; - if turn_has_audio { - let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); - let commit = serde_json::json!({"type":"input_audio_buffer.commit"}); - let final_write = runtime.block_on(tokio::time::timeout( - OPENAI_FINAL_WRITE_TIMEOUT, - socket.send(Message::Text(commit.to_string().into())), + assert!(matches!( + admission + .prepare("delayed".into(), Some(first)) + .await + .unwrap(), + PrepareAssistantSpeechOutcome::Pending )); - if matches!(final_write, Ok(Ok(()))) { - pending_commit_epochs.push_back(observed_mute_epoch); - final_delivery = Some(delivered_tx); - final_delivered = Some(delivered_rx); - } } - if final_delivered.is_none() - && (!committed_turns.is_empty() || !pending_commit_epochs.is_empty()) - { - let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); - final_item_id = pending_commit_epochs - .is_empty() - .then(|| committed_turns.back().map(|turn| turn.item_id.clone())) - .flatten(); - final_delivery = Some(delivered_tx); - final_delivered = Some(delivered_rx); - } - - // Keep receiving until the shutdown commit itself completes. Earlier turns - // are delivered from the queue first, and already committed transcripts are - // drained even when there was no partial turn to commit at shutdown. - let deadline = std::time::Instant::now() + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT; - while std::time::Instant::now() < deadline { - if final_delivered - .as_ref() - .is_some_and(|receiver| receiver.try_recv().is_ok()) - || (final_delivered.is_none() - && committed_turns.is_empty() - && pending_commit_epochs.is_empty()) - { - break; - } - let Some(message) = runtime.block_on(async { - tokio::time::timeout(Duration::from_millis(50), socket.next()) - .await - .ok() - .flatten() - }) else { - continue; - }; - let Ok(Message::Text(text)) = message else { - continue; - }; - let Ok(value) = serde_json::from_str::(&text) else { - continue; - }; - let event_type = value.get("type").and_then(|value| value.as_str()); - match record_openai_transcription_event( - &value, - observed_mute_epoch, - &mut pending_commit_epochs, - &mut committed_turns, - &mut completed_turns, - ) { - Ok(Some(turn)) - if turn.mute_epoch == observed_mute_epoch - && pending_commit_epochs.is_empty() - && final_delivered.is_some() => - { - final_item_id = Some(turn.item_id); - } - Ok(_) => {} - Err(error) => { - let _ = event_tx.blocking_send(SttMessage::Failed(error)); - break; - } - } - deliver_completed_openai_turns( - &mut committed_turns, - &mut completed_turns, - &event_tx, - observed_mute_epoch, - final_item_id.as_deref(), - &mut final_delivery, - ); - if event_type == Some("conversation.item.input_audio_transcription.completed") - && final_item_id.as_deref() == value.get("item_id").and_then(|value| value.as_str()) - && final_delivery.is_none() + + #[tokio::test] + async fn fast_second_delivery_cannot_hide_slow_first_rejection() { + let state = NativeVoiceState::default(); + let admission = Arc::new(BerdAdmissionCoordinator::default()); { - break; + let mut runtime = state.runtime.lock().unwrap(); + runtime.session_id = Some("session-1".into()); + runtime.lifecycle_id = Some("lifecycle-1".into()); + runtime.revision = 4; + runtime.admission = Some(Arc::clone(&admission)); + } + for id in ["first", "second"] { + let reference = transcript_reference(id); + admission.add_final(reference, id.to_string()).unwrap(); + state.pending.lock().unwrap().push_back(PendingTranscript { + session_id: "session-1".into(), + lifecycle_id: "lifecycle-1".into(), + id: id.into(), + text: id.into(), + revision: 4, + delivery_attempts: 0, + }); } - } - // Do not await the peer's WebSocket close handshake here. OpenAI may leave - // it pending beyond the bounded voice-stop window; dropping the socket - // closes the connection after final transcript delivery has been drained. - drop(socket); -} -#[allow(clippy::too_many_arguments)] // Worker boundary keeps channel and mute lifecycle inputs explicit. -fn stt_worker( - model_dir: PathBuf, - audio_rx: Receiver, - event_tx: tokio_mpsc::Sender, - shutdown: Arc, - discard_on_shutdown: Arc, - input_muted: Arc, - input_mute_epoch: Arc, - shutdown_mute_epoch: Arc, - assistant_speaking: Arc, - assistant_vad_threshold: Arc, - speech_vad_threshold: f32, -) { - use rubato::{Fft, FixedSync, Resampler}; - use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; - - let mut resampler = match Fft::::new(48_000, 16_000, 1024, 2, 1, FixedSync::Input) { - Ok(resampler) => resampler, - Err(error) => { - let _ = event_tx.blocking_send(SttMessage::Failed(format!( - "Could not initialize native audio resampling: {error}" - ))); - return; - } - }; - let chunk_in = resampler.input_frames_next(); - let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); - - let mut config = OfflineRecognizerConfig::default(); - config.model_config.nemo_ctc.model = Some( - model_dir - .join("model.int8.onnx") - .to_string_lossy() - .into_owned(), - ); - config.model_config.tokens = Some(model_dir.join("tokens.txt").to_string_lossy().into_owned()); - config.model_config.num_threads = 1; - config.model_config.debug = false; - let Some(recognizer) = OfflineRecognizer::create(&config) else { - let _ = event_tx.blocking_send(SttMessage::Failed( - "Could not load the Parakeet speech model.".to_string(), + state + .acknowledge_transcript("session-1", "second", 4) + .unwrap(); + assert!(matches!( + admission + .prepare("reply".into(), Some(transcript_reference("second"))) + .await + .unwrap(), + PrepareAssistantSpeechOutcome::Pending )); - return; - }; - let mut input_48k = Vec::new(); - let mut leftover_16k = Vec::new(); - let mut speech = Vec::new(); - let mut silence_frames = 0_usize; - let mut in_speech = false; - let mut observed_mute_epoch = input_mute_epoch.load(Ordering::Acquire); - while !shutdown.load(Ordering::Acquire) { - let batch = match audio_rx.recv_timeout(Duration::from_millis(50)) { - Ok(batch) => Some(batch), - Err(mpsc::RecvTimeoutError::Timeout) => None, - Err(mpsc::RecvTimeoutError::Disconnected) => break, - }; - let (shutting_down, current_mute_epoch) = - sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); - if current_mute_epoch != observed_mute_epoch { - observed_mute_epoch = current_mute_epoch; - if clear_buffered_audio( - &mut input_48k, - &mut leftover_16k, - &mut speech, - &mut silence_frames, - &mut in_speech, - ) { - let _ = event_tx.blocking_send(SttMessage::Speaking(false)); - } - } - if shutting_down && (discard_on_shutdown.load(Ordering::Acquire) || batch.is_none()) { - break; - } - if !shutting_down && input_muted.load(Ordering::Acquire) { - continue; - } - let Some(batch) = batch else { - continue; - }; - if batch.mute_epoch != observed_mute_epoch { - continue; + for _ in 0..MAX_TRANSCRIPT_DELIVERY_ATTEMPTS { + state.reject_transcript("session-1", "first", 4).unwrap(); } - input_48k.extend( - batch - .bytes - .chunks_exact(4) - .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])), - ); - while input_48k.len() >= chunk_in { - let chunk: Vec = input_48k.drain(..chunk_in).collect(); - let resampled = resample(&mut resampler, &chunk); - leftover_16k.extend_from_slice(&resampled); - while leftover_16k.len() >= VAD_FRAME_SAMPLES { - let frame: Vec = leftover_16k.drain(..VAD_FRAME_SAMPLES).collect(); - let clamped: Vec = - frame.iter().map(|sample| sample.clamp(-1.0, 1.0)).collect(); - let vad_threshold = active_vad_threshold_for_speech( - &assistant_speaking, - &assistant_vad_threshold, - speech_vad_threshold, - ); - let speaking = vad.predict_f32(&clamped) > vad_threshold; - if speaking { - if !in_speech { - in_speech = true; - log::info!("Native Parakeet detected speech"); - let _ = event_tx.blocking_send(SttMessage::Speaking(true)); - } - silence_frames = 0; - speech.extend_from_slice(&frame); - if speech.len() >= MAX_SPEECH_SAMPLES { - flush_speech( - &speech, - &recognizer, - &event_tx, - None, - &input_mute_epoch, - &shutdown, - &shutdown_mute_epoch, - observed_mute_epoch, - ); - speech.clear(); - in_speech = false; - let _ = event_tx.blocking_send(SttMessage::Speaking(false)); - } - } else if in_speech { - speech.extend_from_slice(&frame); - silence_frames += 1; - if silence_frames >= SILENCE_FLUSH_FRAMES { - flush_speech( - &speech, - &recognizer, - &event_tx, - None, - &input_mute_epoch, - &shutdown, - &shutdown_mute_epoch, - observed_mute_epoch, - ); - speech.clear(); - silence_frames = 0; - in_speech = false; - let _ = event_tx.blocking_send(SttMessage::Speaking(false)); - } - } - } - } - } - if !speech.is_empty() && !discard_on_shutdown.load(Ordering::Acquire) { - let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); - flush_speech( - &speech, - &recognizer, - &event_tx, - Some(delivered_tx), - &input_mute_epoch, - &shutdown, - &shutdown_mute_epoch, - observed_mute_epoch, - ); - let _ = delivered_rx.recv_timeout(FINAL_TRANSCRIPT_DELIVERY_TIMEOUT); + assert!(matches!( + admission + .prepare("reply".into(), Some(transcript_reference("second"))) + .await + .unwrap(), + PrepareAssistantSpeechOutcome::Admitted { .. } + )); } -} -fn sample_effective_mute_epoch( - input_mute_epoch: &AtomicU64, - shutdown: &AtomicBool, - shutdown_mute_epoch: &AtomicU64, -) -> (bool, u64) { - let live_mute_epoch = input_mute_epoch.load(Ordering::Acquire); - let shutting_down = shutdown.load(Ordering::Acquire); - if shutting_down { - (true, shutdown_mute_epoch.load(Ordering::Acquire)) - } else { - (false, live_mute_epoch) - } -} + #[test] + fn superseded_lifecycle_recovery_can_be_acknowledged_without_admission() { + let state = NativeVoiceState::default(); + state.pending.lock().unwrap().push_back(PendingTranscript { + session_id: "old-session".into(), + lifecycle_id: "old-lifecycle".into(), + id: "old-final".into(), + text: "recover me".into(), + revision: 3, + delivery_attempts: 0, + }); -fn clear_buffered_audio( - input_48k: &mut Vec, - leftover_16k: &mut Vec, - speech: &mut Vec, - silence_frames: &mut usize, - in_speech: &mut bool, -) -> bool { - input_48k.clear(); - leftover_16k.clear(); - speech.clear(); - *silence_frames = 0; - std::mem::take(in_speech) -} + state + .acknowledge_transcript("old-session", "old-final", 3) + .unwrap(); -fn resample(resampler: &mut rubato::Fft, samples: &[f32]) -> Vec { - use audioadapter_buffers::direct::InterleavedSlice; - use rubato::Resampler; - let Ok(input) = InterleavedSlice::new(samples, 1, samples.len()) else { - return Vec::new(); - }; - resampler - .process(&input, 0, None) - .map(|output| output.take_data()) - .unwrap_or_default() -} + assert!(state.pending.lock().unwrap().is_empty()); + } -#[allow(clippy::too_many_arguments)] // Recognition needs both live and shutdown mute clocks. -fn flush_speech( - speech: &[f32], - recognizer: &sherpa_onnx::OfflineRecognizer, - event_tx: &tokio_mpsc::Sender, - delivered: Option>, - input_mute_epoch: &AtomicU64, - shutdown: &AtomicBool, - shutdown_mute_epoch: &AtomicU64, - expected_mute_epoch: u64, -) { - if speech.is_empty() { - return; - } - let stream = recognizer.create_stream(); - stream.accept_waveform(16_000, speech); - recognizer.decode(&stream); - let text = stream - .get_result() - .map(|result| result.text.trim().to_string()) - .unwrap_or_default(); - if sample_effective_mute_epoch(input_mute_epoch, shutdown, shutdown_mute_epoch).1 - != expected_mute_epoch - { - if let Some(delivered) = delivered { - let _ = delivered.send(()); + #[test] + fn superseded_lifecycle_recovery_can_be_terminally_rejected_without_admission() { + let state = NativeVoiceState::default(); + state.pending.lock().unwrap().push_back(PendingTranscript { + session_id: "old-session".into(), + lifecycle_id: "old-lifecycle".into(), + id: "old-final".into(), + text: "recover me".into(), + revision: 3, + delivery_attempts: 0, + }); + + for _ in 0..MAX_TRANSCRIPT_DELIVERY_ATTEMPTS { + state + .reject_transcript("old-session", "old-final", 3) + .unwrap(); } - return; + + assert!(state.pending.lock().unwrap().is_empty()); } - log::info!( - "Native Parakeet finalized {} samples into {} text characters", - speech.len(), - text.chars().count() - ); - deliver_recognition_result(text, event_tx, delivered); -} -fn deliver_recognition_result( - text: String, - event_tx: &tokio_mpsc::Sender, - delivered: Option>, -) { - if text.is_empty() { - if let Some(delivered) = delivered { - let _ = delivered.send(()); - } - return; + #[tokio::test] + async fn input_after_prepare_invalidates_the_installed_reservation_before_claim() { + let admission = Arc::new(BerdAdmissionCoordinator::default()); + let PrepareAssistantSpeechOutcome::Admitted { speech_id } = + admission.prepare("reply".into(), None).await.unwrap() + else { + panic!("expected admission") + }; + + admission.set_recognition_pending(true); + let playback_active = Arc::new(AtomicBool::new(true)); + assert!(admission + .claim(speech_id, Arc::clone(&playback_active)) + .unwrap() + .is_none()); + assert!(playback_active.load(Ordering::SeqCst)); } - let _ = event_tx.blocking_send(SttMessage::Final { text, delivered }); -} -#[cfg(test)] -mod tests { - use super::*; + #[tokio::test] + async fn input_after_claim_cancels_playback_until_the_terminal_guard_finishes() { + let admission = Arc::new(BerdAdmissionCoordinator::default()); + let PrepareAssistantSpeechOutcome::Admitted { speech_id } = + admission.prepare("reply".into(), None).await.unwrap() + else { + panic!("expected admission") + }; + let playback_active = Arc::new(AtomicBool::new(true)); + let guard = admission + .claim(speech_id, Arc::clone(&playback_active)) + .unwrap() + .expect("claim reservation"); + + admission.set_user_speaking(true); + assert!(!playback_active.load(Ordering::SeqCst)); + admission.set_user_speaking(false); + assert!(matches!( + admission.prepare("next".into(), None).await.unwrap(), + PrepareAssistantSpeechOutcome::NotAdmitted + )); + drop(guard); + assert!(matches!( + admission.prepare("next".into(), None).await.unwrap(), + PrepareAssistantSpeechOutcome::Admitted { .. } + )); + } + + #[tokio::test] + async fn closing_the_lifecycle_wakes_a_held_prepare() { + let admission = Arc::new(BerdAdmissionCoordinator::default()); + admission.set_user_speaking(true); + let waiting = { + let admission = Arc::clone(&admission); + tokio::spawn(async move { admission.prepare("reply".into(), None).await }) + }; + tokio::task::yield_now().await; + admission.close(); + assert_eq!( + waiting.await.unwrap().unwrap_err(), + "The voice conversation is no longer running." + ); + } #[tokio::test] async fn replacement_revalidates_target_after_waiting_for_stop_serialization() { @@ -3555,15 +3148,6 @@ mod tests { ); } - #[test] - fn worker_shutdown_budget_covers_recognition_and_delivery() { - assert!( - STT_WORKER_SHUTDOWN_TIMEOUT_SECONDS - > mac_speech::RECOGNITION_FINISH_TIMEOUT_SECONDS - + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS - ); - } - #[test] fn apple_status_is_only_queried_when_it_can_change_availability() { assert!(!needs_macos_status(true, false)); @@ -3643,40 +3227,6 @@ mod tests { assert_eq!(status.revision, 2); } - #[cfg(target_os = "macos")] - #[test] - fn finishing_finals_share_one_delivery_deadline() { - let (recognition_tx, mut recognition_rx) = tokio_mpsc::unbounded_channel(); - recognition_tx - .send(mac_speech::RecognitionEvent::Final("first".to_string())) - .expect("queue first final"); - recognition_tx - .send(mac_speech::RecognitionEvent::Final("second".to_string())) - .expect("queue second final"); - recognition_tx - .send(mac_speech::RecognitionEvent::Finished) - .expect("queue finish"); - let (output_tx, mut output_rx) = tokio_mpsc::channel(4); - let started = Instant::now(); - - forward_macos_events( - &mut recognition_rx, - &output_tx, - Some(started + Duration::from_millis(20)), - ) - .expect("drain final events"); - - assert!(started.elapsed() < Duration::from_millis(100)); - assert!(matches!( - output_rx.try_recv(), - Ok(SttMessage::Final { text, .. }) if text == "first" - )); - assert!(matches!( - output_rx.try_recv(), - Ok(SttMessage::Final { text, .. }) if text == "second" - )); - } - #[test] fn native_mute_control_does_not_latch_the_software_fallback() { assert!(!software_microphone_mute(true, true)); @@ -3685,10 +3235,10 @@ mod tests { } #[test] - fn call_target_ownership_survives_focus_changes_and_rejects_other_windows() { + fn replacement_stop_requires_the_target_session_window() { assert!(caller_owns_target("main", None, true)); assert!(!caller_owns_target("main", None, false)); - assert!(!caller_owns_target("main", Some("session:target"), true)); + assert!(!caller_owns_target("main", Some("session:target"), true,)); assert!(caller_owns_target( "session:target", Some("session:target"), @@ -3699,370 +3249,108 @@ mod tests { Some("session:target"), true, )); - assert!(!caller_owns_target("voice-buddy", None, true)); - } - - #[test] - fn interruption_sensitivity_only_changes_vad_while_assistant_speaks() { - let state = NativeVoiceState::default(); - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - VAD_THRESHOLD - ); - - { - let _guard = state.begin_assistant_speech(InterruptionSensitivity::More, false); - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - InterruptionSensitivity::More.vad_threshold() - ); - } - - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - VAD_THRESHOLD - ); - assert_eq!(InterruptionSensitivity::More.vad_threshold(), 0.5); - assert_eq!(InterruptionSensitivity::Balanced.vad_threshold(), 0.65); - assert_eq!(InterruptionSensitivity::Less.vad_threshold(), 0.8); - assert!(InterruptionSensitivity::Less.vad_threshold() > VAD_THRESHOLD); - assert_eq!(InterruptionSensitivity::More.vad_threshold(), VAD_THRESHOLD); - } - - #[test] - fn stale_assistant_speech_guard_does_not_clear_newer_playback() { - let state = NativeVoiceState::default(); - let older = state.begin_assistant_speech(InterruptionSensitivity::Less, false); - let newer = state.begin_assistant_speech(InterruptionSensitivity::More, false); - - drop(older); - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - InterruptionSensitivity::More.vad_threshold() - ); - - drop(newer); - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - VAD_THRESHOLD - ); + assert!(!caller_owns_target("voice-buddy", None, true,)); } #[test] - fn newer_guard_finishing_first_restores_overlapping_playback_policy() { + fn assistant_suppression_uses_the_shared_input_controls() { let state = NativeVoiceState::default(); - let older = state.begin_assistant_speech(InterruptionSensitivity::Less, true); - let newer = state.begin_assistant_speech(InterruptionSensitivity::More, false); + assert!(!state.input_controls.is_muted()); - drop(newer); - assert!(state.capture_is_suppressed()); - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - InterruptionSensitivity::Less.vad_threshold() + let guard = state.begin_assistant_speech( + InterruptionSensitivity::Balanced, + berd_voice::input::InputDuringTtsPolicy::SuppressInput, ); - - drop(older); - assert!(!state.capture_is_suppressed()); - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - VAD_THRESHOLD - ); - } - - #[test] - fn speaker_playback_blocks_vad_ingestion_until_all_guards_finish() { - let state = NativeVoiceState::default(); - assert!(!state.capture_is_suppressed()); - - let first = state.suppress_capture(); - assert!(state.capture_is_suppressed()); - { - let second = state.suppress_capture(); - assert!(state.capture_is_suppressed()); - drop(second); - assert!(state.capture_is_suppressed()); - } - - drop(first); - assert!(!state.capture_is_suppressed()); - } - - #[test] - fn assistant_speech_guard_scopes_capture_suppression_to_playback() { - let state = NativeVoiceState::default(); - assert!(!state.capture_is_suppressed()); - - let guard = state.begin_assistant_speech(InterruptionSensitivity::Balanced, true); - assert!(state.capture_is_suppressed()); + assert!(state.input_controls.is_muted()); drop(guard); - assert!(!state.capture_is_suppressed()); + assert!(!state.input_controls.is_muted()); } #[test] - fn assistant_speech_policy_can_restart_after_a_silent_gap() { - let state = NativeVoiceState::default(); - - let first_burst = state.begin_assistant_speech(InterruptionSensitivity::Less, true); - assert!(state.capture_is_suppressed()); - drop(first_burst); - assert!(!state.capture_is_suppressed()); - - let second_burst = state.begin_assistant_speech(InterruptionSensitivity::More, false); - assert!(!state.capture_is_suppressed()); - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - InterruptionSensitivity::More.vad_threshold() - ); - drop(second_burst); - } - - #[test] - fn assistant_speech_policy_outlives_voice_lifecycle_replacement() { - for suppress_capture in [false, true] { - let state = NativeVoiceState::default(); - { - let mut runtime = state.runtime.lock().expect("lock native runtime"); - runtime.session_id = Some("old-session".to_string()); - runtime.revision = 7; - } - let guard = - state.begin_assistant_speech(InterruptionSensitivity::Less, suppress_capture); - - state - .take_stop_snapshot(Some(("old-session", 7))) - .expect("stop old lifecycle") - .expect("active old lifecycle"); - { - let mut runtime = state.runtime.lock().expect("lock native runtime"); - runtime.session_id = Some("new-session".to_string()); - runtime.revision = 8; - } - - assert_eq!(state.capture_is_suppressed(), suppress_capture); - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - InterruptionSensitivity::Less.vad_threshold() - ); - - drop(guard); - assert!(!state.capture_is_suppressed()); - assert_eq!( - active_vad_threshold(&state.assistant_speaking, &state.assistant_vad_threshold), - VAD_THRESHOLD - ); - } - } - - #[test] - fn assistant_activity_is_bound_to_the_exact_voice_lifecycle() { + fn assistant_input_suppression_outlives_lifecycle_replacement() { let state = NativeVoiceState::default(); { let mut runtime = state.runtime.lock().expect("lock native runtime"); - runtime.session_id = Some("session-1".to_string()); + runtime.session_id = Some("old-session".to_string()); runtime.revision = 7; - runtime.owner = Some(RuntimeOwner { - window_label: "main".to_string(), - }); } - - assert_eq!( - state - .assistant_activity_target("main", "session-1", 7) - .expect("current activity target"), - Some(("main".to_string(), 7)), - ); - assert_eq!( - state - .assistant_activity_target("main", "session-1", 6) - .expect("stale activity is ignored"), - None, - ); - assert!(state - .assistant_activity_target("session:other", "session-1", 7) - .is_err()); - - state.runtime.lock().expect("lock native runtime").revision = 8; - assert_eq!( - state - .assistant_activity_target("main", "session-1", 7) - .expect("prior lifecycle activity is ignored after restart"), - None, + let guard = state.begin_assistant_speech( + InterruptionSensitivity::Less, + berd_voice::input::InputDuringTtsPolicy::SuppressInput, ); - } - - #[test] - fn stale_controls_watchdog_cannot_take_a_restarted_voice_lifecycle() { - let state = NativeVoiceState::default(); + state + .take_stop_snapshot(Some(("old-session", 7))) + .expect("stop old lifecycle") + .expect("active old lifecycle"); { let mut runtime = state.runtime.lock().expect("lock native runtime"); - runtime.session_id = Some("session-b".to_string()); + runtime.session_id = Some("new-session".to_string()); runtime.revision = 8; - runtime.owner = Some(RuntimeOwner { - window_label: "main".to_string(), - }); } - assert!(state - .take_stop_snapshot(Some(("session-a", 7))) - .expect("stale watchdog check") - .is_none()); - assert_eq!( - state.active_session_lifecycle_target(), - Some(("session-b".to_string(), "main".to_string(), 8)), - ); + assert!(state.input_controls.is_muted()); + drop(guard); + assert!(!state.input_controls.is_muted()); } - #[tokio::test] - async fn concurrent_stops_flush_one_final_transcript_once() { + #[test] + fn assistant_activity_is_bound_to_the_exact_voice_lifecycle() { let state = NativeVoiceState::default(); - let (audio_tx, _audio_rx) = mpsc::sync_channel(1); - let shutdown = Arc::new(AtomicBool::new(false)); - let worker_shutdown = Arc::clone(&shutdown); - let pending = Arc::clone(&state.pending); - let worker = thread::spawn(move || { - while !worker_shutdown.load(Ordering::Acquire) { - thread::yield_now(); - } - pending - .lock() - .expect("lock pending transcripts") - .push_back(PendingTranscript { - session_id: "session-1".to_string(), - lifecycle_id: "lifecycle-1".to_string(), - id: "final-1".to_string(), - text: "final words".to_string(), - revision: 4, - delivery_attempts: 0, - }); - }); { let mut runtime = state.runtime.lock().expect("lock native runtime"); runtime.session_id = Some("session-1".to_string()); - runtime.lifecycle_id = Some("lifecycle-1".to_string()); - runtime.revision = 4; + runtime.revision = 7; runtime.owner = Some(RuntimeOwner { window_label: "main".to_string(), }); - runtime.pipeline = Some(SttPipeline { - audio_tx, - audio_seen: AtomicBool::new(false), - shutdown, - discard_on_shutdown: Arc::new(AtomicBool::new(false)), - input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - thread: Some(worker), - }); } - let first_state = state.clone(); - let second_state = state.clone(); - let first = tokio::spawn(async move { - first_state - .stop_lifecycle(Some(("session-1", 4))) - .await - .expect("first stop") - }); - let second = tokio::spawn(async move { - second_state - .stop_lifecycle(Some(("session-1", 4))) - .await - .expect("second stop") - }); - let (first, second) = tokio::join!(first, second); - let completions = [first.expect("join first"), second.expect("join second")]; - assert_eq!( - completions.iter().filter(|result| result.is_some()).count(), - 1 + state + .assistant_activity_target("main", "session-1", 7) + .expect("current activity target"), + Some(("main".to_string(), 7)), ); - let pending = state.pending.lock().expect("lock pending transcripts"); - assert_eq!(pending.len(), 1); assert_eq!( - pending.front().map(|item| item.id.as_str()), - Some("final-1") + state + .assistant_activity_target("main", "session-1", 6) + .expect("stale activity is ignored"), + None, + ); + assert!(state + .assistant_activity_target("session:other", "session-1", 7) + .is_err()); + + state.runtime.lock().expect("lock native runtime").revision = 8; + assert_eq!( + state + .assistant_activity_target("main", "session-1", 7) + .expect("prior lifecycle activity is ignored after restart"), + None, ); } - #[tokio::test] - async fn non_cooperative_worker_cannot_block_stop_or_replacement_lifecycle() { + #[test] + fn stale_controls_watchdog_cannot_take_a_restarted_voice_lifecycle() { let state = NativeVoiceState::default(); - let (audio_tx, _audio_rx) = mpsc::sync_channel(1); - let worker_release = Arc::new(AtomicBool::new(false)); - let release = Arc::clone(&worker_release); - let worker = thread::spawn(move || { - while !release.load(Ordering::Acquire) { - thread::sleep(Duration::from_millis(5)); - } - }); { let mut runtime = state.runtime.lock().expect("lock native runtime"); - runtime.session_id = Some("session-1".to_string()); - runtime.lifecycle_id = Some("lifecycle-1".to_string()); - runtime.revision = 4; + runtime.session_id = Some("session-b".to_string()); + runtime.revision = 8; runtime.owner = Some(RuntimeOwner { window_label: "main".to_string(), }); - runtime.pipeline = Some(SttPipeline { - audio_tx, - audio_seen: AtomicBool::new(false), - shutdown: Arc::new(AtomicBool::new(false)), - discard_on_shutdown: Arc::new(AtomicBool::new(false)), - input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - thread: Some(worker), - }); } - let completion = tokio::time::timeout( - Duration::from_millis(500), - state.stop_lifecycle(Some(("session-1", 4))), - ) - .await - .expect("stop is bounded") - .expect("stop succeeds"); - assert!(completion.is_some()); - - { - let mut runtime = state.runtime.lock().expect("lock native runtime"); - assert!(runtime.session_id.is_none()); - runtime.session_id = Some("session-2".to_string()); - runtime.lifecycle_id = Some("lifecycle-2".to_string()); - runtime.revision = 6; - runtime.owner = Some(RuntimeOwner { - window_label: "main".to_string(), - }); - } + assert!(state + .take_stop_snapshot(Some(("session-a", 7))) + .expect("stale watchdog check") + .is_none()); assert_eq!( state.active_session_lifecycle_target(), - Some(("session-2".to_string(), "main".to_string(), 6)) + Some(("session-b".to_string(), "main".to_string(), 8)), ); - assert!(state - .take_stop_snapshot(Some(("session-1", 4))) - .expect("late stale lifecycle is ignored") - .is_none()); - let (accepted, evicted) = enqueue_transcript_if_active( - &state.runtime, - &state.pending, - "session-1", - 4, - PendingTranscript { - session_id: "session-1".to_string(), - lifecycle_id: "lifecycle-1".to_string(), - id: "late-final".to_string(), - text: "late words".to_string(), - revision: 4, - delivery_attempts: 0, - }, - ) - .expect("late transcript lifecycle check"); - assert!(!accepted); - assert!(evicted.is_none()); - assert!(state.pending.lock().expect("lock pending queue").is_empty()); - - worker_release.store(true, Ordering::Release); } #[test] @@ -4100,32 +3388,13 @@ mod tests { Some("main".to_string()), ); assert!(state.microphone_is_muted()); - } - - #[test] - fn software_microphone_mute_advances_the_audio_epoch() { - let state = NativeVoiceState::default(); - { - let mut runtime = state.runtime.lock().expect("lock native runtime"); - runtime.session_id = Some("session-1".to_string()); - runtime.revision = 4; - runtime.owner = Some(RuntimeOwner { - window_label: "main".to_string(), - }); - runtime.native_microphone_mute_control = false; - } - - assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 0); - state - .set_microphone_muted_target("main", "session-1", 4, true) - .expect("mute"); - assert!(state.input_muted.load(Ordering::Acquire)); - assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 1); + state.microphone_muted.store(false, Ordering::SeqCst); + assert!(state.input_controls.is_host_muted()); state - .set_microphone_muted_target("main", "session-1", 4, false) - .expect("unmute"); - assert!(!state.input_muted.load(Ordering::Acquire)); - assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 2); + .set_microphone_muted_target("main", "session-b", 8, false) + .expect("authoritative unmute repairs a stale UI projection"); + assert!(!state.input_controls.is_host_muted()); + assert!(!state.microphone_is_muted()); } #[test] @@ -4321,261 +3590,38 @@ mod tests { assert!(state.controls_ready_for("session-1", 4)); } - #[test] - fn audio_push_rejects_malformed_batches() { - let (sender, _receiver) = mpsc::sync_channel(1); - let pipeline = SttPipeline { - audio_tx: sender, - shutdown: Arc::new(AtomicBool::new(false)), - discard_on_shutdown: Arc::new(AtomicBool::new(false)), - input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - audio_seen: AtomicBool::new(false), - thread: None, - }; - assert!(pipeline.push(vec![0; 3]).is_err()); - assert!(pipeline.push(vec![0; MAX_AUDIO_BATCH_BYTES + 4]).is_err()); - } - - #[test] - fn audio_push_reports_bounded_queue_overrun() { - let (sender, _receiver) = mpsc::sync_channel(1); - let pipeline = SttPipeline { - audio_tx: sender, - shutdown: Arc::new(AtomicBool::new(false)), - discard_on_shutdown: Arc::new(AtomicBool::new(false)), - input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - audio_seen: AtomicBool::new(false), - thread: None, - }; - - pipeline.push(vec![0; 4]).expect("first batch fits"); - assert!(pipeline - .push(vec![0; 4]) - .expect_err("full queue must report overrun") - .contains("overrun")); - } - - #[test] - fn input_mute_discards_audio_and_unmute_resumes_queueing() { - let (sender, receiver) = mpsc::sync_channel(1); - let input_muted = Arc::new(AtomicBool::new(true)); - let input_mute_epoch = Arc::new(AtomicU64::new(1)); - let pipeline = SttPipeline { - audio_tx: sender, - shutdown: Arc::new(AtomicBool::new(false)), - discard_on_shutdown: Arc::new(AtomicBool::new(false)), - input_muted: Arc::clone(&input_muted), - input_mute_epoch: Arc::clone(&input_mute_epoch), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - audio_seen: AtomicBool::new(false), - thread: None, - }; - - pipeline - .push(vec![0; 4]) - .expect("muted microphone input is accepted and discarded"); - assert!(receiver.try_recv().is_err()); - - input_muted.store(false, Ordering::Release); - pipeline.push(vec![0; 4]).expect("unmuted audio is queued"); - assert_eq!( - receiver.try_recv().expect("unmuted audio").bytes, - vec![0; 4] - ); - } - - #[test] - fn queued_audio_retains_epoch_across_fast_mute_unmute() { - let (sender, receiver) = mpsc::sync_channel(1); - let input_mute_epoch = Arc::new(AtomicU64::new(0)); - let pipeline = SttPipeline { - audio_tx: sender, - shutdown: Arc::new(AtomicBool::new(false)), - discard_on_shutdown: Arc::new(AtomicBool::new(false)), - input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::clone(&input_mute_epoch), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - audio_seen: AtomicBool::new(false), - thread: None, - }; - - pipeline.push(vec![0; 4]).expect("audio queues before mute"); - input_mute_epoch.fetch_add(1, Ordering::AcqRel); - - let batch = receiver.try_recv().expect("queued audio"); - assert_ne!(batch.mute_epoch, input_mute_epoch.load(Ordering::Acquire)); - } - - #[test] - fn input_mute_clears_partial_utterance_even_without_a_new_batch() { - let mut input_48k = vec![0.1]; - let mut leftover_16k = vec![0.2]; - let mut speech = vec![0.3]; - let mut silence_frames = 4; - let mut in_speech = true; - - assert!(clear_buffered_audio( - &mut input_48k, - &mut leftover_16k, - &mut speech, - &mut silence_frames, - &mut in_speech, - )); - assert!(input_48k.is_empty()); - assert!(leftover_16k.is_empty()); - assert!(speech.is_empty()); - assert_eq!(silence_frames, 0); - assert!(!in_speech); - } - - #[test] - fn muted_shutdown_keeps_final_utterance_discarded_after_handler_reset() { - let (sender, _receiver) = mpsc::sync_channel(1); - let input_muted = Arc::new(AtomicBool::new(true)); - let discard_on_shutdown = Arc::new(AtomicBool::new(false)); - let mut pipeline = SttPipeline { - audio_tx: sender, - shutdown: Arc::new(AtomicBool::new(false)), - discard_on_shutdown: Arc::clone(&discard_on_shutdown), - input_muted: Arc::clone(&input_muted), - input_mute_epoch: Arc::new(AtomicU64::new(1)), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - audio_seen: AtomicBool::new(false), - thread: None, - }; - - pipeline.begin_shutdown(); - input_muted.store(false, Ordering::Release); - - assert!(discard_on_shutdown.load(Ordering::Acquire)); - } - - #[test] - fn unmuted_shutdown_keeps_final_utterance_after_later_mute_event() { - let (sender, _receiver) = mpsc::sync_channel(1); - let input_muted = Arc::new(AtomicBool::new(false)); - let input_mute_epoch = Arc::new(AtomicU64::new(0)); - let discard_on_shutdown = Arc::new(AtomicBool::new(false)); - let mut pipeline = SttPipeline { - audio_tx: sender, - shutdown: Arc::new(AtomicBool::new(false)), - discard_on_shutdown: Arc::clone(&discard_on_shutdown), - input_muted: Arc::clone(&input_muted), - input_mute_epoch: Arc::clone(&input_mute_epoch), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - audio_seen: AtomicBool::new(false), - thread: None, - }; - - pipeline.begin_shutdown(); - input_muted.store(true, Ordering::Release); - input_mute_epoch.fetch_add(1, Ordering::AcqRel); - pipeline.signal_shutdown(); - - assert!(!discard_on_shutdown.load(Ordering::Acquire)); - assert_eq!(pipeline.shutdown_mute_epoch.load(Ordering::Acquire), 0); - assert_eq!( - sample_effective_mute_epoch( - &input_mute_epoch, - &AtomicBool::new(false), - &pipeline.shutdown_mute_epoch, - ), - (false, 1), - ); - assert_eq!( - sample_effective_mute_epoch( - &input_mute_epoch, - &pipeline.shutdown, - &pipeline.shutdown_mute_epoch, - ), - (true, 0), - ); - } - #[test] fn only_owning_window_can_inject_audio() { let state = NativeVoiceState::default(); - let (sender, receiver) = mpsc::sync_channel(2); { let mut runtime = state.runtime.lock().expect("lock native runtime"); runtime.owner = Some(RuntimeOwner { window_label: "owner-window".to_string(), }); - runtime.pipeline = Some(SttPipeline { - audio_tx: sender, - shutdown: Arc::new(AtomicBool::new(false)), - discard_on_shutdown: Arc::new(AtomicBool::new(false)), - input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - audio_seen: AtomicBool::new(false), - thread: None, - }); } assert!(push_audio_for_window(&state, "other-window", vec![0; 4]).is_err()); - assert!(receiver.try_recv().is_err()); state.microphone_muted.store(true, Ordering::SeqCst); push_audio_for_window(&state, "owner-window", vec![0; 4]) .expect("muted owner audio is ignored"); - assert!(receiver.try_recv().is_err()); state.microphone_muted.store(false, Ordering::SeqCst); - state.input_muted.store(true, Ordering::SeqCst); push_audio_for_window(&state, "owner-window", vec![0; 4]) - .expect("native-muted owner audio is ignored"); - assert!(receiver.try_recv().is_err()); - state.input_muted.store(false, Ordering::SeqCst); - push_audio_for_window(&state, "owner-window", vec![0; 4]).expect("owner can send audio"); - assert_eq!( - receiver.try_recv().expect("owner audio queued").bytes, - vec![0; 4] - ); + .expect("owner can send audio while no runtime is installed"); } - #[tokio::test] - async fn window_destroy_awaits_bounded_worker_shutdown_off_callback() { - let state = NativeVoiceState::default(); - state.microphone_muted.store(true, Ordering::SeqCst); - let (sender, _receiver) = mpsc::sync_channel(1); - let shutdown = Arc::new(AtomicBool::new(false)); - let worker = thread::spawn(|| thread::sleep(Duration::from_millis(250))); - { - let mut runtime = state.runtime.lock().expect("lock native runtime"); - runtime.session_id = Some("session-1".to_string()); - runtime.owner = Some(RuntimeOwner { - window_label: "owner-window".to_string(), - }); - runtime.pipeline = Some(SttPipeline { - audio_tx: sender, - shutdown: Arc::clone(&shutdown), - discard_on_shutdown: Arc::new(AtomicBool::new(false)), - input_muted: Arc::new(AtomicBool::new(false)), - input_mute_epoch: Arc::new(AtomicU64::new(0)), - shutdown_mute_epoch: Arc::new(AtomicU64::new(0)), - audio_seen: AtomicBool::new(false), - thread: Some(worker), - }); - } + #[test] + fn audio_transport_decodes_only_exact_finite_frames() { + let samples = [0.0_f32; berd_voice::input::INPUT_FRAME_SAMPLES]; + let bytes = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + decode_voice_input_frame(&bytes).expect("exact frame"); + assert!(decode_voice_input_frame(&bytes[..bytes.len() - 4]).is_err()); - let completion = state - .stop_destroyed_owner_lifecycle("owner-window", "session-1", 0) - .await - .expect("stop destroyed owner") - .expect("owned lifecycle stops"); - assert_eq!(completion.controls_revision, 0); - assert_eq!(completion.next_revision, 1); - assert!(shutdown.load(Ordering::Acquire)); - assert!(!state.microphone_muted.load(Ordering::SeqCst)); - assert!(state - .runtime - .lock() - .expect("lock native runtime") - .session_id - .is_none()); + let mut nonfinite = bytes; + nonfinite[..4].copy_from_slice(&f32::NAN.to_le_bytes()); + assert!(decode_voice_input_frame(&nonfinite).is_err()); } #[tokio::test] @@ -4772,412 +3818,16 @@ mod tests { let id = pending.front().expect("retained transcript").id.clone(); for attempts in 1..MAX_TRANSCRIPT_DELIVERY_ATTEMPTS { - let outcome = reject_pending_transcript(&mut pending, "session-1", &id, 2); + let outcome = + reject_pending_transcript(&mut pending, "session-1", &id, 2, None).unwrap(); assert_eq!(outcome.attempts, attempts); assert!(!outcome.terminal); } - let outcome = reject_pending_transcript(&mut pending, "session-1", &id, 2); + let outcome = reject_pending_transcript(&mut pending, "session-1", &id, 2, None).unwrap(); assert!(outcome.terminal); assert!(!pending.iter().any(|item| item.id == id)); } - #[test] - fn empty_recognition_result_releases_stop_waiter() { - let (event_tx, _event_rx) = tokio_mpsc::channel(1); - let (delivered_tx, delivered_rx) = mpsc::sync_channel(1); - - deliver_recognition_result(String::new(), &event_tx, Some(delivered_tx)); - - assert!(delivered_rx.try_recv().is_ok()); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn openai_startup_waits_for_session_updated_before_consuming_audio() { - use futures_util::{SinkExt, StreamExt}; - use tokio::net::TcpListener; - use tokio::sync::oneshot; - use tokio_tungstenite::{accept_async, tungstenite::Message}; - - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let endpoint = format!( - "ws://{}/v1/realtime?intent=transcription", - listener.local_addr().unwrap() - ); - let (update_seen_tx, update_seen_rx) = oneshot::channel(); - let (ack_tx, ack_rx) = oneshot::channel(); - let (finish_tx, finish_rx) = oneshot::channel(); - let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.expect("accept"); - let mut socket = accept_async(stream).await.expect("websocket"); - let update = socket - .next() - .await - .expect("session update") - .expect("session update frame"); - let Message::Text(update) = update else { - panic!("expected text session update"); - }; - let value: serde_json::Value = serde_json::from_str(&update).expect("json update"); - assert_eq!( - value.get("type").and_then(|value| value.as_str()), - Some("session.update") - ); - update_seen_tx.send(()).expect("report update"); - ack_rx.await.expect("release acknowledgement"); - socket - .send(Message::Text( - serde_json::json!({"type":"session.updated"}) - .to_string() - .into(), - )) - .await - .expect("send acknowledgement"); - let _ = finish_rx.await; - }); - - let (audio_tx, audio_rx) = mpsc::sync_channel(1); - audio_tx - .try_send(AudioBatch { - bytes: vec![0; 4], - mute_epoch: 0, - }) - .expect("queue initial audio"); - let (event_tx, _event_rx) = tokio_mpsc::channel(4); - let (startup_tx, startup_rx) = mpsc::sync_channel(0); - let shutdown = Arc::new(AtomicBool::new(false)); - let worker_shutdown = Arc::clone(&shutdown); - let worker = thread::spawn(move || { - openai_stt_worker( - "test-key".to_string(), - endpoint, - "gpt-live-transcribe".to_string(), - audio_rx, - event_tx, - worker_shutdown, - Arc::new(AtomicBool::new(false)), - Arc::new(AtomicBool::new(false)), - Arc::new(AtomicU64::new(0)), - Arc::new(AtomicU64::new(0)), - Arc::new(AtomicBool::new(false)), - Arc::new(AtomicU32::new(VAD_THRESHOLD.to_bits())), - VAD_THRESHOLD, - startup_tx, - ); - }); - - tokio::time::timeout(Duration::from_secs(2), update_seen_rx) - .await - .expect("session update timeout") - .expect("session update signal"); - assert!(matches!( - startup_rx.try_recv(), - Err(mpsc::TryRecvError::Empty) - )); - assert!(matches!( - audio_tx.try_send(AudioBatch { - bytes: vec![0; 4], - mute_epoch: 0, - }), - Err(TrySendError::Full(_)) - )); - - ack_tx.send(()).expect("release acknowledgement"); - let startup = - tokio::task::spawn_blocking(move || startup_rx.recv_timeout(Duration::from_secs(2))) - .await - .expect("join startup wait") - .expect("startup signal"); - assert_eq!(startup, Ok(())); - - shutdown.store(true, Ordering::Release); - let _ = finish_tx.send(()); - drop(audio_tx); - tokio::task::spawn_blocking(move || worker.join().expect("worker")) - .await - .expect("join worker task"); - server.await.expect("server"); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn openai_startup_surfaces_error_before_session_updated() { - use futures_util::{SinkExt, StreamExt}; - use tokio::net::TcpListener; - use tokio_tungstenite::{accept_async, tungstenite::Message}; - - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let endpoint = format!( - "ws://{}/v1/realtime?intent=transcription", - listener.local_addr().unwrap() - ); - let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.expect("accept"); - let mut socket = accept_async(stream).await.expect("websocket"); - let update = socket.next().await.expect("session update").expect("frame"); - assert!(matches!(update, Message::Text(_))); - socket - .send(Message::Text( - serde_json::json!({ - "type":"error", - "error":{"message":"model rejected"} - }) - .to_string() - .into(), - )) - .await - .expect("send error"); - }); - - let (audio_tx, audio_rx) = mpsc::sync_channel(1); - audio_tx - .try_send(AudioBatch { - bytes: vec![0; 4], - mute_epoch: 0, - }) - .expect("queue initial audio"); - let (event_tx, mut event_rx) = tokio_mpsc::channel(4); - let (startup_tx, startup_rx) = mpsc::sync_channel(0); - let worker = thread::spawn(move || { - openai_stt_worker( - "test-key".to_string(), - endpoint, - "bad-model".to_string(), - audio_rx, - event_tx, - Arc::new(AtomicBool::new(false)), - Arc::new(AtomicBool::new(false)), - Arc::new(AtomicBool::new(false)), - Arc::new(AtomicU64::new(0)), - Arc::new(AtomicU64::new(0)), - Arc::new(AtomicBool::new(false)), - Arc::new(AtomicU32::new(VAD_THRESHOLD.to_bits())), - VAD_THRESHOLD, - startup_tx, - ); - }); - - let startup = - tokio::task::spawn_blocking(move || startup_rx.recv_timeout(Duration::from_secs(2))) - .await - .expect("join startup wait") - .expect("startup signal") - .expect_err("startup must fail"); - assert!(startup.contains("model rejected")); - match event_rx.recv().await.expect("failure event") { - SttMessage::Failed(error) => assert!(error.contains("model rejected")), - _ => panic!("expected startup failure event"), - } - - drop(audio_tx); - tokio::task::spawn_blocking(move || worker.join().expect("worker")) - .await - .expect("join worker task"); - server.await.expect("server"); - } - - #[test] - fn openai_transcripts_are_delivered_in_commit_order() { - let mut pending_epochs = VecDeque::from([7, 7]); - let mut committed = VecDeque::new(); - let mut completed = HashMap::new(); - for item_id in ["first", "second"] { - record_openai_transcription_event( - &serde_json::json!({ - "type": "input_audio_buffer.committed", - "item_id": item_id, - }), - 7, - &mut pending_epochs, - &mut committed, - &mut completed, - ) - .expect("commit event"); - } - for (item_id, transcript) in [("second", "two"), ("first", "one")] { - record_openai_transcription_event( - &serde_json::json!({ - "type": "conversation.item.input_audio_transcription.completed", - "item_id": item_id, - "transcript": transcript, - }), - 7, - &mut pending_epochs, - &mut committed, - &mut completed, - ) - .expect("completion event"); - } - let (event_tx, mut event_rx) = tokio_mpsc::channel(4); - - deliver_completed_openai_turns( - &mut committed, - &mut completed, - &event_tx, - 7, - None, - &mut None, - ); - - let texts = [event_rx.try_recv(), event_rx.try_recv()].map(|event| match event { - Ok(SttMessage::Final { text, .. }) => text, - _ => panic!("expected finalized transcript"), - }); - assert_eq!(texts, ["one", "two"]); - } - - #[test] - fn openai_transcription_ignores_commits_from_stale_mute_epochs() { - let mut pending_epochs = VecDeque::from([1]); - let mut committed = VecDeque::new(); - let mut completed = HashMap::new(); - - let turn = record_openai_transcription_event( - &serde_json::json!({ - "type": "input_audio_buffer.committed", - "item_id": "stale", - }), - 2, - &mut pending_epochs, - &mut committed, - &mut completed, - ) - .expect("commit event") - .expect("recorded commit"); - - assert_eq!(turn.mute_epoch, 1); - assert!(committed.is_empty()); - } - - #[test] - fn openai_transcription_discards_completed_turns_after_mute_changes() { - let mut committed = VecDeque::from([OpenAiCommittedTurn { - item_id: "stale".to_string(), - mute_epoch: 1, - }]); - let mut completed = HashMap::from([("stale".to_string(), "ignore me".to_string())]); - let (event_tx, mut event_rx) = tokio_mpsc::channel(1); - - deliver_completed_openai_turns( - &mut committed, - &mut completed, - &event_tx, - 2, - None, - &mut None, - ); - - assert!(committed.is_empty()); - assert!(completed.is_empty()); - assert!(event_rx.try_recv().is_err()); - } - - #[test] - fn openai_transcription_surfaces_protocol_failures() { - let error = record_openai_transcription_event( - &serde_json::json!({ - "type": "conversation.item.input_audio_transcription.failed", - "error": { "message": "bad audio" }, - }), - 0, - &mut VecDeque::new(), - &mut VecDeque::new(), - &mut HashMap::new(), - ) - .expect_err("failure event"); - - assert_eq!(error, "bad audio"); - } - - #[test] - fn openai_transcription_ignores_failures_for_discarded_turns() { - let mut committed = VecDeque::from([OpenAiCommittedTurn { - item_id: "current".to_string(), - mute_epoch: 2, - }]); - - let result = record_openai_transcription_event( - &serde_json::json!({ - "type": "conversation.item.input_audio_transcription.failed", - "item_id": "stale", - "error": { "message": "discarded turn failed" }, - }), - 2, - &mut VecDeque::new(), - &mut committed, - &mut HashMap::new(), - ); - - assert_eq!(result, Ok(None)); - } - - #[test] - fn empty_openai_final_acknowledges_shutdown_delivery() { - let mut committed = VecDeque::from([OpenAiCommittedTurn { - item_id: "final".to_string(), - mute_epoch: 3, - }]); - let mut completed = HashMap::from([("final".to_string(), String::new())]); - let (event_tx, mut event_rx) = tokio_mpsc::channel(1); - let (delivered_tx, delivered_rx) = mpsc::sync_channel(1); - let mut final_delivery = Some(delivered_tx); - - deliver_completed_openai_turns( - &mut committed, - &mut completed, - &event_tx, - 3, - Some("final"), - &mut final_delivery, - ); - - assert!(delivered_rx.try_recv().is_ok()); - assert!(event_rx.try_recv().is_err()); - } - - #[test] - fn stalled_openai_operation_observes_shutdown() { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .expect("runtime"); - let shutdown = Arc::new(AtomicBool::new(false)); - let shutdown_for_thread = shutdown.clone(); - let signal = std::thread::spawn(move || { - std::thread::sleep(Duration::from_millis(30)); - shutdown_for_thread.store(true, Ordering::Release); - }); - - let result = block_on_openai_operation( - &runtime, - shutdown.as_ref(), - std::future::pending::>(), - "stalled operation", - ) - .expect("shutdown is not an error"); - - signal.join().expect("shutdown signal"); - assert_eq!(result, None); - } - - #[test] - fn openai_idle_audio_keeps_only_bounded_pre_roll() { - let mut pre_roll = VecDeque::new(); - for index in 0..(OPENAI_PRE_ROLL_CHUNKS * 4) { - push_openai_pre_roll(&mut pre_roll, vec![index as u8]); - } - - assert_eq!(pre_roll.len(), OPENAI_PRE_ROLL_CHUNKS); - assert_eq!( - pre_roll.front(), - Some(&vec![(OPENAI_PRE_ROLL_CHUNKS * 3) as u8]) - ); - } - - #[test] - fn openai_continuous_speech_has_a_turn_limit() { - assert!(!openai_turn_reached_limit(MAX_SPEECH_SAMPLES - 1)); - assert!(openai_turn_reached_limit(MAX_SPEECH_SAMPLES)); - } - #[test] fn native_voice_events_use_renderer_field_names() { let event = NativeVoiceEvent::User { @@ -5212,6 +3862,23 @@ mod tests { ); } + #[test] + fn prepared_speech_outcomes_match_the_shared_renderer_contract() { + let expected: serde_json::Value = serde_json::from_str(include_str!( + "../../../tests/contracts/voice/prepare-assistant-speech-outcomes.json" + )) + .expect("parse shared prepared speech contract"); + let actual = serde_json::json!([ + serde_json::to_value(PrepareAssistantSpeechOutcome::Pending) + .expect("serialize pending outcome"), + serde_json::to_value(PrepareAssistantSpeechOutcome::NotAdmitted) + .expect("serialize not-admitted outcome"), + serde_json::to_value(PrepareAssistantSpeechOutcome::Admitted { speech_id: 7 }) + .expect("serialize admitted outcome"), + ]); + assert_eq!(actual, expected); + } + #[test] fn input_backend_uses_renderer_wire_values() { assert_eq!( diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index e3b637ea8..58a91ba0b 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -8,24 +8,22 @@ use std::sync::{ use std::time::Duration; #[cfg(target_os = "macos")] -use futures_util::StreamExt; -#[cfg(target_os = "macos")] -use reqwest::header::CONTENT_TYPE; -#[cfg(target_os = "macos")] -use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; +use berd_voice::{ + ConfiguredTtsSlot, DeliveryProgress as VoiceDeliveryProgress, DrainPolicy, OutboundFailure, + OutboundOutcome, OutboundPlayback, PocketAudioPlayer, TtsBackend, TtsConfiguration, +}; use serde::Serialize; use serde_json::json; -use tauri::Emitter; -use tauri::{AppHandle, State}; +use tauri::{AppHandle, Emitter, State}; #[cfg(target_os = "macos")] -use super::{ - native_voice::AssistantSpeechGuard, - pocket_audio_player::PocketAudioPlayer, - pocket_voice::{ - effective_output_device_name, playback_latency_safety_duration, selected_output_device, - should_suppress_capture, - }, +use super::native_voice::AssistantSpeechGuard; +#[cfg(any(test, target_os = "macos"))] +use super::native_voice::{output_latency_grace_elapsed, output_latency_grace_remaining}; +#[cfg(target_os = "macos")] +use super::pocket_voice::{ + effective_output_device_name, playback_latency_safety_duration, + resolve_input_during_tts_policy, selected_output_device, }; use super::{ native_voice::{InterruptionSensitivity, NativeVoiceState}, @@ -33,6 +31,8 @@ use super::{ pocket_voice::VoiceInterruptionMode, voice_capture::VoiceCaptureState, }; +#[cfg(target_os = "macos")] +use berd_voice::input::InputDuringTtsPolicy; #[cfg(any(test, target_os = "macos"))] use std::time::Instant; @@ -54,13 +54,7 @@ const INITIAL_PLAYBACK_BUFFER_FRAMES: usize = TTS_SAMPLE_RATE as usize / 5; #[cfg(target_os = "macos")] const TTS_EVENT: &str = "openai-voice:stream-event"; #[cfg(target_os = "macos")] -const STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(120); -#[cfg(target_os = "macos")] -const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); -#[cfg(target_os = "macos")] const MAX_TTS_INPUT_CHARS: usize = 4096; -#[cfg(target_os = "macos")] -const MAX_FINAL_PLAYBACK_DRAIN: Duration = Duration::from_secs(600); #[derive(Clone, Debug, Default)] pub struct OpenAiVoiceState { @@ -153,24 +147,6 @@ enum OpenAiStreamEventState { Failed, } -#[cfg(target_os = "macos")] -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct VoiceDeliveryProgress { - sample_rate: u32, - segments: Vec, -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct VoiceDeliverySegment { - text: String, - played_frames: u64, - total_frames: u64, - synthesis_complete: bool, -} - fn env_trimmed(name: &str) -> Option { std::env::var(name) .ok() @@ -278,17 +254,6 @@ fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { Ok(url.to_string()) } -#[cfg(target_os = "macos")] -fn authorized_headers(key: &str) -> Result { - let mut headers = HeaderMap::new(); - let bearer = format!("Bearer {key}"); - headers.insert( - AUTHORIZATION, - HeaderValue::from_str(&bearer).map_err(|_| "OpenAI API key is not a valid header value")?, - ); - Ok(headers) -} - fn speed_settings_path() -> Result { Ok(crate::services::goose_config::config_path()? .parent() @@ -320,14 +285,6 @@ fn persist_playback_speed(speed: f32) -> Result<(), String> { .map_err(|error| format!("write OpenAI voice settings: {error}")) } -#[cfg(target_os = "macos")] -fn client() -> Result { - reqwest::Client::builder() - .connect_timeout(CONNECT_TIMEOUT) - .build() - .map_err(|error| format!("create OpenAI HTTP client: {error}")) -} - #[tauri::command] pub async fn get_openai_voice_status( state: State<'_, OpenAiVoiceState>, @@ -461,15 +418,19 @@ pub async fn clear_openai_tts_api_key( } #[tauri::command] +#[allow(clippy::too_many_arguments)] // Tauri injects four runtime dependencies beside the stream payload. pub fn start_openai_voice_stream( app: AppHandle, webview_window: tauri::WebviewWindow, state: State<'_, OpenAiVoiceState>, native_voice: State<'_, NativeVoiceState>, + session_id: String, + expected_revision: u64, + speech_id: u64, stream_id: String, interruption_mode: VoiceInterruptionMode, interruption_sensitivity: InterruptionSensitivity, -) -> Result<(), String> { +) -> Result { #[cfg(not(target_os = "macos"))] { let _ = ( @@ -477,6 +438,9 @@ pub fn start_openai_voice_stream( webview_window, state, native_voice, + session_id, + expected_revision, + speech_id, stream_id, interruption_mode, interruption_sensitivity, @@ -489,9 +453,24 @@ pub fn start_openai_voice_stream( if stream_id.trim().is_empty() { return Err("OpenAI voice stream id cannot be empty".to_string()); } - let key = tts_api_key()?; let (sender, receiver) = mpsc::channel(); let active = Arc::new(AtomicBool::new(true)); + let output_device = selected_output_device(); + let effective_output_device = effective_output_device_name(output_device.as_deref()); + let input_during_tts = + resolve_input_during_tts_policy(interruption_mode, effective_output_device.as_deref()); + let output_latency_grace = + playback_latency_safety_duration(effective_output_device.as_deref()); + let Some(admission) = native_voice.claim_assistant_speech( + &session_id, + expected_revision, + speech_id, + active.clone(), + )? + else { + return Ok(false); + }; + let key = tts_api_key()?; { let mut playback = state .playback @@ -518,6 +497,7 @@ pub fn start_openai_voice_stream( let playback = state.playback.clone(); let native_voice = native_voice.inner().clone(); tauri::async_runtime::spawn_blocking(move || { + let admission_guard = admission; let result = run_openai_voice_stream( &app, &stream_id, @@ -525,8 +505,10 @@ pub fn start_openai_voice_stream( active.clone(), receiver, native_voice, - interruption_mode, interruption_sensitivity, + input_during_tts, + output_device, + output_latency_grace, speed, ); if let Ok(mut playback) = playback.lock() { @@ -550,9 +532,10 @@ pub fn start_openai_voice_stream( failure.delivery, ), }; + drop(admission_guard); emit_openai_stream_event(&app, &stream_id, state, error, delivery); }); - Ok(()) + Ok(true) } } @@ -696,37 +679,34 @@ fn run_openai_voice_stream( active: Arc, receiver: mpsc::Receiver, native_voice: NativeVoiceState, - interruption_mode: VoiceInterruptionMode, interruption_sensitivity: InterruptionSensitivity, + input_during_tts: InputDuringTtsPolicy, + output_device: Option, + output_latency_grace: Duration, speed: f32, ) -> Result { - let client = client()?; - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| format!("Could not initialize OpenAI speech runtime: {error}"))?; - let configured_output_device = selected_output_device(); - let player = PocketAudioPlayer::new(TTS_SAMPLE_RATE, 1.0, configured_output_device.as_deref())?; - let output_device = effective_output_device_name(configured_output_device.as_deref()); - let suppress_capture = should_suppress_capture(interruption_mode, output_device.as_deref()); - let output_latency_grace = playback_latency_safety_duration(output_device.as_deref()); + let tts = ConfiguredTtsSlot::new(TtsConfiguration::openai( + endpoint("audio/speech")?, + key, + speech_model(), + speech_voice(), + speed, + ))?; + let tts = tts.lease()?; + let backend = tts.backend(); + let player = PocketAudioPlayer::new(TTS_SAMPLE_RATE, 1.0, output_device.as_deref())?; + let mut playback = OutboundPlayback::new( + &player, + &active, + TTS_SAMPLE_RATE, + INITIAL_PLAYBACK_BUFFER_FRAMES, + )?; let mut assistant_speech = None::; let mut playback_drained_at = None::; let mut pending = String::new(); - let mut delivery = VoiceDeliveryProgress { - sample_rate: TTS_SAMPLE_RATE, - segments: Vec::new(), - }; - let mut started = false; let mut last_progress = Instant::now(); loop { - if started { - player.ensure_healthy().map_err(|error| StreamFailure { - error, - delivery: Some(snapshot_delivery(&delivery, &player)), - })?; - } update_openai_assistant_speech( player.is_empty(), &mut assistant_speech, @@ -734,133 +714,124 @@ fn run_openai_voice_stream( output_latency_grace, Instant::now(), ); - if !active.load(Ordering::SeqCst) { - player.stop(); + if !playback.poll().map_err(openai_playback_failure)? { return Ok(StreamOutcome { state: OpenAiStreamEventState::Interrupted, - delivery: Some(snapshot_delivery(&delivery, &player)), + delivery: Some(playback.snapshot()), }); } match receiver.recv_timeout(Duration::from_millis(20)) { Ok(OpenAiStreamCommand::Append(text)) => { pending.push_str(&text); - if pending.len() >= 24 && ends_sentence_boundary(&pending) { - speak_pending( - &runtime, + if pending.len() >= 24 && pending.trim_end().ends_with(['.', '!', '?', '\n']) { + match speak_pending( app, stream_id, - &client, - &key, - &active, - &player, + backend.as_ref(), + &mut playback, &mut pending, - &mut delivery, - &mut started, &native_voice, interruption_sensitivity, - suppress_capture, + input_during_tts, &mut assistant_speech, &mut playback_drained_at, - output_latency_grace, - speed, ) - .map_err(|error| StreamFailure { - error, - delivery: Some(snapshot_delivery(&delivery, &player)), - })?; + .map_err(openai_playback_failure)? + { + OutboundOutcome::Interrupted => { + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Interrupted, + delivery: Some(playback.snapshot()), + }) + } + OutboundOutcome::Completed => {} + } } } Ok(OpenAiStreamCommand::Flush) => { - speak_pending( - &runtime, + if speak_pending( app, stream_id, - &client, - &key, - &active, - &player, + backend.as_ref(), + &mut playback, &mut pending, - &mut delivery, - &mut started, &native_voice, interruption_sensitivity, - suppress_capture, + input_during_tts, &mut assistant_speech, &mut playback_drained_at, - output_latency_grace, - speed, ) - .map_err(|error| StreamFailure { - error, - delivery: Some(snapshot_delivery(&delivery, &player)), - })?; + .map_err(openai_playback_failure)? + == OutboundOutcome::Interrupted + { + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Interrupted, + delivery: Some(playback.snapshot()), + }); + } } Ok(OpenAiStreamCommand::Finish) => { - speak_pending( - &runtime, + if speak_pending( app, stream_id, - &client, - &key, - &active, - &player, + backend.as_ref(), + &mut playback, &mut pending, - &mut delivery, - &mut started, &native_voice, interruption_sensitivity, - suppress_capture, + input_during_tts, &mut assistant_speech, &mut playback_drained_at, - output_latency_grace, - speed, ) - .map_err(|error| StreamFailure { - error, - delivery: Some(snapshot_delivery(&delivery, &player)), - })?; - let drain_started = Instant::now(); - while active.load(Ordering::SeqCst) - && (!player.is_empty() || assistant_speech.is_some()) + .map_err(openai_playback_failure)? + == OutboundOutcome::Interrupted { - if drain_started.elapsed() >= MAX_FINAL_PLAYBACK_DRAIN { - player.stop(); - return Err(StreamFailure { - error: "OpenAI voice playback did not finish within 10 minutes" - .to_string(), - delivery: Some(snapshot_delivery(&delivery, &player)), - }); - } - player.ensure_healthy().map_err(|error| StreamFailure { - error, - delivery: Some(snapshot_delivery(&delivery, &player)), - })?; - update_openai_assistant_speech( - player.is_empty(), - &mut assistant_speech, - &mut playback_drained_at, - output_latency_grace, - Instant::now(), - ); - if last_progress.elapsed() >= Duration::from_millis(100) { - emit_openai_stream_event( - app, - stream_id, - OpenAiStreamEventState::Progress, - None, - Some(snapshot_delivery(&delivery, &player)), - ); - last_progress = Instant::now(); - } - std::thread::sleep(Duration::from_millis(20)); + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Interrupted, + delivery: Some(playback.snapshot()), + }); } - if !active.load(Ordering::SeqCst) { - player.stop(); + let post_drain = output_latency_grace_remaining( + assistant_speech.is_some(), + playback_drained_at, + output_latency_grace, + Instant::now(), + ); + let outcome = playback + .finish( + DrainPolicy { + post_drain, + ..DrainPolicy::default() + }, + &mut |delivery| { + update_openai_assistant_speech( + true, + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + Instant::now(), + ); + if last_progress.elapsed() >= Duration::from_millis(100) { + emit_openai_stream_event( + app, + stream_id, + OpenAiStreamEventState::Progress, + None, + Some(delivery.clone()), + ); + last_progress = Instant::now(); + } + Ok(()) + }, + ) + .map_err(openai_playback_failure)?; + if outcome == OutboundOutcome::Interrupted { return Ok(StreamOutcome { state: OpenAiStreamEventState::Interrupted, - delivery: Some(snapshot_delivery(&delivery, &player)), + delivery: Some(playback.snapshot()), }); } + assistant_speech.take(); return Ok(StreamOutcome { state: OpenAiStreamEventState::Completed, delivery: None, @@ -868,20 +839,20 @@ fn run_openai_voice_stream( } Ok(OpenAiStreamCommand::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => { active.store(false, Ordering::SeqCst); - player.stop(); + playback.interrupt().map_err(openai_playback_failure)?; return Ok(StreamOutcome { state: OpenAiStreamEventState::Interrupted, - delivery: Some(snapshot_delivery(&delivery, &player)), + delivery: Some(playback.snapshot()), }); } Err(mpsc::RecvTimeoutError::Timeout) => { - if started && last_progress.elapsed() >= Duration::from_millis(100) { + if playback.started() && last_progress.elapsed() >= Duration::from_millis(100) { emit_openai_stream_event( app, stream_id, OpenAiStreamEventState::Progress, None, - Some(snapshot_delivery(&delivery, &player)), + Some(playback.snapshot()), ); last_progress = Instant::now(); } @@ -893,123 +864,36 @@ fn run_openai_voice_stream( #[cfg(target_os = "macos")] #[allow(clippy::too_many_arguments)] fn speak_pending( - runtime: &tokio::runtime::Runtime, app: &AppHandle, stream_id: &str, - client: &reqwest::Client, - key: &str, - active: &AtomicBool, - player: &PocketAudioPlayer, + backend: &dyn TtsBackend, + playback: &mut OutboundPlayback<'_>, pending: &mut String, - delivery: &mut VoiceDeliveryProgress, - started: &mut bool, native_voice: &NativeVoiceState, interruption_sensitivity: InterruptionSensitivity, - suppress_capture: bool, + input_during_tts: InputDuringTtsPolicy, assistant_speech: &mut Option, playback_drained_at: &mut Option, - output_latency_grace: Duration, - speed: f32, -) -> Result<(), String> { +) -> Result { let text = std::mem::take(pending).trim().to_string(); if text.is_empty() { - return Ok(()); + return Ok(OutboundOutcome::Completed); } for chunk in chunk_text(&text, MAX_TTS_INPUT_CHARS) { - if !active.load(Ordering::SeqCst) { - return Ok(()); - } - let mut segment_frames = 0_u64; - delivery.segments.push(VoiceDeliverySegment { - text: chunk.to_string(), - played_frames: 0, - total_frames: 0, - synthesis_complete: false, - }); - let Some(mut bytes) = runtime.block_on(openai_speech_stream_cancellable( - client, - key, - chunk.to_string(), - speed, - active, - ))? - else { - return Ok(()); - }; - let mut pcm_remainder = Vec::::new(); - let mut initial_samples = Vec::::new(); - let mut last_network_data = Instant::now(); - loop { - update_openai_assistant_speech( - player.is_empty(), - assistant_speech, - playback_drained_at, - output_latency_grace, - Instant::now(), - ); - if !active.load(Ordering::SeqCst) { - return Ok(()); - } - let item = runtime.block_on(async { - tokio::time::timeout(Duration::from_millis(50), bytes.next()).await - }); - let Some(item) = (match item { - Ok(item) => item, - Err(_) if last_network_data.elapsed() < STREAM_IDLE_TIMEOUT => continue, - Err(_) => return Err("OpenAI speech audio stream timed out".to_string()), - }) else { - break; - }; - let item = - item.map_err(|error| format_openai_request_error("stream speech audio", error))?; - last_network_data = Instant::now(); - if !active.load(Ordering::SeqCst) { - return Ok(()); - } - pcm_remainder.extend_from_slice(&item); - let sample_bytes = pcm_remainder.len() / 2 * 2; - let samples = pcm16le_to_f32(&pcm_remainder[..sample_bytes]); - pcm_remainder.drain(..sample_bytes); - if *started { - assistant_speech.get_or_insert_with(|| { - native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture) - }); + let outcome = playback.synthesize_segment( + backend, + chunk, + &mut |_| { *playback_drained_at = None; - player.enqueue(&samples)?; - } else { - initial_samples.extend_from_slice(&samples); - if initial_samples.len() >= INITIAL_PLAYBACK_BUFFER_FRAMES { - assistant_speech.get_or_insert_with(|| { + if assistant_speech.is_none() { + *assistant_speech = Some( native_voice - .begin_assistant_speech(interruption_sensitivity, suppress_capture) - }); - *playback_drained_at = None; - player.enqueue(&initial_samples)?; - initial_samples.clear(); - *started = true; - emit_openai_stream_event( - app, - stream_id, - OpenAiStreamEventState::Started, - None, - None, + .begin_assistant_speech(interruption_sensitivity, input_during_tts), ); } - } - segment_frames = segment_frames.saturating_add(samples.len() as u64); - upsert_delivery_segment(delivery, chunk, segment_frames, false); - } - if !pcm_remainder.is_empty() { - return Err("OpenAI speech returned an incomplete PCM sample".to_string()); - } - if !initial_samples.is_empty() { - assistant_speech.get_or_insert_with(|| { - native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture) - }); - *playback_drained_at = None; - player.enqueue(&initial_samples)?; - if !*started { - *started = true; + Ok(()) + }, + &mut || { emit_openai_stream_event( app, stream_id, @@ -1017,91 +901,25 @@ fn speak_pending( None, None, ); - } - } - upsert_delivery_segment(delivery, chunk, segment_frames, true); - } - Ok(()) -} - -#[cfg(target_os = "macos")] -fn ends_sentence_boundary(text: &str) -> bool { - if text.trim_end_matches([' ', '\t', '\r']).ends_with('\n') { - return true; - } - let trimmed = text.trim_end(); - trimmed - .trim_end_matches(['"', '\'', '”', '’', ')', ']', '}']) - .ends_with(['.', '!', '?']) -} - -#[cfg(target_os = "macos")] -async fn run_while_active( - future: F, - active: &AtomicBool, -) -> Option { - tokio::pin!(future); - loop { - tokio::select! { - result = &mut future => return Some(result), - _ = tokio::time::sleep(Duration::from_millis(20)) => { - if !active.load(Ordering::SeqCst) { - return None; - } - } + Ok(()) + }, + &mut |_| Ok(()), + )?; + if outcome == OutboundOutcome::Interrupted { + return Ok(outcome); } } + Ok(OutboundOutcome::Completed) } #[cfg(target_os = "macos")] -async fn openai_speech_stream_cancellable( - client: &reqwest::Client, - key: &str, - input: String, - speed: f32, - active: &AtomicBool, -) -> Result>>, String> -{ - match run_while_active(openai_speech_stream(client, key, input, speed), active).await { - Some(result) => result.map(Some), - None => Ok(None), +fn openai_playback_failure(failure: OutboundFailure) -> StreamFailure { + StreamFailure { + error: failure.message, + delivery: Some(failure.delivery), } } -#[cfg(target_os = "macos")] -async fn openai_speech_stream( - client: &reqwest::Client, - key: &str, - input: String, - speed: f32, -) -> Result>, String> { - let response = client - .post(endpoint("audio/speech")?) - .headers(authorized_headers(key)?) - .header(CONTENT_TYPE, "application/json") - .json(&json!({ - "model": speech_model(), - "voice": speech_voice(), - "input": input, - "speed": speed, - "response_format": "pcm", - "stream_format": "audio" - })) - .send() - .await - .map_err(|error| format_openai_request_error("start speech audio", error))?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(format_openai_response_error( - "start speech audio", - status, - &body, - )); - } - Ok(response.bytes_stream()) -} - #[cfg(target_os = "macos")] fn chunk_text(text: &str, max_chars: usize) -> Vec<&str> { let mut chunks = Vec::new(); @@ -1114,15 +932,6 @@ fn chunk_text(text: &str, max_chars: usize) -> Vec<&str> { if end == start { end = text.len(); } - if end < text.len() { - if let Some((offset, _)) = text[start..end] - .char_indices() - .rev() - .find(|(offset, character)| *offset > 0 && character.is_whitespace()) - { - end = start + offset; - } - } chunks.push(text[start..end].trim()); start = end; } @@ -1132,30 +941,6 @@ fn chunk_text(text: &str, max_chars: usize) -> Vec<&str> { .collect() } -#[cfg(target_os = "macos")] -fn pcm16le_to_f32(bytes: &[u8]) -> Vec { - bytes - .chunks_exact(2) - .map(|sample| i16::from_le_bytes([sample[0], sample[1]]) as f32 / i16::MAX as f32) - .collect() -} - -#[cfg(any(test, target_os = "macos"))] -fn openai_assistant_speech_grace_elapsed( - playback_drained: bool, - guard_active: bool, - playback_drained_at: &mut Option, - output_latency_grace: Duration, - now: Instant, -) -> bool { - if !guard_active || !playback_drained { - *playback_drained_at = None; - return false; - } - let drained_at = *playback_drained_at.get_or_insert(now); - now.saturating_duration_since(drained_at) >= output_latency_grace -} - #[cfg(target_os = "macos")] fn update_openai_assistant_speech( playback_drained: bool, @@ -1164,7 +949,7 @@ fn update_openai_assistant_speech( output_latency_grace: Duration, now: Instant, ) { - if openai_assistant_speech_grace_elapsed( + if output_latency_grace_elapsed( playback_drained, assistant_speech.is_some(), playback_drained_at, @@ -1175,45 +960,6 @@ fn update_openai_assistant_speech( } } -#[cfg(target_os = "macos")] -fn upsert_delivery_segment( - delivery: &mut VoiceDeliveryProgress, - _text: &str, - total_frames: u64, - synthesis_complete: bool, -) { - if let Some(segment) = delivery.segments.last_mut() { - segment.total_frames = total_frames; - segment.synthesis_complete = synthesis_complete; - } -} - -#[cfg(target_os = "macos")] -fn snapshot_delivery( - delivery: &VoiceDeliveryProgress, - player: &PocketAudioPlayer, -) -> VoiceDeliveryProgress { - let mut remaining_played = player.played_frames(); - let segments = delivery - .segments - .iter() - .map(|segment| { - let played_frames = remaining_played.min(segment.total_frames); - remaining_played = remaining_played.saturating_sub(played_frames); - VoiceDeliverySegment { - text: segment.text.clone(), - played_frames, - total_frames: segment.total_frames, - synthesis_complete: segment.synthesis_complete, - } - }) - .collect(); - VoiceDeliveryProgress { - sample_rate: delivery.sample_rate, - segments, - } -} - #[cfg(target_os = "macos")] fn emit_openai_stream_event( app: &AppHandle, @@ -1233,23 +979,6 @@ fn emit_openai_stream_event( ); } -#[cfg(target_os = "macos")] -fn format_openai_request_error(action: &str, error: reqwest::Error) -> String { - if error.is_timeout() { - format!("OpenAI voice could not {action}: the request timed out") - } else if error.is_connect() { - format!("OpenAI voice could not {action}: check your network connection") - } else { - format!("OpenAI voice could not {action}: {error}") - } -} - -#[cfg(target_os = "macos")] -fn format_openai_response_error(action: &str, status: reqwest::StatusCode, body: &str) -> String { - let preview: String = body.chars().take(500).collect(); - format!("OpenAI voice could not {action}: HTTP {status}: {preview}") -} - #[cfg(test)] mod tests { use super::*; @@ -1326,21 +1055,21 @@ mod tests { let mut drained_at = None; let grace = Duration::from_millis(100); - assert!(!openai_assistant_speech_grace_elapsed( + assert!(!output_latency_grace_elapsed( true, true, &mut drained_at, grace, started, )); - assert!(openai_assistant_speech_grace_elapsed( + assert!(output_latency_grace_elapsed( true, true, &mut drained_at, grace, started + grace, )); - assert!(!openai_assistant_speech_grace_elapsed( + assert!(!output_latency_grace_elapsed( false, true, &mut drained_at, @@ -1348,6 +1077,19 @@ mod tests { started + grace, )); assert_eq!(drained_at, None); + assert_eq!( + output_latency_grace_remaining( + true, + Some(started), + grace, + started + Duration::from_millis(40), + ), + Duration::from_millis(60) + ); + assert_eq!( + output_latency_grace_remaining(false, Some(started), grace, started), + Duration::ZERO + ); } #[test] @@ -1365,41 +1107,5 @@ mod tests { fn chunks_tts_text_on_char_boundaries() { assert_eq!(chunk_text("hello", 10), vec!["hello"]); assert_eq!(chunk_text("ééé", 3), vec!["é", "é", "é"]); - assert_eq!( - chunk_text("hello wide world", 8), - vec!["hello", "wide", "world"] - ); - } - - #[cfg(target_os = "macos")] - #[test] - fn recognizes_sentence_boundaries_before_streaming() { - assert!(ends_sentence_boundary("Hello world.\n")); - assert!(ends_sentence_boundary("Did it work?” ")); - assert!(ends_sentence_boundary("It did!)")); - assert!(!ends_sentence_boundary("Still speaking,")); - } - - #[cfg(target_os = "macos")] - #[test] - fn cancels_a_stalled_speech_request() { - let active = Arc::new(AtomicBool::new(true)); - let active_for_thread = active.clone(); - let cancellation = std::thread::spawn(move || { - std::thread::sleep(Duration::from_millis(30)); - active_for_thread.store(false, Ordering::SeqCst); - }); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .expect("runtime"); - - let result = runtime.block_on(run_while_active( - std::future::pending::<()>(), - active.as_ref(), - )); - - cancellation.join().expect("cancellation thread"); - assert_eq!(result, None); } } diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index 68228d50b..12db25fd5 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -2,21 +2,28 @@ use std::collections::VecDeque; use std::fs; -use std::io::Read; #[cfg(target_os = "macos")] use std::io::Write; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; #[cfg(target_os = "macos")] use std::sync::mpsc; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant, SystemTime}; +use berd_voice::input::InputDuringTtsPolicy; +use berd_voice::local_assets::{self, LocalAssetRoots, LocalInstallPhase}; +#[cfg(any(test, target_os = "macos"))] +use berd_voice::DeliveryProgress as VoiceDeliveryProgress; #[cfg(target_os = "macos")] use berd_voice::SAMPLE_RATE; #[cfg(target_os = "macos")] -use berd_voice::{load_text_to_speech, load_voice_style, PocketTts, VoiceStyle}; -use futures_util::StreamExt; +use berd_voice::{ + load_pocket_voice_style, load_text_to_speech, ConfiguredTtsSlot, DrainPolicy, + DrainTimeoutOutcome, OutboundFailure, OutboundOutcome, OutboundPlayback, TtsBackend, + TtsConfiguration, +}; +use berd_voice::{parakeet_assets, pocket_assets}; #[cfg(target_os = "macos")] use objc2_core_audio::{ kAudioDevicePropertyScopeOutput, kAudioDevicePropertyStreams, kAudioDeviceTransportTypeBuiltIn, @@ -27,29 +34,25 @@ use objc2_core_audio::{ #[cfg(target_os = "macos")] use rodio::DeviceTrait; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; use tauri::{AppHandle, Emitter, Manager, State}; #[cfg(target_os = "macos")] use super::native_voice::AssistantSpeechGuard; -#[cfg(target_os = "macos")] -use super::pocket_audio_player::PocketAudioPlayer; +#[cfg(any(test, target_os = "macos"))] +use super::native_voice::{output_latency_grace_elapsed, output_latency_grace_remaining}; use super::{ native_voice::{InterruptionSensitivity, NativeVoiceState}, voice_capture::VoiceCaptureState, }; -use tokio::io::AsyncWriteExt; +#[cfg(target_os = "macos")] +use berd_voice::PocketAudioPlayer; -const CACHE_VERSION: &str = "native-voice-v2"; -const VERIFIED_MARKER: &str = ".verified"; +const CACHE_VERSION: &str = pocket_assets::MODEL_ID; const POCKET_EVENT: &str = "pocket-voice:event"; #[cfg(target_os = "macos")] const POCKET_STREAM_EVENT: &str = "pocket-voice:stream-event"; const DEFAULT_VOICE: &str = "mary"; const DOWNLOAD_PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(100); -const DOWNLOAD_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); -const DOWNLOAD_READ_TIMEOUT: Duration = Duration::from_secs(30); -const DOWNLOAD_TOTAL_TIMEOUT: Duration = Duration::from_secs(30 * 60); #[cfg(target_os = "macos")] const STREAMING_EMIT_FRAMES: usize = 12; #[cfg(target_os = "macos")] @@ -64,8 +67,6 @@ const AIRPLAY_PLAYBACK_LATENCY_SAFETY_DURATION: Duration = Duration::from_secs(2 const UNKNOWN_PLAYBACK_LATENCY_SAFETY_DURATION: Duration = Duration::from_secs(2); #[cfg(any(test, target_os = "macos"))] const POCKET_SOURCE_COMPLETION_TIMEOUT: Duration = Duration::from_secs(2); -#[cfg(any(test, target_os = "macos"))] -const MIN_POCKET_PLAYBACK_SPEED: f32 = 0.75; #[cfg(target_os = "macos")] fn playback_latency_safety_duration_for_transport(transport: Option) -> Duration { @@ -100,86 +101,14 @@ pub(crate) fn playback_latency_safety_duration(output_device: Option<&str>) -> D device_id.and_then(|id| get_device_transport_type(id).ok()), ) } -const PARAKEET_ARCHIVE: Artifact = Artifact { - filename: "parakeet.tar.bz2", - size: 104_337_827, - sha256: "17f945007b52ccd8b7200ffc7c5652e9e8e961dfdf479cefcabd06cf5703630b", - url: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8.tar.bz2", -}; -const PARAKEET_ARCHIVE_DIR: &str = "sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8"; -const PARAKEET_MODEL_SIZE: u64 = 131_652_171; -const PARAKEET_MODEL_SHA256: &str = - "9177a9146cf32ee0cc8152276ef95116f312018d316be37ccf57f7efea81fc1a"; -const PARAKEET_TOKENS_SIZE: u64 = 9_953; -const PARAKEET_TOKENS_SHA256: &str = - "450e56bd2f036fe5b6aa821865838cc5aa9d8b0106134ce9a9ba0664abe6cd10"; -const PARAKEET_LICENSE: &str = "\ -NVIDIA Parakeet TDT-CTC 110M (English) -© NVIDIA Corporation. - -Licensed under the Creative Commons Attribution 4.0 International License: -https://creativecommons.org/licenses/by/4.0/ - -Original model: https://huggingface.co/nvidia/parakeet-tdt_ctc-110m -ONNX conversion: https://github.com/k2-fsa/sherpa-onnx -"; - -#[derive(Clone, Copy)] -struct Artifact { - filename: &'static str, - size: u64, - sha256: &'static str, - url: &'static str, -} - -struct DownloadSpec<'a> { - url: &'a str, - destination: &'a Path, - expected_size: u64, - expected_sha256: &'a str, -} - -const MODEL_ARTIFACTS: &[Artifact] = &[ - Artifact { filename: "bundle.json", size: 24_381, sha256: "bab643150f437f37df080a710520ff39ed9ebd9a339f8ebdc739f7eddfc28b3f", url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/bundle.json" }, - Artifact { filename: "bos_before_voice.npy", size: 4_224, sha256: "f46edf4f7007b7ba4ea58831f49d003e59e167b4641c44bb3addfe9231a780b1", url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/bos_before_voice.npy" }, - Artifact { filename: "tokenizer.model", size: 59_339, sha256: "d461765ae179566678c93091c5fa6f2984c31bbe990bf1aa62d92c64d91bc3f6", url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/tokenizer.model" }, - Artifact { filename: "flow_lm_main_int8.onnx", size: 76_341_079, sha256: "f9bd8106b79a0192c1c43399ab938fb24900a95c1c599870d75a884e99000116", url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/flow_lm_main_int8.onnx" }, - Artifact { filename: "flow_lm_flow_int8.onnx", size: 9_962_530, sha256: "3dd781ee5abee9e195320bf0106bebd6372a852b3b36352524ee78b40554635d", url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/flow_lm_flow_int8.onnx" }, - Artifact { filename: "mimi_decoder_int8.onnx", size: 22_684_077, sha256: "3630450a3297a101792a6ac66619ebc70ab916b265e6220c2afaef8b1673f925", url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/mimi_decoder_int8.onnx" }, - Artifact { filename: "mimi_encoder.onnx", size: 39_768_446, sha256: "853e2ca623b8782d94c3745ec6133bfdff7ce33d9b11128bd29ea03f28d76e3d", url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/mimi_encoder.onnx" }, - Artifact { filename: "text_conditioner.onnx", size: 16_388_344, sha256: "4ecee995fb69f85c7a7493d11f7b5ee15d9950facc7ab3f5c9c49ef1e03847bb", url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/english_2026-04/text_conditioner.onnx" }, - Artifact { filename: "LICENSE", size: 18_655, sha256: "fe7b4ce83b8381cc5b216bbb4af73c570688d1b819c73bbaed8ca401f4677cd6", url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/58a6d00cf13d239b6748cb0769f35c580a8f606c/onnx/LICENSE" }, -]; +type PocketVoice = pocket_assets::PocketVoiceDescriptor; #[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct PocketVoice { +struct PocketVoiceOption { id: &'static str, name: &'static str, - #[serde(skip_serializing)] - filename: &'static str, - #[serde(skip_serializing)] - size_bytes: u64, - #[serde(skip_serializing)] - sha256: &'static str, - #[serde(skip_serializing)] - url: &'static str, -} - -const VOICES: &[PocketVoice] = &[ - PocketVoice { id: "anna", name: "Anna", filename: "anna.wav", size_bytes: 804_630, sha256: "0a6de25cf12bf1540beb85979f306a92be81fecc051c547c5395e7e5237a3856", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p228_023_enhanced.wav" }, - PocketVoice { id: "vera", name: "Vera", filename: "vera.wav", size_bytes: 691_416, sha256: "309cf91a895830f15842b398f69a4962cb1f7e0bfab10e25dd27838e826c204b", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p229_023_enhanced.wav" }, - PocketVoice { id: "fantine", name: "Fantine", filename: "fantine.wav", size_bytes: 674_852, sha256: "5f07d4e2a3f20a15572aae885156b43ef3fc12ef3812996fd135680d9956448b", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p244_023_enhanced.wav" }, - PocketVoice { id: "charles", name: "Charles", filename: "charles.wav", size_bytes: 639_272, sha256: "6b681a429198f16e378d53bccb08d06939da7b00144a7696111d4f8f76be7756", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p254_023_enhanced.wav" }, - PocketVoice { id: "paul", name: "Paul", filename: "paul.wav", size_bytes: 717_182, sha256: "7aba504fe0b3b16478b69eb27ce6007e3cb42b0c1915b5f1c6a6024ae37d679b", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p259_023_enhanced.wav" }, - PocketVoice { id: "eponine", name: "Eponine", filename: "eponine.wav", size_bytes: 716_330, sha256: "a13c27fb47627b05223691a0ef2974358a18c886e6c2f9d2762ff1d02c20926b", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p262_023_enhanced.wav" }, - PocketVoice { id: "azelma", name: "Azelma", filename: "azelma.wav", size_bytes: 823_852, sha256: "60e3d26cdf2efdec5df712152c839928f4d5522821e6554ae11fd96c57ab1026", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p303_023_enhanced.wav" }, - PocketVoice { id: "george", name: "George", filename: "george.wav", size_bytes: 642_692, sha256: "29a41f93bf5236e5b21501091d7774c255d5f3d4e62fa4f9fdf0a92a793c84ae", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p315_023_enhanced.wav" }, - PocketVoice { id: "mary", name: "Mary", filename: "mary.wav", size_bytes: 639_084, sha256: "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p333_023_enhanced.wav" }, - PocketVoice { id: "jane", name: "Jane", filename: "jane.wav", size_bytes: 759_340, sha256: "2f12e7f155eb3118f55425394f1b049e5b1b67bdc9b3932c8ba4521420aeb84a", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p339_023_enhanced.wav" }, - PocketVoice { id: "michael", name: "Michael", filename: "michael.wav", size_bytes: 751_140, sha256: "b6743e9195e5e3fd34fe9d1633ae93f7ffab787b249e45f6467d7d6f7a6ee6ad", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p360_023_enhanced.wav" }, - PocketVoice { id: "eve", name: "Eve", filename: "eve.wav", size_bytes: 671_872, sha256: "396e7cbd066b0f3fb6d67fa26e7904076958239d736d4390f15b5fe88feb14cd", url: "https://huggingface.co/kyutai/tts-voices/resolve/323332d33f997de8394f24a193e1a76df720e01a/vctk/p361_023_enhanced.wav" }, -]; +} #[derive(Clone, Debug, Default)] pub struct PocketVoiceState { @@ -191,14 +120,13 @@ pub struct PocketVoiceState { #[derive(Debug, Default)] struct PlaybackRuntime { active: Option>, - playback_rate: Option>, #[cfg(target_os = "macos")] stream: Option, } struct PlaybackSession { active: Arc, - playback_rate: Arc, + playback_rate: f32, } #[cfg(target_os = "macos")] @@ -238,87 +166,6 @@ struct PocketStreamEvent { delivery: Option, } -#[cfg(any(test, target_os = "macos"))] -#[derive(Clone, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct VoiceDeliverySegment { - text: String, - played_frames: u64, - total_frames: u64, - synthesis_complete: bool, -} - -#[cfg(any(test, target_os = "macos"))] -#[derive(Clone, Debug, Serialize)] -struct VoiceDeliveryProgress { - #[serde(rename = "sampleRate")] - sample_rate: u32, - segments: Vec, -} - -#[cfg(any(test, target_os = "macos"))] -#[derive(Debug, Default)] -struct PlaybackDeliveryLedger { - segments: Vec<(String, u64, bool)>, -} - -#[cfg(any(test, target_os = "macos"))] -impl PlaybackDeliveryLedger { - fn begin_segment(&mut self, text: String) { - self.segments.push((text, 0, false)); - } - - fn append_frames(&mut self, frames: usize) { - let frames = frames as u64; - if frames == 0 { - return; - } - if let Some((_, total, synthesis_complete)) = self.segments.last_mut() { - if !*synthesis_complete { - *total = total.saturating_add(frames); - } - } - } - - fn complete_segment(&mut self, final_total_frames: u64) { - if let Some((_, total, synthesis_complete)) = self.segments.last_mut() { - *total = (*total).max(final_total_frames); - *synthesis_complete = true; - } - } - - fn total_frames(&self) -> u64 { - self.segments - .iter() - .map(|(_, total_frames, _)| *total_frames) - .sum() - } - - fn snapshot_consumed_frames(&self, consumed_frames: u64) -> VoiceDeliveryProgress { - let mut segment_start = 0_u64; - let segments = self - .segments - .iter() - .map(|(text, total_frames, synthesis_complete)| { - let played_frames = consumed_frames - .saturating_sub(segment_start) - .min(*total_frames); - segment_start = segment_start.saturating_add(*total_frames); - VoiceDeliverySegment { - text: text.clone(), - played_frames, - total_frames: *total_frames, - synthesis_complete: *synthesis_complete, - } - }) - .collect(); - VoiceDeliveryProgress { - sample_rate: berd_voice::SAMPLE_RATE, - segments, - } - } -} - #[cfg(target_os = "macos")] struct PocketStreamOutcome { state: PocketStreamEventState, @@ -429,7 +276,7 @@ pub struct PocketVoiceStatus { error: Option, selected_voice: String, playback_speed: f32, - voices: &'static [PocketVoice], + voices: Vec, } fn default_playback_speed() -> f32 { @@ -447,32 +294,32 @@ fn settings(base: &Path) -> PocketSettings { } fn pocket_download_bytes() -> u64 { - MODEL_ARTIFACTS.iter().map(|item| item.size).sum::() - + VOICES.iter().map(|item| item.size_bytes).sum::() + pocket_assets::download_bytes() } fn parakeet_download_bytes() -> u64 { - PARAKEET_ARCHIVE.size + parakeet_assets::download_bytes() } +#[cfg(test)] fn pocket_published_bytes() -> u64 { pocket_download_bytes() } #[cfg(test)] fn parakeet_published_bytes() -> u64 { - PARAKEET_MODEL_SIZE + PARAKEET_TOKENS_SIZE + PARAKEET_LICENSE.len() as u64 + parakeet_assets::published_bytes() } fn pocket_disk_bytes(base: &Path) -> Option { let version = base.join(CACHE_VERSION); - MODEL_ARTIFACTS + pocket_assets::model_artifacts() .iter() - .map(|item| version.join(item.filename)) + .map(|item| version.join(item.relative_path)) .chain( - VOICES + pocket_assets::voices() .iter() - .map(|voice| version.join("voices").join(voice.filename)), + .map(|voice| version.join(voice.relative_path)), ) .try_fold(0_u64, |total, path| { total.checked_add(fs::metadata(path).ok()?.len()) @@ -481,20 +328,12 @@ fn pocket_disk_bytes(base: &Path) -> Option { fn parakeet_disk_bytes(base: &Path) -> Option { let stt = base.join(CACHE_VERSION).join("stt"); - [ - stt.join("model.int8.onnx"), - stt.join("tokens.txt"), - stt.join("MODEL_LICENSE.txt"), - ] - .into_iter() - .try_fold(0_u64, |total, path| { - total.checked_add(fs::metadata(path).ok()?.len()) - }) -} - -#[cfg(test)] -fn total_bytes() -> u64 { - pocket_download_bytes() + parakeet_download_bytes() + parakeet_assets::published_assets() + .iter() + .map(|asset| stt.join(asset.relative_path)) + .try_fold(0_u64, |total, path| { + total.checked_add(fs::metadata(path).ok()?.len()) + }) } fn cache_base(app: &AppHandle) -> Result { @@ -504,9 +343,18 @@ fn cache_base(app: &AppHandle) -> Result { .map_err(|error| format!("resolve Pocket TTS data directory: {error}")) } +fn local_asset_roots(base: &Path) -> Result { + LocalAssetRoots::new( + base, + base.join(CACHE_VERSION), + base.join(CACHE_VERSION).join("stt"), + ) + .map_err(|error| error.to_string()) +} + fn selected_voice(base: &Path) -> String { Some(settings(base).selected_voice) - .filter(|id| VOICES.iter().any(|voice| voice.id == id)) + .filter(|id| pocket_assets::voices().iter().any(|voice| voice.id == id)) .unwrap_or_else(|| DEFAULT_VOICE.to_string()) } @@ -654,23 +502,23 @@ pub(crate) enum VoiceInterruptionMode { } #[cfg_attr(not(any(test, target_os = "macos")), allow(dead_code))] -pub(crate) fn should_suppress_capture( +pub(crate) fn resolve_input_during_tts_policy( mode: VoiceInterruptionMode, output_device: Option<&str>, -) -> bool { +) -> InputDuringTtsPolicy { match mode { // Automatic is best-effort because macOS cannot classify every external route. // Prevent feedback remains the reliable fallback when this heuristic misses one. - VoiceInterruptionMode::Automatic => output_device_uses_speakers(output_device), - VoiceInterruptionMode::AllowInterruptions => false, - VoiceInterruptionMode::PreventFeedback => true, + VoiceInterruptionMode::Automatic if output_device_uses_speakers(output_device) => { + InputDuringTtsPolicy::SuppressInput + } + VoiceInterruptionMode::Automatic | VoiceInterruptionMode::AllowInterruptions => { + InputDuringTtsPolicy::AllowBargeIn + } + VoiceInterruptionMode::PreventFeedback => InputDuringTtsPolicy::SuppressInput, } } -fn file_has_size(path: &Path, size: u64) -> bool { - fs::metadata(path).is_ok_and(|metadata| metadata.len() == size) -} - #[derive(Clone, Debug, PartialEq, Eq)] struct InstallationFingerprint(Vec<(PathBuf, u64, SystemTime)>); @@ -714,21 +562,10 @@ fn pocket_installation_valid(base: &Path) -> bool { pocket_installation_fingerprint(base), &POCKET_INSTALLATION_VALIDATION, || { - MODEL_ARTIFACTS.iter().all(|item| { - verify_file( - &base.join(CACHE_VERSION).join(item.filename), - item.size, - item.sha256, - ) - .is_ok() - }) && VOICES.iter().all(|voice| { - verify_file( - &base.join(CACHE_VERSION).join("voices").join(voice.filename), - voice.size_bytes, - voice.sha256, - ) - .is_ok() - }) + matches!( + pocket_assets::inspect(&base.join(CACHE_VERSION)), + Ok(pocket_assets::PocketAssetStatus::Ready { .. }) + ) }, ) } @@ -739,59 +576,52 @@ fn parakeet_installation_valid(base: &Path) -> bool { parakeet_installation_fingerprint(base), &PARAKEET_INSTALLATION_VALIDATION, || { - verify_file( - &base.join(CACHE_VERSION).join("stt").join("model.int8.onnx"), - PARAKEET_MODEL_SIZE, - PARAKEET_MODEL_SHA256, + matches!( + parakeet_assets::inspect(&base.join(CACHE_VERSION).join("stt")), + Ok(parakeet_assets::ParakeetAssetStatus::Ready { .. }) ) - .is_ok() - && verify_file( - &base.join(CACHE_VERSION).join("stt").join("tokens.txt"), - PARAKEET_TOKENS_SIZE, - PARAKEET_TOKENS_SHA256, - ) - .is_ok() }, ) } -fn verified_version(base: &Path) -> Option { +fn lock_local_assets_for_read(base: &Path) -> Result { + let roots = local_asset_roots(base)?; + local_assets::try_lock_for_read(&roots).map_err(|error| error.to_string()) +} + +fn version_root(base: &Path) -> Option { let version = base.join(CACHE_VERSION); - if !matches!( - fs::read_to_string(version.join(VERIFIED_MARKER)).as_deref(), - Ok(CACHE_VERSION) - ) { + if !version.is_dir() { return None; } Some(version) } fn pocket_installation_fingerprint(base: &Path) -> Option { - let version = verified_version(base)?; - let mut files: Vec<(PathBuf, u64)> = MODEL_ARTIFACTS + let version = version_root(base)?; + let mut files: Vec<(PathBuf, u64)> = pocket_assets::model_artifacts() .iter() - .map(|item| (version.join(item.filename), item.size)) + .map(|item| (version.join(item.relative_path), item.size_bytes)) .collect(); - files.extend(VOICES.iter().map(|voice| { - ( - version.join("voices").join(voice.filename), - voice.size_bytes, - ) - })); + files.extend( + pocket_assets::voices() + .iter() + .map(|voice| (version.join(voice.relative_path), voice.size_bytes)), + ); fingerprint_files(files) } fn parakeet_installation_fingerprint(base: &Path) -> Option { - let version = verified_version(base)?; - let mut files = vec![ - ( - version.join("stt").join("model.int8.onnx"), - PARAKEET_MODEL_SIZE, - ), - (version.join("stt").join("tokens.txt"), PARAKEET_TOKENS_SIZE), - ]; - let license = version.join("stt").join("MODEL_LICENSE.txt"); - files.push((license.clone(), fs::metadata(&license).ok()?.len())); + let version = version_root(base)?; + let files = parakeet_assets::published_assets() + .iter() + .map(|asset| { + ( + version.join("stt").join(asset.relative_path), + asset.size_bytes, + ) + }) + .collect::>(); fingerprint_files(files) } @@ -822,6 +652,7 @@ fn pocket_voice_status( state: &PocketVoiceState, ) -> Result { let base = cache_base(app)?; + let _assets = lock_local_assets_for_read(&base)?; let runtime = state .install .lock() @@ -866,13 +697,22 @@ fn pocket_voice_status( }), selected_voice: selected_voice(&base), playback_speed: playback_speed(&base), - voices: VOICES, + voices: pocket_assets::voices() + .iter() + .map(|voice| PocketVoiceOption { + id: voice.id, + name: voice.name, + }) + .collect(), }) } #[tauri::command] pub fn select_pocket_voice(app: AppHandle, voice_id: String) -> Result<(), String> { - if !VOICES.iter().any(|voice| voice.id == voice_id) { + if !pocket_assets::voices() + .iter() + .any(|voice| voice.id == voice_id) + { return Err(format!("Unknown Pocket voice: {voice_id}")); } let base = cache_base(&app)?; @@ -889,11 +729,7 @@ pub fn select_pocket_voice(app: AppHandle, voice_id: String) -> Result<(), Strin } #[tauri::command] -pub fn set_pocket_playback_speed( - app: AppHandle, - state: State<'_, PocketVoiceState>, - speed: f32, -) -> Result<(), String> { +pub fn set_pocket_playback_speed(app: AppHandle, speed: f32) -> Result<(), String> { if !speed.is_finite() || !(0.75..=2.0).contains(&speed) { return Err("Pocket playback speed must be between 0.75 and 2.0".to_string()); } @@ -907,19 +743,7 @@ pub fn set_pocket_playback_speed( let temporary = base.join("settings.json.tmp"); fs::write(&temporary, data).map_err(|error| format!("write Pocket settings: {error}"))?; fs::rename(&temporary, base.join("settings.json")) - .map_err(|error| format!("publish Pocket settings: {error}"))?; - update_active_playback_speed(&state, speed) -} - -fn update_active_playback_speed(state: &PocketVoiceState, speed: f32) -> Result<(), String> { - let playback = state - .playback - .lock() - .map_err(|_| "Pocket TTS playback state lock was poisoned".to_string())?; - if let Some(playback_rate) = playback.playback_rate.as_ref() { - playback_rate.store(speed.to_bits(), Ordering::SeqCst); - } - Ok(()) + .map_err(|error| format!("publish Pocket settings: {error}")) } #[tauri::command] @@ -930,12 +754,15 @@ pub async fn preview_pocket_voice( voice_id: String, ) -> Result<(), String> { let base = cache_base(&app)?; - let voice = VOICES + let voice = pocket_assets::voices() .iter() .find(|voice| voice.id == voice_id) .copied() .ok_or_else(|| format!("Unknown Pocket voice: {voice_id}"))?; - if !pocket_installation_valid(&base) { + let assets = lock_local_assets_for_read(&base)?; + let installed = pocket_installation_valid(&base); + drop(assets); + if !installed { return Err("Pocket TTS must be downloaded before previewing a voice".to_string()); } let session = begin_playback( @@ -945,16 +772,19 @@ pub async fn preview_pocket_voice( )?; let output_device = selected_output_device(); let effective_output_device = effective_output_device_name(output_device.as_deref()); - let capture_suppression = + let assistant_speech = output_device_uses_speakers(effective_output_device.as_deref()).then(|| { log::info!("[voice-echo-guard] speaker output detected"); - native_voice.suppress_capture() + native_voice.begin_assistant_speech( + InterruptionSensitivity::Balanced, + InputDuringTtsPolicy::SuppressInput, + ) }); let playback = state.playback.clone(); let playback_active = session.active.clone(); tauri::async_runtime::spawn_blocking(move || { - let _capture_suppression = capture_suppression; + let _assistant_speech = assistant_speech; let result = synthesize_and_stream( &base, voice, @@ -982,11 +812,14 @@ pub async fn speak_pocket_voice( return Ok(()); } let base = cache_base(&app)?; - if !pocket_installation_valid(&base) { + let assets = lock_local_assets_for_read(&base)?; + let installed = pocket_installation_valid(&base); + drop(assets); + if !installed { return Err("Pocket TTS installation is incomplete or corrupt".to_string()); } let voice_id = selected_voice(&base); - let voice = VOICES + let voice = pocket_assets::voices() .iter() .find(|voice| voice.id == voice_id) .copied() @@ -994,16 +827,19 @@ pub async fn speak_pocket_voice( let session = begin_playback(&state, "Pocket voice playback is already active", &base)?; let output_device = selected_output_device(); let effective_output_device = effective_output_device_name(output_device.as_deref()); - let capture_suppression = + let assistant_speech = output_device_uses_speakers(effective_output_device.as_deref()).then(|| { log::info!("[voice-echo-guard] speaker output detected"); - native_voice.suppress_capture() + native_voice.begin_assistant_speech( + InterruptionSensitivity::Balanced, + InputDuringTtsPolicy::SuppressInput, + ) }); let playback = state.playback.clone(); let playback_active = session.active.clone(); tauri::async_runtime::spawn_blocking(move || { - let _capture_suppression = capture_suppression; + let _assistant_speech = assistant_speech; let result = synthesize_and_stream( &base, voice, @@ -1020,20 +856,27 @@ pub async fn speak_pocket_voice( } #[tauri::command] +#[allow(clippy::too_many_arguments)] // Tauri injects three runtime dependencies beside the stream payload. pub fn start_pocket_voice_stream( app: AppHandle, state: State<'_, PocketVoiceState>, native_voice: State<'_, NativeVoiceState>, + session_id: String, + expected_revision: u64, + speech_id: u64, stream_id: String, interruption_mode: VoiceInterruptionMode, interruption_sensitivity: InterruptionSensitivity, -) -> Result<(), String> { +) -> Result { #[cfg(not(target_os = "macos"))] { let _ = ( app, state, native_voice, + session_id, + expected_revision, + speech_id, stream_id, interruption_mode, interruption_sensitivity, @@ -1047,11 +890,14 @@ pub fn start_pocket_voice_stream( return Err("Pocket voice stream id cannot be empty".to_string()); } let base = cache_base(&app)?; - if !pocket_installation_valid(&base) { + let assets = lock_local_assets_for_read(&base)?; + let installed = pocket_installation_valid(&base); + drop(assets); + if !installed { return Err("Pocket TTS installation is incomplete or corrupt".to_string()); } let voice_id = selected_voice(&base); - let voice = VOICES + let voice = pocket_assets::voices() .iter() .find(|voice| voice.id == voice_id) .copied() @@ -1059,8 +905,8 @@ pub fn start_pocket_voice_stream( let session = begin_playback(&state, "Pocket voice playback is already active", &base)?; let output_device = selected_output_device(); let effective_output_device = effective_output_device_name(output_device.as_deref()); - let suppress_capture = - should_suppress_capture(interruption_mode, effective_output_device.as_deref()); + let input_during_tts = + resolve_input_during_tts_policy(interruption_mode, effective_output_device.as_deref()); let (sender, receiver) = mpsc::channel(); { let mut playback = state @@ -1077,10 +923,21 @@ pub fn start_pocket_voice_stream( active, playback_rate, } = session; + let Some(admission) = native_voice.claim_assistant_speech( + &session_id, + expected_revision, + speech_id, + active.clone(), + )? + else { + finish_playback(&state.playback, &active); + return Ok(false); + }; let playback = state.playback.clone(); let playback_active = active.clone(); let native_voice_state = native_voice.inner().clone(); tauri::async_runtime::spawn_blocking(move || { + let admission_guard = admission; let result = run_with_playback_cleanup(&playback, &playback_active, || { run_pocket_voice_stream( &app, @@ -1093,7 +950,7 @@ pub fn start_pocket_voice_stream( receiver, native_voice_state, interruption_sensitivity, - suppress_capture, + input_during_tts, ) }); let (event_state, error, delivery) = match result { @@ -1111,9 +968,10 @@ pub fn start_pocket_voice_stream( // A terminal event hands stream ownership back to the renderer, // which may immediately start a replacement stream. Release the // backend playback token before publishing that handoff. + drop(admission_guard); emit_pocket_stream_event(&app, &stream_id, event_state, error, delivery); }); - Ok(()) + Ok(true) } } @@ -1343,6 +1201,36 @@ async fn wait_for_install_idle(state: &PocketVoiceState) -> Result<(), String> { } fn remove_cached_model(base: &Path, model: VoiceModelKind) -> Result<(), String> { + remove_cached_model_with( + base, + model, + pocket_installation_valid, + parakeet_installation_valid, + |mutation, source, destination| { + pocket_assets::stage_verified_bundle(mutation, source, destination) + .map_err(|error| error.to_string()) + }, + |mutation, source, destination| { + parakeet_assets::stage_verified_bundle(mutation, source, destination) + .map_err(|error| error.to_string()) + }, + ) +} + +fn remove_cached_model_with( + base: &Path, + model: VoiceModelKind, + pocket_ready: impl Fn(&Path) -> bool, + parakeet_ready: impl Fn(&Path) -> bool, + stage_pocket: impl Fn(&local_assets::LocalAssetMutationGuard, &Path, &Path) -> Result<(), String>, + stage_parakeet: impl Fn(&local_assets::LocalAssetMutationGuard, &Path, &Path) -> Result<(), String>, +) -> Result<(), String> { + let roots = local_asset_roots(base)?; + let mutation = + local_assets::lock_for_mutation_blocking(&roots).map_err(|error| error.to_string())?; + mutation + .recover_interrupted_publication() + .map_err(|error| error.to_string())?; let final_dir = base.join(CACHE_VERSION); if !final_dir.exists() { return Ok(()); @@ -1350,34 +1238,27 @@ fn remove_cached_model(base: &Path, model: VoiceModelKind) -> Result<(), String> let operation_id = uuid::Uuid::new_v4(); let staging = base.join(format!("{CACHE_VERSION}.remove-{operation_id}")); - let previous = base.join(format!("{CACHE_VERSION}.removed-{operation_id}")); + // Use the shared transaction prefix so a later mutation can recover if this + // process exits after retiring the live bundle. + let previous = base.join(format!(".voice-backup-{operation_id}")); fs::create_dir_all(&staging) .map_err(|error| format!("stage retained voice model assets: {error}"))?; - let retained_paths: Vec = match model { - VoiceModelKind::Pocket => vec![PathBuf::from("stt")], - VoiceModelKind::Parakeet => MODEL_ARTIFACTS - .iter() - .map(|artifact| PathBuf::from(artifact.filename)) - .chain(std::iter::once(PathBuf::from("voices"))) - .collect(), + let retained_any = match model { + VoiceModelKind::Pocket => parakeet_ready(base), + VoiceModelKind::Parakeet => pocket_ready(base), }; - let mut retained_any = false; - let stage_result = (|| { - for relative in retained_paths { - let source = final_dir.join(&relative); - if !source.exists() { - continue; - } - retained_any = true; - clone_cache_path(&source, &staging.join(relative))?; - } - if retained_any { - fs::write(staging.join(VERIFIED_MARKER), CACHE_VERSION) - .map_err(|error| format!("verify retained voice model cache: {error}"))?; + let stage_result = match model { + VoiceModelKind::Pocket if retained_any => stage_parakeet( + &mutation, + roots.parakeet_bundle_root(), + &staging.join("stt"), + ), + VoiceModelKind::Parakeet if retained_any => { + stage_pocket(&mutation, roots.pocket_bundle_root(), &staging) } - Ok::<(), String>(()) - })(); + _ => Ok(()), + }; if let Err(error) = stage_result { let _ = fs::remove_dir_all(&staging); return Err(error); @@ -1391,11 +1272,40 @@ fn remove_cached_model(base: &Path, model: VoiceModelKind) -> Result<(), String> .map_err(|error| format!("retire voice model cache atomically: {error}"))?; if retained_any { if let Err(error) = fs::rename(&staging, &final_dir) { - let _ = fs::rename(&previous, &final_dir); - let _ = fs::remove_dir_all(&staging); - return Err(format!( - "publish retained voice model cache atomically: {error}" - )); + if let Err(rollback_error) = fs::rename(&previous, &final_dir) { + return Err(format!( + "publish retained voice model cache failed ({error}); restoring the prior cache also failed ({rollback_error}); recovery data remains at {} and {}", + previous.display(), + staging.display(), + )); + } + fs::remove_dir_all(&staging) + .map_err(|cleanup| format!("clean failed removal staging cache: {cleanup}"))?; + return Err(format!("publish retained voice model cache: {error}")); + } + let retained_ready = match model { + VoiceModelKind::Pocket => parakeet_ready(base), + VoiceModelKind::Parakeet => pocket_ready(base), + }; + if !retained_ready { + let failed = base.join(format!("{CACHE_VERSION}.remove-failed-{operation_id}")); + fs::rename(&final_dir, &failed).map_err(|error| { + format!( + "preserve invalid retained model cache: {error}; recovery data remains at {} and {}", + final_dir.display(), + previous.display(), + ) + })?; + fs::rename(&previous, &final_dir).map_err(|error| { + format!( + "restore prior model cache after verification failure: {error}; recovery data remains at {} and {}", + previous.display(), + failed.display(), + ) + })?; + fs::remove_dir_all(&failed) + .map_err(|error| format!("clean invalid retained model cache: {error}"))?; + return Err("Retained voice model cache failed pinned-file verification".to_string()); } } fs::remove_dir_all(&previous) @@ -1403,23 +1313,6 @@ fn remove_cached_model(base: &Path, model: VoiceModelKind) -> Result<(), String> Ok(()) } -fn clone_cache_path(source: &Path, destination: &Path) -> Result<(), String> { - if source.is_dir() { - fs::create_dir_all(destination) - .map_err(|error| format!("create retained cache directory: {error}"))?; - for entry in fs::read_dir(source) - .map_err(|error| format!("read retained cache directory: {error}"))? - { - let entry = entry.map_err(|error| format!("read retained cache entry: {error}"))?; - clone_cache_path(&entry.path(), &destination.join(entry.file_name()))?; - } - return Ok(()); - } - fs::hard_link(source, destination) - .or_else(|_| fs::copy(source, destination).map(|_| ())) - .map_err(|error| format!("retain voice model asset {}: {error}", source.display())) -} - fn begin_playback( state: &State<'_, PocketVoiceState>, already_active: &str, @@ -1448,9 +1341,8 @@ fn begin_playback_runtime( return Err(already_active.to_string()); } let active = Arc::new(AtomicBool::new(true)); - let playback_rate = Arc::new(AtomicU32::new(current_playback_speed().to_bits())); + let playback_rate = current_playback_speed(); playback.active = Some(active.clone()); - playback.playback_rate = Some(playback_rate.clone()); drop(install); Ok(PlaybackSession { active, @@ -1486,7 +1378,6 @@ fn finish_playback(playback: &std::sync::Mutex, completed: &Arc .is_some_and(|active| Arc::ptr_eq(active, completed)) { playback.active = None; - playback.playback_rate = None; #[cfg(target_os = "macos")] { playback.stream = None; @@ -1526,6 +1417,7 @@ fn queue_model_install( model: VoiceModelKind, ) -> Result { let base = cache_base(app)?; + let _assets = lock_local_assets_for_read(&base)?; let already_installed = match model { VoiceModelKind::Pocket => pocket_installation_valid(&base), VoiceModelKind::Parakeet => parakeet_installation_valid(&base), @@ -1766,6 +1658,7 @@ fn advance_model_progress( Ok(true) } +#[cfg(test)] fn increment_model_progress( runtime: &mut InstallRuntime, model: VoiceModelKind, @@ -1838,341 +1731,132 @@ async fn install_one_model( attempt_id: u64, ) -> Result<(), String> { let base = cache_base(app)?; - fs::create_dir_all(&base).map_err(|error| format!("create Pocket cache: {error}"))?; - let staging = base.join(format!("{CACHE_VERSION}.partial-{}", uuid::Uuid::new_v4())); - fs::create_dir_all(&staging) - .map_err(|error| format!("create voice model staging directory: {error}"))?; - let current = base.join(CACHE_VERSION); - if current.exists() { - for entry in fs::read_dir(¤t) - .map_err(|error| format!("read current voice model cache: {error}"))? - { - let entry = entry.map_err(|error| format!("read voice model cache entry: {error}"))?; - if entry.file_name() == VERIFIED_MARKER { - continue; - } - clone_cache_path(&entry.path(), &staging.join(entry.file_name()))?; - } - } - match model { - VoiceModelKind::Pocket => { - for artifact in MODEL_ARTIFACTS { - let _ = fs::remove_file(staging.join(artifact.filename)); - } - let _ = fs::remove_dir_all(staging.join("voices")); + let roots = local_asset_roots(&base)?; + let mut callback_error = None; + let mut last_phase = None; + let mut on_progress = |progress: local_assets::LocalInstallProgress| { + if callback_error.is_some() { + return; } - VoiceModelKind::Parakeet => { - let _ = fs::remove_dir_all(staging.join("stt")); + let phase = match progress.phase { + LocalInstallPhase::Downloading => VoiceModelDownloadPhase::Downloading, + LocalInstallPhase::Extracting => VoiceModelDownloadPhase::Extracting, + LocalInstallPhase::Verifying => VoiceModelDownloadPhase::Verifying, + LocalInstallPhase::Publishing => VoiceModelDownloadPhase::Publishing, + LocalInstallPhase::Complete => VoiceModelDownloadPhase::Complete, + }; + if let Err(error) = set_model_progress( + state, + model, + attempt_id, + phase, + Some(progress.downloaded_bytes), + ) { + callback_error = Some(error); + return; } - } - let client = voice_download_client( - DOWNLOAD_CONNECT_TIMEOUT, - DOWNLOAD_READ_TIMEOUT, - DOWNLOAD_TOTAL_TIMEOUT, - )?; - let install_result = async { - match model { - VoiceModelKind::Parakeet => { - let archive = staging.join(PARAKEET_ARCHIVE.filename); - download_artifact( - app, - state, - model, - attempt_id, - &client, - DownloadSpec { - url: PARAKEET_ARCHIVE.url, - destination: &archive, - expected_size: PARAKEET_ARCHIVE.size, - expected_sha256: PARAKEET_ARCHIVE.sha256, - }, - ) - .await?; - set_model_progress( - state, - model, - attempt_id, - VoiceModelDownloadPhase::Extracting, - Some(PARAKEET_ARCHIVE.size), - )?; - emit_pocket_status(app, state); - extract_parakeet(&archive, &staging).await?; - tokio::fs::remove_file(&archive) - .await - .map_err(|error| format!("remove Parakeet archive: {error}"))?; - set_model_progress( - state, - model, - attempt_id, - VoiceModelDownloadPhase::Verifying, - Some(parakeet_download_bytes()), - )?; - emit_pocket_status(app, state); - } - VoiceModelKind::Pocket => { - tokio::fs::create_dir_all(staging.join("voices")) - .await - .map_err(|error| format!("create Pocket staging directory: {error}"))?; - for item in MODEL_ARTIFACTS { - download_artifact( - app, - state, - model, - attempt_id, - &client, - DownloadSpec { - url: item.url, - destination: &staging.join(item.filename), - expected_size: item.size, - expected_sha256: item.sha256, - }, - ) - .await?; - } - for voice in VOICES { - download_artifact( - app, - state, + let phase_changed = last_phase.replace(phase) != Some(phase); + let should_emit = if phase == VoiceModelDownloadPhase::Downloading && !phase_changed { + state + .install + .lock() + .map(|mut runtime| { + should_emit_download_progress_at( + &mut runtime, model, attempt_id, - &client, - DownloadSpec { - url: voice.url, - destination: &staging.join("voices").join(voice.filename), - expected_size: voice.size_bytes, - expected_sha256: voice.sha256, - }, + Instant::now(), ) - .await?; - } - set_model_progress( - state, - model, - attempt_id, - VoiceModelDownloadPhase::Verifying, - Some(pocket_published_bytes()), - )?; - emit_pocket_status(app, state); - } - } - tokio::fs::write(staging.join(VERIFIED_MARKER), CACHE_VERSION) - .await - .map_err(|error| format!("mark verified voice model installation: {error}"))?; - set_model_progress( - state, - model, - attempt_id, - VoiceModelDownloadPhase::Publishing, - None, - )?; - emit_pocket_status(app, state); - publish_staging(&base, &staging)?; - let published = match model { - VoiceModelKind::Pocket => pocket_installation_valid(&base), - VoiceModelKind::Parakeet => parakeet_installation_valid(&base), + }) + .unwrap_or(false) + } else { + true }; - if !published { - return Err("Published voice model failed pinned-file verification".to_string()); + if should_emit { + emit_pocket_status(app, state); } - set_model_progress( - state, - model, - attempt_id, - VoiceModelDownloadPhase::Complete, - Some(match model { - VoiceModelKind::Pocket => pocket_download_bytes(), - VoiceModelKind::Parakeet => parakeet_download_bytes(), + }; + match model { + VoiceModelKind::Pocket => { + pocket_assets::install(&roots, &mut on_progress) + .await + .map(|outcome| { + if let pocket_assets::PocketInstallOutcome::Installed { + cleanup_pending: Some(path), + .. + } = outcome + { + log::warn!( + "Pocket assets installed; prior backup cleanup remains at {}", + path.display() + ); + } + }) + } + VoiceModelKind::Parakeet => parakeet_assets::install(&roots, &mut on_progress) + .await + .map(|outcome| { + if let parakeet_assets::ParakeetInstallOutcome::Installed { + cleanup_pending: Some(path), + .. + } = outcome + { + log::warn!( + "Parakeet assets installed; prior backup cleanup remains at {}", + path.display() + ); + } }), - )?; - emit_pocket_status(app, state); - Ok::<(), String>(()) } - .await; - if let Err(error) = install_result { - let _ = tokio::fs::remove_dir_all(&staging).await; + .map_err(|error| error.to_string())?; + if let Some(error) = callback_error { return Err(error); } + normalize_successful_install(state, model, attempt_id)?; + emit_pocket_status(app, state); Ok(()) } -fn voice_download_client( - connect_timeout: Duration, - read_timeout: Duration, - total_timeout: Duration, -) -> Result { - reqwest::Client::builder() - .connect_timeout(connect_timeout) - .read_timeout(read_timeout) - .timeout(total_timeout) - .build() - .map_err(|error| format!("create Pocket download client: {error}")) -} - -async fn extract_parakeet(archive: &Path, staging: &Path) -> Result<(), String> { - let archive = archive.to_path_buf(); - let staging = staging.to_path_buf(); - tauri::async_runtime::spawn_blocking(move || { - let extraction = staging.join("parakeet-extract"); - fs::create_dir_all(&extraction) - .map_err(|error| format!("create Parakeet extraction directory: {error}"))?; - let compressed = - fs::File::open(&archive).map_err(|error| format!("open Parakeet archive: {error}"))?; - let decoder = bzip2::read::BzDecoder::new(compressed); - let mut archive = tar::Archive::new(decoder); - archive - .unpack(&extraction) - .map_err(|error| format!("extract Parakeet archive: {error}"))?; - let source = extraction.join(PARAKEET_ARCHIVE_DIR); - verify_file( - &source.join("model.int8.onnx"), - PARAKEET_MODEL_SIZE, - PARAKEET_MODEL_SHA256, - )?; - verify_file( - &source.join("tokens.txt"), - PARAKEET_TOKENS_SIZE, - PARAKEET_TOKENS_SHA256, - )?; - let destination = staging.join("stt"); - fs::create_dir_all(&destination) - .map_err(|error| format!("create Parakeet staging directory: {error}"))?; - fs::rename( - source.join("model.int8.onnx"), - destination.join("model.int8.onnx"), - ) - .map_err(|error| format!("stage Parakeet model: {error}"))?; - fs::rename(source.join("tokens.txt"), destination.join("tokens.txt")) - .map_err(|error| format!("stage Parakeet tokens: {error}"))?; - fs::write(destination.join("MODEL_LICENSE.txt"), PARAKEET_LICENSE) - .map_err(|error| format!("write Parakeet attribution: {error}"))?; - fs::remove_dir_all(&extraction) - .map_err(|error| format!("remove Parakeet extraction directory: {error}")) - }) - .await - .map_err(|error| format!("Parakeet extraction task failed: {error}"))? -} - -fn verify_file(path: &Path, expected_size: u64, expected_sha256: &str) -> Result<(), String> { - if !file_has_size(path, expected_size) { - return Err(format!("Voice asset size mismatch for {}", path.display())); - } - let mut file = - fs::File::open(path).map_err(|error| format!("open {}: {error}", path.display()))?; - let mut hasher = Sha256::new(); - let mut buffer = [0_u8; 64 * 1024]; - loop { - let read = file - .read(&mut buffer) - .map_err(|error| format!("read {}: {error}", path.display()))?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - } - let actual = format!("{:x}", hasher.finalize()); - if actual != expected_sha256 { - return Err(format!( - "Voice asset checksum mismatch for {}: expected {expected_sha256}, got {actual}", - path.display() - )); +fn normalize_successful_install( + state: &PocketVoiceState, + model: VoiceModelKind, + attempt_id: u64, +) -> Result<(), String> { + let mut runtime = state + .install + .lock() + .map_err(|_| "Pocket TTS install state lock was poisoned".to_string())?; + let Some(progress) = model_progress_mut(&mut runtime, model) else { + return Err("Voice model progress was not initialized".to_string()); + }; + if progress.attempt_id != attempt_id { + return Ok(()); } + // AlreadyReady may be discovered during either locked recheck without any + // network transfer by this attempt. Complete the host projection while + // retaining the honest downloaded byte count instead of fabricating it. + progress.phase = VoiceModelDownloadPhase::Complete; Ok(()) } pub fn parakeet_model_dir(app: &AppHandle) -> Result { let base = cache_base(app)?; + let _assets = lock_local_assets_for_read(&base)?; if !parakeet_installation_valid(&base) { return Err("Native voice installation is incomplete or corrupt".to_string()); } Ok(base.join(CACHE_VERSION).join("stt")) } -fn publish_staging(base: &Path, staging: &Path) -> Result<(), String> { - let final_dir = base.join(CACHE_VERSION); - let previous = base.join(format!("{CACHE_VERSION}.previous")); - let _ = fs::remove_dir_all(&previous); - if final_dir.exists() { - fs::rename(&final_dir, &previous) - .map_err(|error| format!("retire incomplete Pocket cache: {error}"))?; - } - if let Err(error) = fs::rename(staging, &final_dir) { - if previous.exists() { - let _ = fs::rename(&previous, &final_dir); - } - return Err(format!("publish Pocket cache atomically: {error}")); - } - let _ = fs::remove_dir_all(previous); - Ok(()) -} - -async fn download_artifact( +pub fn parakeet_model_for_loading( app: &AppHandle, - state: &PocketVoiceState, - model: VoiceModelKind, - attempt_id: u64, - client: &reqwest::Client, - spec: DownloadSpec<'_>, -) -> Result<(), String> { - let DownloadSpec { - url, - destination, - expected_size, - expected_sha256, - } = spec; - let response = client - .get(url) - .send() - .await - .map_err(|error| format!("download {url}: {error}"))? - .error_for_status() - .map_err(|error| format!("download {url}: {error}"))?; - let mut file = tokio::fs::File::create(destination) - .await - .map_err(|error| format!("create {}: {error}", destination.display()))?; - let mut stream = response.bytes_stream(); - let mut size = 0_u64; - let mut hasher = Sha256::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|error| format!("read {url}: {error}"))?; - size = size - .checked_add(chunk.len() as u64) - .ok_or_else(|| format!("download size overflow for {url}"))?; - if size > expected_size { - return Err(format!("download exceeded pinned size for {url}")); - } - hasher.update(&chunk); - file.write_all(&chunk) - .await - .map_err(|error| format!("write {}: {error}", destination.display()))?; - let should_emit = { - let mut runtime = state - .install - .lock() - .map_err(|_| "Pocket TTS install state lock was poisoned".to_string())?; - increment_model_progress(&mut runtime, model, attempt_id, chunk.len() as u64)? - && should_emit_download_progress_at(&mut runtime, model, attempt_id, Instant::now()) - }; - if should_emit { - emit_pocket_status(app, state); - } - } - file.flush() - .await - .map_err(|error| format!("flush {}: {error}", destination.display()))?; - if size != expected_size { - return Err(format!( - "size mismatch for {}: expected {expected_size}, got {size}", - destination.display() - )); - } - let actual_sha256 = format!("{:x}", hasher.finalize()); - if actual_sha256 != expected_sha256 { - return Err(format!( - "checksum mismatch for {}: expected {expected_sha256}, got {actual_sha256}", - destination.display() - )); +) -> Result<(PathBuf, local_assets::LocalAssetReadGuard), String> { + let base = cache_base(app)?; + let assets = lock_local_assets_for_read(&base)?; + if !parakeet_installation_valid(&base) { + return Err("Native voice installation is incomplete or corrupt".to_string()); } - Ok(()) + Ok((base.join(CACHE_VERSION).join("stt"), assets)) } #[cfg(target_os = "macos")] @@ -2203,36 +1887,33 @@ fn run_pocket_voice_stream( voice: PocketVoice, output_device: Option<&str>, active: Arc, - playback_rate: Arc, + playback_rate: f32, receiver: mpsc::Receiver, native_voice: NativeVoiceState, interruption_sensitivity: InterruptionSensitivity, - suppress_capture: bool, + input_during_tts: InputDuringTtsPolicy, ) -> Result { let version = base.join(CACHE_VERSION); - let engine = load_text_to_speech( - version - .to_str() - .ok_or_else(|| "Pocket model path is not valid UTF-8".to_string())?, - )?; - let style = load_voice_style(&version.join("voices").join(voice.filename))?; - let mut applied_rate_bits = playback_rate.load(Ordering::SeqCst); - let player = PocketAudioPlayer::new( - SAMPLE_RATE, - f32::from_bits(applied_rate_bits), - output_device, - )?; + let _assets = lock_local_assets_for_read(base)?; + let tts = ConfiguredTtsSlot::new(TtsConfiguration::pocket( + version, + CACHE_VERSION.into(), + voice.id.into(), + playback_rate, + ))?; + let tts = tts.lease()?; + drop(_assets); + let backend = tts.backend(); + let player = PocketAudioPlayer::new(SAMPLE_RATE, playback_rate, output_device)?; + let mut playback = OutboundPlayback::new(&player, &active, SAMPLE_RATE, 0)?; let mut pending = String::new(); let mut first_chunk_pending = true; - let mut playback_started = false; let mut assistant_speech = None::; let mut playback_drained_at = None; let output_latency_grace = playback_latency_safety_duration(output_device); - let mut delivery_ledger = PlaybackDeliveryLedger::default(); let mut last_progress_emit = Instant::now(); let result: Result = (|| loop { - sync_pocket_playback_rate(&player, &playback_rate, &mut applied_rate_bits)?; update_pocket_assistant_speech( player.is_empty(), &mut assistant_speech, @@ -2240,15 +1921,12 @@ fn run_pocket_voice_stream( output_latency_grace, Instant::now(), ); - if !active.load(Ordering::SeqCst) { - let delivery = pocket_delivery_snapshot(&delivery_ledger, &player); - player.stop(); + if !playback.poll().map_err(|failure| failure.message)? { return Ok(PocketStreamOutcome { state: PocketStreamEventState::Interrupted, - delivery: Some(delivery), + delivery: Some(playback.snapshot()), }); } - player.ensure_healthy()?; let command = receiver.recv_timeout(Duration::from_millis(20)); match command { Ok(PocketStreamCommand::Append(text)) => { @@ -2256,31 +1934,21 @@ fn run_pocket_voice_stream( if !synthesize_pocket_stream_ready( app, stream_id, - &engine, - &style, - &active, - &player, - &playback_rate, - &mut applied_rate_bits, + backend.as_ref(), + &mut playback, &mut pending, &mut first_chunk_pending, - &mut playback_started, &native_voice, interruption_sensitivity, - suppress_capture, + input_during_tts, &mut assistant_speech, &mut playback_drained_at, - &mut delivery_ledger, &mut last_progress_emit, false, )? { - let delivery = capture_before_stop( - || pocket_delivery_snapshot(&delivery_ledger, &player), - || player.stop(), - ); return Ok(PocketStreamOutcome { state: PocketStreamEventState::Interrupted, - delivery: Some(delivery), + delivery: Some(playback.snapshot()), }); } } @@ -2288,31 +1956,21 @@ fn run_pocket_voice_stream( if !synthesize_pocket_stream_ready( app, stream_id, - &engine, - &style, - &active, - &player, - &playback_rate, - &mut applied_rate_bits, + backend.as_ref(), + &mut playback, &mut pending, &mut first_chunk_pending, - &mut playback_started, &native_voice, interruption_sensitivity, - suppress_capture, + input_during_tts, &mut assistant_speech, &mut playback_drained_at, - &mut delivery_ledger, &mut last_progress_emit, true, )? { - let delivery = capture_before_stop( - || pocket_delivery_snapshot(&delivery_ledger, &player), - || player.stop(), - ); return Ok(PocketStreamOutcome { state: PocketStreamEventState::Interrupted, - delivery: Some(delivery), + delivery: Some(playback.snapshot()), }); } } @@ -2320,104 +1978,93 @@ fn run_pocket_voice_stream( if !synthesize_pocket_stream_ready( app, stream_id, - &engine, - &style, - &active, - &player, - &playback_rate, - &mut applied_rate_bits, + backend.as_ref(), + &mut playback, &mut pending, &mut first_chunk_pending, - &mut playback_started, &native_voice, interruption_sensitivity, - suppress_capture, + input_during_tts, &mut assistant_speech, &mut playback_drained_at, - &mut delivery_ledger, &mut last_progress_emit, true, )? { - let delivery = capture_before_stop( - || pocket_delivery_snapshot(&delivery_ledger, &player), - || player.stop(), - ); return Ok(PocketStreamOutcome { state: PocketStreamEventState::Interrupted, - delivery: Some(delivery), + delivery: Some(playback.snapshot()), }); } // Playback speed can change while buffers drain. Use the slowest // supported rate so a later slowdown cannot truncate valid audio. let drain_timeout = pocket_native_drain_timeout( - delivery_ledger.total_frames(), + playback + .snapshot() + .segments + .iter() + .map(|segment| segment.total_frames) + .sum(), player.completed_source_frames(), - MIN_POCKET_PLAYBACK_SPEED, + playback_rate, ); - let drain_started = Instant::now(); - let mut completion_timed_out = false; - loop { - if !active.load(Ordering::SeqCst) { - let delivery = pocket_delivery_snapshot(&delivery_ledger, &player); - player.stop(); - return Ok(PocketStreamOutcome { - state: PocketStreamEventState::Interrupted, - delivery: Some(delivery), - }); - } - sync_pocket_playback_rate_before_timeout(completion_timed_out, || { - sync_pocket_playback_rate(&player, &playback_rate, &mut applied_rate_bits) - })?; - if !completion_timed_out { - player.ensure_healthy()?; - match pocket_native_drain_status( - player.is_empty(), - drain_started.elapsed(), - drain_timeout, - ) { - PocketNativeDrainStatus::Waiting => {} - PocketNativeDrainStatus::Drained => { - player.ensure_healthy()?; - } - PocketNativeDrainStatus::TimedOut => { - log::warn!("Pocket native buffer completion bookkeeping timed out"); - player.stop(); - reset_pocket_drain_grace(&mut playback_drained_at); - completion_timed_out = true; + let post_drain = output_latency_grace_remaining( + assistant_speech.is_some(), + playback_drained_at, + output_latency_grace, + Instant::now(), + ); + let outcome = playback + .finish( + DrainPolicy { + poll_interval: Duration::from_millis(10), + timeout: Some(drain_timeout), + timeout_outcome: DrainTimeoutOutcome::Complete, + post_drain, + }, + &mut |delivery| { + update_pocket_assistant_speech( + player.is_empty(), + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + Instant::now(), + ); + if last_progress_emit.elapsed() >= PLAYBACK_PROGRESS_EMIT_INTERVAL { + emit_pocket_stream_event( + app, + stream_id, + PocketStreamEventState::Progress, + None, + Some(delivery.clone()), + ); + last_progress_emit = Instant::now(); } - } - } - let playback_drained = completion_timed_out || player.is_empty(); - if playback_drained { - update_pocket_assistant_speech( - playback_drained, - &mut assistant_speech, - &mut playback_drained_at, - output_latency_grace, - Instant::now(), - ); - if assistant_speech.is_none() { - break; - } - } - std::thread::sleep(Duration::from_millis(10)); + Ok(()) + }, + ) + .map_err(|failure| failure.message)?; + if outcome == OutboundOutcome::Interrupted { + return Ok(PocketStreamOutcome { + state: PocketStreamEventState::Interrupted, + delivery: Some(playback.snapshot()), + }); } + assistant_speech.take(); return Ok(PocketStreamOutcome { state: PocketStreamEventState::Completed, delivery: None, }); } Ok(PocketStreamCommand::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => { - let delivery = pocket_delivery_snapshot(&delivery_ledger, &player); active.store(false, Ordering::SeqCst); - player.stop(); + playback.interrupt().map_err(|failure| failure.message)?; return Ok(PocketStreamOutcome { state: PocketStreamEventState::Interrupted, - delivery: Some(delivery), + delivery: Some(playback.snapshot()), }); } Err(mpsc::RecvTimeoutError::Timeout) => { - if playback_started + if playback.started() && last_progress_emit.elapsed() >= PLAYBACK_PROGRESS_EMIT_INTERVAL { emit_pocket_stream_event( @@ -2425,7 +2072,7 @@ fn run_pocket_voice_stream( stream_id, PocketStreamEventState::Progress, None, - Some(pocket_delivery_snapshot(&delivery_ledger, &player)), + Some(playback.snapshot()), ); last_progress_emit = Instant::now(); } @@ -2435,32 +2082,12 @@ fn run_pocket_voice_stream( assistant_speech.take(); result.map_err(|error| { - let delivery = delivery_with_played_audio(capture_before_stop( - || pocket_delivery_snapshot(&delivery_ledger, &player), - || player.stop(), - )); + let delivery = delivery_with_played_audio(playback.snapshot()); + let _ = playback.interrupt(); PocketStreamFailure { error, delivery } }) } -#[cfg(target_os = "macos")] -fn pocket_delivery_snapshot( - ledger: &PlaybackDeliveryLedger, - player: &PocketAudioPlayer, -) -> VoiceDeliveryProgress { - ledger.snapshot_consumed_frames(player.played_frames()) -} - -#[cfg(any(test, target_os = "macos"))] -fn capture_before_stop( - snapshot: impl FnOnce() -> VoiceDeliveryProgress, - stop: impl FnOnce(), -) -> VoiceDeliveryProgress { - let delivery = snapshot(); - stop(); - delivery -} - #[cfg(target_os = "macos")] fn update_pocket_assistant_speech( playback_drained: bool, @@ -2469,7 +2096,7 @@ fn update_pocket_assistant_speech( output_latency_grace: Duration, now: Instant, ) { - if pocket_assistant_speech_grace_elapsed( + if output_latency_grace_elapsed( playback_drained, assistant_speech.is_some(), playback_drained_at, @@ -2480,22 +2107,6 @@ fn update_pocket_assistant_speech( } } -#[cfg(any(test, target_os = "macos"))] -fn pocket_assistant_speech_grace_elapsed( - playback_drained: bool, - guard_active: bool, - playback_drained_at: &mut Option, - output_latency_grace: Duration, - now: Instant, -) -> bool { - if !guard_active || !playback_drained { - *playback_drained_at = None; - return false; - } - let drained_at = *playback_drained_at.get_or_insert(now); - now.saturating_duration_since(drained_at) >= output_latency_grace -} - #[cfg(any(test, target_os = "macos"))] fn reset_pocket_drain_grace(playback_drained_at: &mut Option) { *playback_drained_at = None; @@ -2537,56 +2148,14 @@ fn pocket_native_drain_timeout( .saturating_add(POCKET_SOURCE_COMPLETION_TIMEOUT) } -#[cfg(any(test, target_os = "macos"))] -fn sync_pocket_playback_rate_before_timeout( - completion_timed_out: bool, - sync: impl FnOnce() -> Result<(), String>, -) -> Result<(), String> { - if completion_timed_out { - Ok(()) - } else { - sync() - } -} - -#[cfg(target_os = "macos")] -fn sync_pocket_playback_rate( - player: &PocketAudioPlayer, - playback_rate: &AtomicU32, - applied_rate_bits: &mut u32, -) -> Result<(), String> { - let requested_rate_bits = playback_rate.load(Ordering::SeqCst); - if requested_rate_bits != *applied_rate_bits { - player.set_rate(f32::from_bits(requested_rate_bits))?; - *applied_rate_bits = requested_rate_bits; - } - Ok(()) -} - #[cfg(target_os = "macos")] #[allow(clippy::too_many_arguments)] -fn mark_pocket_playback_started( - app: &AppHandle, - stream_id: &str, - native_voice: &NativeVoiceState, - interruption_sensitivity: InterruptionSensitivity, - suppress_capture: bool, - playback_started: &mut bool, - assistant_speech: &mut Option, -) -> Result<(), String> { - if assistant_speech.is_none() { - *assistant_speech = - Some(native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture)); - } - if !*playback_started { - *playback_started = true; - emit_pocket_stream_event(app, stream_id, PocketStreamEventState::Started, None, None); - println!("VOICE_CONVERSATION_PLAYBACK_STARTED"); - std::io::stdout() - .flush() - .map_err(|error| format!("signal Pocket playback start: {error}"))?; - } - Ok(()) +fn mark_pocket_playback_started(app: &AppHandle, stream_id: &str) -> Result<(), String> { + emit_pocket_stream_event(app, stream_id, PocketStreamEventState::Started, None, None); + println!("VOICE_CONVERSATION_PLAYBACK_STARTED"); + std::io::stdout() + .flush() + .map_err(|error| format!("signal Pocket playback start: {error}")) } #[cfg(target_os = "macos")] @@ -2594,91 +2163,56 @@ fn mark_pocket_playback_started( fn synthesize_pocket_stream_ready( app: &AppHandle, stream_id: &str, - engine: &PocketTts, - style: &VoiceStyle, - active: &Arc, - player: &PocketAudioPlayer, - playback_rate: &AtomicU32, - applied_rate_bits: &mut u32, + backend: &dyn TtsBackend, + playback: &mut OutboundPlayback<'_>, pending: &mut String, first_chunk_pending: &mut bool, - playback_started: &mut bool, native_voice: &NativeVoiceState, interruption_sensitivity: InterruptionSensitivity, - suppress_capture: bool, + input_during_tts: InputDuringTtsPolicy, assistant_speech: &mut Option, playback_drained_at: &mut Option, - delivery_ledger: &mut PlaybackDeliveryLedger, last_progress_emit: &mut Instant, flush: bool, ) -> Result { - let split = engine.take_streaming_text_chunks(pending, *first_chunk_pending, flush)?; + let split = berd_voice::take_streaming_text_chunks(pending, *first_chunk_pending, flush)?; *pending = split.pending; *first_chunk_pending = split.first_chunk_pending; for text in split.ready { - if !active.load(Ordering::SeqCst) { - return Ok(false); - } let text = text.trim().to_string(); - delivery_ledger.begin_segment(text.clone()); - let mut segment_frames = 0_u64; - let mut callback_error = None; - let completed = - engine.synth_chunk_streaming(&text, style, STREAMING_EMIT_FRAMES, &mut |samples| { - if !active.load(Ordering::SeqCst) { - return false; - } - if samples.is_empty() { - return true; - } - if let Err(error) = - sync_pocket_playback_rate(player, playback_rate, applied_rate_bits) - { - callback_error = Some(error); - return false; - } - if let Err(error) = player.ensure_healthy() { - callback_error = Some(error); - return false; - } - if let Err(error) = mark_pocket_playback_started( - app, - stream_id, - native_voice, - interruption_sensitivity, - suppress_capture, - playback_started, - assistant_speech, - ) { - callback_error = Some(error); - return false; - } - if let Err(error) = player.enqueue(&samples) { - callback_error = Some(error); - return false; - } - reset_pocket_drain_grace(playback_drained_at); - segment_frames = segment_frames.saturating_add(samples.len() as u64); - delivery_ledger.append_frames(samples.len()); - if last_progress_emit.elapsed() >= PLAYBACK_PROGRESS_EMIT_INTERVAL { - emit_pocket_stream_event( - app, - stream_id, - PocketStreamEventState::Progress, - None, - Some(pocket_delivery_snapshot(delivery_ledger, player)), - ); - *last_progress_emit = Instant::now(); - } - true - })?; - if let Some(error) = callback_error { - return Err(error); - } - if !completed { + let outcome = playback + .synthesize_segment( + backend, + &text, + &mut |_| { + if assistant_speech.is_none() { + *assistant_speech = Some( + native_voice + .begin_assistant_speech(interruption_sensitivity, input_during_tts), + ); + } + reset_pocket_drain_grace(playback_drained_at); + Ok(()) + }, + &mut || mark_pocket_playback_started(app, stream_id), + &mut |delivery| { + if last_progress_emit.elapsed() >= PLAYBACK_PROGRESS_EMIT_INTERVAL { + emit_pocket_stream_event( + app, + stream_id, + PocketStreamEventState::Progress, + None, + Some(delivery.clone()), + ); + *last_progress_emit = Instant::now(); + } + Ok(()) + }, + ) + .map_err(|failure: OutboundFailure| failure.message)?; + if outcome == OutboundOutcome::Interrupted { return Ok(false); } - delivery_ledger.complete_segment(segment_frames); } Ok(true) } @@ -2690,24 +2224,21 @@ fn synthesize_and_stream( text: &str, output_device: Option<&str>, active: Arc, - playback_rate: Arc, + playback_rate: f32, ) -> Result<(), String> { use std::sync::Mutex; use std::time::Duration; let version = base.join(CACHE_VERSION); + let assets = lock_local_assets_for_read(base)?; let engine = load_text_to_speech( version .to_str() .ok_or_else(|| "Pocket model path is not valid UTF-8".to_string())?, )?; - let style = load_voice_style(&version.join("voices").join(voice.filename))?; - let mut applied_rate_bits = playback_rate.load(Ordering::SeqCst); - let player = PocketAudioPlayer::new( - SAMPLE_RATE, - f32::from_bits(applied_rate_bits), - output_device, - )?; + let style = load_pocket_voice_style(&version, voice.id)?; + drop(assets); + let player = PocketAudioPlayer::new(SAMPLE_RATE, playback_rate, output_device)?; let callback_error = Arc::new(Mutex::new(None::)); let playback_started = Arc::new(AtomicBool::new(false)); let mut total_source_frames = 0_u64; @@ -2722,15 +2253,7 @@ fn synthesize_and_stream( if samples.is_empty() { return true; } - if let Err(error) = - sync_pocket_playback_rate(&player, &playback_rate, &mut applied_rate_bits) - { - if let Ok(mut callback_error) = callback_error_slot.lock() { - *callback_error = Some(error); - } - return false; - } - if let Err(error) = player.ensure_healthy() { + if let Err(error) = player.check_health() { if let Ok(mut callback_error) = callback_error_slot.lock() { *callback_error = Some(error); } @@ -2772,21 +2295,20 @@ fn synthesize_and_stream( let drain_timeout = pocket_native_drain_timeout( total_source_frames, player.completed_source_frames(), - MIN_POCKET_PLAYBACK_SPEED, + playback_rate, ); let drain_started = Instant::now(); loop { - sync_pocket_playback_rate(&player, &playback_rate, &mut applied_rate_bits)?; if !active.load(Ordering::SeqCst) { player.stop(); break; } - player.ensure_healthy()?; + player.check_health()?; match pocket_native_drain_status(player.is_empty(), drain_started.elapsed(), drain_timeout) { PocketNativeDrainStatus::Waiting => {} PocketNativeDrainStatus::Drained => { - player.ensure_healthy()?; + player.check_health()?; break; } PocketNativeDrainStatus::TimedOut => { @@ -2807,7 +2329,7 @@ fn synthesize_and_stream( _text: &str, _output_device: Option<&str>, _active: Arc, - _playback_rate: Arc, + _playback_rate: f32, ) -> Result<(), String> { Err("Pocket voice playback is currently supported on macOS only".to_string()) } @@ -2815,91 +2337,20 @@ fn synthesize_and_stream( #[cfg(test)] mod tests { use super::*; + use berd_voice::DeliverySegment as VoiceDeliverySegment; #[test] - fn active_playback_observes_live_speed_changes() { + fn playback_snapshots_speed_when_the_utterance_begins() { let state = PocketVoiceState::default(); let session = begin_playback_runtime(&state, "already active", || 1.0).expect("start playback"); - assert_eq!( - f32::from_bits(session.playback_rate.load(Ordering::SeqCst)), - 1.0 - ); - - update_active_playback_speed(&state, 1.75).expect("update active playback"); - assert_eq!( - f32::from_bits(session.playback_rate.load(Ordering::SeqCst)), - 1.75 - ); + assert_eq!(session.playback_rate, 1.0); finish_playback(&state.playback, &session.active); - update_active_playback_speed(&state, 0.75).expect("ignore completed playback"); - assert_eq!( - f32::from_bits(session.playback_rate.load(Ordering::SeqCst)), - 1.75 - ); - } - - #[test] - fn playback_ledger_maps_consumed_frames_to_text_segments_conservatively() { - let mut ledger = PlaybackDeliveryLedger::default(); - ledger.begin_segment("First sentence.".to_string()); - ledger.append_frames(4_800); - assert!(!ledger.snapshot_consumed_frames(0).segments[0].synthesis_complete); - ledger.complete_segment(4_800); - ledger.begin_segment("Second sentence.".to_string()); - ledger.append_frames(4_800); - ledger.complete_segment(4_800); - - let progress = ledger.snapshot_consumed_frames(3_600); - assert_eq!(progress.segments[0].played_frames, 3_600); - assert_eq!(progress.segments[0].total_frames, 4_800); - assert!(progress.segments[0].synthesis_complete); - assert_eq!(progress.segments[1].played_frames, 0); - assert_eq!(progress.segments[1].total_frames, 4_800); - assert!(progress.segments[1].synthesis_complete); - } - - #[test] - fn playback_ledger_maps_native_consumed_frames_across_segments() { - let mut ledger = PlaybackDeliveryLedger::default(); - ledger.begin_segment("First sentence.".to_string()); - ledger.append_frames(4_800); - ledger.complete_segment(4_800); - ledger.begin_segment("Second sentence.".to_string()); - ledger.append_frames(4_800); - ledger.complete_segment(4_800); - - let progress = ledger.snapshot_consumed_frames(7_200); - assert_eq!(progress.segments[0].played_frames, 4_800); - assert_eq!(progress.segments[1].played_frames, 2_400); - } - - #[test] - fn interruption_and_failure_capture_delivery_before_stopping_playback() { - use std::cell::RefCell; - - let mut ledger = PlaybackDeliveryLedger::default(); - ledger.begin_segment("Played piece.".to_string()); - ledger.append_frames(4_800); - ledger.complete_segment(4_800); - ledger.begin_segment("Queued audio.".to_string()); - ledger.append_frames(4_800); - ledger.append_frames(4_800); - ledger.complete_segment(9_600); - let calls = RefCell::new(Vec::new()); - - let delivery = capture_before_stop( - || { - calls.borrow_mut().push("snapshot"); - ledger.snapshot_consumed_frames(3_600) - }, - || calls.borrow_mut().push("stop"), - ); - - assert_eq!(&*calls.borrow(), &["snapshot", "stop"]); - assert_eq!(delivery.segments[0].played_frames, 3_600); - assert_eq!(delivery.segments[1].played_frames, 0); + let next = + begin_playback_runtime(&state, "already active", || 1.75).expect("next playback"); + assert_eq!(session.playback_rate, 1.0); + assert_eq!(next.playback_rate, 1.75); } #[test] @@ -2944,28 +2395,12 @@ mod tests { assert!(!PocketVoiceState::default().stop_for_window_destroyed()); } - #[test] - fn manifest_has_unique_paths_and_expected_total() { - let mut names = std::collections::HashSet::new(); - for artifact in MODEL_ARTIFACTS { - assert!(names.insert(artifact.filename)); - assert_eq!(artifact.url.matches("/resolve/").count(), 1); - } - for voice in VOICES { - assert!(names.insert(voice.filename)); - assert_eq!(voice.url.matches("/resolve/").count(), 1); - } - assert_eq!(VOICES.len(), 12); - assert_eq!(total_bytes(), 278_120_564); - } - #[test] fn invalid_install_rejects_missing_and_corrupt_files() { let directory = tempfile::tempdir().expect("temporary directory"); assert!(!installation_valid(directory.path())); let version = directory.path().join(CACHE_VERSION); fs::create_dir_all(version.join("voices")).expect("create fixture"); - fs::write(version.join(VERIFIED_MARKER), CACHE_VERSION).expect("write verified marker"); fs::write(version.join("bundle.json"), b"wrong").expect("write corrupt fixture"); assert!(!installation_valid(directory.path())); } @@ -2978,16 +2413,16 @@ mod tests { fs::create_dir_all(version.join("stt")).expect("create Parakeet fixture"); let mut expected_pocket_bytes = 0; - for artifact in MODEL_ARTIFACTS { - let contents = vec![b'p'; artifact.filename.len()]; + for artifact in pocket_assets::model_artifacts() { + let contents = vec![b'p'; artifact.relative_path.len()]; expected_pocket_bytes += contents.len() as u64; - fs::write(version.join(artifact.filename), contents) + fs::write(version.join(artifact.relative_path), contents) .expect("write Pocket artifact fixture"); } - for voice in VOICES { - let contents = vec![b'v'; voice.filename.len()]; + for voice in pocket_assets::voices() { + let contents = vec![b'v'; voice.relative_path.len()]; expected_pocket_bytes += contents.len() as u64; - fs::write(version.join("voices").join(voice.filename), contents) + fs::write(version.join(voice.relative_path), contents) .expect("write Pocket voice fixture"); } @@ -3005,64 +2440,6 @@ mod tests { assert_eq!(parakeet_disk_bytes(directory.path()), Some(18)); } - #[test] - fn verification_rejects_same_length_corruption() { - let directory = tempfile::tempdir().expect("temporary directory"); - let path = directory.path().join("asset.bin"); - fs::write(&path, b"same-size-a").expect("write original fixture"); - let expected_sha256 = format!("{:x}", Sha256::digest(b"same-size-a")); - verify_file(&path, 11, &expected_sha256).expect("verify original fixture"); - - fs::write(&path, b"same-size-b").expect("write corrupt fixture"); - assert!(verify_file(&path, 11, &expected_sha256) - .expect_err("same-length corruption must fail") - .contains("checksum mismatch")); - } - - #[tokio::test] - async fn voice_download_client_times_out_a_stalled_partial_body() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind partial response server"); - let address = listener.local_addr().expect("server address"); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("accept request"); - socket - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nx") - .await - .expect("write partial response"); - tokio::time::sleep(Duration::from_secs(1)).await; - }); - let client = voice_download_client( - Duration::from_millis(100), - Duration::from_millis(100), - Duration::from_secs(1), - ) - .expect("build timeout client"); - let response = client - .get(format!("http://{address}/model")) - .send() - .await - .expect("receive response headers"); - let mut stream = response.bytes_stream(); - assert_eq!( - stream - .next() - .await - .expect("first body chunk") - .expect("read first body chunk") - .as_ref(), - b"x" - ); - let error = stream - .next() - .await - .expect("stalled body must terminate") - .expect_err("stalled body must time out"); - assert!(error.is_timeout(), "unexpected error: {error}"); - server.abort(); - } - #[test] fn failed_download_attempt_releases_worker_and_preserves_error() { let mut runtime = InstallRuntime::default(); @@ -3106,25 +2483,6 @@ mod tests { assert_ne!(original, replacement); } - #[test] - fn failed_atomic_publication_restores_previous_cache() { - let directory = tempfile::tempdir().expect("temporary directory"); - let final_dir = directory.path().join(CACHE_VERSION); - fs::create_dir_all(&final_dir).expect("create previous cache"); - fs::write(final_dir.join("sentinel"), b"previous").expect("write previous cache"); - - let missing_staging = directory.path().join("missing-staging"); - assert!(publish_staging(directory.path(), &missing_staging).is_err()); - assert_eq!( - fs::read(final_dir.join("sentinel")).expect("restored cache"), - b"previous" - ); - assert!(!directory - .path() - .join(format!("{CACHE_VERSION}.previous")) - .exists()); - } - #[test] fn pocket_multi_file_progress_is_monotonic_for_one_attempt() { let mut runtime = InstallRuntime::default(); @@ -3133,10 +2491,10 @@ mod tests { .expect("begin Pocket attempt"); let mut observed = vec![0]; - for size in MODEL_ARTIFACTS + for size in pocket_assets::model_artifacts() .iter() - .map(|artifact| artifact.size) - .chain(VOICES.iter().map(|voice| voice.size_bytes)) + .map(|artifact| artifact.size_bytes) + .chain(pocket_assets::voices().iter().map(|voice| voice.size_bytes)) { assert!(increment_model_progress( &mut runtime, @@ -3277,35 +2635,50 @@ mod tests { } #[test] - fn interruption_mode_selects_capture_suppression_policy() { - assert!(should_suppress_capture( - VoiceInterruptionMode::Automatic, - Some("MacBook Pro Speakers"), - )); - assert!(!should_suppress_capture( - VoiceInterruptionMode::Automatic, - Some("AirPods Pro"), - )); - assert!(!should_suppress_capture( - VoiceInterruptionMode::Automatic, - Some("USB Headphones"), - )); - assert!(!should_suppress_capture( - VoiceInterruptionMode::Automatic, - Some("Studio Display Audio"), - )); - assert!(!should_suppress_capture( - VoiceInterruptionMode::Automatic, - None, - )); - assert!(!should_suppress_capture( - VoiceInterruptionMode::AllowInterruptions, - Some("MacBook Pro Speakers"), - )); - assert!(should_suppress_capture( - VoiceInterruptionMode::PreventFeedback, - Some("AirPods Pro"), - )); + fn interruption_mode_resolves_shared_input_policy() { + assert_eq!( + resolve_input_during_tts_policy( + VoiceInterruptionMode::Automatic, + Some("MacBook Pro Speakers"), + ), + InputDuringTtsPolicy::SuppressInput + ); + assert_eq!( + resolve_input_during_tts_policy(VoiceInterruptionMode::Automatic, Some("AirPods Pro"),), + InputDuringTtsPolicy::AllowBargeIn + ); + assert_eq!( + resolve_input_during_tts_policy( + VoiceInterruptionMode::Automatic, + Some("USB Headphones"), + ), + InputDuringTtsPolicy::AllowBargeIn + ); + assert_eq!( + resolve_input_during_tts_policy( + VoiceInterruptionMode::Automatic, + Some("Studio Display Audio"), + ), + InputDuringTtsPolicy::AllowBargeIn + ); + assert_eq!( + resolve_input_during_tts_policy(VoiceInterruptionMode::Automatic, None,), + InputDuringTtsPolicy::AllowBargeIn + ); + assert_eq!( + resolve_input_during_tts_policy( + VoiceInterruptionMode::AllowInterruptions, + Some("MacBook Pro Speakers"), + ), + InputDuringTtsPolicy::AllowBargeIn + ); + assert_eq!( + resolve_input_during_tts_policy( + VoiceInterruptionMode::PreventFeedback, + Some("AirPods Pro"), + ), + InputDuringTtsPolicy::SuppressInput + ); } #[cfg(target_os = "macos")] @@ -3348,7 +2721,7 @@ mod tests { let grace = Duration::from_millis(500); let mut drained_at = None; - assert!(!pocket_assistant_speech_grace_elapsed( + assert!(!output_latency_grace_elapsed( true, true, &mut drained_at, @@ -3358,21 +2731,21 @@ mod tests { reset_pocket_drain_grace(&mut drained_at); assert_eq!(drained_at, None); - assert!(!pocket_assistant_speech_grace_elapsed( + assert!(!output_latency_grace_elapsed( true, true, &mut drained_at, grace, started + Duration::from_millis(600), )); - assert!(!pocket_assistant_speech_grace_elapsed( + assert!(!output_latency_grace_elapsed( true, true, &mut drained_at, grace, started + Duration::from_millis(900), )); - assert!(pocket_assistant_speech_grace_elapsed( + assert!(output_latency_grace_elapsed( true, true, &mut drained_at, @@ -3380,7 +2753,7 @@ mod tests { started + Duration::from_millis(1_100), )); - assert!(!pocket_assistant_speech_grace_elapsed( + assert!(!output_latency_grace_elapsed( true, false, &mut drained_at, @@ -3390,6 +2763,36 @@ mod tests { assert_eq!(drained_at, None); } + #[test] + fn playback_drain_grace_never_starts_before_output_is_empty() { + let started = Instant::now(); + let grace = Duration::from_millis(100); + let mut drained_at = None; + + assert!(!output_latency_grace_elapsed( + false, + true, + &mut drained_at, + grace, + started + Duration::from_secs(10), + )); + assert_eq!(drained_at, None); + assert!(!output_latency_grace_elapsed( + true, + true, + &mut drained_at, + grace, + started + Duration::from_secs(10), + )); + assert!(output_latency_grace_elapsed( + true, + true, + &mut drained_at, + grace, + started + Duration::from_secs(10) + grace, + )); + } + #[test] fn native_drain_times_out_after_expected_remaining_audio() { let timeout = pocket_native_drain_timeout(72_000, 24_000, 2.0); @@ -3409,66 +2812,49 @@ mod tests { } #[test] - fn native_drain_timeout_covers_a_live_slowdown() { + fn native_drain_timeout_uses_the_snapshotted_playback_rate() { let fastest_timeout = pocket_native_drain_timeout(72_000, 24_000, 2.0); - let live_rate_timeout = - pocket_native_drain_timeout(72_000, 24_000, MIN_POCKET_PLAYBACK_SPEED); - assert!(live_rate_timeout > fastest_timeout); + let snapshotted_rate = 0.75; + let snapshotted_timeout = pocket_native_drain_timeout(72_000, 24_000, snapshotted_rate); + assert!(snapshotted_timeout > fastest_timeout); assert_eq!( - live_rate_timeout, - Duration::from_secs_f64(2.0 / f64::from(MIN_POCKET_PLAYBACK_SPEED)) + snapshotted_timeout, + Duration::from_secs_f64(2.0 / f64::from(snapshotted_rate)) .saturating_add(POCKET_SOURCE_COMPLETION_TIMEOUT) ); } #[test] - fn post_timeout_grace_ignores_live_rate_changes() { - let mut sync_count = 0; - sync_pocket_playback_rate_before_timeout(false, || { - sync_count += 1; - Ok(()) - }) - .expect("sync while native playback is active"); - sync_pocket_playback_rate_before_timeout(true, || { - sync_count += 1; - Err("stopped player rejected rate change".to_string()) - }) - .expect("ignore rate change after native timeout"); - assert_eq!(sync_count, 1); - } - - #[cfg(target_os = "macos")] - #[test] - fn native_drain_timeout_releases_guard_after_route_grace() { - let native_voice = NativeVoiceState::default(); - let mut assistant_speech = - Some(native_voice.begin_assistant_speech(InterruptionSensitivity::More, false)); - let mut drained_at = Some(Instant::now()); + fn native_drain_timeout_preserves_remaining_route_grace() { let timed_out_at = Instant::now(); let route_grace = Duration::from_millis(500); - reset_pocket_drain_grace(&mut drained_at); - update_pocket_assistant_speech( - true, - &mut assistant_speech, - &mut drained_at, - route_grace, - timed_out_at, + assert_eq!( + output_latency_grace_remaining(true, None, route_grace, timed_out_at,), + route_grace ); - assert!(assistant_speech.is_some()); - - update_pocket_assistant_speech( - true, - &mut assistant_speech, - &mut drained_at, - route_grace, - timed_out_at + route_grace, + assert_eq!( + output_latency_grace_remaining( + true, + Some(timed_out_at - Duration::from_millis(200)), + route_grace, + timed_out_at, + ), + Duration::from_millis(300) + ); + assert_eq!( + output_latency_grace_remaining( + true, + Some(timed_out_at - route_grace), + route_grace, + timed_out_at, + ), + Duration::ZERO + ); + assert_eq!( + output_latency_grace_remaining(false, None, route_grace, timed_out_at,), + Duration::ZERO ); - assert!(assistant_speech.is_none()); - - assistant_speech = - Some(native_voice.begin_assistant_speech(InterruptionSensitivity::More, false)); - assert!(assistant_speech.is_some()); } #[test] @@ -3493,7 +2879,7 @@ mod tests { &mut runtime, VoiceModelKind::Parakeet, attempt_id, - PARAKEET_ARCHIVE.size - first_chunk, + parakeet_assets::ARCHIVE.size_bytes - first_chunk, ) .expect("finish Parakeet archive"); advance_model_progress( @@ -3501,7 +2887,7 @@ mod tests { VoiceModelKind::Parakeet, attempt_id, VoiceModelDownloadPhase::Extracting, - Some(PARAKEET_ARCHIVE.size), + Some(parakeet_assets::ARCHIVE.size_bytes), ) .expect("extract Parakeet"); let after_archive = runtime @@ -3666,51 +3052,76 @@ mod tests { } } - fn write_removal_fixture(base: &Path) { + #[test] + fn initial_already_ready_is_normalized_to_complete_without_fake_download_bytes() { + let state = PocketVoiceState::default(); + let attempt_id = { + let mut runtime = state.install.lock().expect("install state"); + begin_model_attempt(&mut runtime, VoiceModelKind::Pocket, 100, true) + .expect("begin install") + }; + normalize_successful_install(&state, VoiceModelKind::Pocket, attempt_id) + .expect("normalize success"); + let runtime = state.install.lock().expect("install state"); + let progress = model_progress(&runtime, VoiceModelKind::Pocket).expect("progress"); + assert_eq!(progress.phase, VoiceModelDownloadPhase::Complete); + assert_eq!(progress.downloaded_bytes, 0); + assert_eq!(progress.total_bytes, 100); + } + + fn remove_fixture_model(base: &Path, model: VoiceModelKind) { let version = base.join(CACHE_VERSION); - fs::create_dir_all(version.join("voices")).expect("create Pocket fixture"); - fs::create_dir_all(version.join("stt")).expect("create Parakeet fixture"); - for artifact in MODEL_ARTIFACTS { - fs::write(version.join(artifact.filename), b"pocket") - .expect("write Pocket artifact fixture"); - } - fs::write(version.join("voices").join("mary.wav"), b"voice").expect("write voice fixture"); - fs::write(version.join("stt").join("model.int8.onnx"), b"parakeet") - .expect("write Parakeet fixture"); - fs::write(version.join("stt").join("tokens.txt"), b"tokens") - .expect("write Parakeet token fixture"); - fs::write(version.join("stt").join("MODEL_LICENSE.txt"), b"license") - .expect("write Parakeet license fixture"); - fs::write(version.join(VERIFIED_MARKER), CACHE_VERSION).expect("write verified marker"); + fs::create_dir_all(version.join("stt")).expect("create combined fixture"); + fs::write(version.join("pocket-ready"), b"pocket").expect("write Pocket fixture"); + fs::write(version.join("stt/parakeet-ready"), b"parakeet").expect("write Parakeet fixture"); + remove_cached_model_with( + base, + model, + |base| base.join(CACHE_VERSION).join("pocket-ready").is_file(), + |base| { + base.join(CACHE_VERSION) + .join("stt/parakeet-ready") + .is_file() + }, + |_mutation, source, destination| { + fs::create_dir_all(destination) + .map_err(|error| format!("create Pocket stage: {error}"))?; + fs::copy( + source.join("pocket-ready"), + destination.join("pocket-ready"), + ) + .map_err(|error| format!("copy Pocket stage: {error}"))?; + Ok(()) + }, + |_mutation, source, destination| { + fs::create_dir_all(destination) + .map_err(|error| format!("create Parakeet stage: {error}"))?; + fs::copy( + source.join("parakeet-ready"), + destination.join("parakeet-ready"), + ) + .map_err(|error| format!("copy Parakeet stage: {error}"))?; + Ok(()) + }, + ) + .expect("remove fixture model"); } #[test] - fn pocket_removal_atomically_preserves_parakeet_assets() { + fn pocket_removal_preserves_only_the_ready_parakeet_counterpart() { let directory = tempfile::tempdir().expect("temporary directory"); - write_removal_fixture(directory.path()); - - remove_cached_model(directory.path(), VoiceModelKind::Pocket) - .expect("remove Pocket assets"); - + remove_fixture_model(directory.path(), VoiceModelKind::Pocket); let version = directory.path().join(CACHE_VERSION); - assert!(version.join("stt").join("model.int8.onnx").exists()); - assert!(version.join(VERIFIED_MARKER).exists()); - assert!(!version.join("voices").exists()); - assert!(!version.join(MODEL_ARTIFACTS[0].filename).exists()); + assert!(version.join("stt/parakeet-ready").is_file()); + assert!(!version.join("pocket-ready").exists()); } #[test] - fn parakeet_removal_atomically_preserves_pocket_assets() { + fn parakeet_removal_preserves_only_the_ready_pocket_counterpart() { let directory = tempfile::tempdir().expect("temporary directory"); - write_removal_fixture(directory.path()); - - remove_cached_model(directory.path(), VoiceModelKind::Parakeet) - .expect("remove Parakeet assets"); - + remove_fixture_model(directory.path(), VoiceModelKind::Parakeet); let version = directory.path().join(CACHE_VERSION); - assert!(version.join("voices").join("mary.wav").exists()); - assert!(version.join(MODEL_ARTIFACTS[0].filename).exists()); - assert!(version.join(VERIFIED_MARKER).exists()); + assert!(version.join("pocket-ready").is_file()); assert!(!version.join("stt").exists()); } diff --git a/src-tauri/src/commands/siri_voice.rs b/src-tauri/src/commands/siri_voice.rs index 141950ae5..af737878c 100644 --- a/src-tauri/src/commands/siri_voice.rs +++ b/src-tauri/src/commands/siri_voice.rs @@ -7,7 +7,7 @@ use std::fs; use std::os::raw::c_char; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -#[cfg(any(test, target_os = "macos"))] +#[cfg(target_os = "macos")] use std::sync::mpsc; use std::sync::{Arc, Mutex}; #[cfg(any(test, target_os = "macos"))] @@ -20,12 +20,32 @@ use tauri::{AppHandle, Manager}; #[cfg(target_os = "macos")] use super::native_voice::AssistantSpeechGuard; +#[cfg(any(test, target_os = "macos"))] +use super::native_voice::{output_latency_grace_elapsed, output_latency_grace_remaining}; use super::native_voice::{InterruptionSensitivity, NativeVoiceState}; use super::pocket_voice::VoiceInterruptionMode; #[cfg(target_os = "macos")] use super::pocket_voice::{ effective_output_device_name, output_device_uses_speakers, playback_latency_safety_duration, - should_suppress_capture, + resolve_input_during_tts_policy, selected_output_device, +}; +#[cfg(target_os = "macos")] +use berd_voice::input::InputDuringTtsPolicy; +#[cfg(target_os = "macos")] +use berd_voice::siri::{ + download_voice as download_managed_siri_voice, SiriDownloadAvailabilityWait, +}; +use berd_voice::siri::{ + load_voice_catalog, validate_installed_voice, SiriVoice, SiriVoiceIdentity, +}; +#[cfg(any(test, target_os = "macos"))] +use berd_voice::DeliveryProgress as VoiceDeliveryProgress; +#[cfg(test)] +use berd_voice::DeliverySegment as VoiceDeliverySegment; +#[cfg(target_os = "macos")] +use berd_voice::{ + ConfiguredTtsSlot, DrainPolicy, OutboundFailure, OutboundOutcome, OutboundPlayback, + PcmAudioOutput, PocketAudioPlayer, TtsBackend, TtsConfiguration, }; #[derive(Clone, Debug, Default)] @@ -78,24 +98,6 @@ struct SiriStreamEvent { delivery: Option, } -#[cfg(any(test, target_os = "macos"))] -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -struct VoiceDeliverySegment { - text: String, - played_frames: u64, - total_frames: u64, - synthesis_complete: bool, -} - -#[cfg(any(test, target_os = "macos"))] -#[derive(Clone, Debug, Deserialize, Serialize)] -struct VoiceDeliveryProgress { - #[serde(rename = "sampleRate")] - sample_rate: u32, - segments: Vec, -} - #[cfg(target_os = "macos")] struct SiriStreamOutcome { state: SiriStreamEventState, @@ -128,17 +130,10 @@ fn delivery_with_played_audio(delivery: VoiceDeliveryProgress) -> Option(snapshot: impl FnOnce() -> T, cancel: impl FnOnce()) -> T { - let delivery = snapshot(); - cancel(); - delivery -} - #[cfg(target_os = "macos")] const SIRI_STREAM_EVENT: &str = "siri-voice:stream-event"; -#[cfg(target_os = "macos")] -const SIRI_STREAM_STALL_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(any(test, target_os = "macos"))] +const SIRI_OUTPUT_DRAIN_MARGIN: Duration = Duration::from_secs(60); #[cfg(target_os = "macos")] const PLAYBACK_PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(100); const MIN_PLAYBACK_SPEED: f32 = 0.5; @@ -146,191 +141,7 @@ const MAX_PLAYBACK_SPEED: f32 = 2.0; static SIRI_SETTINGS_LOCK: Mutex<()> = Mutex::new(()); static SIRI_SETTINGS_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); -#[cfg(any(test, target_os = "macos"))] -struct SiriPlaybackLifetimeState { - guard: Option, - generation: u64, - cancelled: bool, -} - -#[cfg(any(test, target_os = "macos"))] -struct SiriPlaybackLifetime { - state: Mutex>, -} - -#[cfg(any(test, target_os = "macos"))] -impl Default for SiriPlaybackLifetime { - fn default() -> Self { - Self { - state: Mutex::new(SiriPlaybackLifetimeState { - guard: None, - generation: 0, - cancelled: false, - }), - } - } -} - -#[cfg(any(test, target_os = "macos"))] -impl SiriPlaybackLifetime { - fn start(&self, create_guard: impl FnOnce() -> T) { - let mut state = self - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if state.cancelled { - return; - } - state.generation = state.generation.wrapping_add(1); - if state.guard.is_none() { - state.guard = Some(create_guard()); - } - } - - fn begin_drain(&self) -> Option { - let mut state = self - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if state.cancelled || state.guard.is_none() { - return None; - } - state.generation = state.generation.wrapping_add(1); - Some(state.generation) - } - - fn release_if_current(&self, generation: u64) { - let mut state = self - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if !state.cancelled && state.generation == generation { - state.guard.take(); - } - } - - fn is_active(&self) -> bool { - self.state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .guard - .is_some() - } - - fn cancel(&self) { - let mut state = self - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - state.cancelled = true; - state.generation = state.generation.wrapping_add(1); - state.guard.take(); - } -} - -#[cfg(any(test, target_os = "macos"))] -enum SiriPlaybackMonitorEvent { - Started, - Drain(u64), - Shutdown, -} - -#[cfg(any(test, target_os = "macos"))] -fn run_siri_playback_monitor( - receiver: mpsc::Receiver, - lifetime: Arc>, - playback_latency_safety_duration: Duration, -) { - let mut pending_release: Option<(u64, Instant)> = None; - loop { - let event = if let Some((_, deadline)) = pending_release { - match receiver.recv_timeout(deadline.saturating_duration_since(Instant::now())) { - Ok(event) => Some(event), - Err(mpsc::RecvTimeoutError::Timeout) => { - if let Some((generation, _)) = pending_release.take() { - lifetime.release_if_current(generation); - } - continue; - } - Err(mpsc::RecvTimeoutError::Disconnected) => return, - } - } else { - match receiver.recv() { - Ok(event) => Some(event), - Err(mpsc::RecvError) => return, - } - }; - - match event { - Some(SiriPlaybackMonitorEvent::Started) => pending_release = None, - Some(SiriPlaybackMonitorEvent::Drain(generation)) => { - pending_release = Some(( - generation, - Instant::now() + playback_latency_safety_duration, - )); - } - Some(SiriPlaybackMonitorEvent::Shutdown) | None => return, - } - } -} - -#[cfg(any(test, target_os = "macos"))] -fn spawn_siri_playback_monitor_with( - task: impl FnOnce() + Send + 'static, - spawn: F, -) -> std::io::Result> -where - F: FnOnce(Box) -> std::io::Result>, -{ - spawn(Box::new(task)) -} - -#[cfg(target_os = "macos")] -fn spawn_siri_playback_monitor( - receiver: mpsc::Receiver, - lifetime: Arc>, - playback_latency_safety_duration: Duration, - failed: Arc, -) -> std::io::Result> { - spawn_siri_playback_monitor_with( - move || { - let monitor_lifetime = Arc::clone(&lifetime); - if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - run_siri_playback_monitor( - receiver, - monitor_lifetime, - playback_latency_safety_duration, - ); - })) - .is_err() - { - failed.store(true, Ordering::SeqCst); - lifetime.cancel(); - } - }, - |task| { - std::thread::Builder::new() - .name("siri-playback-monitor".to_string()) - .spawn(task) - }, - ) -} - -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct SiriVoice { - name: String, - language: String, - size_bytes: u64, - installed: bool, -} - -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct SiriVoiceSelection { - name: String, - language: String, -} +pub type SiriVoiceSelection = SiriVoiceIdentity; #[derive(Clone, Debug, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] @@ -413,17 +224,6 @@ fn update_settings( #[cfg(target_os = "macos")] unsafe extern "C" { - fn berd_siri_tts_catalog_json( - language_prefix: *const c_char, - error_out: *mut *mut c_char, - ) -> *mut c_char; - fn berd_siri_tts_languages_json(error_out: *mut *mut c_char) -> *mut c_char; - fn berd_siri_tts_download_voice( - language: *const c_char, - voice_name: *const c_char, - timeout_seconds: f64, - error_out: *mut *mut c_char, - ) -> bool; fn berd_siri_tts_play_sample( voice_name: *const c_char, language: *const c_char, @@ -442,27 +242,6 @@ unsafe extern "C" { context: *mut std::ffi::c_void, error_out: *mut *mut c_char, ) -> bool; - fn berd_siri_tts_stream_create( - language: *const c_char, - voice_name: *const c_char, - rate: f32, - playback_started: Option, - playback_stopped: Option, - context: *mut std::ffi::c_void, - error_out: *mut *mut c_char, - ) -> *mut std::ffi::c_void; - fn berd_siri_tts_stream_enqueue( - stream: *mut std::ffi::c_void, - text: *const c_char, - error_out: *mut *mut c_char, - ) -> bool; - fn berd_siri_tts_stream_finish(stream: *mut std::ffi::c_void); - fn berd_siri_tts_stream_is_finished(stream: *mut std::ffi::c_void) -> bool; - fn berd_siri_tts_stream_progress(stream: *mut std::ffi::c_void) -> u64; - fn berd_siri_tts_stream_copy_delivery_json(stream: *mut std::ffi::c_void) -> *mut c_char; - fn berd_siri_tts_stream_copy_error(stream: *mut std::ffi::c_void) -> *mut c_char; - fn berd_siri_tts_stream_cancel(stream: *mut std::ffi::c_void); - fn berd_siri_tts_stream_release(stream: *mut std::ffi::c_void); fn berd_siri_tts_free_string(value: *mut c_char); } @@ -510,32 +289,6 @@ fn finish_playback(state: &SiriVoiceState, completed: &Arc) { } } -#[cfg(target_os = "macos")] -#[derive(Debug)] -struct SiriStreamWatchdog { - progress: u64, - last_progress_at: Instant, -} - -#[cfg(target_os = "macos")] -impl SiriStreamWatchdog { - fn new(progress: u64, now: Instant) -> Self { - Self { - progress, - last_progress_at: now, - } - } - - fn observe(&mut self, progress: u64, now: Instant) -> bool { - if progress != self.progress { - self.progress = progress; - self.last_progress_at = now; - return false; - } - now.duration_since(self.last_progress_at) >= SIRI_STREAM_STALL_TIMEOUT - } -} - #[cfg(target_os = "macos")] fn take_bridge_string(value: *mut c_char) -> Option { if value.is_null() { @@ -555,63 +308,6 @@ fn bridge_error(error: *mut c_char, fallback: &str) -> String { take_bridge_string(error).unwrap_or_else(|| fallback.to_string()) } -#[cfg(target_os = "macos")] -struct SiriStreamCallbackContext { - app: AppHandle, - stream_id: String, - native_voice: NativeVoiceState, - interruption_sensitivity: InterruptionSensitivity, - suppress_capture: bool, - playback_started: AtomicBool, - playback_lifetime: Arc>, - playback_monitor_sender: mpsc::Sender, -} - -#[cfg(target_os = "macos")] -unsafe extern "C" fn siri_playback_started(context: *mut std::ffi::c_void) { - if context.is_null() { - return; - } - // SAFETY: The stream worker owns this boxed context until after the native - // player has completed and been released. - let context = unsafe { &*(context.cast::()) }; - context.playback_lifetime.start(|| { - context - .native_voice - .begin_assistant_speech(context.interruption_sensitivity, context.suppress_capture) - }); - let _ = context - .playback_monitor_sender - .send(SiriPlaybackMonitorEvent::Started); - if !context.playback_started.swap(true, Ordering::AcqRel) { - let _ = context.app.emit( - SIRI_STREAM_EVENT, - SiriStreamEvent { - stream_id: context.stream_id.clone(), - state: SiriStreamEventState::Started, - error: None, - delivery: None, - }, - ); - } -} - -#[cfg(target_os = "macos")] -unsafe extern "C" fn siri_playback_stopped(context: *mut std::ffi::c_void) { - if context.is_null() { - return; - } - // SAFETY: The stream worker owns this boxed context until after the native - // player has completed and been released. - let context = unsafe { &*(context.cast::()) }; - let Some(generation) = context.playback_lifetime.begin_drain() else { - return; - }; - let _ = context - .playback_monitor_sender - .send(SiriPlaybackMonitorEvent::Drain(generation)); -} - #[cfg(target_os = "macos")] fn emit_stream_event( app: &AppHandle, @@ -633,25 +329,8 @@ fn emit_stream_event( #[cfg(target_os = "macos")] fn discover_voices(language_prefix: &str) -> Result, String> { - let prefix = CString::new(language_prefix) - .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; - let mut error = std::ptr::null_mut(); - // SAFETY: The bridge copies the input string synchronously and returns - // owned strings through its documented allocation contract. - let json = unsafe { berd_siri_tts_catalog_json(prefix.as_ptr(), &mut error) }; - let json = take_bridge_string(json) - .ok_or_else(|| bridge_error(error, "Could not load the Siri voice catalog"))?; - serde_json::from_str(&json).map_err(|error| format!("decode Siri voice catalog: {error}")) -} - -#[cfg(target_os = "macos")] -fn discover_languages() -> Result, String> { - let mut error = std::ptr::null_mut(); - // SAFETY: Returned strings follow the bridge allocation contract. - let json = unsafe { berd_siri_tts_languages_json(&mut error) }; - let json = take_bridge_string(json) - .ok_or_else(|| bridge_error(error, "Could not load Siri voice languages"))?; - serde_json::from_str(&json).map_err(|error| format!("decode Siri voice languages: {error}")) + load_voice_catalog((!language_prefix.is_empty()).then_some(language_prefix)) + .map(|catalog| catalog.voices) } #[cfg(not(target_os = "macos"))] @@ -659,34 +338,11 @@ fn discover_voices(_language_prefix: &str) -> Result, String> { Ok(Vec::new()) } -#[cfg(not(target_os = "macos"))] -fn discover_languages() -> Result, String> { - Ok(Vec::new()) -} - -fn normalize_language(value: &str) -> String { - value.replace('_', "-").to_lowercase() -} - -fn find_voice<'a>( - voices: &'a [SiriVoice], - selection: &SiriVoiceSelection, -) -> Option<&'a SiriVoice> { - let language = normalize_language(&selection.language); - voices.iter().find(|voice| { - voice.name.eq_ignore_ascii_case(&selection.name) - && normalize_language(&voice.language) == language - }) -} - fn first_installed_voice(voices: &[SiriVoice]) -> Option { voices .iter() .find(|voice| voice.installed) - .map(|voice| SiriVoiceSelection { - name: voice.name.clone(), - language: voice.language.clone(), - }) + .map(SiriVoice::identity) } fn resolve_voice_selection( @@ -695,7 +351,11 @@ fn resolve_voice_selection( load_all_voices: impl FnOnce() -> Result, String>, ) -> Result<(Option, bool), String> { if let Some(selection) = selected_voice { - if find_voice(preferred_voices, selection).is_some_and(|voice| voice.installed) { + if preferred_voices + .iter() + .find(|voice| voice.matches(selection)) + .is_some_and(|voice| voice.installed) + { return Ok((Some(selection.clone()), true)); } } else if let Some(selection) = first_installed_voice(preferred_voices) { @@ -704,7 +364,11 @@ fn resolve_voice_selection( let all_voices = load_all_voices()?; if let Some(selection) = selected_voice { - if find_voice(&all_voices, selection).is_some_and(|voice| voice.installed) { + if all_voices + .iter() + .find(|voice| voice.matches(selection)) + .is_some_and(|voice| voice.installed) + { return Ok((Some(selection.clone()), true)); } } @@ -722,7 +386,11 @@ fn resolve_stream_voice( load_all_voices: impl FnOnce() -> Result, String>, ) -> Result { let voices = load_all_voices()?; - if find_voice(&voices, selection).is_some_and(|voice| voice.installed) { + if voices + .iter() + .find(|voice| voice.matches(selection)) + .is_some_and(|voice| voice.installed) + { return Ok(selection.clone()); } @@ -732,8 +400,9 @@ fn resolve_stream_voice( } fn status(app: &AppHandle, language_prefix: &str) -> Result { - let voices = discover_voices(language_prefix)?; - let available_languages = discover_languages()?; + let catalog = load_voice_catalog((!language_prefix.is_empty()).then_some(language_prefix))?; + let voices = catalog.voices; + let available_languages = catalog.available_languages; let path = settings_path(app)?; let previous_selection = read_settings(&path).selected_voice; let (resolved_selection, resolved_selection_installed) = @@ -755,10 +424,15 @@ fn status(app: &AppHandle, language_prefix: &str) -> Result Result<(), String> { - let prefix = voice.language.clone(); let candidate = voice.clone(); - let installed = tauri::async_runtime::spawn_blocking(move || { - let voices = discover_voices(&prefix)?; - Ok::<_, String>(find_voice(&voices, &candidate).is_some_and(|voice| voice.installed)) - }) - .await - .map_err(|error| format!("Siri voice validation task failed: {error}"))??; - if !installed { - return Err(format!( - "Siri voice {} ({}) must be downloaded before selection", - voice.name, voice.language - )); - } + tauri::async_runtime::spawn_blocking(move || validate_installed_voice(&candidate)) + .await + .map_err(|error| format!("Siri voice validation task failed: {error}"))??; update_settings(&settings_path(&app)?, |settings| { settings.selected_voice = Some(voice); true @@ -832,20 +496,10 @@ pub async fn download_siri_voice(app: AppHandle, voice: SiriVoiceSelection) -> R #[cfg(target_os = "macos")] { - let language = CString::new(voice.language.clone()) - .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; - let name = CString::new(voice.name.clone()) - .map_err(|_| "Siri voice name cannot contain NUL bytes".to_string())?; tauri::async_runtime::spawn_blocking(move || { - let mut error = std::ptr::null_mut(); - // SAFETY: Inputs stay alive for the blocking call and returned - // errors follow the bridge string ownership contract. - let downloaded = unsafe { - berd_siri_tts_download_voice(language.as_ptr(), name.as_ptr(), 300.0, &mut error) - }; - downloaded - .then_some(()) - .ok_or_else(|| bridge_error(error, "Siri voice download failed")) + download_managed_siri_voice(&voice, SiriDownloadAvailabilityWait::default()) + .map(|_| ()) + .map_err(|error| error.to_string()) }) .await .map_err(|error| format!("Siri voice download task failed: {error}"))??; @@ -855,22 +509,129 @@ pub async fn download_siri_voice(app: AppHandle, voice: SiriVoiceSelection) -> R } #[cfg(target_os = "macos")] -fn enqueue_native_stream(stream: *mut std::ffi::c_void, text: &str) -> Result<(), String> { - let text = - CString::new(text).map_err(|_| "Siri speech text cannot contain NUL bytes".to_string())?; - let mut error = std::ptr::null_mut(); - // SAFETY: The native stream remains owned by the worker for this call and - // the bridge copies the text before returning. - let accepted = unsafe { berd_siri_tts_stream_enqueue(stream, text.as_ptr(), &mut error) }; - accepted - .then_some(()) - .ok_or_else(|| bridge_error(error, "Siri stream rejected text")) +#[allow(clippy::too_many_arguments)] +fn synthesize_siri_stream_ready( + app: &AppHandle, + stream_id: &str, + backend: &dyn TtsBackend, + playback: &mut OutboundPlayback<'_>, + player: &PocketAudioPlayer, + output_latency_grace: Duration, + pending: &mut String, + first_chunk_pending: &mut bool, + native_voice: &NativeVoiceState, + interruption_sensitivity: InterruptionSensitivity, + input_during_tts: InputDuringTtsPolicy, + assistant_speech: &mut Option, + playback_drained_at: &mut Option, + last_progress_emit: &mut Instant, + last_progress: &mut Option, + flush: bool, +) -> Result { + let split = berd_voice::take_streaming_text_chunks(pending, *first_chunk_pending, flush)?; + *pending = split.pending; + *first_chunk_pending = split.first_chunk_pending; + for text in split.ready { + // The coordinator invokes these callbacks serially, but Rust cannot + // infer that two callback values never overlap. Interior borrows keep + // the single host-owned guard state shared without duplicating it. + let assistant_speech_cell = std::cell::RefCell::new(&mut *assistant_speech); + let playback_drained_at_cell = std::cell::RefCell::new(&mut *playback_drained_at); + let outcome = playback + .synthesize_segment( + backend, + text.trim(), + &mut |_| { + let mut assistant_speech = assistant_speech_cell.borrow_mut(); + if assistant_speech.is_none() { + **assistant_speech = Some( + native_voice + .begin_assistant_speech(interruption_sensitivity, input_during_tts), + ); + } + **playback_drained_at_cell.borrow_mut() = None; + Ok(()) + }, + &mut || { + emit_stream_event(app, stream_id, SiriStreamEventState::Started, None, None); + Ok(()) + }, + &mut |delivery| { + let mut assistant_speech = assistant_speech_cell.borrow_mut(); + let mut playback_drained_at = playback_drained_at_cell.borrow_mut(); + update_siri_assistant_speech( + player.is_drained(), + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + Instant::now(), + ); + emit_siri_progress_if_changed( + app, + stream_id, + delivery, + last_progress_emit, + last_progress, + ); + Ok(()) + }, + ) + .map_err(|failure: OutboundFailure| failure.message)?; + if outcome == OutboundOutcome::Interrupted { + return Ok(false); + } + } + Ok(true) } #[cfg(target_os = "macos")] -fn siri_delivery_progress(stream: *mut std::ffi::c_void) -> Option { - let json = take_bridge_string(unsafe { berd_siri_tts_stream_copy_delivery_json(stream) })?; - serde_json::from_str(&json).ok() +fn emit_siri_progress_if_changed( + app: &AppHandle, + stream_id: &str, + delivery: &VoiceDeliveryProgress, + last_progress_emit: &mut Instant, + last_progress: &mut Option, +) { + if last_progress_emit.elapsed() < PLAYBACK_PROGRESS_EMIT_INTERVAL + || last_progress.as_ref() == Some(delivery) + { + return; + } + emit_stream_event( + app, + stream_id, + SiriStreamEventState::Progress, + None, + Some(delivery.clone()), + ); + *last_progress_emit = Instant::now(); + *last_progress = Some(delivery.clone()); +} + +#[cfg(target_os = "macos")] +fn update_siri_assistant_speech( + playback_drained: bool, + assistant_speech: &mut Option, + playback_drained_at: &mut Option, + output_latency_grace: Duration, + now: Instant, +) { + if output_latency_grace_elapsed( + playback_drained, + assistant_speech.is_some(), + playback_drained_at, + output_latency_grace, + now, + ) { + assistant_speech.take(); + } +} + +#[cfg(any(test, target_os = "macos"))] +fn siri_drain_timeout(total_frames: u64, completed_frames: u64, sample_rate: u32) -> Duration { + let remaining_frames = total_frames.saturating_sub(completed_frames); + Duration::from_secs_f64(remaining_frames as f64 / f64::from(sample_rate)) + .saturating_add(SIRI_OUTPUT_DRAIN_MARGIN) } #[cfg(target_os = "macos")] @@ -884,191 +645,203 @@ fn run_siri_stream( receiver: mpsc::Receiver, native_voice: NativeVoiceState, interruption_sensitivity: InterruptionSensitivity, - suppress_capture: bool, - playback_latency_safety_duration: Duration, + input_during_tts: InputDuringTtsPolicy, + output_device: Option<&str>, + output_latency_grace: Duration, ) -> Result { - let language = CString::new(selection.language) - .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; - let name = CString::new(selection.name) - .map_err(|_| "Siri voice name cannot contain NUL bytes".to_string())?; - let playback_lifetime = Arc::new(SiriPlaybackLifetime::default()); - let playback_monitor_failed = Arc::new(AtomicBool::new(false)); - let (playback_monitor_sender, playback_monitor_receiver) = mpsc::channel(); - let playback_monitor = spawn_siri_playback_monitor( - playback_monitor_receiver, - Arc::clone(&playback_lifetime), - playback_latency_safety_duration, - Arc::clone(&playback_monitor_failed), - ) - .map_err(|error| format!("Could not start Siri playback monitor: {error}"))?; - let callback_context = Box::new(SiriStreamCallbackContext { - app: app.clone(), - stream_id: stream_id.clone(), - native_voice, - interruption_sensitivity, - suppress_capture, - playback_started: AtomicBool::new(false), - playback_lifetime: Arc::clone(&playback_lifetime), - playback_monitor_sender: playback_monitor_sender.clone(), - }); - let callback_context = Box::into_raw(callback_context); - let mut error = std::ptr::null_mut(); - // SAFETY: Strings remain alive through creation. The callback context is - // released only after the native stream has finished and is released. - let stream = unsafe { - berd_siri_tts_stream_create( - language.as_ptr(), - name.as_ptr(), - speed, - Some(siri_playback_started), - Some(siri_playback_stopped), - callback_context.cast(), - &mut error, - ) - }; - if stream.is_null() { - // SAFETY: Native creation failed, so no callback retained the box. - unsafe { drop(Box::from_raw(callback_context)) }; - playback_lifetime.cancel(); - let _ = playback_monitor_sender.send(SiriPlaybackMonitorEvent::Shutdown); - let _ = playback_monitor.join(); - return Err(bridge_error(error, "Could not start Siri voice stream").into()); - } - - let result = (|| { - let mut pending = String::new(); - let mut first_chunk_pending = true; - let mut finishing = false; - let mut watchdog: Option = None; - let mut last_progress_emit = Instant::now(); - let mut last_delivery_json = String::new(); - loop { - if playback_monitor_failed.load(Ordering::SeqCst) { - return Err("Siri playback monitor failed".to_string()); - } - if !active.load(Ordering::SeqCst) { - let delivery = siri_delivery_progress(stream); - unsafe { berd_siri_tts_stream_cancel(stream) }; - return Ok(SiriStreamOutcome { - state: SiriStreamEventState::Interrupted, - delivery, - }); - } - if finishing && unsafe { berd_siri_tts_stream_is_finished(stream) } { - let native_error = - take_bridge_string(unsafe { berd_siri_tts_stream_copy_error(stream) }); - if let Some(error) = native_error { - return Err(error); - } - if !playback_lifetime.is_active() { + let tts = ConfiguredTtsSlot::new(TtsConfiguration::siri( + selection.name().to_owned(), + selection.language().to_owned(), + speed, + ))?; + let tts = tts.lease()?; + let backend = tts.backend(); + let pcm_spec = backend.pcm_spec(); + let player = + PocketAudioPlayer::new(pcm_spec.sample_rate, pcm_spec.playback_rate, output_device)?; + let mut playback = OutboundPlayback::new(&player, &active, pcm_spec.sample_rate, 0)?; + let mut pending = String::new(); + let mut first_chunk_pending = true; + let mut assistant_speech = None::; + let mut playback_drained_at = None; + let mut last_progress_emit = Instant::now(); + let mut last_progress = None; + + let result: Result = (|| loop { + update_siri_assistant_speech( + player.is_drained(), + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + Instant::now(), + ); + if !playback.poll().map_err(|failure| failure.message)? { + return Ok(SiriStreamOutcome { + state: SiriStreamEventState::Interrupted, + delivery: Some(playback.snapshot()), + }); + } + let command = receiver.recv_timeout(Duration::from_millis(10)); + match command { + Ok(SiriStreamCommand::Append(text)) => { + pending.push_str(&text); + if !synthesize_siri_stream_ready( + &app, + &stream_id, + backend.as_ref(), + &mut playback, + &player, + output_latency_grace, + &mut pending, + &mut first_chunk_pending, + &native_voice, + interruption_sensitivity, + input_during_tts, + &mut assistant_speech, + &mut playback_drained_at, + &mut last_progress_emit, + &mut last_progress, + false, + )? { return Ok(SiriStreamOutcome { - state: SiriStreamEventState::Completed, - delivery: None, + state: SiriStreamEventState::Interrupted, + delivery: Some(playback.snapshot()), }); } } - if let Some(watchdog) = watchdog.as_mut() { - let progress = unsafe { berd_siri_tts_stream_progress(stream) }; - if watchdog.observe(progress, Instant::now()) { - return Err("Siri synthesis stopped making progress".to_string()); - } - } - if last_progress_emit.elapsed() >= PLAYBACK_PROGRESS_EMIT_INTERVAL { - if let Some(delivery) = siri_delivery_progress(stream) { - let delivery_json = serde_json::to_string(&delivery).unwrap_or_default(); - if delivery_json != last_delivery_json { - emit_stream_event( - &app, - &stream_id, - SiriStreamEventState::Progress, - None, - Some(delivery), - ); - last_delivery_json = delivery_json; - } + Ok(SiriStreamCommand::Flush) => { + if !synthesize_siri_stream_ready( + &app, + &stream_id, + backend.as_ref(), + &mut playback, + &player, + output_latency_grace, + &mut pending, + &mut first_chunk_pending, + &native_voice, + interruption_sensitivity, + input_during_tts, + &mut assistant_speech, + &mut playback_drained_at, + &mut last_progress_emit, + &mut last_progress, + true, + )? { + return Ok(SiriStreamOutcome { + state: SiriStreamEventState::Interrupted, + delivery: Some(playback.snapshot()), + }); } - last_progress_emit = Instant::now(); } - - let command = match receiver.recv_timeout(Duration::from_millis(10)) { - Ok(command) => command, - Err(mpsc::RecvTimeoutError::Timeout) => continue, - Err(mpsc::RecvTimeoutError::Disconnected) => SiriStreamCommand::Stop, - }; - match command { - SiriStreamCommand::Append(text) if !finishing => { - pending.push_str(&text); - let split = berd_voice::take_streaming_text_chunks( - &pending, - first_chunk_pending, - false, - )?; - pending = split.pending; - first_chunk_pending = split.first_chunk_pending; - for ready in split.ready { - enqueue_native_stream(stream, ready.trim())?; - } - } - SiriStreamCommand::Flush if !finishing => { - let split = berd_voice::take_streaming_text_chunks( - &pending, - first_chunk_pending, - true, - )?; - pending = split.pending; - first_chunk_pending = split.first_chunk_pending; - for ready in split.ready { - enqueue_native_stream(stream, ready.trim())?; - } - } - SiriStreamCommand::Finish if !finishing => { - let split = berd_voice::take_streaming_text_chunks( - &pending, - first_chunk_pending, - true, - )?; - for ready in split.ready { - enqueue_native_stream(stream, ready.trim())?; - } - pending.clear(); - finishing = true; - unsafe { berd_siri_tts_stream_finish(stream) }; - watchdog = Some(SiriStreamWatchdog::new( - unsafe { berd_siri_tts_stream_progress(stream) }, - Instant::now(), - )); + Ok(SiriStreamCommand::Finish) => { + if !synthesize_siri_stream_ready( + &app, + &stream_id, + backend.as_ref(), + &mut playback, + &player, + output_latency_grace, + &mut pending, + &mut first_chunk_pending, + &native_voice, + interruption_sensitivity, + input_during_tts, + &mut assistant_speech, + &mut playback_drained_at, + &mut last_progress_emit, + &mut last_progress, + true, + )? { + return Ok(SiriStreamOutcome { + state: SiriStreamEventState::Interrupted, + delivery: Some(playback.snapshot()), + }); } - SiriStreamCommand::Stop => { - let delivery = siri_delivery_progress(stream); - active.store(false, Ordering::SeqCst); - unsafe { berd_siri_tts_stream_cancel(stream) }; + let total_frames = playback + .snapshot() + .segments + .iter() + .map(|segment| segment.total_frames) + .sum(); + let post_drain = output_latency_grace_remaining( + assistant_speech.is_some(), + playback_drained_at, + output_latency_grace, + Instant::now(), + ); + let outcome = playback + .finish( + DrainPolicy { + poll_interval: Duration::from_millis(10), + timeout: Some(siri_drain_timeout( + total_frames, + player.completed_source_frames(), + pcm_spec.sample_rate, + )), + post_drain, + ..DrainPolicy::default() + }, + &mut |delivery| { + update_siri_assistant_speech( + player.is_drained(), + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + Instant::now(), + ); + emit_siri_progress_if_changed( + &app, + &stream_id, + delivery, + &mut last_progress_emit, + &mut last_progress, + ); + Ok(()) + }, + ) + .map_err(|failure| failure.message)?; + if outcome == OutboundOutcome::Interrupted { return Ok(SiriStreamOutcome { state: SiriStreamEventState::Interrupted, - delivery, + delivery: Some(playback.snapshot()), }); } - _ => {} + assistant_speech.take(); + return Ok(SiriStreamOutcome { + state: SiriStreamEventState::Completed, + delivery: None, + }); + } + Ok(SiriStreamCommand::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => { + active.store(false, Ordering::SeqCst); + playback.interrupt().map_err(|failure| failure.message)?; + return Ok(SiriStreamOutcome { + state: SiriStreamEventState::Interrupted, + delivery: Some(playback.snapshot()), + }); + } + Err(mpsc::RecvTimeoutError::Timeout) => { + if playback.started() { + let delivery = playback.snapshot(); + emit_siri_progress_if_changed( + &app, + &stream_id, + &delivery, + &mut last_progress_emit, + &mut last_progress, + ); + } } } })(); - let result = result.map_err(|error| { - let delivery = capture_before_cancel( - || siri_delivery_progress(stream), - || unsafe { berd_siri_tts_stream_cancel(stream) }, - ) - .and_then(delivery_with_played_audio); + assistant_speech.take(); + result.map_err(|error| { + let delivery = delivery_with_played_audio(playback.snapshot()); + let _ = playback.interrupt(); SiriStreamFailure { error, delivery } - }); - - unsafe { berd_siri_tts_stream_release(stream) }; - unsafe { - drop(Box::from_raw(callback_context)); - } - playback_lifetime.cancel(); - let _ = playback_monitor_sender.send(SiriPlaybackMonitorEvent::Shutdown); - let _ = playback_monitor.join(); - result + }) } #[tauri::command] @@ -1078,11 +851,14 @@ pub fn start_siri_voice_stream( webview_window: tauri::WebviewWindow, state: tauri::State<'_, SiriVoiceState>, native_voice: tauri::State<'_, NativeVoiceState>, + session_id: String, + expected_revision: u64, + speech_id: u64, stream_id: String, voice: SiriVoiceSelection, interruption_mode: VoiceInterruptionMode, interruption_sensitivity: InterruptionSensitivity, -) -> Result<(), String> { +) -> Result { #[cfg(not(target_os = "macos"))] { let _ = ( @@ -1090,6 +866,9 @@ pub fn start_siri_voice_stream( webview_window, state, native_voice, + session_id, + expected_revision, + speech_id, stream_id, voice, interruption_mode, @@ -1106,11 +885,22 @@ pub fn start_siri_voice_stream( let voice = resolve_stream_voice(&voice, || discover_voices(""))?; let settings = read_settings(&settings_path(&app)?); let active = begin_playback(&state, webview_window.label())?; - let effective_output_device = effective_output_device_name(None); - let suppress_capture = - should_suppress_capture(interruption_mode, effective_output_device.as_deref()); + let output_device = selected_output_device(); + let effective_output_device = effective_output_device_name(output_device.as_deref()); + let input_during_tts = + resolve_input_during_tts_policy(interruption_mode, effective_output_device.as_deref()); let playback_latency_safety_duration = playback_latency_safety_duration(effective_output_device.as_deref()); + let Some(admission) = native_voice.claim_assistant_speech( + &session_id, + expected_revision, + speech_id, + active.clone(), + )? + else { + finish_playback(state.inner(), &active); + return Ok(false); + }; let (sender, receiver) = mpsc::channel(); { let mut runtime = state @@ -1126,6 +916,7 @@ pub fn start_siri_voice_stream( let playback_active = active.clone(); let native_voice_state = native_voice.inner().clone(); tauri::async_runtime::spawn_blocking(move || { + let admission_guard = admission; let result = run_siri_stream( app.clone(), stream_id.clone(), @@ -1137,7 +928,8 @@ pub fn start_siri_voice_stream( receiver, native_voice_state, interruption_sensitivity, - suppress_capture, + input_during_tts, + output_device.as_deref(), playback_latency_safety_duration, ); let (event_state, error, delivery) = match result { @@ -1155,9 +947,10 @@ pub fn start_siri_voice_stream( // A terminal event hands stream ownership back to the renderer, // which may immediately start a replacement stream. Release the // backend playback token before publishing that handoff. + drop(admission_guard); emit_stream_event(&app, &stream_id, event_state, error, delivery); }); - Ok(()) + Ok(true) } } @@ -1239,20 +1032,23 @@ pub async fn preview_siri_voice( .playback_speed .clamp(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED); let text = CString::new("Hello. This is a preview of my voice.").expect("static preview"); - let language = CString::new(voice.language.clone()) + let language = CString::new(voice.language()) .map_err(|_| "Siri voice language cannot contain NUL bytes".to_string())?; - let name = CString::new(voice.name.clone()) + let name = CString::new(voice.name()) .map_err(|_| "Siri voice name cannot contain NUL bytes".to_string())?; let active = begin_playback(&state, webview_window.label())?; - let capture_suppression = + let assistant_speech = output_device_uses_speakers(effective_output_device_name(None).as_deref()).then(|| { log::info!("[voice-echo-guard] speaker output detected"); - native_voice.suppress_capture() + native_voice.begin_assistant_speech( + InterruptionSensitivity::Balanced, + InputDuringTtsPolicy::SuppressInput, + ) }); let playback_state = state.inner().clone(); let playback_active = active.clone(); tauri::async_runtime::spawn_blocking(move || { - let _capture_suppression = capture_suppression; + let _assistant_speech = assistant_speech; let result = (|| { let mut error = std::ptr::null_mut(); // SAFETY: The bridge copies all strings synchronously. The Arc @@ -1273,8 +1069,7 @@ pub async fn preview_siri_voice( } let sample_error = bridge_error(error, "No system preview is available"); - let voices = discover_voices(&voice.language)?; - if !find_voice(&voices, &voice).is_some_and(|candidate| candidate.installed) { + if validate_installed_voice(&voice).is_err() { return Err(sample_error); } @@ -1356,18 +1151,22 @@ mod tests { use super::*; #[test] - fn voice_lookup_normalizes_language_and_name_case() { + fn voice_lookup_normalizes_language_but_preserves_exact_name() { let voices = vec![SiriVoice { name: "Aaron".to_string(), - language: "en_US".to_string(), + language: "en-US".to_string(), size_bytes: 10, installed: true, }]; - let selected = SiriVoiceSelection { - name: "aaron".to_string(), - language: "EN-us".to_string(), - }; - assert_eq!(find_voice(&voices, &selected), voices.first()); + let selected = SiriVoiceSelection::new("Aaron", "en-US").unwrap(); + assert_eq!( + voices.iter().find(|voice| voice.matches(&selected)), + voices.first() + ); + assert!(voices + .iter() + .find(|voice| { voice.matches(&SiriVoiceSelection::new("aaron", "en-US").unwrap()) }) + .is_none()); } #[test] @@ -1393,10 +1192,7 @@ mod tests { release_selection_rx .recv() .expect("release selection write"); - settings.selected_voice = Some(SiriVoiceSelection { - name: "Aaron".to_string(), - language: "en-US".to_string(), - }); + settings.selected_voice = Some(SiriVoiceSelection::new("Aaron", "en-US").unwrap()); true }) .expect("write selected voice"); @@ -1427,10 +1223,7 @@ mod tests { let settings = read_settings(&path); assert_eq!( settings.selected_voice, - Some(SiriVoiceSelection { - name: "Aaron".to_string(), - language: "en-US".to_string(), - }) + Some(SiriVoiceSelection::new("Aaron", "en-US").unwrap()) ); assert_eq!(settings.playback_speed, 1.5); serde_json::from_slice::(&fs::read(&*path).expect("settings JSON")) @@ -1457,10 +1250,7 @@ mod tests { assert_eq!( resolve_voice_selection(&voices, None, || Ok(Vec::new())), Ok(( - Some(SiriVoiceSelection { - name: "Aaron".to_string(), - language: "en-US".to_string(), - }), + Some(SiriVoiceSelection::new("Aaron", "en-US").unwrap()), true, )) ); @@ -1485,10 +1275,7 @@ mod tests { }]) }), Ok(( - Some(SiriVoiceSelection { - name: "Catherine".to_string(), - language: "en-AU".to_string(), - }), + Some(SiriVoiceSelection::new("Catherine", "en-AU").unwrap()), true, )) ); @@ -1496,10 +1283,7 @@ mod tests { #[test] fn unavailable_selection_falls_back_to_an_installed_siri_voice() { - let selected = SiriVoiceSelection { - name: "Aaron".to_string(), - language: "en-US".to_string(), - }; + let selected = SiriVoiceSelection::new("Aaron", "en-US").unwrap(); let preferred_voices = vec![ SiriVoice { name: "Aaron".to_string(), @@ -1528,10 +1312,7 @@ mod tests { ]) }), Ok(( - Some(SiriVoiceSelection { - name: "Catherine".to_string(), - language: "en-AU".to_string(), - }), + Some(SiriVoiceSelection::new("Catherine", "en-AU").unwrap()), true, )) ); @@ -1539,10 +1320,7 @@ mod tests { #[test] fn unavailable_selection_is_preserved_when_no_siri_voice_is_installed() { - let selected = SiriVoiceSelection { - name: "Aaron".to_string(), - language: "en-US".to_string(), - }; + let selected = SiriVoiceSelection::new("Aaron", "en-US").unwrap(); let voices = vec![SiriVoice { name: "Aaron".to_string(), language: "en-US".to_string(), @@ -1558,17 +1336,17 @@ mod tests { #[test] fn stream_voice_ingress_re_resolves_a_voice_removed_after_status() { - let selected = SiriVoiceSelection { - name: "Aaron".to_string(), - language: "en-US".to_string(), - }; + let selected = SiriVoiceSelection::new("Aaron", "en-US").unwrap(); let status_catalog = vec![SiriVoice { - name: selected.name.clone(), - language: selected.language.clone(), + name: selected.name().to_string(), + language: selected.language().to_string(), size_bytes: 10, installed: true, }]; - assert!(find_voice(&status_catalog, &selected).is_some_and(|voice| voice.installed)); + assert!(status_catalog + .iter() + .find(|voice| voice.matches(&selected)) + .is_some_and(|voice| voice.installed)); let current_catalog = vec![ SiriVoice { @@ -1585,25 +1363,19 @@ mod tests { assert_eq!( resolve_stream_voice(&selected, || Ok(current_catalog)), - Ok(SiriVoiceSelection { - name: "Catherine".to_string(), - language: "en-AU".to_string(), - }) + Ok(SiriVoiceSelection::new("Catherine", "en-AU").unwrap()) ); } #[test] fn stream_voice_ingress_rejects_when_no_siri_voice_is_installed() { - let selected = SiriVoiceSelection { - name: "Aaron".to_string(), - language: "en-US".to_string(), - }; + let selected = SiriVoiceSelection::new("Aaron", "en-US").unwrap(); assert_eq!( resolve_stream_voice(&selected, || { Ok(vec![SiriVoice { - name: selected.name.clone(), - language: selected.language.clone(), + name: selected.name().to_string(), + language: selected.language().to_string(), size_bytes: 10, installed: false, }]) @@ -1631,167 +1403,52 @@ mod tests { } #[test] - fn siri_playback_monitor_reschedules_many_drain_gaps_on_one_thread() { - struct DropSignal(mpsc::SyncSender<()>); - impl Drop for DropSignal { - fn drop(&mut self) { - let _ = self.0.send(()); - } - } - - let (drop_sender, drop_receiver) = mpsc::sync_channel(1); - let lifetime = Arc::new(SiriPlaybackLifetime::default()); - lifetime.start(|| DropSignal(drop_sender)); - let (event_sender, event_receiver) = mpsc::channel(); - let monitor_lifetime = Arc::clone(&lifetime); - let monitor = std::thread::Builder::new() - .name("siri-playback-monitor-test".to_string()) - .spawn(move || { - run_siri_playback_monitor( - event_receiver, - monitor_lifetime, - Duration::from_millis(20), - ); - }) - .expect("start monitor"); - - for _ in 0..100 { - let generation = lifetime.begin_drain().expect("drain active guard"); - event_sender - .send(SiriPlaybackMonitorEvent::Drain(generation)) - .expect("schedule drain"); - lifetime.start(|| panic!("resumed buffering must retain the existing guard")); - event_sender - .send(SiriPlaybackMonitorEvent::Started) - .expect("cancel pending drain"); - } - - let final_drain = lifetime.begin_drain().expect("final drain"); - event_sender - .send(SiriPlaybackMonitorEvent::Drain(final_drain)) - .expect("schedule final drain"); - drop_receiver - .recv_timeout(Duration::from_secs(1)) - .expect("final drain releases guard"); - assert!(!lifetime.is_active()); - assert!(drop_receiver.try_recv().is_err()); - - event_sender - .send(SiriPlaybackMonitorEvent::Shutdown) - .expect("stop monitor"); - monitor.join().expect("join monitor"); - } - - #[test] - fn cancelling_siri_playback_invalidates_a_pending_grace_release() { - struct DropCounter(Arc); - impl Drop for DropCounter { - fn drop(&mut self) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - - let drops = Arc::new(AtomicU64::new(0)); - let lifetime = Arc::new(SiriPlaybackLifetime::default()); - lifetime.start(|| DropCounter(Arc::clone(&drops))); - let drain = lifetime.begin_drain().expect("drain before cancellation"); - let (event_sender, event_receiver) = mpsc::channel(); - let monitor_lifetime = Arc::clone(&lifetime); - let monitor = std::thread::Builder::new() - .name("siri-playback-cancel-test".to_string()) - .spawn(move || { - run_siri_playback_monitor( - event_receiver, - monitor_lifetime, - Duration::from_secs(60), - ); - }) - .expect("start monitor"); - event_sender - .send(SiriPlaybackMonitorEvent::Drain(drain)) - .expect("schedule drain"); - - lifetime.cancel(); - assert!(!lifetime.is_active()); - assert_eq!(drops.load(Ordering::SeqCst), 1); - event_sender - .send(SiriPlaybackMonitorEvent::Shutdown) - .expect("stop monitor"); - monitor.join().expect("join monitor"); - assert_eq!(drops.load(Ordering::SeqCst), 1); - } - - #[test] - fn siri_playback_monitor_spawn_failure_is_reported_without_running_the_task() { - let ran = Arc::new(AtomicBool::new(false)); - let task_ran = Arc::clone(&ran); - let result = spawn_siri_playback_monitor_with( - move || task_ran.store(true, Ordering::SeqCst), - |_task| Err(std::io::Error::other("injected spawn failure")), + fn siri_drain_bound_covers_remaining_pcm_and_stall_margin() { + assert_eq!( + siri_drain_timeout(144_000, 48_000, 48_000), + SIRI_OUTPUT_DRAIN_MARGIN + Duration::from_secs(2) ); - assert_eq!( - result.expect_err("spawn must fail").to_string(), - "injected spawn failure" + siri_drain_timeout(48_000, 96_000, 48_000), + SIRI_OUTPUT_DRAIN_MARGIN ); - assert!(!ran.load(Ordering::SeqCst)); } - #[cfg(target_os = "macos")] #[test] - fn siri_playback_monitor_panic_cancels_the_lifetime_and_reports_failure() { - struct DropSignal(mpsc::SyncSender<()>); - impl Drop for DropSignal { - fn drop(&mut self) { - let _ = self.0.send(()); - } - } - - let (drop_sender, drop_receiver) = mpsc::sync_channel(1); - let lifetime = Arc::new(SiriPlaybackLifetime::default()); - lifetime.start(|| DropSignal(drop_sender)); - let drain = lifetime.begin_drain().expect("drain active guard"); - let failed = Arc::new(AtomicBool::new(false)); - let (event_sender, event_receiver) = mpsc::channel(); - let monitor = spawn_siri_playback_monitor( - event_receiver, - Arc::clone(&lifetime), - Duration::MAX, - Arc::clone(&failed), - ) - .expect("start monitor"); - - event_sender - .send(SiriPlaybackMonitorEvent::Drain(drain)) - .expect("trigger monitor overflow panic"); - drop_receiver - .recv_timeout(Duration::from_secs(1)) - .expect("monitor panic cancels playback lifetime"); - monitor.join().expect("panic is contained by monitor"); - - assert!(failed.load(Ordering::SeqCst)); - assert!(!lifetime.is_active()); - } + fn siri_route_grace_preserves_only_the_unelapsed_tail() { + let now = Instant::now(); + let grace = Duration::from_millis(500); + assert_eq!( + output_latency_grace_remaining(true, None, grace, now), + grace + ); + assert_eq!( + output_latency_grace_remaining( + true, + Some(now - Duration::from_millis(200)), + grace, + now, + ), + Duration::from_millis(300) + ); + assert_eq!( + output_latency_grace_remaining(false, None, grace, now), + Duration::ZERO + ); - #[cfg(target_os = "macos")] - #[test] - fn stream_watchdog_times_out_only_after_progress_stalls() { - let started = Instant::now(); - let mut watchdog = SiriStreamWatchdog::new(1, started); - - assert!(!watchdog.observe(2, started + SIRI_STREAM_STALL_TIMEOUT)); - assert!(!watchdog.observe( - 2, - started + SIRI_STREAM_STALL_TIMEOUT + Duration::from_millis(1), + let mut drained_at = None; + assert!(!output_latency_grace_elapsed( + false, + true, + &mut drained_at, + grace, + now + Duration::from_secs(10), )); - assert!(watchdog.observe(2, started + SIRI_STREAM_STALL_TIMEOUT * 2,)); + assert_eq!(drained_at, None); } #[test] fn failed_stream_retains_only_delivery_with_played_audio() { - use std::cell::RefCell; - - let calls = RefCell::new(Vec::new()); let progress = VoiceDeliveryProgress { sample_rate: 24_000, segments: vec![VoiceDeliverySegment { @@ -1801,14 +1458,6 @@ mod tests { synthesis_complete: true, }], }; - let progress = capture_before_cancel( - || { - calls.borrow_mut().push("snapshot"); - progress - }, - || calls.borrow_mut().push("cancel"), - ); - assert_eq!(&*calls.borrow(), &["snapshot", "cancel"]); assert_eq!( delivery_with_played_audio(progress) .expect("played audio is evidence") diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index f8086c2f1..ddbda54f4 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -690,6 +690,8 @@ pub fn run() { commands::native_voice::get_native_voice_conversation_status, commands::native_voice::block_native_voice_conversation_starts, commands::native_voice::release_native_voice_conversation_start_block, + commands::native_voice::prepare_native_voice_assistant_speech, + commands::native_voice::cancel_native_voice_assistant_speech, commands::native_voice::set_native_voice_microphone_muted, commands::native_voice::set_native_voice_assistant_speaking, commands::native_voice::drain_native_voice_conversation_transcripts, diff --git a/src-tauri/swift/BerdAirPodsBridge/Package.swift b/src-tauri/swift/BerdAirPodsBridge/Package.swift index fe55a5d69..39e07ca01 100644 --- a/src-tauri/swift/BerdAirPodsBridge/Package.swift +++ b/src-tauri/swift/BerdAirPodsBridge/Package.swift @@ -21,8 +21,6 @@ let package = Package( dependencies: ["BerdObjCExceptionCatch"], linkerSettings: [ .linkedFramework("AVFAudio"), - .linkedFramework("AVFoundation"), - .linkedFramework("Speech"), ] ) ] diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index 8fe696a63..cd15c25ad 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -55,11 +55,17 @@ export function listenToOpenAiVoiceSettings( } export function startOpenAiVoiceStream( + sessionId: string, + expectedRevision: number, + speechId: number, streamId: string, interruptionMode: VoiceInterruptionMode, interruptionSensitivity: VoiceInterruptionSensitivity, -): Promise { - return invoke("start_openai_voice_stream", { +): Promise { + return invoke("start_openai_voice_stream", { + sessionId, + expectedRevision, + speechId, streamId, interruptionMode, interruptionSensitivity, diff --git a/src/features/voice-conversation/api/pocketVoice.test.ts b/src/features/voice-conversation/api/pocketVoice.test.ts index f8d8d9c6c..66f03ac9e 100644 --- a/src/features/voice-conversation/api/pocketVoice.test.ts +++ b/src/features/voice-conversation/api/pocketVoice.test.ts @@ -88,7 +88,14 @@ describe("Pocket voice API", () => { it("uses the streaming utterance commands", async () => { mocks.invoke.mockResolvedValue(undefined); - await startPocketVoiceStream("stream-1", "allowInterruptions", "more"); + await startPocketVoiceStream( + "session-1", + 4, + 7, + "stream-1", + "allowInterruptions", + "more", + ); await appendPocketVoiceStream("stream-1", "Hello"); await flushPocketVoiceStream("stream-1"); await finishPocketVoiceStream("stream-1"); @@ -97,6 +104,9 @@ describe("Pocket voice API", () => { 1, "start_pocket_voice_stream", { + sessionId: "session-1", + expectedRevision: 4, + speechId: 7, streamId: "stream-1", interruptionMode: "allowInterruptions", interruptionSensitivity: "more", diff --git a/src/features/voice-conversation/api/pocketVoice.ts b/src/features/voice-conversation/api/pocketVoice.ts index 46b030492..93969f286 100644 --- a/src/features/voice-conversation/api/pocketVoice.ts +++ b/src/features/voice-conversation/api/pocketVoice.ts @@ -105,11 +105,17 @@ export function speakPocketVoice(text: string): Promise { } export function startPocketVoiceStream( + sessionId: string, + expectedRevision: number, + speechId: number, streamId: string, interruptionMode: VoiceInterruptionMode, interruptionSensitivity: VoiceInterruptionSensitivity, -): Promise { - return invoke("start_pocket_voice_stream", { +): Promise { + return invoke("start_pocket_voice_stream", { + sessionId, + expectedRevision, + speechId, streamId, interruptionMode, interruptionSensitivity, diff --git a/src/features/voice-conversation/api/siriVoice.test.ts b/src/features/voice-conversation/api/siriVoice.test.ts index 0e9873d25..9e7b5ba8f 100644 --- a/src/features/voice-conversation/api/siriVoice.test.ts +++ b/src/features/voice-conversation/api/siriVoice.test.ts @@ -48,6 +48,9 @@ describe("Siri voice API", () => { const voice = { name: "Samantha", language: "en-US" }; await startSiriVoiceStream( + "session-1", + 4, + 7, "stream-1", voice, "allowInterruptions", @@ -55,6 +58,9 @@ describe("Siri voice API", () => { ); expect(mocks.invoke).toHaveBeenCalledWith("start_siri_voice_stream", { + sessionId: "session-1", + expectedRevision: 4, + speechId: 7, streamId: "stream-1", voice, interruptionMode: "allowInterruptions", diff --git a/src/features/voice-conversation/api/siriVoice.ts b/src/features/voice-conversation/api/siriVoice.ts index 3fc8d4ab5..a7d5433e3 100644 --- a/src/features/voice-conversation/api/siriVoice.ts +++ b/src/features/voice-conversation/api/siriVoice.ts @@ -69,12 +69,18 @@ export interface SiriVoiceStreamEvent { } export function startSiriVoiceStream( + sessionId: string, + expectedRevision: number, + speechId: number, streamId: string, voice: SiriVoiceSelection, interruptionMode: VoiceInterruptionMode, interruptionSensitivity: VoiceInterruptionSensitivity, -): Promise { - return invoke("start_siri_voice_stream", { +): Promise { + return invoke("start_siri_voice_stream", { + sessionId, + expectedRevision, + speechId, streamId, voice, interruptionMode, diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index 3fc0e7bd2..55b5ba2dd 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import preparedSpeechOutcomes from "../../../../tests/contracts/voice/prepare-assistant-speech-outcomes.json"; const mocks = vi.hoisted(() => ({ invoke: vi.fn(), @@ -37,6 +38,9 @@ import { getVoiceConversationStatus, listenToVoiceConversation, openVoiceConversationSession, + prepareAssistantSpeechOutcomeSchema, + prepareVoiceConversationAssistantSpeech, + cancelVoiceConversationAssistantSpeech, reconcileVoiceConversationMicrophone, releaseNativeVoiceConversationStartBlock, resetVoiceConversationForegroundSessionForTest, @@ -67,6 +71,72 @@ describe("voice conversation API", () => { mocks.stopMicrophone.mockReset(); }); + it("shares the real prepared-speech wire contract and rejects drift", async () => { + const outcomes = prepareAssistantSpeechOutcomeSchema + .array() + .parse(preparedSpeechOutcomes); + expect(outcomes).toEqual(preparedSpeechOutcomes); + + mocks.invoke.mockResolvedValueOnce({ + outcome: "admitted", + speech_id: 7, + }); + await expect( + prepareVoiceConversationAssistantSpeech("session-1", 1, "hello", null), + ).rejects.toThrow(); + }); + + it("prepares and cancels assistant speech through the active renderer", async () => { + mocks.invoke + .mockResolvedValueOnce({ outcome: "admitted", speechId: 9 }) + .mockResolvedValueOnce(true); + const acknowledgement = { + lifecycleId: "lifecycle-1", + id: "utterance-1", + revision: 4, + }; + + await expect( + prepareVoiceConversationAssistantSpeech( + "session-1", + 4, + "hello", + acknowledgement, + ), + ).resolves.toEqual({ outcome: "admitted", speechId: 9 }); + await expect( + cancelVoiceConversationAssistantSpeech("session-1", 4, 9), + ).resolves.toBe(true); + + expect(mocks.invoke).toHaveBeenNthCalledWith( + 1, + "prepare_native_voice_assistant_speech", + { + request: { + sessionId: "session-1", + expectedRevision: 4, + text: "hello", + acknowledgement, + rendererId: "renderer-test", + rendererEpoch: 7, + }, + }, + ); + expect(mocks.invoke).toHaveBeenNthCalledWith( + 2, + "cancel_native_voice_assistant_speech", + { + request: { + sessionId: "session-1", + expectedRevision: 4, + speechId: 9, + rendererId: "renderer-test", + rendererEpoch: 7, + }, + }, + ); + }); + it("uses the typed native command surface", async () => { const status = { available: true, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index 9c4846053..c06c825c1 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -1,6 +1,7 @@ import { invoke } from "@tauri-apps/api/core"; import { listen, type UnlistenFn } from "@tauri-apps/api/event"; import { getCurrentWindow } from "@tauri-apps/api/window"; +import { z } from "zod"; import { getRendererInstance } from "@/shared/lib/rendererInstance"; import { startNativeMicrophone, @@ -487,6 +488,70 @@ export interface PendingVoiceTranscript { deliveryAttempts: number; } +export interface VoiceTranscriptReference { + lifecycleId: string; + id: string; + revision: number; +} + +export const prepareAssistantSpeechOutcomeSchema = z.discriminatedUnion( + "outcome", + [ + z.object({ outcome: z.literal("pending") }).strict(), + z.object({ outcome: z.literal("notAdmitted") }).strict(), + z + .object({ + outcome: z.literal("admitted"), + speechId: z.number().int().nonnegative(), + }) + .strict(), + ], +); + +export type PrepareAssistantSpeechOutcome = z.infer< + typeof prepareAssistantSpeechOutcomeSchema +>; + +export async function prepareVoiceConversationAssistantSpeech( + sessionId: string, + expectedRevision: number, + text: string, + acknowledgement: VoiceTranscriptReference | null, +): Promise { + const { rendererId, rendererEpoch } = await getRendererInstance(); + const outcome = await invoke( + "prepare_native_voice_assistant_speech", + { + request: { + sessionId, + expectedRevision, + text, + acknowledgement, + rendererId, + rendererEpoch, + }, + }, + ); + return prepareAssistantSpeechOutcomeSchema.parse(outcome); +} + +export async function cancelVoiceConversationAssistantSpeech( + sessionId: string, + expectedRevision: number, + speechId: number, +): Promise { + const { rendererId, rendererEpoch } = await getRendererInstance(); + return invoke("cancel_native_voice_assistant_speech", { + request: { + sessionId, + expectedRevision, + speechId, + rendererId, + rendererEpoch, + }, + }); +} + export interface VoiceTranscriptRejection { attempts: number; terminal: boolean; diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 8a04b6b27..49592a695 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -33,6 +33,8 @@ const mocks = vi.hoisted(() => ({ speaking: boolean, ) => Promise >(), + prepareAssistantSpeech: vi.fn(), + cancelAssistantSpeech: vi.fn(), streamHandler: null as ((event: PocketVoiceStreamEvent) => void) | null, siriStart: vi.fn< @@ -70,14 +72,22 @@ const mocks = vi.hoisted(() => ({ })); vi.mock("../api/voiceConversation", () => ({ setVoiceConversationAssistantSpeaking: mocks.setAssistantSpeaking, + prepareVoiceConversationAssistantSpeech: mocks.prepareAssistantSpeech, + cancelVoiceConversationAssistantSpeech: mocks.cancelAssistantSpeech, })); vi.mock("../api/pocketVoice", () => ({ startPocketVoiceStream: ( + _sessionId: string, + _expectedRevision: number, + _speechId: number, streamId: string, interruptionMode: typeof mocks.interruptionMode, interruptionSensitivity: "less" | "balanced" | "more", - ) => mocks.start(streamId, interruptionMode, interruptionSensitivity), + ) => + mocks + .start(streamId, interruptionMode, interruptionSensitivity) + .then(() => true), appendPocketVoiceStream: (streamId: string, text: string) => mocks.append(streamId, text), flushPocketVoiceStream: (streamId: string) => mocks.flush(streamId), @@ -93,10 +103,16 @@ vi.mock("../api/pocketVoice", () => ({ vi.mock("../api/openAiVoice", () => ({ startOpenAiVoiceStream: ( + _sessionId: string, + _expectedRevision: number, + _speechId: number, streamId: string, interruptionMode: typeof mocks.interruptionMode, interruptionSensitivity: "less" | "balanced" | "more", - ) => mocks.openAiStart(streamId, interruptionMode, interruptionSensitivity), + ) => + mocks + .openAiStart(streamId, interruptionMode, interruptionSensitivity) + .then(() => true), appendOpenAiVoiceStream: (streamId: string, text: string) => mocks.openAiAppend(streamId, text), flushOpenAiVoiceStream: (streamId: string) => mocks.openAiFlush(streamId), @@ -112,12 +128,17 @@ vi.mock("../api/openAiVoice", () => ({ vi.mock("../api/siriVoice", () => ({ startSiriVoiceStream: ( + _sessionId: string, + _expectedRevision: number, + _speechId: number, streamId: string, voice: { name: string; language: string }, interruptionMode: typeof mocks.interruptionMode, interruptionSensitivity: "less" | "balanced" | "more", ) => - mocks.siriStart(streamId, voice, interruptionMode, interruptionSensitivity), + mocks + .siriStart(streamId, voice, interruptionMode, interruptionSensitivity) + .then(() => true), appendSiriVoiceStream: (streamId: string, text: string) => mocks.siriAppend(streamId, text), flushSiriVoiceStream: (streamId: string) => mocks.siriFlush(streamId), @@ -209,6 +230,10 @@ describe("native assistant speech stream", () => { mocks.finish.mockReset().mockResolvedValue(); mocks.stop.mockReset().mockResolvedValue(true); mocks.setAssistantSpeaking.mockReset().mockResolvedValue(undefined); + mocks.prepareAssistantSpeech + .mockReset() + .mockResolvedValue({ outcome: "admitted", speechId: 7 }); + mocks.cancelAssistantSpeech.mockReset().mockResolvedValue(true); mocks.streamHandler = null; mocks.siriStart.mockReset().mockResolvedValue(); mocks.siriAppend.mockReset().mockResolvedValue(); @@ -283,6 +308,196 @@ describe("native assistant speech stream", () => { }); }); + it("waits for speakable text before requesting admission", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [assistant([{ type: "text", text: " " }])]); + + await new Promise((resolve) => window.setTimeout(resolve, 10)); + expect(mocks.prepareAssistantSpeech).not.toHaveBeenCalled(); + expect(mocks.start).not.toHaveBeenCalled(); + + useChatStore + .getState() + .appendStreamingText("session-1", "assistant-1", "Hello."); + await vi.waitFor(() => + expect(mocks.prepareAssistantSpeech).toHaveBeenCalledWith( + "session-1", + 1, + " Hello.", + null, + ), + ); + expect(mocks.append).toHaveBeenCalledWith( + mocks.start.mock.calls[0]?.[0], + "Hello.", + ); + }); + + it("completes whitespace-only output without addressing a native stream", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore.getState().setMessages("session-1", [ + assistant( + [ + { type: "text", text: " " }, + { + type: "toolRequest", + id: "tool-1", + name: "noop", + arguments: {}, + status: "completed", + }, + ], + "completed", + ), + ]); + + await new Promise((resolve) => window.setTimeout(resolve, 10)); + expect(mocks.prepareAssistantSpeech).not.toHaveBeenCalled(); + expect(mocks.flush).not.toHaveBeenCalled(); + expect(mocks.finish).not.toHaveBeenCalled(); + + useChatStore.getState().setMessages("session-1", [ + assistant( + [ + { type: "text", text: " " }, + { + type: "toolRequest", + id: "tool-1", + name: "noop", + arguments: {}, + status: "completed", + }, + ], + "completed", + ), + assistant( + [{ type: "text", text: "The next response speaks." }], + "completed", + "assistant-2", + ), + ]); + await vi.waitFor(() => + expect(mocks.append).toHaveBeenCalledWith( + mocks.start.mock.calls[0]?.[0], + "The next response speaks.", + ), + ); + }); + + it("retires whitespace-only output held during user speech", async () => { + vi.useFakeTimers(); + try { + useVoiceConversationStore.setState({ userSpeaking: true }); + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: " " }], "completed"), + ]); + + useVoiceConversationStore.setState({ userSpeaking: false }); + await vi.advanceTimersByTimeAsync(250); + await vi.runAllTimersAsync(); + finalizeVoiceTranscript("voice-after-inert-output"); + await vi.runAllTimersAsync(); + + expect(mocks.prepareAssistantSpeech).not.toHaveBeenCalled(); + expect(mocks.start).not.toHaveBeenCalled(); + expect(takeVoicePlaybackNotices("session-1")).toBeNull(); + } finally { + vi.useRealTimers(); + } + }); + + it("prepares playback with the assistant response's exact causal transcript", async () => { + finalizeVoiceTranscript("voice-k1"); + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + voiceUser("voice-k1"), + assistant([{ type: "text", text: "Causal reply." }]), + ]); + + await vi.waitFor(() => + expect(mocks.prepareAssistantSpeech).toHaveBeenCalledWith( + "session-1", + 1, + "Causal reply.", + { + lifecycleId: "lifecycle-1", + id: "voice-k1", + revision: 1, + }, + ), + ); + expect(mocks.start).toHaveBeenCalled(); + }); + + it("does not start a response rejected by authoritative pending input", async () => { + mocks.prepareAssistantSpeech.mockResolvedValueOnce({ outcome: "pending" }); + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Stale reply." }]), + ]); + + await vi.waitFor(() => + expect(mocks.prepareAssistantSpeech).toHaveBeenCalled(), + ); + expect(mocks.start).not.toHaveBeenCalled(); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ speech: { status: "notSpoken" } }); + }); + + it("does not reinterpret a late pending result after the utterance was interrupted", async () => { + let resolvePrepare: ((outcome: { outcome: "pending" }) => void) | undefined; + mocks.prepareAssistantSpeech.mockImplementationOnce( + () => + new Promise((resolve) => { + resolvePrepare = resolve; + }), + ); + const terminalStatuses: string[] = []; + let previousStatus: string | undefined; + const unsubscribe = useChatStore.subscribe((state) => { + const content = state.messagesBySession["session-1"]?.[0]?.content[0]; + const status = + content && "speech" in content ? content.speech?.status : undefined; + if (status && status !== previousStatus) terminalStatuses.push(status); + previousStatus = status; + }); + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Held reply." }]), + ]); + + await vi.waitFor(() => expect(resolvePrepare).toBeDefined()); + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toMatchObject({ speech: { status: "interrupted" } }), + ); + const terminalStatus = + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0]; + + resolvePrepare?.({ outcome: "pending" }); + await vi.waitFor(() => expect(mocks.start).not.toHaveBeenCalled()); + + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.content[0], + ).toEqual(terminalStatus); + expect(terminalStatuses).toEqual(["interrupted"]); + unsubscribe(); + }); + it("routes ordering and cancellation through OpenAI when selected", async () => { mocks.backend = "openai"; startNativeAssistantSpeech("session-1", vi.fn()); @@ -2898,7 +3113,7 @@ describe("native assistant speech stream", () => { ); }); - it("hydrates causal ownership from an existing voice transcript", async () => { + it("does not reuse a transcript reference from a prior voice lifecycle", async () => { useChatStore.getState().setMessages("session-1", [voiceUser("voice-k1")]); startNativeAssistantSpeech("session-1", vi.fn()); useChatStore @@ -2912,15 +3127,12 @@ describe("native assistant speech stream", () => { ), ]); - await vi.waitFor(() => - expect(mocks.append).toHaveBeenCalledWith( - mocks.start.mock.calls[0]?.[0], - "Recovered-turn reply.", - ), - ); + await new Promise((resolve) => window.setTimeout(resolve, 10)); + expect(mocks.prepareAssistantSpeech).not.toHaveBeenCalled(); + expect(mocks.append).not.toHaveBeenCalled(); expect( useVoiceConversationStore.getState().latestFinalizedTranscriptKey, - ).toBe(["session-1", "lifecycle-1", "1", "voice-k1"].join("\0")); + ).toBeNull(); }); it("plays each completed reply held during user speech in its own stream", async () => { diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index ba5c44c98..c0e0a9c76 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -33,7 +33,12 @@ import { type SiriVoiceStreamEvent, type SiriVoiceSelection, } from "../api/siriVoice"; -import { setVoiceConversationAssistantSpeaking } from "../api/voiceConversation"; +import { + cancelVoiceConversationAssistantSpeech, + prepareVoiceConversationAssistantSpeech, + setVoiceConversationAssistantSpeaking, + type VoiceTranscriptReference, +} from "../api/voiceConversation"; import { FIXED_INTERRUPTION_SENSITIVITY, getVoiceInterruptionPreference, @@ -68,13 +73,16 @@ type ActiveUtterance = { targetSpans: SpeechTargetSpan[]; text: string; finishing: boolean; + nativeStartQueued: boolean; nativeStartInvoked: boolean; + speechId: number | null; interruptionRequested: boolean; resumptionDiscarded: boolean; interruptionFallback: ReturnType | null; interruptionCause: InterruptionCause | null; latestDelivery: VoiceDeliveryProgress | null; causalTranscriptKey: string | null; + causalTranscriptReference: VoiceTranscriptReference | null; status: SpeechStatus | null; onFailure: SpeechFailureHandler; onInterrupted: ( @@ -171,6 +179,30 @@ function voiceTranscriptKeyForMessage( ].join("\0"); } +function voiceTranscriptReferenceForMessage( + message: ReturnType< + typeof useChatStore.getState + >["messagesBySession"][string][number], +): VoiceTranscriptReference | null { + const metadata = message.metadata; + if ( + metadata?.origin !== "voice_conversation" || + typeof metadata.voiceConversationLifecycleId !== "string" || + metadata.voiceConversationLifecycleId.length === 0 || + typeof metadata.voiceConversationRevision !== "number" || + !Number.isInteger(metadata.voiceConversationRevision) || + typeof metadata.voiceUtteranceId !== "string" || + metadata.voiceUtteranceId.length === 0 + ) { + return null; + } + return { + lifecycleId: metadata.voiceConversationLifecycleId, + id: metadata.voiceUtteranceId, + revision: metadata.voiceConversationRevision, + }; +} + function reportAssistantActivity( sessionId: string, expectedRevision: number, @@ -753,6 +785,13 @@ function interruptActiveUtterance( if (utterance && !utterance.interruptionRequested) { utterance.interruptionRequested = true; utterance.interruptionCause = cause; + if (utterance.speechId !== null) { + void cancelVoiceConversationAssistantSpeech( + utterance.sessionId, + utterance.voiceRevision, + utterance.speechId, + ).catch(() => false); + } } if (utterance && !terminalEventExpected) { finalizeInterruptedUtterance( @@ -879,7 +918,9 @@ export function startNativeAssistantSpeech( outputBackend === "siri" ? { start: ( + speechId: number, streamId: string, + voiceRevision: number, interruptionMode: VoiceInterruptionMode, interruptionSensitivity: VoiceInterruptionSensitivity, ) => { @@ -889,6 +930,9 @@ export function startNativeAssistantSpeech( ); } return startSiriVoiceStream( + sessionId, + voiceRevision, + speechId, streamId, activeSiriVoice, interruptionMode, @@ -903,7 +947,21 @@ export function startNativeAssistantSpeech( } : outputBackend === "openai" ? { - start: startOpenAiVoiceStream, + start: ( + speechId: number, + streamId: string, + voiceRevision: number, + interruptionMode: VoiceInterruptionMode, + interruptionSensitivity: VoiceInterruptionSensitivity, + ) => + startOpenAiVoiceStream( + sessionId, + voiceRevision, + speechId, + streamId, + interruptionMode, + interruptionSensitivity, + ), append: appendOpenAiVoiceStream, flush: flushOpenAiVoiceStream, finish: finishOpenAiVoiceStream, @@ -911,7 +969,21 @@ export function startNativeAssistantSpeech( listen: listenToOpenAiVoiceStream, } : { - start: startPocketVoiceStream, + start: ( + speechId: number, + streamId: string, + voiceRevision: number, + interruptionMode: VoiceInterruptionMode, + interruptionSensitivity: VoiceInterruptionSensitivity, + ) => + startPocketVoiceStream( + sessionId, + voiceRevision, + speechId, + streamId, + interruptionMode, + interruptionSensitivity, + ), append: appendPocketVoiceStream, flush: flushPocketVoiceStream, finish: finishPocketVoiceStream, @@ -933,6 +1005,7 @@ export function startNativeAssistantSpeech( const toolCountByMessage = new Map(); const consumedTextBySlot = new Map(); const causalTranscriptKeyByMessage = new Map(); + const transcriptReferenceByKey = new Map(); const invalidatedMessages = new Set(); const completedMessages = new Set(); const interruptedMessages = new Set(); @@ -947,6 +1020,9 @@ export function startNativeAssistantSpeech( ); if (voiceTranscriptKey !== null) { precedingTranscriptKey = voiceTranscriptKey; + const reference = voiceTranscriptReferenceForMessage(message); + if (reference) + transcriptReferenceByKey.set(voiceTranscriptKey, reference); } } else if (message.role === "assistant") { causalTranscriptKeyByMessage.set(message.id, precedingTranscriptKey); @@ -969,16 +1045,6 @@ export function startNativeAssistantSpeech( textOrdinal += 1; } } - if ( - initialVoice.latestFinalizedTranscriptKey === null && - precedingTranscriptKey !== null && - precedingTranscriptKey !== MALFORMED_VOICE_TRANSCRIPT_KEY - ) { - useVoiceConversationStore.setState({ - latestFinalizedTranscriptKey: precedingTranscriptKey, - }); - } - let heldSpeech: HeldSpeech | null = null; let resumableInterruption: ResumableInterruption | null = null; let heldReleaseReady = false; @@ -1001,6 +1067,9 @@ export function startNativeAssistantSpeech( ); if (voiceTranscriptKey !== null) { causalTranscriptKey = voiceTranscriptKey; + const reference = voiceTranscriptReferenceForMessage(message); + if (reference) + transcriptReferenceByKey.set(voiceTranscriptKey, reference); } } else if ( message.role === "assistant" && @@ -1204,13 +1273,19 @@ export function startNativeAssistantSpeech( targetSpans: [], text: "", finishing: false, + nativeStartQueued: false, nativeStartInvoked: false, + speechId: null, interruptionRequested: false, resumptionDiscarded: false, interruptionFallback: null, interruptionCause: null, latestDelivery: null, causalTranscriptKey, + causalTranscriptReference: + causalTranscriptKey === null + ? null + : (transcriptReferenceByKey.get(causalTranscriptKey) ?? null), status: null, onFailure: (text, error) => { for (const utteranceTarget of utterance.targets) { @@ -1250,6 +1325,12 @@ export function startNativeAssistantSpeech( onTerminal: () => queueMicrotask(inspect), }; activeUtterance = utterance; + return utterance; + }; + + const queueNativeStart = (utterance: ActiveUtterance) => { + if (utterance.nativeStartQueued) return; + utterance.nativeStartQueued = true; queueStreamCommand( utterance, async () => { @@ -1260,23 +1341,82 @@ export function startNativeAssistantSpeech( ) { return; } - utterance.nativeStartInvoked = true; - await streamBackend.start( - utterance.id, - utterance.interruptionMode, - utterance.interruptionSensitivity, + const prepared = await prepareVoiceConversationAssistantSpeech( + utterance.sessionId, + utterance.voiceRevision, + utterance.text, + utterance.causalTranscriptReference, ); if ( utterance.interruptionRequested || activeUtterance?.id !== utterance.id ) { - await streamBackend.stop(); + if (prepared.outcome === "admitted") { + await cancelVoiceConversationAssistantSpeech( + utterance.sessionId, + utterance.voiceRevision, + prepared.speechId, + ); + } + return; + } + if (prepared.outcome === "pending") { + commandEpoch += 1; + for (const target of utterance.targets) { + const content = targetContent(utterance.sessionId, target); + suppressTarget( + targetKey(target), + target, + content?.text ?? utterance.text, + ); + } + if (activeUtterance?.id === utterance.id) { + activeUtterance = null; + } + restoreListeningIfConversationIsRunning(utterance); + utterance.onTerminal(); return; } + if (prepared.outcome === "notAdmitted") { + interruptActiveUtterance(false, "voiceStopped"); + return; + } + utterance.speechId = prepared.speechId; + let started: boolean; + try { + // Once the native start command has been invoked, interruption must + // preserve terminal ownership until that command resolves or emits + // its stream terminal event. + utterance.nativeStartInvoked = true; + started = await streamBackend.start( + prepared.speechId, + utterance.id, + utterance.voiceRevision, + utterance.interruptionMode, + utterance.interruptionSensitivity, + ); + } catch (error) { + await cancelVoiceConversationAssistantSpeech( + utterance.sessionId, + utterance.voiceRevision, + prepared.speechId, + ).catch(() => false); + throw error; + } + if (!started) { + utterance.nativeStartInvoked = false; + interruptActiveUtterance(false, "userSpeaking"); + return; + } + if ( + utterance.interruptionRequested || + activeUtterance?.id !== utterance.id + ) { + await streamBackend.stop(); + } }, onFailure, ); - return utterance; }; const inspectNow = () => { @@ -1446,7 +1586,8 @@ export function startNativeAssistantSpeech( if ( invalidatedMessages.has(message.id) || - causalTranscriptKey !== finalizedTranscriptKey + (causalTranscriptKey !== null && + causalTranscriptKey !== finalizedTranscriptKey) ) { suppressTarget(slot, target, content.text); continue; @@ -1479,6 +1620,8 @@ export function startNativeAssistantSpeech( targetEnd: targetStart + delta.length, }); } + if (utterance.text.trim().length === 0) continue; + queueNativeStart(utterance); queueStreamCommand( utterance, () => streamBackend.append(utterance.id, delta), @@ -1509,6 +1652,7 @@ export function startNativeAssistantSpeech( crossedToolBoundary && utterance && utteranceOwnsMessage && + utterance.nativeStartQueued && !utterance.finishing ) { queueStreamCommand( @@ -1517,6 +1661,22 @@ export function startNativeAssistantSpeech( onFailure, ); } + if ( + completed && + utterance && + utteranceOwnsMessage && + !utterance.nativeStartQueued + ) { + for (const target of utterance.targets) { + heldSpeech?.targets.delete(targetKey(target)); + } + if (heldSpeech?.targets.size === 0) { + heldSpeech = null; + heldReleaseReady = false; + } + activeUtterance = null; + continue; + } if ( completed && utterance && diff --git a/tests/contracts/voice/prepare-assistant-speech-outcomes.json b/tests/contracts/voice/prepare-assistant-speech-outcomes.json new file mode 100644 index 000000000..0b260b76d --- /dev/null +++ b/tests/contracts/voice/prepare-assistant-speech-outcomes.json @@ -0,0 +1,5 @@ +[ + { "outcome": "pending" }, + { "outcome": "notAdmitted" }, + { "outcome": "admitted", "speechId": 7 } +]