From 3c6dd7b96886d0dea56ac930af93b39d35e148e9 Mon Sep 17 00:00:00 2001 From: aaldrich <134402+aaldrich@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:35:00 +0000 Subject: [PATCH 1/2] feat(audio): filter low-level background audio before ASR --- Sources/Fluid/Persistence/BackupService.swift | 4 +- Sources/Fluid/Persistence/SettingsStore.swift | 15 ++ Sources/Fluid/Services/ASRService.swift | 177 +++++++++++++++++- Sources/Fluid/UI/SettingsSearch.swift | 7 + Sources/Fluid/UI/SettingsView.swift | 11 ++ .../DirectAudioReliabilityTests.swift | 101 ++++++++++ .../HotkeyShortcutTests.swift | 30 +++ 7 files changed, 336 insertions(+), 9 deletions(-) diff --git a/Sources/Fluid/Persistence/BackupService.swift b/Sources/Fluid/Persistence/BackupService.swift index 0dfdd165c..e4f8a104b 100644 --- a/Sources/Fluid/Persistence/BackupService.swift +++ b/Sources/Fluid/Persistence/BackupService.swift @@ -62,8 +62,10 @@ struct SettingsBackupPayload: Codable, Equatable { let experimentalParakeetUnifiedFinalEnabled: Bool? // Optional so backups created before History performance details still decode. let showHistoryPerformanceMetrics: Bool? - // Optional so backups created before the silence filter still decode. + // Optional so backups created before silent-recording detection still decode. let skipSilentRecordingsEnabled: Bool? + // Optional so backups created before low-level background filtering still decode. + let lowLevelBackgroundAudioFilterEnabled: Bool? let enableAIStreaming: Bool let copyTranscriptionToClipboard: Bool let textInsertionMode: SettingsStore.TextInsertionMode diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index cd10fcdec..2184766ec 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -1865,6 +1865,16 @@ final class SettingsStore: ObservableObject { } } + /// Filters sustained low-level background audio before streaming and final ASR. + /// Opt-in so unusually quiet speech keeps the existing behavior by default. + var lowLevelBackgroundAudioFilterEnabled: Bool { + get { self.defaults.object(forKey: Keys.lowLevelBackgroundAudioFilterEnabled) as? Bool ?? false } + set { + objectWillChange.send() + self.defaults.set(newValue, forKey: Keys.lowLevelBackgroundAudioFilterEnabled) + } + } + var enableAIStreaming: Bool { get { let value = self.defaults.object(forKey: Keys.enableAIStreaming) @@ -3299,6 +3309,7 @@ final class SettingsStore: ObservableObject { experimentalParakeetUnifiedFinalEnabled: self.experimentalParakeetUnifiedFinalEnabled, showHistoryPerformanceMetrics: self.showHistoryPerformanceMetrics, skipSilentRecordingsEnabled: self.skipSilentRecordingsEnabled, + lowLevelBackgroundAudioFilterEnabled: self.lowLevelBackgroundAudioFilterEnabled, enableAIStreaming: self.enableAIStreaming, copyTranscriptionToClipboard: self.copyTranscriptionToClipboard, textInsertionMode: self.textInsertionMode, @@ -3438,6 +3449,9 @@ final class SettingsStore: ObservableObject { if let skipSilentRecordingsEnabled = payload.skipSilentRecordingsEnabled { self.skipSilentRecordingsEnabled = skipSilentRecordingsEnabled } + if let lowLevelBackgroundAudioFilterEnabled = payload.lowLevelBackgroundAudioFilterEnabled { + self.lowLevelBackgroundAudioFilterEnabled = lowLevelBackgroundAudioFilterEnabled + } self.enableAIStreaming = payload.enableAIStreaming self.copyTranscriptionToClipboard = payload.copyTranscriptionToClipboard self.textInsertionMode = payload.textInsertionMode @@ -5450,6 +5464,7 @@ private extension SettingsStore { static let experimentalParakeetUnifiedFinalEnabled = "ExperimentalParakeetUnifiedFinalEnabled" static let showHistoryPerformanceMetrics = "ShowHistoryPerformanceMetrics" static let skipSilentRecordingsEnabled = "SkipSilentRecordingsEnabled" + static let lowLevelBackgroundAudioFilterEnabled = "LowLevelBackgroundAudioFilterEnabled" static let enableAIStreaming = "EnableAIStreaming" static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard" static let textInsertionMode = "TextInsertionMode" diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index 6d90b906c..c837356e3 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -1551,7 +1551,7 @@ final class ASRService: ObservableObject { private let audioBuffer = ThreadSafeAudioBuffer() private var lastCompletedAudioSnapshot: DictationAudioSnapshot? - // Streaming transcription state (no VAD) + // Streaming transcription state private let streamingTaskLifecycle = StreamingTaskLifecycle() private var streamingWorkState = StreamingTranscriptionWorkState() private var streamingSchedulingSessionID: Int? @@ -1560,6 +1560,7 @@ final class ASRService: ObservableObject { private var resetProviderAfterStreamingRecovery = false private var streamingHealthCheckCount: Int = 0 private var streamingHealthLastBufferCount: Int = 0 + private var streamingActivityGateEnabled: Bool = false private var lastProcessedSampleCount: Int = 0 private var isProcessingChunk: Bool = false private var skipNextChunk: Bool = false @@ -2275,11 +2276,14 @@ final class ASRService: ObservableObject { self.benchmarkCompletedStreamingChunks = 0 self.benchmarkLastChunkSampleCount = 0 (self.transcriptionProvider as? FluidAudioProvider)?.resetStreamingPreviewCache() + self.streamingActivityGateEnabled = + SettingsStore.shared.lowLevelBackgroundAudioFilterEnabled && !forDictionaryTraining self.audioCapturePipeline.setRecordingEnabled( true, sessionID: captureSessionID, attemptID: readinessAttemptID, - startHostTime: mach_absolute_time() + startHostTime: mach_absolute_time(), + activityGateEnabled: self.streamingActivityGateEnabled ) self.refreshWordBoostStatus() let dims = self.currentTranscriptionAnalyticsDimensions() @@ -2628,7 +2632,8 @@ final class ASRService: ObservableObject { true, sessionID: sessionID, attemptID: attemptID, - startHostTime: mach_absolute_time() + startHostTime: mach_absolute_time(), + activityGateEnabled: self.streamingActivityGateEnabled ) DebugLogger.shared.info( "Retrying direct audio startup in the same session after \(reason) " + @@ -4210,7 +4215,8 @@ final class ASRService: ObservableObject { true, sessionID: self.benchmarkSessionID, attemptID: readinessAttemptID, - startHostTime: mach_absolute_time() + startHostTime: mach_absolute_time(), + activityGateEnabled: self.streamingActivityGateEnabled ) do { @@ -5377,7 +5383,8 @@ final class ASRService: ObservableObject { if self.streamingHealthCheckCount >= 3 { let currentBufferCount = self.audioBuffer.count if currentBufferCount == self.streamingHealthLastBufferCount, - currentBufferCount < 16_000 + currentBufferCount < 16_000, + self.streamingActivityGateEnabled == false { DebugLogger.shared.warning( "Audio buffer not growing after three streaming intervals (count: \(currentBufferCount)). " + @@ -6072,6 +6079,127 @@ private extension ASRService { } } +// MARK: - Streaming speech activity gate + +/// Removes sustained low-level background audio before it reaches streaming ASR. +/// +/// The gate deliberately uses the same amplitude limits as the existing clear- +/// silence assessment. A short attack rejects isolated clicks, while pre-roll +/// and hangover retain complete words when speech opens or closes the gate. +struct StreamingSpeechActivityGate: Sendable { + struct Configuration: Sendable { + let frameSampleCount: Int + let rmsThreshold: Float + let peakThreshold: Float + let activationFrameCount: Int + let preRollFrameCount: Int + let hangoverFrameCount: Int + + static let dictation = Configuration( + frameSampleCount: 320, // 20 ms at FluidVoice's 16 kHz ASR rate + rmsThreshold: 0.002, + peakThreshold: 0.01, + activationFrameCount: 2, + preRollFrameCount: 8, // 160 ms + hangoverFrameCount: 18 // 360 ms + ) + } + + private let configuration: Configuration + private var pendingSamples: [Float] = [] + private var preRollFrames: [[Float]] = [] + private var consecutiveActiveFrames = 0 + private var remainingHangoverFrames = 0 + private(set) var isOpen = false + + init(configuration: Configuration = .dictation) { + precondition(configuration.frameSampleCount > 0) + precondition(configuration.activationFrameCount > 0) + precondition(configuration.preRollFrameCount >= configuration.activationFrameCount) + precondition(configuration.hangoverFrameCount >= 0) + self.configuration = configuration + self.pendingSamples.reserveCapacity(configuration.frameSampleCount * 2) + self.preRollFrames.reserveCapacity(configuration.preRollFrameCount) + } + + mutating func process(_ samples: [Float]) -> [Float] { + guard samples.isEmpty == false else { return [] } + self.pendingSamples.append(contentsOf: samples) + + var output: [Float] = [] + var consumedSampleCount = 0 + while self.pendingSamples.count - consumedSampleCount >= self.configuration.frameSampleCount { + let frameEnd = consumedSampleCount + self.configuration.frameSampleCount + let frame = Array(self.pendingSamples[consumedSampleCount.. 0 { + self.pendingSamples.removeFirst(consumedSampleCount) + } + return output + } + + /// Flushes only a partial frame that belongs to already-accepted speech. + /// Pending low-level audio and inactive pre-roll remain rejected. + mutating func finish() -> [Float] { + let output = self.isOpen ? self.pendingSamples : [] + self.reset() + return output + } + + mutating func reset() { + self.pendingSamples.removeAll(keepingCapacity: true) + self.preRollFrames.removeAll(keepingCapacity: true) + self.consecutiveActiveFrames = 0 + self.remainingHangoverFrames = 0 + self.isOpen = false + } + + private mutating func processFrame(_ frame: [Float], into output: inout [Float]) { + let isActive = self.isActive(frame) + if self.isOpen { + output.append(contentsOf: frame) + if isActive { + self.remainingHangoverFrames = self.configuration.hangoverFrameCount + } else if self.remainingHangoverFrames > 1 { + self.remainingHangoverFrames -= 1 + } else { + self.remainingHangoverFrames = 0 + self.isOpen = false + self.consecutiveActiveFrames = 0 + } + return + } + + self.preRollFrames.append(frame) + if self.preRollFrames.count > self.configuration.preRollFrameCount { + self.preRollFrames.removeFirst() + } + + self.consecutiveActiveFrames = isActive ? self.consecutiveActiveFrames + 1 : 0 + guard self.consecutiveActiveFrames >= self.configuration.activationFrameCount else { return } + + self.isOpen = true + self.remainingHangoverFrames = self.configuration.hangoverFrameCount + for bufferedFrame in self.preRollFrames { + output.append(contentsOf: bufferedFrame) + } + self.preRollFrames.removeAll(keepingCapacity: true) + } + + private func isActive(_ frame: [Float]) -> Bool { + var squareSum: Double = 0 + var peak: Float = 0 + for sample in frame { + squareSum += Double(sample) * Double(sample) + peak = max(peak, abs(sample)) + } + let rms = Float(sqrt(squareSum / Double(frame.count))) + return rms >= self.configuration.rmsThreshold || peak >= self.configuration.peakThreshold + } +} + // MARK: - Audio capture pipeline // @@ -6095,6 +6223,8 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { private var recordingAttemptID: UInt64 = 0 private var recordingStartHostTime: UInt64 = 0 private var recordingStopHostTime: UInt64? + private var activityGateEnabled: Bool = false + private var activityGate = StreamingSpeechActivityGate() private var resampleSourceRate: Double = 0 private var resampleSourceFrameCursor: Int64 = 0 private var resampleNextSourcePosition: Double = 0 @@ -6134,7 +6264,8 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { _ enabled: Bool, sessionID: Int = 0, attemptID: UInt64 = 0, - startHostTime: UInt64 = 0 + startHostTime: UInt64 = 0, + activityGateEnabled: Bool = false ) { self.lock.lock() if enabled { @@ -6143,6 +6274,8 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { self.recordingAttemptID = attemptID self.recordingStartHostTime = startHostTime == 0 ? mach_absolute_time() : startHostTime self.recordingStopHostTime = nil + self.activityGateEnabled = activityGateEnabled + self.activityGate.reset() self.resetResamplerLocked() self.lastInputSampleEnd = nil self.resetCaptureHealthLocked() @@ -6154,6 +6287,8 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { self.recordingAttemptID = 0 self.recordingStartHostTime = 0 self.recordingStopHostTime = nil + self.activityGateEnabled = false + self.activityGate.reset() self.resetResamplerLocked() self.lastInputSampleEnd = nil self.resetCaptureHealthLocked() @@ -6197,7 +6332,26 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { } func finishRecording() { - self.setRecordingEnabled(false) + self.lock.lock() + if self.recordingEnabled, self.activityGateEnabled { + let acceptedTail = self.activityGate.finish() + if acceptedTail.isEmpty == false { + self.audioBuffer.append(acceptedTail) + } + } + self.recordingEnabled = false + self.recordingSessionID = 0 + self.recordingAttemptID = 0 + self.recordingStartHostTime = 0 + self.recordingStopHostTime = nil + self.activityGateEnabled = false + self.activityGate.reset() + self.resetResamplerLocked() + self.lastInputSampleEnd = nil + self.resetCaptureHealthLocked() + self.levelHistory.removeAll(keepingCapacity: true) + self.smoothedLevel = 0.0 + self.lock.unlock() self.onLevel(0.0) } @@ -6317,10 +6471,16 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { self.firstAudioReported = true } + let acceptedForTranscription = self.activityGateEnabled + ? self.activityGate.process(mono16k) + : mono16k + // Keep append and first-audio attribution inside the capture lock. // Disabling an attempt therefore returns only after every accepted // callback has committed its PCM and queued its attempt-scoped signal. - self.audioBuffer.append(mono16k) + if acceptedForTranscription.isEmpty == false { + self.audioBuffer.append(acceptedForTranscription) + } if shouldReportFirstAudio { let acceptedHostTime = Self.hostTime( inputHostTime, @@ -6423,6 +6583,7 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { self.resampleSourceFrameCursor = 0 self.resampleNextSourcePosition = 0 self.resamplePreviousSample = nil + self.activityGate.reset() } private func resetCaptureHealthLocked() { diff --git a/Sources/Fluid/UI/SettingsSearch.swift b/Sources/Fluid/UI/SettingsSearch.swift index 83767ca77..da03153d0 100644 --- a/Sources/Fluid/UI/SettingsSearch.swift +++ b/Sources/Fluid/UI/SettingsSearch.swift @@ -35,6 +35,7 @@ enum SettingsSearchTarget: Hashable { case audioHistory case usageStreak case skipSilentRecordings + case lowLevelBackgroundAudioFilter case pauseMedia case dictionarySuggestions case accessibilityPermission @@ -93,6 +94,7 @@ enum SettingsSearchTarget: Hashable { .audioStorage, .usageStreak, .skipSilentRecordings, + .lowLevelBackgroundAudioFilter, .pauseMedia, .dictionarySuggestions, .accessibilityPermission, @@ -258,6 +260,11 @@ enum SettingsSearchIndex { title: "Skip Silent Recordings", terms: ["silence quiet speech avoid transcription"] ), + .init( + target: .lowLevelBackgroundAudioFilter, + title: "Filter Low-Level Background Audio", + terms: ["noise television TV background streaming gate speech activity faint audio"] + ), .init( target: .pauseMedia, title: "Pause Media During Transcription", diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index 49ccd8574..ea2176fc1 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -948,6 +948,17 @@ struct SettingsView: View { .settingsSearchTarget(.skipSilentRecordings) Divider().opacity(0.2) + self.optionToggleRow( + title: "Filter Low-Level Background Audio", + description: "Prevent faint background audio from reaching live or final transcription. May suppress unusually quiet speech.", + isOn: Binding( + get: { SettingsStore.shared.lowLevelBackgroundAudioFilterEnabled }, + set: { SettingsStore.shared.lowLevelBackgroundAudioFilterEnabled = $0 } + ) + ) + .settingsSearchTarget(.lowLevelBackgroundAudioFilter) + Divider().opacity(0.2) + self.optionToggleRow( title: "Pause Media During Transcription", description: "Automatically pause currently playing audio/video when transcription starts. Resumes only if FluidVoice paused it.", diff --git a/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift b/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift index 100f39c17..72b94396d 100644 --- a/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift +++ b/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift @@ -4,6 +4,107 @@ import CoreAudio import Foundation import XCTest +final class StreamingSpeechActivityGateTests: XCTestCase { + private let frameSampleCount = 320 + + func testRejectsSustainedFaintBackgroundAudio() { + var gate = StreamingSpeechActivityGate() + // A representative faint background signal remains comfortably beneath + // both production activity limits for every 20 ms frame. + let televisionFrame = (0..= 0.019 }) + + XCTAssertEqual(firstSpeechSample, 6 * self.frameSampleCount) + XCTAssertEqual(output.filter { abs($0) >= 0.019 }.count, speech.count) + XCTAssertEqual(output.count, 34 * self.frameSampleCount) + } + + func testRejectsSingleLoudTransientWithoutOpening() { + var gate = StreamingSpeechActivityGate() + let quiet = self.frames(count: 20, amplitude: 0.0005) + let click = self.frames(count: 1, amplitude: 0.02) + + let output = gate.process(quiet + click + quiet) + gate.finish() + + XCTAssertTrue(output.isEmpty) + } + + func testLongPauseIsCompressedButBothSpeechBurstsRemainComplete() { + var gate = StreamingSpeechActivityGate() + let speech = self.frames(count: 10, amplitude: 0.02) + let longPause = self.frames(count: 100, amplitude: 0.0005) + + let output = gate.process(speech + longPause + speech) + gate.finish() + + XCTAssertEqual(output.filter { abs($0) >= 0.019 }.count, speech.count * 2) + XCTAssertLessThan(output.count, speech.count * 2 + longPause.count) + } + + func testArbitraryCallbackBoundariesProduceSameResult() { + let input = + self.frames(count: 20, amplitude: 0.0005) + + self.frames(count: 12, amplitude: 0.02) + + self.frames(count: 30, amplitude: 0.0005) + var contiguousGate = StreamingSpeechActivityGate() + let contiguousOutput = contiguousGate.process(input) + contiguousGate.finish() + + var chunkedGate = StreamingSpeechActivityGate() + var chunkedOutput: [Float] = [] + var index = 0 + let chunkSizes = [137, 503, 61, 997, 211] + var chunkIndex = 0 + while index < input.count { + let end = min(index + chunkSizes[chunkIndex % chunkSizes.count], input.count) + chunkedOutput.append(contentsOf: chunkedGate.process(Array(input[index.. [Float] { + Array(repeating: amplitude, count: count * self.frameSampleCount) + } +} + final class DirectAudioReliabilityTests: XCTestCase { func testPipelineCorrelationIsInheritedAndRestoredAcrossConcurrentRequests() async { let original = DebugLogger.pipelineID diff --git a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift index 422837874..e0bb01b5f 100644 --- a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift +++ b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift @@ -266,6 +266,36 @@ final class HotkeyShortcutTests: XCTestCase { XCTAssertNil(decoded.settings.skipSilentRecordingsEnabled) } + @MainActor + func testLowLevelBackgroundFilterSettingIsIndependentAndLegacySafe() async throws { + let settingsStore = SettingsStore.shared + let originalSilentValue = settingsStore.skipSilentRecordingsEnabled + let originalFilterValue = settingsStore.lowLevelBackgroundAudioFilterEnabled + defer { + settingsStore.skipSilentRecordingsEnabled = originalSilentValue + settingsStore.lowLevelBackgroundAudioFilterEnabled = originalFilterValue + } + + settingsStore.skipSilentRecordingsEnabled = false + settingsStore.lowLevelBackgroundAudioFilterEnabled = true + XCTAssertFalse(settingsStore.skipSilentRecordingsEnabled) + XCTAssertTrue(settingsStore.lowLevelBackgroundAudioFilterEnabled) + + let document = try await BackupService.shared.makeBackupDocument() + XCTAssertEqual(document.settings.skipSilentRecordingsEnabled, false) + XCTAssertEqual(document.settings.lowLevelBackgroundAudioFilterEnabled, true) + + let encoded = try BackupService.shared.encode(document) + var root = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + var settings = try XCTUnwrap(root["settings"] as? [String: Any]) + settings.removeValue(forKey: "lowLevelBackgroundAudioFilterEnabled") + root["settings"] = settings + + let legacyData = try JSONSerialization.data(withJSONObject: root) + let decoded = try BackupService.shared.decode(legacyData) + XCTAssertNil(decoded.settings.lowLevelBackgroundAudioFilterEnabled) + } + @MainActor func testIncrementalParakeetDefaultsOnAndRoundTripsWithoutBreakingLegacyBackups() async throws { let defaults = UserDefaults.standard From 1e5293685450b10e52a22d6a55aa0e306b29e2e3 Mon Sep 17 00:00:00 2001 From: aaldrich <134402+aaldrich@users.noreply.github.com> Date: Sun, 20 Sep 2026 22:50:00 +0000 Subject: [PATCH 2/2] fix(audio): retain capture stall diagnostics with filtering --- Sources/Fluid/Services/ASRService.swift | 48 +++++++++++++++++-- .../DirectAudioReliabilityTests.swift | 32 +++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index c837356e3..40a3b37f6 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -1560,6 +1560,7 @@ final class ASRService: ObservableObject { private var resetProviderAfterStreamingRecovery = false private var streamingHealthCheckCount: Int = 0 private var streamingHealthLastBufferCount: Int = 0 + private var streamingHealthLastInputSampleCount: Int = 0 private var streamingActivityGateEnabled: Bool = false private var lastProcessedSampleCount: Int = 0 private var isProcessingChunk: Bool = false @@ -2254,6 +2255,7 @@ final class ASRService: ObservableObject { self.streamingWorkState.beginSession(self.benchmarkSessionID) self.streamingHealthCheckCount = 0 self.streamingHealthLastBufferCount = 0 + self.streamingHealthLastInputSampleCount = 0 self.silentPCMRecoveryWatchdog = AudioCaptureIdlePolicy.SilentPCMRecoveryWatchdog() let captureSessionID = self.benchmarkSessionID // Start media work alongside microphone startup; never await it on the @@ -5382,17 +5384,26 @@ final class ASRService: ObservableObject { self.streamingHealthCheckCount += 1 if self.streamingHealthCheckCount >= 3 { let currentBufferCount = self.audioBuffer.count - if currentBufferCount == self.streamingHealthLastBufferCount, - currentBufferCount < 16_000, - self.streamingActivityGateEnabled == false - { + let currentCaptureInputSampleCount = self.audioCapturePipeline.captureInputSampleCount( + sessionID: sessionID + ) + if StreamingCaptureHealthAssessment.isStalled( + currentBufferCount: currentBufferCount, + previousBufferCount: self.streamingHealthLastBufferCount, + currentCaptureInputSampleCount: currentCaptureInputSampleCount, + previousCaptureInputSampleCount: self.streamingHealthLastInputSampleCount, + activityGateEnabled: self.streamingActivityGateEnabled + ) { DebugLogger.shared.warning( - "Audio buffer not growing after three streaming intervals (count: \(currentBufferCount)). " + + "Audio capture not progressing after three streaming intervals " + + "(bufferCount: \(currentBufferCount), " + + "captureInputSamples: \(currentCaptureInputSampleCount)). " + "Audio capture may have failed. Check if engine is running and tap is installed.", source: "ASRService" ) } self.streamingHealthLastBufferCount = currentBufferCount + self.streamingHealthLastInputSampleCount = currentCaptureInputSampleCount self.streamingHealthCheckCount = 0 } @@ -6079,6 +6090,24 @@ private extension ASRService { } } +// MARK: - Streaming capture health + +struct StreamingCaptureHealthAssessment { + static func isStalled( + currentBufferCount: Int, + previousBufferCount: Int, + currentCaptureInputSampleCount: Int, + previousCaptureInputSampleCount: Int, + activityGateEnabled: Bool + ) -> Bool { + guard currentBufferCount < 16_000 else { return false } + if activityGateEnabled { + return currentCaptureInputSampleCount == previousCaptureInputSampleCount + } + return currentBufferCount == previousBufferCount + } +} + // MARK: - Streaming speech activity gate /// Removes sustained low-level background audio before it reaches streaming ASR. @@ -6355,6 +6384,15 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable { self.onLevel(0.0) } + func captureInputSampleCount(sessionID: Int) -> Int { + self.lock.lock() + defer { self.lock.unlock() } + guard self.recordingEnabled, + self.recordingSessionID == sessionID + else { return 0 } + return self.captureHealthTotalSampleCount + } + /// Compatibility for capture teardown paths. Session-scoped timestamps /// replace the old cross-session preroll buffer, so there is nothing to clear. func clearPreroll() { diff --git a/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift b/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift index 72b94396d..ee8bafade 100644 --- a/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift +++ b/Tests/FluidDictationIntegrationTests/DirectAudioReliabilityTests.swift @@ -4,6 +4,38 @@ import CoreAudio import Foundation import XCTest +final class StreamingCaptureHealthAssessmentTests: XCTestCase { + func testFilteredSilenceUsesRawCaptureProgressInsteadOfAcceptedBuffer() { + XCTAssertFalse(StreamingCaptureHealthAssessment.isStalled( + currentBufferCount: 0, + previousBufferCount: 0, + currentCaptureInputSampleCount: 16_000, + previousCaptureInputSampleCount: 0, + activityGateEnabled: true + )) + } + + func testFilteredSessionStillDetectsStoppedRawCapture() { + XCTAssertTrue(StreamingCaptureHealthAssessment.isStalled( + currentBufferCount: 0, + previousBufferCount: 0, + currentCaptureInputSampleCount: 16_000, + previousCaptureInputSampleCount: 16_000, + activityGateEnabled: true + )) + } + + func testUnfilteredSessionRetainsAcceptedBufferDiagnostic() { + XCTAssertTrue(StreamingCaptureHealthAssessment.isStalled( + currentBufferCount: 2_000, + previousBufferCount: 2_000, + currentCaptureInputSampleCount: 4_000, + previousCaptureInputSampleCount: 2_000, + activityGateEnabled: false + )) + } +} + final class StreamingSpeechActivityGateTests: XCTestCase { private let frameSampleCount = 320