Skip to content

Filter low-level background audio before streaming ASR #993

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aaldrich wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Filter low-level background audio before streaming ASR #993

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3c6dd7b
feat(audio): filter low-level background audio before ASR
aaldrich Sep 20, 2026
1e52936
fix(audio): retain capture stall diagnostics with filtering
aaldrich Sep 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion Sources/Fluid/Persistence/BackupService.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,10 @@ struct SettingsBackupPayload: Codable, Equatable {
let experimentalParakeetUnifiedFinalEnabled: Bool?
// Optional so backups created before History performance details still decode.
let showHistoryPerformanceMetrics: Bool?
// Optional so backups created before the silence filter still decode.
// Optional so backups created before silent-recording detection still decode.
let skipSilentRecordingsEnabled: Bool?
// Optional so backups created before low-level background filtering still decode.
let lowLevelBackgroundAudioFilterEnabled: Bool?
let enableAIStreaming: Bool
let copyTranscriptionToClipboard: Bool
let textInsertionMode: SettingsStore.TextInsertionMode
Expand Down
15 changes: 15 additions & 0 deletions Sources/Fluid/Persistence/SettingsStore.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1865,6 +1865,16 @@ final class SettingsStore: ObservableObject {
}
}

/// Filters sustained low-level background audio before streaming and final ASR.
/// Opt-in so unusually quiet speech keeps the existing behavior by default.
var lowLevelBackgroundAudioFilterEnabled: Bool {
get { self.defaults.object(forKey: Keys.lowLevelBackgroundAudioFilterEnabled) as? Bool ?? false }
set {
objectWillChange.send()
self.defaults.set(newValue, forKey: Keys.lowLevelBackgroundAudioFilterEnabled)
}
}

var enableAIStreaming: Bool {
get {
let value = self.defaults.object(forKey: Keys.enableAIStreaming)
Expand Down Expand Up @@ -3299,6 +3309,7 @@ final class SettingsStore: ObservableObject {
experimentalParakeetUnifiedFinalEnabled: self.experimentalParakeetUnifiedFinalEnabled,
showHistoryPerformanceMetrics: self.showHistoryPerformanceMetrics,
skipSilentRecordingsEnabled: self.skipSilentRecordingsEnabled,
lowLevelBackgroundAudioFilterEnabled: self.lowLevelBackgroundAudioFilterEnabled,
enableAIStreaming: self.enableAIStreaming,
copyTranscriptionToClipboard: self.copyTranscriptionToClipboard,
textInsertionMode: self.textInsertionMode,
Expand Down Expand Up @@ -3438,6 +3449,9 @@ final class SettingsStore: ObservableObject {
if let skipSilentRecordingsEnabled = payload.skipSilentRecordingsEnabled {
self.skipSilentRecordingsEnabled = skipSilentRecordingsEnabled
}
if let lowLevelBackgroundAudioFilterEnabled = payload.lowLevelBackgroundAudioFilterEnabled {
self.lowLevelBackgroundAudioFilterEnabled = lowLevelBackgroundAudioFilterEnabled
}
self.enableAIStreaming = payload.enableAIStreaming
self.copyTranscriptionToClipboard = payload.copyTranscriptionToClipboard
self.textInsertionMode = payload.textInsertionMode
Expand Down Expand Up @@ -5450,6 +5464,7 @@ private extension SettingsStore {
static let experimentalParakeetUnifiedFinalEnabled = "ExperimentalParakeetUnifiedFinalEnabled"
static let showHistoryPerformanceMetrics = "ShowHistoryPerformanceMetrics"
static let skipSilentRecordingsEnabled = "SkipSilentRecordingsEnabled"
static let lowLevelBackgroundAudioFilterEnabled = "LowLevelBackgroundAudioFilterEnabled"
static let enableAIStreaming = "EnableAIStreaming"
static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard"
static let textInsertionMode = "TextInsertionMode"
Expand Down
221 changes: 210 additions & 11 deletions Sources/Fluid/Services/ASRService.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1551,7 +1551,7 @@ final class ASRService: ObservableObject {
private let audioBuffer = ThreadSafeAudioBuffer()
private var lastCompletedAudioSnapshot: DictationAudioSnapshot?

// Streaming transcription state (no VAD)
// Streaming transcription state
private let streamingTaskLifecycle = StreamingTaskLifecycle()
private var streamingWorkState = StreamingTranscriptionWorkState()
private var streamingSchedulingSessionID: Int?
Expand All @@ -1560,6 +1560,8 @@ final class ASRService: ObservableObject {
private var resetProviderAfterStreamingRecovery = false
private var streamingHealthCheckCount: Int = 0
private var streamingHealthLastBufferCount: Int = 0
private var streamingHealthLastInputSampleCount: Int = 0
private var streamingActivityGateEnabled: Bool = false
private var lastProcessedSampleCount: Int = 0
private var isProcessingChunk: Bool = false
private var skipNextChunk: Bool = false
Expand Down Expand Up @@ -2253,6 +2255,7 @@ final class ASRService: ObservableObject {
self.streamingWorkState.beginSession(self.benchmarkSessionID)
self.streamingHealthCheckCount = 0
self.streamingHealthLastBufferCount = 0
self.streamingHealthLastInputSampleCount = 0
self.silentPCMRecoveryWatchdog = AudioCaptureIdlePolicy.SilentPCMRecoveryWatchdog()
let captureSessionID = self.benchmarkSessionID
// Start media work alongside microphone startup; never await it on the
Expand All @@ -2275,11 +2278,14 @@ final class ASRService: ObservableObject {
self.benchmarkCompletedStreamingChunks = 0
self.benchmarkLastChunkSampleCount = 0
(self.transcriptionProvider as? FluidAudioProvider)?.resetStreamingPreviewCache()
self.streamingActivityGateEnabled =
SettingsStore.shared.lowLevelBackgroundAudioFilterEnabled && !forDictionaryTraining
self.audioCapturePipeline.setRecordingEnabled(
true,
sessionID: captureSessionID,
attemptID: readinessAttemptID,
startHostTime: mach_absolute_time()
startHostTime: mach_absolute_time(),
activityGateEnabled: self.streamingActivityGateEnabled
)
self.refreshWordBoostStatus()
let dims = self.currentTranscriptionAnalyticsDimensions()
Expand Down Expand Up @@ -2628,7 +2634,8 @@ final class ASRService: ObservableObject {
true,
sessionID: sessionID,
attemptID: attemptID,
startHostTime: mach_absolute_time()
startHostTime: mach_absolute_time(),
activityGateEnabled: self.streamingActivityGateEnabled
)
DebugLogger.shared.info(
"Retrying direct audio startup in the same session after \(reason) " +
Expand Down Expand Up @@ -4210,7 +4217,8 @@ final class ASRService: ObservableObject {
true,
sessionID: self.benchmarkSessionID,
attemptID: readinessAttemptID,
startHostTime: mach_absolute_time()
startHostTime: mach_absolute_time(),
activityGateEnabled: self.streamingActivityGateEnabled
)

do {
Expand Down Expand Up @@ -5376,16 +5384,26 @@ final class ASRService: ObservableObject {
self.streamingHealthCheckCount += 1
if self.streamingHealthCheckCount >= 3 {
let currentBufferCount = self.audioBuffer.count
if currentBufferCount == self.streamingHealthLastBufferCount,
currentBufferCount < 16_000
{
let currentCaptureInputSampleCount = self.audioCapturePipeline.captureInputSampleCount(
sessionID: sessionID
)
if StreamingCaptureHealthAssessment.isStalled(
currentBufferCount: currentBufferCount,
previousBufferCount: self.streamingHealthLastBufferCount,
currentCaptureInputSampleCount: currentCaptureInputSampleCount,
previousCaptureInputSampleCount: self.streamingHealthLastInputSampleCount,
activityGateEnabled: self.streamingActivityGateEnabled
) {
DebugLogger.shared.warning(
"Audio buffer not growing after three streaming intervals (count: \(currentBufferCount)). " +
"Audio capture not progressing after three streaming intervals " +
"(bufferCount: \(currentBufferCount), " +
"captureInputSamples: \(currentCaptureInputSampleCount)). " +
"Audio capture may have failed. Check if engine is running and tap is installed.",
source: "ASRService"
)
}
self.streamingHealthLastBufferCount = currentBufferCount
self.streamingHealthLastInputSampleCount = currentCaptureInputSampleCount
self.streamingHealthCheckCount = 0
}

Expand Down Expand Up @@ -6072,6 +6090,145 @@ private extension ASRService {
}
}

// MARK: - Streaming capture health

struct StreamingCaptureHealthAssessment {
static func isStalled(
currentBufferCount: Int,
previousBufferCount: Int,
currentCaptureInputSampleCount: Int,
previousCaptureInputSampleCount: Int,
activityGateEnabled: Bool
) -> Bool {
guard currentBufferCount < 16_000 else { return false }
if activityGateEnabled {
return currentCaptureInputSampleCount == previousCaptureInputSampleCount
}
return currentBufferCount == previousBufferCount
}
}

// MARK: - Streaming speech activity gate

/// Removes sustained low-level background audio before it reaches streaming ASR.
///
/// The gate deliberately uses the same amplitude limits as the existing clear-
/// silence assessment. A short attack rejects isolated clicks, while pre-roll
/// and hangover retain complete words when speech opens or closes the gate.
struct StreamingSpeechActivityGate: Sendable {
struct Configuration: Sendable {
let frameSampleCount: Int
let rmsThreshold: Float
let peakThreshold: Float
let activationFrameCount: Int
let preRollFrameCount: Int
let hangoverFrameCount: Int

static let dictation = Configuration(
frameSampleCount: 320, // 20 ms at FluidVoice's 16 kHz ASR rate
rmsThreshold: 0.002,
peakThreshold: 0.01,
activationFrameCount: 2,
preRollFrameCount: 8, // 160 ms
hangoverFrameCount: 18 // 360 ms
)
}

private let configuration: Configuration
private var pendingSamples: [Float] = []
private var preRollFrames: [[Float]] = []
private var consecutiveActiveFrames = 0
private var remainingHangoverFrames = 0
private(set) var isOpen = false

init(configuration: Configuration = .dictation) {
precondition(configuration.frameSampleCount > 0)
precondition(configuration.activationFrameCount > 0)
precondition(configuration.preRollFrameCount >= configuration.activationFrameCount)
precondition(configuration.hangoverFrameCount >= 0)
self.configuration = configuration
self.pendingSamples.reserveCapacity(configuration.frameSampleCount * 2)
self.preRollFrames.reserveCapacity(configuration.preRollFrameCount)
}

mutating func process(_ samples: [Float]) -> [Float] {
guard samples.isEmpty == false else { return [] }
self.pendingSamples.append(contentsOf: samples)

var output: [Float] = []
var consumedSampleCount = 0
while self.pendingSamples.count - consumedSampleCount >= self.configuration.frameSampleCount {
let frameEnd = consumedSampleCount + self.configuration.frameSampleCount
let frame = Array(self.pendingSamples[consumedSampleCount..<frameEnd])
consumedSampleCount = frameEnd
self.processFrame(frame, into: &output)
}
if consumedSampleCount > 0 {
self.pendingSamples.removeFirst(consumedSampleCount)
}
return output
}

/// Flushes only a partial frame that belongs to already-accepted speech.
/// Pending low-level audio and inactive pre-roll remain rejected.
mutating func finish() -> [Float] {
let output = self.isOpen ? self.pendingSamples : []
self.reset()
return output
}

mutating func reset() {
self.pendingSamples.removeAll(keepingCapacity: true)
self.preRollFrames.removeAll(keepingCapacity: true)
self.consecutiveActiveFrames = 0
self.remainingHangoverFrames = 0
self.isOpen = false
}

private mutating func processFrame(_ frame: [Float], into output: inout [Float]) {
let isActive = self.isActive(frame)
if self.isOpen {
output.append(contentsOf: frame)
if isActive {
self.remainingHangoverFrames = self.configuration.hangoverFrameCount
} else if self.remainingHangoverFrames > 1 {
self.remainingHangoverFrames -= 1
} else {
self.remainingHangoverFrames = 0
self.isOpen = false
self.consecutiveActiveFrames = 0
}
return
}

self.preRollFrames.append(frame)
if self.preRollFrames.count > self.configuration.preRollFrameCount {
self.preRollFrames.removeFirst()
}

self.consecutiveActiveFrames = isActive ? self.consecutiveActiveFrames + 1 : 0
guard self.consecutiveActiveFrames >= self.configuration.activationFrameCount else { return }

self.isOpen = true
self.remainingHangoverFrames = self.configuration.hangoverFrameCount
for bufferedFrame in self.preRollFrames {
output.append(contentsOf: bufferedFrame)
}
self.preRollFrames.removeAll(keepingCapacity: true)
}

private func isActive(_ frame: [Float]) -> Bool {
var squareSum: Double = 0
var peak: Float = 0
for sample in frame {
squareSum += Double(sample) * Double(sample)
peak = max(peak, abs(sample))
}
let rms = Float(sqrt(squareSum / Double(frame.count)))
return rms >= self.configuration.rmsThreshold || peak >= self.configuration.peakThreshold
}
}

// MARK: - Audio capture pipeline

//
Expand All @@ -6095,6 +6252,8 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable {
private var recordingAttemptID: UInt64 = 0
private var recordingStartHostTime: UInt64 = 0
private var recordingStopHostTime: UInt64?
private var activityGateEnabled: Bool = false
private var activityGate = StreamingSpeechActivityGate()
private var resampleSourceRate: Double = 0
private var resampleSourceFrameCursor: Int64 = 0
private var resampleNextSourcePosition: Double = 0
Expand Down Expand Up @@ -6134,7 +6293,8 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable {
_ enabled: Bool,
sessionID: Int = 0,
attemptID: UInt64 = 0,
startHostTime: UInt64 = 0
startHostTime: UInt64 = 0,
activityGateEnabled: Bool = false
) {
self.lock.lock()
if enabled {
Expand All @@ -6143,6 +6303,8 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable {
self.recordingAttemptID = attemptID
self.recordingStartHostTime = startHostTime == 0 ? mach_absolute_time() : startHostTime
self.recordingStopHostTime = nil
self.activityGateEnabled = activityGateEnabled
self.activityGate.reset()
self.resetResamplerLocked()
self.lastInputSampleEnd = nil
self.resetCaptureHealthLocked()
Expand All @@ -6154,6 +6316,8 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable {
self.recordingAttemptID = 0
self.recordingStartHostTime = 0
self.recordingStopHostTime = nil
self.activityGateEnabled = false
self.activityGate.reset()
self.resetResamplerLocked()
self.lastInputSampleEnd = nil
self.resetCaptureHealthLocked()
Expand Down Expand Up @@ -6197,10 +6361,38 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable {
}

func finishRecording() {
self.setRecordingEnabled(false)
self.lock.lock()
if self.recordingEnabled, self.activityGateEnabled {
let acceptedTail = self.activityGate.finish()
if acceptedTail.isEmpty == false {
self.audioBuffer.append(acceptedTail)
}
}
self.recordingEnabled = false
self.recordingSessionID = 0
self.recordingAttemptID = 0
self.recordingStartHostTime = 0
self.recordingStopHostTime = nil
self.activityGateEnabled = false
self.activityGate.reset()
self.resetResamplerLocked()
self.lastInputSampleEnd = nil
self.resetCaptureHealthLocked()
self.levelHistory.removeAll(keepingCapacity: true)
self.smoothedLevel = 0.0
self.lock.unlock()
self.onLevel(0.0)
}

func captureInputSampleCount(sessionID: Int) -> Int {
self.lock.lock()
defer { self.lock.unlock() }
guard self.recordingEnabled,
self.recordingSessionID == sessionID
else { return 0 }
return self.captureHealthTotalSampleCount
}

/// Compatibility for capture teardown paths. Session-scoped timestamps
/// replace the old cross-session preroll buffer, so there is nothing to clear.
func clearPreroll() {
Expand Down Expand Up @@ -6317,10 +6509,16 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable {
self.firstAudioReported = true
}

let acceptedForTranscription = self.activityGateEnabled
? self.activityGate.process(mono16k)
: mono16k

// Keep append and first-audio attribution inside the capture lock.
// Disabling an attempt therefore returns only after every accepted
// callback has committed its PCM and queued its attempt-scoped signal.
self.audioBuffer.append(mono16k)
if acceptedForTranscription.isEmpty == false {
self.audioBuffer.append(acceptedForTranscription)
}
if shouldReportFirstAudio {
let acceptedHostTime = Self.hostTime(
inputHostTime,
Expand Down Expand Up @@ -6423,6 +6621,7 @@ private final nonisolated class AudioCapturePipeline: @unchecked Sendable {
self.resampleSourceFrameCursor = 0
self.resampleNextSourcePosition = 0
self.resamplePreviousSample = nil
self.activityGate.reset()
}

private func resetCaptureHealthLocked() {
Expand Down
Loading
Loading