diff --git a/Documentation/TTS/LuxTts.md b/Documentation/TTS/LuxTts.md index 6832868f7..4d08c1d13 100644 --- a/Documentation/TTS/LuxTts.md +++ b/Documentation/TTS/LuxTts.md @@ -71,8 +71,41 @@ The ANE path is ~0.5 dB softer, not bit-identical, but the Parakeet round-trip transcript matches the input text exactly. The much smaller jetsam-visible footprint is why iOS uses it. -All shapes are fixed: ≤ 255 tokens (+1 pad slot), ≤ 1024 mel frames -total, ≤ 555 generated frames (~5.9 s per call; chunking is phase 2). +All graph shapes are fixed: ≤ 255 tokens (+1 pad slot), ≤ 1024 mel frames +total, ≤ 555 generated frames (~5.9 s) per flow-matching pass. + +Text that fits one pass (≤ 102 target tokens, token bucket, 1024 frames, +555-frame vocoder bucket) is rendered in one pass; the 102 cap is measured, +not a graph limit — see `LuxTtsConstants.continuationSpanTokens`. Longer text is handled by `LuxTtsManager` +with continuation prompting rather than a bigger graph: target tokens are +split into balanced spans of ≤ 102 tokens at word/punctuation boundaries, +each span is generated with the previous span's audio and tokens as its +prompt, the vocoder's onset/tail padding is trimmed around sustained +speech (a span ending in punctuation keeps its tail), and spans are joined +with a 30 ms crossfade. Spans are also bounded by the prompt's +frames-per-token ratio so none generates more than ~5 s (468 frames), +which keeps every span short enough to prompt the next one untruncated; +a prompt whose ratio exceeds 12 frames per token (transcript covering only +part of the clip) or would force spans under 8 tokens is rejected with +`inputTooLong`. `speed` is applied to the first span only; later spans +inherit the rate from their prompt. For multi-span output +`generatedFrames`/`featuresLength` are sums over spans and `samples.count` +is shorter than `(generatedFrames − 1) × 512`. + +The model also drops the occasional spurious mid-phrase pause (issue #937): +where it lands depends on the exact length/noise draw — the PyTorch +reference does it too, and a small `speed` change moves or removes it — +so longer text simply has more chances to draw one. Every pass is scanned +for gaps ≥ 80 ms between runs of sustained speech (≤ −45 dB relative to +peak; natural stop closures stay ≤ 60 ms); when they outnumber the span's +pause punctuation (`, . ; : ! ? -`, plus ellipses and quotes/brackets the +G2P renders silently when going through the text API) the span is re-drawn +with the next seed, up to `maxRedraws` times (default 3), keeping the +cleanest pass. Re-draws keep the duration estimate: compressing it (speed +× 1.03–1.06) removed some pauses but squeezed the final word into the last +frames. This applies to single-pass text too, so a given seed is +only bit-reproducible with `maxRedraws: 0` (CLI `--redraws 0`); the result +reports `redraws` and `residualPauses`. Each re-draw is one extra pass. ## Quick Start @@ -204,8 +237,9 @@ corpus-level gate is scored with `luxtts-g2p-dump` + `validate.py score` ## Remaining TODOs -- Long-input chunking across multiple vocoder windows (> 555 generated - frames currently errors; mel truncation is not allowed). +- Continuation spans are bounded by the *first* prompt's frames-per-token + ratio; a prompt whose pace differs a lot from the model's own pace can + still push a later span past the 555-frame vocoder bucket and error. - Optional VAD-based automatic prompt-silence trimming. - Non-English text (the G2P is `en-us` only; Mandarin pinyin tokens exist in `tokens.txt` but have no frontend). diff --git a/Sources/FluidAudio/TTS/LuxTts/LuxTtsConstants.swift b/Sources/FluidAudio/TTS/LuxTts/LuxTtsConstants.swift index 6a7aa6664..615942b4e 100644 --- a/Sources/FluidAudio/TTS/LuxTts/LuxTtsConstants.swift +++ b/Sources/FluidAudio/TTS/LuxTts/LuxTtsConstants.swift @@ -45,6 +45,8 @@ public enum LuxTtsConstants { /// Prompt duration cap in seconds. Frames beyond this would eat too much /// of the 1024-frame bucket (~10.9 s total at 93.75 frames/s). public static let maxPromptSeconds: Double = 5.0 + /// `maxPromptSeconds` in 24 kHz samples. + public static let maxPromptSamples = Int(maxPromptSeconds * Double(melSampleRate)) /// Published fixed-shape vocoder buckets (generated frames). public static let vocoderBuckets = [282, 555] @@ -54,4 +56,46 @@ public enum LuxTtsConstants { /// Default synthesis noise seed (matches the Python reference scripts). public static let defaultSeed: UInt64 = 42 + + /// Largest target-token span sent through one flow-matching pass; longer + /// text is continuation-prompted in balanced spans of at most this size. + /// Measured on the issue #937 text (106 tokens, six seeds): as one pass + /// it paused on 6 of 8 raw draws and the re-draw ladder still failed one + /// seed; as two ~53-token spans every seed was clean with two re-draws + /// in total. Shorter passes draw pauses far less often. + public static let continuationSpanTokens = 102 + /// Smallest span worth rendering. A prompt whose frames-per-token ratio + /// forces spans below this (silence-heavy clip, transcript that does not + /// match) is rejected instead of turning into dozens of passes. + public static let minimumSpanTokens = 8 + /// Highest plausible prompt frames-per-token ratio for continuation + /// synthesis. Natural speech sits around 4–6 (93.75 frames/s); a ratio + /// beyond this means the transcript covers only part of the clip (or + /// the clip is mostly silence), and every span would inherit the error. + public static let maxPromptFramesPerToken = 12.0 + + /// Re-seed attempts for a pass whose mid-speech silences outnumber the + /// span's pause punctuation. The model drops such pauses stochastically + /// (position depends on the exact length/noise draw; the PyTorch + /// reference does the same), so a fresh seed is the fix. One full pass + /// each. + public static let spuriousPauseRetries = 3 + /// Silence floor (dB relative to the pass's peak) below which audio + /// counts as padding or pause, and the minimum gap between sustained + /// speech that counts as a pause. Natural stop closures stay ≤ 60 ms at + /// this floor; the reported pauses measure 100–160 ms. + public static let pauseFloorDb: Float = -45 + /// Minimum gap between sustained-speech runs that counts as a pause. + public static let pauseMinimumSeconds = 0.08 + /// Seed stride between continuation spans; leaves room for re-seeds. + public static let continuationSeedStride: UInt64 = 64 + + /// Generated-frame budget per continuation span. A span must fit inside + /// `maxPromptSeconds` so its untruncated audio can prompt the next span + /// with a transcript that still matches it. + public static let continuationSpanFrameBudget = + Int(maxPromptSeconds * Double(melSampleRate)) / hopLength + + /// Overlap used when joining continuation-prompted spans. + public static let continuationCrossfadeSeconds = 0.03 } diff --git a/Sources/FluidAudio/TTS/LuxTts/LuxTtsContinuation.swift b/Sources/FluidAudio/TTS/LuxTts/LuxTtsContinuation.swift new file mode 100644 index 000000000..a0551d8f6 --- /dev/null +++ b/Sources/FluidAudio/TTS/LuxTts/LuxTtsContinuation.swift @@ -0,0 +1,330 @@ +import Accelerate +import Foundation + +/// Host-side long-utterance helpers for LuxTTS. +enum LuxTtsContinuation { + + private static let analysisWindowsPerSecond = 100 + /// Activity shorter than this is a click or breath, not speech. Every + /// vocoder pass opens with a ~20 ms transient followed by 150–300 ms of + /// silence; anchoring on sustained speech keeps both trim and pause + /// detection from latching onto it. + private static let sustainedSpeechSeconds = 0.05 + private static let onsetPrerollSeconds = 0.03 + private static let tailPostrollSeconds = 0.03 + + /// Runs of sustained speech, in `windowSize`-sample analysis windows. + struct SpeechRuns { + let runs: [Range] + let windowSize: Int + } + + /// Whether the whole text can go through one pass unchanged: at most + /// `continuationSpanTokens`, and within the token bucket, the 1024-frame + /// graph, and the largest vocoder bucket. + static func fitsSinglePass( + textTokenCount: Int, promptFrames: Int, promptTokenCount: Int, speed: Double + ) -> Bool { + guard textTokenCount <= LuxTtsConstants.continuationSpanTokens, + promptTokenCount + textTokenCount + 1 <= LuxTtsConstants.maxTokens + else { return false } + guard promptFrames > 0, promptTokenCount > 0, speed > 0 else { return true } + let featuresLength = LuxTtsSolver.featuresLength( + promptFrames: promptFrames, + promptTokenCount: promptTokenCount, + textTokenCount: textTokenCount, + speed: speed) + return featuresLength <= LuxTtsConstants.maxFrames + && featuresLength - promptFrames <= (LuxTtsConstants.vocoderBuckets.max() ?? 0) + } + + /// Largest span (in target tokens) that fits the token bucket next to + /// this prompt's transcript and generates at most + /// `continuationSpanFrameBudget` frames for its frames-per-token ratio + /// at `speed`, capped at `continuationSpanTokens`. + static func maxSpanTokens(promptFrames: Int, promptTokenCount: Int, speed: Double) -> Int { + let cap = min( + LuxTtsConstants.continuationSpanTokens, + LuxTtsConstants.maxTokens - 1 - promptTokenCount) + guard promptFrames > 0, promptTokenCount > 0, speed > 0 else { return max(1, cap) } + let framesPerToken = Double(promptFrames) / Double(promptTokenCount) + let budgetTokens = Int( + (Double(LuxTtsConstants.continuationSpanFrameBudget) * speed / framesPerToken) + .rounded(.down)) + return max(1, min(cap, budgetTokens)) + } + + /// Split a token sequence into balanced spans, preferring word/pause + /// boundaries nearest each ideal split point. + static func chunks( + tokenIds: [Int], maxTokens: Int, boundaryTokenIds: Set + ) -> [[Int]] { + precondition(maxTokens > 0, "maxTokens must be positive") + guard tokenIds.count > maxTokens else { return tokenIds.isEmpty ? [] : [tokenIds] } + + let chunkCount = (tokenIds.count + maxTokens - 1) / maxTokens + var chunks: [[Int]] = [] + chunks.reserveCapacity(chunkCount) + + var start = 0 + for chunkIndex in 0..<(chunkCount - 1) { + let remainingChunks = chunkCount - chunkIndex + let remainingTokens = tokenIds.count - start + let idealLength = Int( + (Double(remainingTokens) / Double(remainingChunks)).rounded()) + let idealEnd = start + idealLength + let minimumEnd = max( + start + 1, tokenIds.count - (remainingChunks - 1) * maxTokens) + let maxEnd = min(start + maxTokens, tokenIds.count - (remainingChunks - 1)) + let end = nearestBoundaryEnd( + in: tokenIds, + minimumEnd: minimumEnd, + idealEnd: idealEnd, + maxEnd: maxEnd, + searchRadius: max(1, maxTokens / 4), + boundaryTokenIds: boundaryTokenIds) + + chunks.append(Array(tokenIds[start.., boundaryTokenIds: Set + ) -> Int { + var end = tokenIds.count + while end > 0, boundaryTokenIds.contains(tokenIds[end - 1]) { end -= 1 } + return tokenIds[.. Int { + textPausePositions(text).count + } + + /// Relative positions (0..<1, by character) of the silent breaks + /// `textPauseAllowance` counts, so each span is only allowed the breaks + /// that fall inside it. + static func textPausePositions(_ text: String) -> [Double] { + let characters = Array(text) + guard !characters.isEmpty else { return [] } + var positions: [Double] = [] + var index = 0 + while index < characters.count { + let character = characters[index] + if character == ".", index + 2 < characters.count, + characters[index + 1] == ".", characters[index + 2] == "." + { + positions.append(Double(index) / Double(characters.count)) + index += 3 + continue + } + if "…\"()[]«»".contains(character) { + positions.append(Double(index) / Double(characters.count)) + } + index += 1 + } + return positions + } + + /// Silent breaks each span may contain: positions are mapped onto spans + /// by their share of the token sequence. + static func spanPauseAllowances(spanLengths: [Int], positions: [Double]) -> [Int] { + let total = spanLengths.reduce(0, +) + guard total > 0 else { return spanLengths.map { _ in 0 } } + var allowances = [Int](repeating: 0, count: spanLengths.count) + for position in positions { + let token = Int(position * Double(total)) + var end = 0 + for (index, length) in spanLengths.enumerated() { + end += length + if token < end || index == spanLengths.count - 1 { + allowances[index] += 1 + break + } + } + } + return allowances + } + + /// Zero-pad or truncate 24 kHz audio so the mel extractor counts exactly + /// `frames` frames. + static func fitPromptLength(_ audio: inout [Float], frames: Int) { + let target = max(0, frames) * LuxTtsConstants.hopLength + if audio.count > target { + audio.removeLast(audio.count - target) + } else { + audio.append(contentsOf: repeatElement(0, count: target - audio.count)) + } + } + + /// Whether the span's last spoken token is pause punctuation (trailing + /// spaces ignored). + static func endsWithPausePunctuation( + _ tokenIds: [Int], pauseTokenIds: Set, boundaryTokenIds: Set + ) -> Bool { + let spaceTokenIds = boundaryTokenIds.subtracting(pauseTokenIds) + guard let last = tokenIds.last(where: { !spaceTokenIds.contains($0) }) else { return false } + return pauseTokenIds.contains(last) + } + + /// Runs of at least `sustainedSpeechSeconds` above `pauseFloorDb` + /// relative to the clip's peak. + static func speechRuns(_ samples: [Float], sampleRate: Int) -> SpeechRuns { + guard sampleRate > 0, !samples.isEmpty else { return SpeechRuns(runs: [], windowSize: 1) } + let windowSize = max(1, sampleRate / analysisWindowsPerSecond) + let windowCount = samples.count / windowSize + guard windowCount > 0 else { return SpeechRuns(runs: [], windowSize: windowSize) } + + var peak: Float = 0 + vDSP_maxmgv(samples, 1, &peak, vDSP_Length(samples.count)) + guard peak > 0 else { return SpeechRuns(runs: [], windowSize: windowSize) } + let floorMeanSquare = peak * peak * powf(10, LuxTtsConstants.pauseFloorDb / 10) + let minimumWindows = max( + 1, Int((sustainedSpeechSeconds * Double(sampleRate) / Double(windowSize)).rounded(.up))) + + var runs: [Range] = [] + var runStart: Int? + samples.withUnsafeBufferPointer { buffer in + guard let base = buffer.baseAddress else { return } + for window in 0...windowCount { + var active = false + if window < windowCount { + var meanSquare: Float = 0 + vDSP_measqv(base + window * windowSize, 1, &meanSquare, vDSP_Length(windowSize)) + active = meanSquare > floorMeanSquare + } + if active { + if runStart == nil { runStart = window } + } else if let start = runStart { + if window - start >= minimumWindows { runs.append(start.. Int { + guard speech.runs.count > 1 else { return 0 } + let minimumWindows = max( + 1, + Int( + (LuxTtsConstants.pauseMinimumSeconds * Double(sampleRate) / Double(speech.windowSize)) + .rounded(.up))) + return zip(speech.runs, speech.runs.dropFirst()).reduce(0) { count, pair in + count + (pair.1.lowerBound - pair.0.upperBound >= minimumWindows ? 1 : 0) + } + } + + static func innerPauseCount(_ samples: [Float], sampleRate: Int) -> Int { + innerPauseCount(speechRuns(samples, sampleRate: sampleRate), sampleRate: sampleRate) + } + + /// Sample range to keep from a span: the vocoder's onset padding is cut + /// (leaving a short preroll so unvoiced consonants are not clipped) and + /// the tail padding likewise (leaving a short postroll). Clips without + /// sustained speech are kept whole. + static func speechSlice( + _ speech: SpeechRuns, sampleCount: Int, sampleRate: Int, trimLeading: Bool, trimTrailing: Bool + ) -> Range { + guard let first = speech.runs.first, let last = speech.runs.last else { return 0.. [Float] { + let slice = speechSlice( + speechRuns(samples, sampleRate: sampleRate), sampleCount: samples.count, + sampleRate: sampleRate, trimLeading: true, trimTrailing: false) + return Array(samples[slice]) + } + + static func trimmingTrailingPadding(_ samples: [Float], sampleRate: Int) -> [Float] { + let slice = speechSlice( + speechRuns(samples, sampleRate: sampleRate), sampleCount: samples.count, + sampleRate: sampleRate, trimLeading: false, trimTrailing: true) + return Array(samples[slice]) + } + + /// Join two mono clips with a linear crossfade. + static func appendWithCrossfade( + _ next: ArraySlice, to output: inout [Float], crossfadeSamples: Int + ) { + guard !output.isEmpty else { + output = Array(next) + return + } + guard !next.isEmpty else { return } + + let overlap = min(crossfadeSamples, output.count, next.count) + guard overlap > 0 else { + output.append(contentsOf: next) + return + } + + let outputStart = output.count - overlap + let nextStart = next.startIndex + if overlap == 1 { + output[outputStart] = (output[outputStart] + next[nextStart]) * 0.5 + } else { + for index in 0.. + ) -> Int { + var bestEnd: Int? + var bestDistance = Int.max + let searchStart = max(minimumEnd, idealEnd - searchRadius) + let searchEnd = min(maxEnd, idealEnd + searchRadius) + guard searchStart <= searchEnd else { + return min(max(idealEnd, minimumEnd), maxEnd) + } + + for end in searchStart...searchEnd { + guard boundaryTokenIds.contains(tokenIds[end - 1]) else { continue } + let distance = abs(end - idealEnd) + if distance < bestDistance { + bestEnd = end + bestDistance = distance + } + } + return bestEnd ?? min(max(idealEnd, minimumEnd), maxEnd) + } +} diff --git a/Sources/FluidAudio/TTS/LuxTts/LuxTtsManager.swift b/Sources/FluidAudio/TTS/LuxTts/LuxTtsManager.swift index 786d53bc1..888271f67 100644 --- a/Sources/FluidAudio/TTS/LuxTts/LuxTtsManager.swift +++ b/Sources/FluidAudio/TTS/LuxTts/LuxTtsManager.swift @@ -99,18 +99,22 @@ public actor LuxTtsManager { promptAudio: URL, promptText: String, speed: Float = LuxTtsConstants.defaultSpeed, - seed: UInt64 = LuxTtsConstants.defaultSeed + seed: UInt64 = LuxTtsConstants.defaultSeed, + maxRedraws: Int = LuxTtsConstants.spuriousPauseRetries ) async throws -> LuxTtsSynthesisResult { // Fail fast before the (potentially expensive) G2P lexicon load and - // phonemization; the phonemes path guards on the same store below. - guard store != nil else { throw LuxTtsError.notInitialized } + // phonemization. + guard let store else { throw LuxTtsError.notInitialized } let g2p = try englishG2p() + let tokenizer = try await store.tokenizer() return try await synthesize( - phonemes: g2p.phonemize(text: text), + tokenIds: tokenizer.tokenIds(phonemes: g2p.phonemize(text: text)), promptAudio: promptAudio, - promptPhonemes: g2p.phonemize(text: promptText), + promptTokenIds: tokenizer.tokenIds(phonemes: g2p.phonemize(text: promptText)), speed: speed, - seed: seed) + seed: seed, + maxRedraws: maxRedraws, + extraPausePositions: LuxTtsContinuation.textPausePositions(text)) } /// The bundled espeak-parity English G2P (loaded lazily; ~4 MB of @@ -135,12 +139,19 @@ public actor LuxTtsManager { /// - speed: Speech-rate divisor for the generated span. Keep 1.0 /// (upstream's hidden 1.3 clips sentence onsets). /// - seed: Noise seed for the flow-matching init. + /// - maxRedraws: Re-draw budget per span for the spurious-pause + /// detector (see `synthesize(tokenIds:...)`); 0 pins the raw pass. + /// - extraPauseAllowance: Pauses the phonemes call for beyond their + /// punctuation tokens (e.g. ellipses the G2P dropped). Long text + /// spreads them evenly over its spans. public func synthesize( phonemes: String, promptAudio: URL, promptPhonemes: String, speed: Float = LuxTtsConstants.defaultSpeed, - seed: UInt64 = LuxTtsConstants.defaultSeed + seed: UInt64 = LuxTtsConstants.defaultSeed, + maxRedraws: Int = LuxTtsConstants.spuriousPauseRetries, + extraPauseAllowance: Int = 0 ) async throws -> LuxTtsSynthesisResult { guard let store = store else { throw LuxTtsError.notInitialized } let tokenizer = try await store.tokenizer() @@ -149,36 +160,263 @@ public actor LuxTtsManager { promptAudio: promptAudio, promptTokenIds: tokenizer.tokenIds(phonemes: promptPhonemes), speed: speed, - seed: seed) + seed: seed, + maxRedraws: maxRedraws, + extraPauseAllowance: extraPauseAllowance) } /// Synthesize from pre-computed token ids (callers running their own /// espeak frontend against `tokens.txt`). + /// + /// Text that fits one flow-matching pass is rendered in one pass; longer + /// text is split into continuation-prompted spans (see + /// `Documentation/TTS/LuxTts.md`). Every pass is checked for mid-phrase + /// pauses beyond the text's punctuation (issue #937) and re-drawn up to + /// `maxRedraws` times; pass 0 to keep the raw pass for a given seed. The + /// result reports `redraws` and `residualPauses`. `extraPauseAllowance` + /// is spread evenly over the spans. public func synthesize( tokenIds: [Int], promptAudio: URL, promptTokenIds: [Int], speed: Float = LuxTtsConstants.defaultSpeed, - seed: UInt64 = LuxTtsConstants.defaultSeed + seed: UInt64 = LuxTtsConstants.defaultSeed, + maxRedraws: Int = LuxTtsConstants.spuriousPauseRetries, + extraPauseAllowance: Int = 0 + ) async throws -> LuxTtsSynthesisResult { + let count = max(0, extraPauseAllowance) + return try await synthesize( + tokenIds: tokenIds, + promptAudio: promptAudio, + promptTokenIds: promptTokenIds, + speed: speed, + seed: seed, + maxRedraws: maxRedraws, + extraPausePositions: (0.. LuxTtsSynthesisResult { - guard let synthesizer = synthesizer else { throw LuxTtsError.notInitialized } + guard let store, let synthesizer else { throw LuxTtsError.notInitialized } + let converter = AudioConverter(sampleRate: Double(LuxTtsConstants.melSampleRate)) let prompt24k: [Float] do { - let converter = AudioConverter( - sampleRate: Double(LuxTtsConstants.melSampleRate)) prompt24k = try converter.resampleAudioFile(promptAudio) } catch { throw LuxTtsError.invalidPromptAudio( "cannot load \(promptAudio.path): \(error.localizedDescription)") } + if prompt24k.count > LuxTtsConstants.maxPromptSamples { + let seconds = Double(prompt24k.count) / Double(LuxTtsConstants.melSampleRate) + logger.warning( + "LuxTTS prompt is \(String(format: "%.1f", seconds)) s; only the first " + + "\(LuxTtsConstants.maxPromptSeconds) s condition the model while the whole " + + "transcript sets the duration ratio — trim the clip and transcript to match") + } - return try await synthesizer.synthesize( - promptTokenIds: promptTokenIds, - textTokenIds: tokenIds, - promptAudio24k: prompt24k, - speed: speed, - seed: seed) + let tokenizer = try await store.tokenizer() + let pauseTokens = Set([",", ".", ";", ":", "!", "?", "-"].compactMap { tokenizer.tokenToId[$0] }) + let boundaryTokens = pauseTokens.union([tokenizer.tokenToId[" "]].compactMap { $0 }) + let promptFrames = LuxTtsMelExtractor().frameCount( + sampleCount: min(prompt24k.count, LuxTtsConstants.maxPromptSamples)) + + let spans: [[Int]] + if LuxTtsContinuation.fitsSinglePass( + textTokenCount: tokenIds.count, + promptFrames: promptFrames, + promptTokenCount: promptTokenIds.count, + speed: Double(speed)) + { + spans = [tokenIds] + } else { + let maxSpanTokens = LuxTtsContinuation.maxSpanTokens( + promptFrames: promptFrames, + promptTokenCount: promptTokenIds.count, + speed: Double(speed)) + let ratio = Double(promptFrames) / Double(max(1, promptTokenIds.count)) + guard ratio <= LuxTtsConstants.maxPromptFramesPerToken, + maxSpanTokens >= LuxTtsConstants.minimumSpanTokens + else { + throw LuxTtsError.inputTooLong( + "prompt yields \(String(format: "%.1f", ratio)) mel frames per token (plausible " + + "≤ \(Int(LuxTtsConstants.maxPromptFramesPerToken))) at speed \(speed), " + + "leaving spans of \(maxSpanTokens) tokens (minimum " + + "\(LuxTtsConstants.minimumSpanTokens)); trim prompt silence or check that " + + "the transcript matches the clip") + } + spans = LuxTtsContinuation.chunks( + tokenIds: tokenIds, + maxTokens: maxSpanTokens, + boundaryTokenIds: boundaryTokens) + logger.info( + "LuxTTS continuation synthesis: \(tokenIds.count) target tokens in " + + "\(spans.count) balanced spans (≤ \(maxSpanTokens) tokens each)") + } + + let crossfadeSamples = Int( + LuxTtsConstants.continuationCrossfadeSeconds + * Double(LuxTtsConstants.outputSampleRate)) + var currentPromptAudio = prompt24k + var currentPromptTokens = promptTokenIds + var samples: [Float] = [] + var originalPromptFrames = 0 + var totalGeneratedFrames = 0 + var totalRedraws = 0 + var totalResidualPauses = 0 + let spanExtraPauses = LuxTtsContinuation.spanPauseAllowances( + spanLengths: spans.map(\.count), positions: extraPausePositions) + + for (index, span) in spans.enumerated() { + // A continuation prompt already speaks at the requested rate; + // applying `speed` again would compound it on every span. + let pass = try await synthesizeSpan( + synthesizer, + textTokenIds: span, + promptTokenIds: currentPromptTokens, + promptAudio24k: currentPromptAudio, + speed: index == 0 ? speed : 1.0, + seed: seed &+ UInt64(index) &* LuxTtsConstants.continuationSeedStride, + maxRedraws: maxRedraws, + allowedPauses: spanExtraPauses[index] + + LuxTtsContinuation.expectedPauseCount( + in: span, pauseTokenIds: pauseTokens, boundaryTokenIds: boundaryTokens), + label: "span \(index + 1)/\(spans.count)") + totalRedraws += pass.redraws + totalResidualPauses += pass.residualPauses + let result = pass.result + if spans.count == 1 { + return LuxTtsSynthesisResult( + samples: result.samples, + sampleRate: result.sampleRate, + promptFrames: result.promptFrames, + generatedFrames: result.generatedFrames, + featuresLength: result.featuresLength, + redraws: totalRedraws, + residualPauses: totalResidualPauses) + } + + if index == 0 { originalPromptFrames = result.promptFrames } + totalGeneratedFrames += result.generatedFrames + + // Cut onset padding on continuation spans and tail padding on + // spans another span follows, unless the span ends in pause + // punctuation: that tail is the sentence break the text asked for. + let hasNextSpan = index + 1 < spans.count + let keepTail = + !hasNextSpan + || LuxTtsContinuation.endsWithPausePunctuation( + span, pauseTokenIds: pauseTokens, boundaryTokenIds: boundaryTokens) + let slice = LuxTtsContinuation.speechSlice( + pass.speech, + sampleCount: result.samples.count, + sampleRate: result.sampleRate, + trimLeading: index > 0, + trimTrailing: !keepTail) + LuxTtsContinuation.appendWithCrossfade( + result.samples[slice], to: &samples, crossfadeSamples: crossfadeSamples) + + guard hasNextSpan else { continue } + // Prompt with the untrimmed span, sized to exactly the frames the + // host allotted for these tokens so every span keeps the first + // prompt's frames-per-token ratio. The vocoder emits one hop less + // than `generatedFrames`; left as is, each span would shave a + // frame off the next span's ratio and speech would speed up. + do { + currentPromptAudio = try converter.resample( + result.samples, from: Double(result.sampleRate)) + } catch { + throw LuxTtsError.inferenceFailed( + stage: "continuation prompt resample", underlying: "\(error)") + } + LuxTtsContinuation.fitPromptLength( + ¤tPromptAudio, frames: result.generatedFrames) + currentPromptTokens = span + } + + return LuxTtsSynthesisResult( + samples: samples, + sampleRate: LuxTtsConstants.outputSampleRate, + promptFrames: originalPromptFrames, + generatedFrames: totalGeneratedFrames, + featuresLength: originalPromptFrames + totalGeneratedFrames, + redraws: totalRedraws, + residualPauses: totalResidualPauses) + } + + private struct SpanPass { + let result: LuxTtsSynthesisResult + let speech: LuxTtsContinuation.SpeechRuns + let redraws: Int + let residualPauses: Int + } + + /// One flow-matching pass with a bounded re-draw ladder. The model can + /// drop a spurious mid-phrase pause whose position depends on the exact + /// (length, noise) draw (issue #937; the PyTorch reference does the + /// same), so a pass whose silences exceed the span's punctuation is + /// re-drawn with the next seed. The cleanest attempt is kept. Re-draws + /// keep the duration: compressing it (speed × 1.03–1.06) traded the pause + /// for a clipped final word. + private func synthesizeSpan( + _ synthesizer: LuxTtsSynthesizer, + textTokenIds: [Int], + promptTokenIds: [Int], + promptAudio24k: [Float], + speed: Float, + seed: UInt64, + maxRedraws: Int, + allowedPauses: Int, + label: String + ) async throws -> SpanPass { + func render(_ attempt: Int) async throws -> SpanPass { + try Task.checkCancellation() + let result = try await synthesizer.synthesize( + promptTokenIds: promptTokenIds, + textTokenIds: textTokenIds, + promptAudio24k: promptAudio24k, + speed: speed, + seed: seed &+ UInt64(attempt)) + let speech = LuxTtsContinuation.speechRuns(result.samples, sampleRate: result.sampleRate) + let pauses = LuxTtsContinuation.innerPauseCount(speech, sampleRate: result.sampleRate) + return SpanPass( + result: result, speech: speech, redraws: attempt, + residualPauses: max(0, pauses - allowedPauses)) + } + + var best = try await render(0) + guard best.residualPauses > 0, maxRedraws > 0 else { return best } + + var redraws = 0 + for attempt in 1...maxRedraws { + logger.info( + "LuxTTS \(label): \(best.residualPauses) spurious pause(s) beyond the " + + "\(allowedPauses) the text allows; re-drawing (attempt \(attempt))") + let candidate = try await render(attempt) + redraws = attempt + if candidate.residualPauses < best.residualPauses { + best = candidate + } + if candidate.residualPauses == 0 { break } + } + if best.residualPauses > 0 { + logger.warning( + "LuxTTS \(label): \(best.residualPauses) spurious pause(s) remain after " + + "\(redraws) re-draws; keeping the cleanest pass") + } + return SpanPass( + result: best.result, speech: best.speech, redraws: redraws, + residualPauses: best.residualPauses) } public func cleanup() async { diff --git a/Sources/FluidAudio/TTS/LuxTts/LuxTtsSynthesizer.swift b/Sources/FluidAudio/TTS/LuxTts/LuxTtsSynthesizer.swift index 9736d3749..324e2d317 100644 --- a/Sources/FluidAudio/TTS/LuxTts/LuxTtsSynthesizer.swift +++ b/Sources/FluidAudio/TTS/LuxTts/LuxTtsSynthesizer.swift @@ -11,10 +11,22 @@ public struct LuxTtsSynthesisResult: Sendable { public let sampleRate: Int /// Prompt conditioning length in 24 kHz mel frames. public let promptFrames: Int - /// Generated mel frames (`featuresLength - promptFrames`). + /// Generated mel frames. For a single pass this is + /// `featuresLength - promptFrames` and `samples.count == (generatedFrames - 1) * 512`; + /// for continuation-prompted text it is the sum over spans, whose + /// trimmed padding and crossfades make `samples.count` shorter. public let generatedFrames: Int - /// Total flow-matching sequence length (prompt + generated frames). + /// Flow-matching sequence length (prompt + generated frames), summed + /// over spans for continuation-prompted text (then it can exceed the + /// per-pass 1024-frame graph). public let featuresLength: Int + /// Extra flow-matching passes spent re-drawing spans the pause detector + /// flagged (0 = every span was clean on its first pass). See + /// `LuxTtsManager.synthesize(tokenIds:...maxRedraws:)`. + public let redraws: Int + /// Pauses beyond the text's punctuation still present in the kept + /// passes after the re-draw budget was spent. + public let residualPauses: Int } /// Drives the LuxTTS (ZipVoice-Distill) CoreML stages end-to-end, mirroring @@ -119,8 +131,8 @@ struct LuxTtsSynthesizer { throw LuxTtsError.tokenizerFailed( "estimated only \(genFrames) generated frames — text too short") } - // TODO(phase 2): chunk long inputs across multiple vocoder windows - // instead of erroring; mel truncation is NOT allowed. + // Long inputs are split into continuation-prompted spans by + // `LuxTtsManager` before reaching here; mel truncation is NOT allowed. guard let bucket = LuxTtsConstants.vocoderBuckets.first(where: { $0 >= genFrames }) else { throw LuxTtsError.inputTooLong( "generated frames \(genFrames) exceed the largest vocoder bucket " @@ -295,7 +307,9 @@ struct LuxTtsSynthesizer { sampleRate: LuxTtsConstants.outputSampleRate, promptFrames: promptFrames, generatedFrames: genFrames, - featuresLength: featuresLength) + featuresLength: featuresLength, + redraws: 0, + residualPauses: 0) } // MARK: - Helpers diff --git a/Sources/FluidAudioCLI/Commands/TTSCommand.swift b/Sources/FluidAudioCLI/Commands/TTSCommand.swift index 8d7a41d2c..42ab33839 100644 --- a/Sources/FluidAudioCLI/Commands/TTSCommand.swift +++ b/Sources/FluidAudioCLI/Commands/TTSCommand.swift @@ -106,6 +106,7 @@ public struct TTS { var luxttsPromptText: String? = nil var luxttsSpeed: Float = LuxTtsConstants.defaultSpeed var luxttsSeed: UInt64 = LuxTtsConstants.defaultSeed + var luxttsRedraws: Int = LuxTtsConstants.spuriousPauseRetries var neuttsSeed: UInt64 = 1234 var neuttsEmotion = NeuTtsConstants.defaultEmotion var chatterboxSeed: UInt64 = 42 @@ -235,6 +236,11 @@ public struct TTS { luxttsPromptText = arguments[i + 1] i += 1 } + case "--redraws": + if i + 1 < arguments.count, let v = Int(arguments[i + 1]), v >= 0 { + luxttsRedraws = v + i += 1 + } case "--temperature": if i + 1 < arguments.count, let v = Float(arguments[i + 1]) { pocketTemperature = v @@ -408,7 +414,7 @@ public struct TTS { promptAudioPath: luxttsPromptAudioPath, promptText: luxttsPromptText, treatAsPhonemes: treatAsPhonemes, - speed: luxttsSpeed, seed: luxttsSeed, + speed: luxttsSpeed, seed: luxttsSeed, redraws: luxttsRedraws, metricsPath: metricsPath) case .neuTts: await runNeuTts( @@ -519,7 +525,7 @@ public struct TTS { text: String, output: String, promptAudioPath: String?, promptText: String?, treatAsPhonemes: Bool, - speed: Float, seed: UInt64, + speed: Float, seed: UInt64, redraws: Int, metricsPath: String? ) async { guard let promptAudioPath else { @@ -570,14 +576,16 @@ public struct TTS { promptAudio: promptURL, promptPhonemes: resolvedPromptText, speed: speed, - seed: seed) + seed: seed, + maxRedraws: redraws) } else { result = try await manager.synthesize( text: text, promptAudio: promptURL, promptText: resolvedPromptText, speed: speed, - seed: seed) + seed: seed, + maxRedraws: redraws) } let tSynth1 = Date() @@ -610,6 +618,7 @@ public struct TTS { logger.info( " Frames: prompt=\(result.promptFrames) " + "generated=\(result.generatedFrames) total=\(result.featuresLength)") + logger.info(" Re-draws: \(result.redraws), residual pauses: \(result.residualPauses)") logger.info(" RMS: \(String(format: "%.5f", rms))") logger.info(" RTFx: \(String(format: "%.2f", rtfx))x") logger.info(" Total: \(String(format: "%.3f", totalS))s") @@ -1549,6 +1558,9 @@ public struct TTS { --prompt-text are espeak IPA (en-us) --speed 1.0 speech-rate divisor (default 1.0) --seed N flow-matching noise seed (default 42) + --redraws N re-draw budget per span for the + mid-phrase pause detector (default 3; + 0 = raw pass for the given seed) --lexicon, -l Custom pronunciation lexicon file (KokoroAne --variant zh only): word pinyin1 pinyin2 (e.g. zi4 jie2) word @bopomofo1 (escape: @-prefixed, diff --git a/Tests/FluidAudioTests/TTS/LuxTts/LuxTtsContinuationTests.swift b/Tests/FluidAudioTests/TTS/LuxTts/LuxTtsContinuationTests.swift new file mode 100644 index 000000000..9a176d5f4 --- /dev/null +++ b/Tests/FluidAudioTests/TTS/LuxTts/LuxTtsContinuationTests.swift @@ -0,0 +1,255 @@ +import XCTest + +@testable import FluidAudio + +final class LuxTtsContinuationTests: XCTestCase { + + func testChunksStayBalancedAndBreakAtBoundaries() { + let space = 0 + let tokens = Array(repeating: [1, 2, 3, 4, space], count: 22).flatMap { $0 } + let chunks = LuxTtsContinuation.chunks( + tokenIds: tokens, maxTokens: 102, boundaryTokenIds: [space]) + + XCTAssertEqual(chunks.count, 2) + XCTAssertEqual(chunks.flatMap { $0 }, tokens) + XCTAssertTrue(chunks.allSatisfy { $0.count <= 102 }) + XCTAssertTrue(chunks.dropLast().allSatisfy { $0.last == space }) + XCTAssertLessThanOrEqual(abs(chunks[0].count - chunks[1].count), 5) + } + + func testChunksKeepStableIssue937BoundarySinglePass() { + XCTAssertEqual( + LuxTtsContinuation.chunks( + tokenIds: Array(0..<102), maxTokens: 102, boundaryTokenIds: [] + ).count, + 1) + XCTAssertEqual( + LuxTtsContinuation.chunks( + tokenIds: Array(0..<106), maxTokens: 102, boundaryTokenIds: [] + ).map(\.count), + [53, 53]) + } + + func testChunksNeverLeaveAnOversizedRemainder() { + let tokens = Array(0..<205) + let chunks = LuxTtsContinuation.chunks( + tokenIds: tokens, maxTokens: 102, boundaryTokenIds: [0, 1]) + + XCTAssertEqual(chunks.flatMap { $0 }, tokens) + XCTAssertTrue(chunks.allSatisfy { !$0.isEmpty && $0.count <= 102 }) + XCTAssertEqual(chunks.map(\.count), [68, 69, 68]) + } + + func testMaxSpanTokensIsBoundedBySpanSizeTokenBucketAndFrameBudget() { + // Fast prompt: the 102-token span size wins. + XCTAssertEqual( + LuxTtsContinuation.maxSpanTokens(promptFrames: 300, promptTokenCount: 100, speed: 1.0), + 102) + // Long transcript: the 256-token encoder bucket (minus pad slot) wins. + XCTAssertEqual( + LuxTtsContinuation.maxSpanTokens(promptFrames: 300, promptTokenCount: 200, speed: 1.0), + 55) + // Issue #937 prompt (100 tokens / 5 s): 468 frames ÷ 4.69 frames/token. + XCTAssertEqual( + LuxTtsContinuation.maxSpanTokens(promptFrames: 469, promptTokenCount: 100, speed: 1.0), + 99) + // Slower speech needs proportionally fewer tokens per span. + XCTAssertEqual( + LuxTtsContinuation.maxSpanTokens(promptFrames: 469, promptTokenCount: 100, speed: 0.5), + 49) + // Degenerate prompts fall back to the cap (the synthesizer rejects them). + XCTAssertEqual( + LuxTtsContinuation.maxSpanTokens(promptFrames: 0, promptTokenCount: 0, speed: 1.0), + 102) + // Pathological ratios floor at 1; the manager rejects anything under + // `minimumSpanTokens` instead of rendering dozens of passes. + XCTAssertEqual( + LuxTtsContinuation.maxSpanTokens(promptFrames: 469, promptTokenCount: 4, speed: 1.0), + 3) + } + + func testSpanFrameBudgetFitsInsidePromptCap() { + // A span at the budget yields (budget - 1) * hop48k samples at 48 kHz; + // at 24 kHz that must fit under the synthesizer's prompt cap so the + // next span's prompt is not truncated away from its transcript. + let budget = LuxTtsConstants.continuationSpanFrameBudget + let samples24k = (budget - 1) * LuxTtsConstants.hop48k / 2 + XCTAssertLessThanOrEqual( + samples24k, + Int(LuxTtsConstants.maxPromptSeconds * Double(LuxTtsConstants.melSampleRate))) + XCTAssertLessThanOrEqual(budget, LuxTtsConstants.vocoderBuckets.max() ?? 0) + } + + func testFitsSinglePassUsesSpanCapAndGraphLimits() { + // Reporter's prompt: 100 tokens over 5 s (469 frames), speed 1.0. + XCTAssertTrue( + LuxTtsContinuation.fitsSinglePass( + textTokenCount: 102, promptFrames: 469, promptTokenCount: 100, speed: 1.0)) + // 106 tokens would fit the graph (498 frames) but exceed the span cap. + XCTAssertFalse( + LuxTtsContinuation.fitsSinglePass( + textTokenCount: 106, promptFrames: 469, promptTokenCount: 100, speed: 1.0)) + // 102 tokens at a slow prompt → 613 generated frames, past the 555 bucket. + XCTAssertFalse( + LuxTtsContinuation.fitsSinglePass( + textTokenCount: 102, promptFrames: 600, promptTokenCount: 100, speed: 1.0)) + // Half speed doubles the frame estimate past the 555 bucket. + XCTAssertFalse( + LuxTtsContinuation.fitsSinglePass( + textTokenCount: 102, promptFrames: 469, promptTokenCount: 100, speed: 0.5)) + // Prompt transcript + text must leave the pad slot in the 256 bucket. + XCTAssertFalse( + LuxTtsContinuation.fitsSinglePass( + textTokenCount: 60, promptFrames: 300, promptTokenCount: 196, speed: 1.0)) + XCTAssertTrue( + LuxTtsContinuation.fitsSinglePass( + textTokenCount: 59, promptFrames: 300, promptTokenCount: 196, speed: 1.0)) + } + + func testTextPauseAllowanceCountsSilentBreaksOnly() { + XCTAssertEqual(LuxTtsContinuation.textPauseAllowance("He paused... then said \"no\"."), 3) + XCTAssertEqual(LuxTtsContinuation.textPauseAllowance("Wait… (really)"), 3) + XCTAssertEqual(LuxTtsContinuation.textPauseAllowance("don't, won't; can't."), 0) + } + + func testTextPausePositionsLocateSilentBreaks() { + let text = "ab... \"c\"" + XCTAssertEqual( + LuxTtsContinuation.textPausePositions(text), + [2.0 / 9.0, 6.0 / 9.0, 8.0 / 9.0]) + XCTAssertEqual(LuxTtsContinuation.textPausePositions(""), []) + } + + func testSpanPauseAllowancesStayInTheirSpan() { + // Quotes clustered in the last span must not excuse pauses elsewhere. + XCTAssertEqual( + LuxTtsContinuation.spanPauseAllowances( + spanLengths: [50, 50, 50, 50], positions: [0.8, 0.85, 0.9, 0.95]), + [0, 0, 0, 4]) + XCTAssertEqual( + LuxTtsContinuation.spanPauseAllowances( + spanLengths: [30, 70], positions: [0.0, 0.29, 0.3, 0.99]), + [2, 2]) + XCTAssertEqual( + LuxTtsContinuation.spanPauseAllowances(spanLengths: [40], positions: [0.1, 0.5]), + [2]) + } + + func testFitPromptLengthMatchesAllottedFrames() { + let extractor = LuxTtsMelExtractor() + let generatedFrames = 300 + // The vocoder emits (gen - 1) hops, which the extractor counts as gen - 1. + var audio = [Float](repeating: 0.1, count: (generatedFrames - 1) * LuxTtsConstants.hopLength) + XCTAssertEqual(extractor.frameCount(sampleCount: audio.count), generatedFrames - 1) + LuxTtsContinuation.fitPromptLength(&audio, frames: generatedFrames) + XCTAssertEqual(extractor.frameCount(sampleCount: audio.count), generatedFrames) + XCTAssertEqual(audio.last, 0) + + var long = [Float](repeating: 0.1, count: generatedFrames * LuxTtsConstants.hopLength + 77) + LuxTtsContinuation.fitPromptLength(&long, frames: generatedFrames) + XCTAssertEqual(extractor.frameCount(sampleCount: long.count), generatedFrames) + } + + func testExpectedPauseCountIgnoresTrailingBoundaryTokens() { + let space = 0 + let comma = 1 + let period = 2 + let pauses: Set = [comma, period] + let boundaries: Set = [space, comma, period] + XCTAssertEqual( + LuxTtsContinuation.expectedPauseCount( + in: [5, 6, comma, space, 7, 8, period, space], + pauseTokenIds: pauses, boundaryTokenIds: boundaries), + 1) + XCTAssertEqual( + LuxTtsContinuation.expectedPauseCount( + in: [5, 6, 7], pauseTokenIds: pauses, boundaryTokenIds: boundaries), + 0) + } + + func testInnerPauseCountFindsOnlyMidSpeechSilences() { + let sampleRate = 1_000 + let speech = Array(repeating: Float(0.5), count: 300) + let lead = Array(repeating: Float.zero, count: 200) + let shortClosure = Array(repeating: Float.zero, count: 50) + let pause = Array(repeating: Float.zero, count: 150) + // The vocoder's onset click: loud but far shorter than sustained speech. + let click = Array(repeating: Float(1), count: 20) + + XCTAssertEqual( + LuxTtsContinuation.innerPauseCount(lead + speech + lead, sampleRate: sampleRate), 0) + XCTAssertEqual( + LuxTtsContinuation.innerPauseCount(click + lead + speech + lead, sampleRate: sampleRate), + 0) + XCTAssertEqual( + LuxTtsContinuation.innerPauseCount( + lead + speech + shortClosure + speech + lead, sampleRate: sampleRate), + 0) + XCTAssertEqual( + LuxTtsContinuation.innerPauseCount( + lead + speech + pause + speech + shortClosure + speech + pause + speech, + sampleRate: sampleRate), + 2) + XCTAssertEqual(LuxTtsContinuation.innerPauseCount([], sampleRate: sampleRate), 0) + } + + func testCrossfadePreservesEdgesAndExpectedLength() { + var output: [Float] = [1, 1, 1, 1] + LuxTtsContinuation.appendWithCrossfade( + [0, 0, 0, 0], to: &output, crossfadeSamples: 3) + + XCTAssertEqual(output.count, 5) + XCTAssertEqual(output.first, 1) + XCTAssertEqual(output.last, 0) + XCTAssertEqual(output[1], 1) + XCTAssertEqual(output[2], 0.5) + XCTAssertEqual(output[3], 0) + } + + func testLeadingPaddingTrimKeepsOnsetPreroll() { + let samples = + Array(repeating: Float.zero, count: 50) + + Array(repeating: Float(1), count: 50) + + let trimmed = LuxTtsContinuation.trimmingLeadingPadding(samples, sampleRate: 1_000) + + XCTAssertEqual(trimmed.count, 80) + XCTAssertEqual(Array(trimmed.prefix(30)), Array(repeating: Float.zero, count: 30)) + XCTAssertEqual(trimmed[30], 1) + } + + func testLeadingPaddingTrimSkipsOnsetClick() { + let click = Array(repeating: Float(1), count: 20) + let padding = Array(repeating: Float.zero, count: 200) + let speech = Array(repeating: Float(0.5), count: 100) + + let trimmed = LuxTtsContinuation.trimmingLeadingPadding( + click + padding + speech, sampleRate: 1_000) + + // 30 ms preroll before the sustained onset at sample 220. + XCTAssertEqual(trimmed.count, 130) + XCTAssertEqual(trimmed[30], 0.5) + } + + func testTrailingPaddingTrimKeepsPostroll() { + let speech = Array(repeating: Float(1), count: 50) + let padding = Array(repeating: Float.zero, count: 200) + + let trimmed = LuxTtsContinuation.trimmingTrailingPadding(speech + padding, sampleRate: 1_000) + + XCTAssertEqual(trimmed.count, 80) + XCTAssertEqual(trimmed[49], 1) + XCTAssertEqual(trimmed[50], 0) + } + + func testEndsWithPausePunctuationIgnoresTrailingSpaces() { + let space = 0 + let comma = 1 + XCTAssertTrue( + LuxTtsContinuation.endsWithPausePunctuation( + [5, 6, comma, space], pauseTokenIds: [comma], boundaryTokenIds: [space, comma])) + XCTAssertFalse( + LuxTtsContinuation.endsWithPausePunctuation( + [5, comma, 6, space], pauseTokenIds: [comma], boundaryTokenIds: [space, comma])) + } +} diff --git a/Tests/FluidAudioTests/TTS/LuxTts/LuxTtsE2ETests.swift b/Tests/FluidAudioTests/TTS/LuxTts/LuxTtsE2ETests.swift index 5464e113a..879a3675a 100644 --- a/Tests/FluidAudioTests/TTS/LuxTts/LuxTtsE2ETests.swift +++ b/Tests/FluidAudioTests/TTS/LuxTts/LuxTtsE2ETests.swift @@ -46,7 +46,10 @@ final class LuxTtsE2ETests: XCTestCase { promptAudio: promptURL, promptPhonemes: fixtures.prompt.phonemeString, speed: Float(fixtures.e2e.speed), - seed: UInt64(fixtures.e2e.seed)) + seed: UInt64(fixtures.e2e.seed), + // Pin the raw pass: the pause detector would otherwise re-draw + // this fixture (one breath beyond its two commas) with another seed. + maxRedraws: 0) // Frame accounting must be identical to Python (pure integer math // over fixture-pinned token ids and mel frame counts). @@ -55,6 +58,7 @@ final class LuxTtsE2ETests: XCTestCase { XCTAssertEqual(result.featuresLength, fixtures.e2e.featuresLen) XCTAssertEqual(result.generatedFrames, fixtures.e2e.genFrames) XCTAssertEqual(result.samples.count, fixtures.e2e.wavSamples) + XCTAssertEqual(result.redraws, 0) // Loudness within 1 dB of the Python CoreML pipeline, and non-silent. let sumSquares = result.samples.reduce(Double(0)) { $0 + Double($1) * Double($1) }