From b4a8ac067a4b823c0398cb09cdb336066559a591 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 19 Oct 2026 11:13:20 -0700 Subject: [PATCH 1/2] Fix silent truncation of long phoneme segments split_phonemes measured chunks in bytes and left over-long unpunctuated stretches whole; _create_audio then cut them at 510 bytes, dropping audio and possibly splitting a UTF-8 character. Chunks are now at most 510 UTF-8 characters (<= 510 tokens), split at punctuation, then spaces, then after the last tone mark or at exactly 510 characters. The byte cut is replaced by a reported token cap. Closes ISSUES.md #1. --- ARCHITECTURE.md | 14 ++-- CMakeLists.txt | 1 + ISSUES.md | 25 +------ src/Kokoro.cpp | 15 ++-- src/Kokoro.h | 3 +- src/PhonemeChunker.cpp | 152 +++++++++++++++++++++++++++++++-------- src/PhonemeChunker.h | 6 +- src/PhonemeEncoder.cpp | 16 +---- src/Utf8.cpp | 28 ++++++++ src/Utf8.h | 17 +++++ tests/test_chunker.cpp | 115 +++++++++++++++++++++++++++-- tests/test_synthesis.cpp | 53 ++++++++++++++ 12 files changed, 357 insertions(+), 88 deletions(-) create mode 100644 src/Utf8.cpp create mode 100644 src/Utf8.h diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 13b2635..788acc7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -81,10 +81,14 @@ flowchart TD 3. **Phonemize.** Unless the input is already phonemes, `Phonemizer::phonemize(text, language)` dispatches to that language's `PhonemizerBase`. It passes along the number language set by `kokoro_set_number_language`. The result is one UTF-8 phoneme string in Kokoro's symbol set. -4. **Chunk.** `split_phonemes` (`src/PhonemeChunker.cpp`) splits at `.,!?;`, trims the segments, and packs them into - chunks shorter than `MAX_PHONEME_LENGTH` = 510 **bytes**. A single over-long segment stays one chunk. -5. **Encode.** `Kokoro::_create_audio` truncates the chunk to 510 bytes. Then `PhonemeEncoder::encode` maps each UTF-8 - character to its id from `dict/vocab.txt` and **silently drops characters that are not in the vocab**. +4. **Chunk.** `split_phonemes` (`src/PhonemeChunker.cpp`) splits at `.,!?;`, keeping each mark with the text before + it, trims the segments, and packs them into chunks of at most `MAX_PHONEME_LENGTH` = 510 UTF-8 characters, counted + as `PhonemeEncoder` splits them (so at most 510 tokens). A segment over the limit is split at spaces. A space-free + word over the limit (unpunctuated Chinese) is cut after its last tone mark (`→ ↗ ↘ ↓`) within the limit, else at + exactly 510 characters. Nothing is dropped. +5. **Encode.** `PhonemeEncoder::encode` maps each UTF-8 character to its id from `dict/vocab.txt` and **silently drops + characters that are not in the vocab**. More than 510 tokens (unreachable after chunking) are capped at 510 with a + `kokoro:` warning on stderr. - If the result is empty, no audio is returned and the model is not run. This mirrors upstream, which skips empty phoneme strings. - Otherwise token 0 is added at both ends. @@ -325,7 +329,5 @@ The full contract is in [docs/thread-safety.md](docs/thread-safety.md). ## Known sharp edges - `trim_audio` (`src/Kokoro.cpp`) is a no-op, so `trim=true` changes nothing. -- A single phoneme segment longer than 510 bytes is truncated to 510 bytes. The limit counts bytes, and the encoder - drops a UTF-8 character cut at the boundary. - Models with an `input_ids` input receive the speed as int32, so its fractional part is lost. - Phonemizer output symbols that are missing from `vocab.txt` vanish without a warning. diff --git a/CMakeLists.txt b/CMakeLists.txt index 30d51b5..897a31f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,7 @@ add_library(kokoro_core STATIC src/chinese_english/ZHFrontend.cpp src/NeuralG2P.cpp src/PhonemeText.cpp + src/Utf8.cpp src/german/GermanG2P.cpp src/german/GermanLexicon.cpp src/german/GermanPhonemizer.cpp diff --git a/ISSUES.md b/ISSUES.md index 68d5830..88424ca 100644 --- a/ISSUES.md +++ b/ISSUES.md @@ -1,26 +1,3 @@ # Known issues -## 1. Chunk limit and truncation count UTF-8 bytes, not phonemes - -**Where:** -- `split_phonemes`, `src/PhonemeChunker.cpp:20`: `current_batch.length() + part.length() + 1 >= max_length`. -- `Kokoro::_create_audio`, `src/Kokoro.cpp:218-219`: `phonemes.substr(0, MAX_PHONEME_LENGTH)`. - -`MAX_PHONEME_LENGTH` (510, `src/Kokoro.h:20`) is meant to be a count of phonemes: the model takes at most 510 phoneme tokens plus two padding tokens, and a voice has one style row per length up to 510. Upstream Kokoro counts `len(ps)`, which is Python code points. Both places above use `std::string::length()` and `substr`, which count **bytes**. - -**Consequences:** -- **Chunks are much shorter than they need to be.** Most IPA symbols take more than one byte in UTF-8 (`ʈ ʂ ɕ ŋ` take 2 bytes, tone arrows `→ ↗ ↘ ↓` take 3). For Chinese a 510-byte chunk holds far fewer than 510 phonemes, so text is split into more, shorter chunks than upstream would use. That changes prosody at chunk boundaries and selects style rows for shorter lengths. -- **Truncation can split a character.** `substr(0, 510)` can cut a multi-byte UTF-8 sequence in half. The tokenizer (`Tokenizer::tokenize` / `split_utf8`) then sees an invalid trailing fragment, which is not in the vocabulary and gets dropped. Truncation only happens when a single chunk is longer than 510 bytes, which is only possible for a segment that `split_phonemes` could not split (one with no `[.,!?;]` for 509+ bytes). -- **The two limits disagree.** The row index (fixed in `fbf9a6b`) uses the token count, while chunking uses bytes. - -**Suggested fix:** -- Measure chunk length in phoneme tokens (or UTF-8 code points, matching upstream `len(ps)`) instead of bytes, both when packing in `split_phonemes` and when truncating in `_create_audio`. -- Truncate on a code-point boundary. It is simplest to tokenize first and cap the token vector at `MAX_PHONEME_LENGTH`. -- A segment longer than the limit should be split at word boundaries (spaces) before any hard truncation. That is what current kokoro-onnx does: its `split_phonemes` "prefer[s] splitting at punctuation marks, then at word boundaries" (`kokoro_onnx/chunker.py`). - -**Testing notes:** -- `tests/test_chunker.cpp` can assert, for a long Chinese phoneme string: - - every chunk has at most 510 code points or tokens; - - chunks are not cut short at 510 bytes; - - no chunk ends in a partial UTF-8 sequence. -- Expect the `g2p` goldens to be unaffected, because chunking happens after G2P. `synthesis` durations for long inputs will change. +None open. diff --git a/src/Kokoro.cpp b/src/Kokoro.cpp index 1b49b34..3186480 100644 --- a/src/Kokoro.cpp +++ b/src/Kokoro.cpp @@ -193,16 +193,19 @@ std::pair, int> Kokoro::_create_audio( const std::vector& voice, float speed ) { - std::string truncated_phonemes = phonemes; - if (phonemes.length() > MAX_PHONEME_LENGTH) { - truncated_phonemes = phonemes.substr(0, MAX_PHONEME_LENGTH); - } - - std::vector tokens_raw = encoder_->encode(truncated_phonemes); + std::vector tokens_raw = encoder_->encode(phonemes); // Nothing to say: skip the model rather than run it on the padding tokens alone, as upstream // Kokoro skips empty phoneme strings (`if not ps: continue`). if (tokens_raw.empty()) return {{}, SAMPLE_RATE}; + // split_phonemes keeps chunks within MAX_PHONEME_LENGTH characters, at most one token each: a last resort. + const auto max_tokens = static_cast(MAX_PHONEME_LENGTH); + if (tokens_raw.size() > max_tokens) { + std::fprintf(stderr, "kokoro: chunk of %zu phoneme tokens exceeds %zu; dropping the excess\n", + tokens_raw.size(), max_tokens); + tokens_raw.resize(max_tokens); + } + // Add start and end tokens (0) std::vector tokens = {0}; for (int t : tokens_raw) tokens.push_back(t); diff --git a/src/Kokoro.h b/src/Kokoro.h index 9b7f301..1222111 100644 --- a/src/Kokoro.h +++ b/src/Kokoro.h @@ -17,7 +17,8 @@ class PhonemeEncoder; struct KoKoroConfig; // Constants from config -const int MAX_PHONEME_LENGTH = 510; // Example value +// MAX_PHONEME_LENGTH: phoneme tokens per model call; split_phonemes keeps chunks within this many UTF-8 characters. +const int MAX_PHONEME_LENGTH = 510; const int SAMPLE_RATE = 24000; // Example value const int STYLE_DIM = 256; // floats per style row; a voice is a table of these rows diff --git a/src/PhonemeChunker.cpp b/src/PhonemeChunker.cpp index e56c7b2..13ae076 100644 --- a/src/PhonemeChunker.cpp +++ b/src/PhonemeChunker.cpp @@ -1,36 +1,132 @@ #include "PhonemeChunker.h" +#include "Utf8.h" + +#include +#include #include +#include -std::vector split_phonemes(const std::string& phonemes, std::size_t max_length) { - std::vector batches; - std::regex re("([.,!?;])"); - std::sregex_token_iterator it(phonemes.begin(), phonemes.end(), re, {-1, 0}); // -1 for non-match, 0 for match - std::sregex_token_iterator end; - - std::string current_batch; - - for (; it != end; ++it) { - std::string part = *it; - // Removing leading/trailing whitespace - part = std::regex_replace(part, std::regex("^\\s+|\\s+$"), ""); - - if (part.empty()) continue; - - if (current_batch.length() + part.length() + 1 >= max_length) { - if (!current_batch.empty()) batches.push_back(current_batch); - current_batch = part; - } else { - if (std::string(".,!?;").find(part) != std::string::npos) { - current_batch += part; - } else { - if (!current_batch.empty()) current_batch += " "; - current_batch += part; - } +namespace { + +const std::string kPunctuation = ".,!?;"; +const std::array kToneMarks = {"→", "↗", "↘", "↓"}; + +// Regex parts of phonemes split at [.,!?;] (marks kept as their own parts), trimmed, empties dropped. +std::vector segments(const std::string& phonemes) { + static const std::regex split_re("([.,!?;])"); + static const std::regex trim_re("^\\s+|\\s+$"); + std::vector parts; + for (std::sregex_token_iterator it(phonemes.begin(), phonemes.end(), split_re, {-1, 0}), end; it != end; ++it) { + std::string part = std::regex_replace(it->str(), trim_re, ""); + if (!part.empty()) parts.push_back(std::move(part)); + } + return parts; +} + +bool is_punctuation(const std::string& segment) { + return segment.size() == 1 && kPunctuation.find(segment[0]) != std::string::npos; +} + +// Each punctuation segment joins the group before it; a leading one opens its own group. +std::vector attach_punctuation(const std::vector& segments) { + std::vector groups; + for (const std::string& segment : segments) { + if (is_punctuation(segment) && !groups.empty()) groups.back() += segment; + else groups.push_back(segment); + } + return groups; +} + +bool is_tone_mark(const std::string& word, std::size_t pos, std::size_t next) { + const std::string_view character(word.data() + pos, next - pos); + return std::find(kToneMarks.begin(), kToneMarks.end(), character) != kToneMarks.end(); +} + +// Byte end of the piece of word starting at begin: the rest if it fits in max_length characters, else just past the +// last tone mark within the window, else exactly max_length characters. A cut that would leave only the trailing +// punctuation (which starts after last_core_start) moves back to last_core_start. +std::size_t hard_cut(const std::string& word, std::size_t begin, std::size_t max_length, std::size_t last_core_start) { + std::size_t end = begin; + std::size_t after_tone = begin; + for (std::size_t n = 0; n < max_length && end < word.size(); ++n) { + const std::size_t next = utf8_next(word, end); + if (is_tone_mark(word, end, next)) after_tone = next; + end = next; + } + if (end == word.size()) return end; + if (after_tone > begin) end = after_tone; + if (end > last_core_start && last_core_start > begin) end = last_core_start; + return end; +} + +// Byte offset where the last non-punctuation character of word starts; npos if word is all punctuation. +std::size_t last_core_char_start(const std::string& word) { + const std::size_t core_end = word.find_last_not_of(kPunctuation); + if (core_end == std::string::npos) return std::string::npos; + std::size_t start = core_end; + while (start > 0 && (static_cast(word[start]) & 0xC0) == 0x80) --start; + return start; +} + +// word cut into pieces of at most max_length characters by hard_cut. +std::vector hard_split(const std::string& word, std::size_t max_length) { + const std::size_t core_start = last_core_char_start(word); + std::vector pieces; + for (std::size_t begin = 0; begin < word.size();) { + const std::size_t end = hard_cut(word, begin, max_length, core_start); + pieces.push_back(word.substr(begin, end - begin)); + begin = end; + } + return pieces; +} + +// group if it fits; else its space-separated words (empty ones dropped), each word over the limit hard-split. +std::vector pieces(const std::string& group, std::size_t max_length) { + if (utf8_length(group) <= max_length) return {group}; + std::vector out; + std::size_t begin = 0; + while (begin <= group.size()) { + const std::size_t space = std::min(group.find(' ', begin), group.size()); + const std::string word = group.substr(begin, space - begin); + if (utf8_length(word) > max_length) { + const std::vector parts = hard_split(word, max_length); + out.insert(out.end(), parts.begin(), parts.end()); + } else if (!word.empty()) { + out.push_back(word); } + begin = space + 1; } - if (!current_batch.empty()) { - batches.push_back(current_batch); + return out; +} + +// Joins pieces with single spaces, starting a new chunk when the next piece would push it past max_length. +std::vector pack(const std::vector& pieces, std::size_t max_length) { + std::vector chunks; + std::string current; + std::size_t current_length = 0; + for (const std::string& piece : pieces) { + const std::size_t piece_length = utf8_length(piece); + if (!current.empty() && current_length + 1 + piece_length <= max_length) { + current += ' ' + piece; + current_length += 1 + piece_length; + continue; + } + if (!current.empty()) chunks.push_back(std::move(current)); + current = piece; + current_length = piece_length; + } + if (!current.empty()) chunks.push_back(std::move(current)); + return chunks; +} + +} // namespace + +std::vector split_phonemes(const std::string& phonemes, std::size_t max_length) { + std::vector all_pieces; + for (const std::string& group : attach_punctuation(segments(phonemes))) { + const std::vector group_pieces = pieces(group, max_length); + all_pieces.insert(all_pieces.end(), group_pieces.begin(), group_pieces.end()); } - return batches; + return pack(all_pieces, max_length); } diff --git a/src/PhonemeChunker.h b/src/PhonemeChunker.h index cca2c31..35c3d6d 100644 --- a/src/PhonemeChunker.h +++ b/src/PhonemeChunker.h @@ -4,6 +4,8 @@ #include #include -// Splits a phoneme string at [.,!?;] and packs the trimmed segments into chunks, starting a new chunk -// when the current one would reach max_length bytes. A single segment longer than that stays one chunk. +// Splits a phoneme string into chunks of at most max_length UTF-8 characters (as PhonemeEncoder counts them, so at +// most max_length tokens) without dropping any phoneme. Splits at [.,!?;], keeping each mark with the text before it, +// then at spaces; a space-free word still too long is cut after its last tone mark (→ ↗ ↘ ↓) within the limit, else +// at exactly max_length characters. Segments are trimmed and joined with single spaces. std::vector split_phonemes(const std::string& phonemes, std::size_t max_length); diff --git a/src/PhonemeEncoder.cpp b/src/PhonemeEncoder.cpp index cf2b387..01672ef 100644 --- a/src/PhonemeEncoder.cpp +++ b/src/PhonemeEncoder.cpp @@ -1,4 +1,5 @@ #include "PhonemeEncoder.h" +#include "Utf8.h" #include #include @@ -7,21 +8,6 @@ namespace { -std::vector split_utf8(const std::string& str) { - std::vector chars; - for (size_t i = 0; i < str.length();) { - const auto c = static_cast(str[i]); - size_t char_len = 1; - if ((c & 0xE0) == 0xC0) char_len = 2; - else if ((c & 0xF0) == 0xE0) char_len = 3; - else if ((c & 0xF8) == 0xF0) char_len = 4; - if (i + char_len > str.length()) char_len = str.length() - i; - chars.push_back(str.substr(i, char_len)); - i += char_len; - } - return chars; -} - void unescape(std::string& token, const std::string& escape, const char* plain) { for (size_t pos = token.find(escape); pos != std::string::npos; pos = token.find(escape, pos + 1)) { token.replace(pos, escape.size(), plain); diff --git a/src/Utf8.cpp b/src/Utf8.cpp new file mode 100644 index 0000000..269f706 --- /dev/null +++ b/src/Utf8.cpp @@ -0,0 +1,28 @@ +#include "Utf8.h" + +#include + +std::size_t utf8_next(const std::string& s, std::size_t pos) { + const auto c = static_cast(s[pos]); + std::size_t len = 1; + if ((c & 0xE0) == 0xC0) len = 2; + else if ((c & 0xF0) == 0xE0) len = 3; + else if ((c & 0xF8) == 0xF0) len = 4; + return std::min(pos + len, s.size()); +} + +std::size_t utf8_length(const std::string& s) { + std::size_t count = 0; + for (std::size_t pos = 0; pos < s.size(); pos = utf8_next(s, pos)) ++count; + return count; +} + +std::vector split_utf8(const std::string& s) { + std::vector chars; + for (std::size_t pos = 0; pos < s.size();) { + const std::size_t next = utf8_next(s, pos); + chars.push_back(s.substr(pos, next - pos)); + pos = next; + } + return chars; +} diff --git a/src/Utf8.h b/src/Utf8.h new file mode 100644 index 0000000..6b9580b --- /dev/null +++ b/src/Utf8.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +// UTF-8 characters as PhonemeEncoder tokenizes them: the lead byte alone sets the length (1-4 bytes), sequences are +// not validated, and a sequence truncated by the end of the string ends there. + +// Byte offset just past the character starting at pos (pos < s.size()). +std::size_t utf8_next(const std::string& s, std::size_t pos); + +// Number of characters in s. +std::size_t utf8_length(const std::string& s); + +// s split into its characters. +std::vector split_utf8(const std::string& s); diff --git a/tests/test_chunker.cpp b/tests/test_chunker.cpp index c627a0f..28d194c 100644 --- a/tests/test_chunker.cpp +++ b/tests/test_chunker.cpp @@ -5,6 +5,8 @@ #include "PhonemeChunker.h" #include +#include +#include #include #include @@ -23,13 +25,52 @@ std::string repeat(const std::string& unit, std::size_t count) { return out; } +bool is_continuation(char c) { return (static_cast(c) & 0xC0) == 0x80; } + +std::size_t code_points(const std::string& s) { + return static_cast(std::count_if(s.begin(), s.end(), [](char c) { return !is_continuation(c); })); +} + +// Every lead byte announces 1-4 bytes and that many continuation bytes follow inside the string. +bool is_valid_utf8(const std::string& s) { + for (std::size_t i = 0; i < s.size();) { + const auto c = static_cast(s[i]); + const std::size_t len = + c < 0x80 ? 1 : (c & 0xE0) == 0xC0 ? 2 : (c & 0xF0) == 0xE0 ? 3 : (c & 0xF8) == 0xF0 ? 4 : 0; + if (len == 0 || i + len > s.size()) return false; + if (!std::all_of(s.begin() + i + 1, s.begin() + i + len, is_continuation)) return false; + i += len; + } + return true; +} + +std::string join(const std::vector& chunks) { + std::string out; + for (const std::string& c : chunks) out += c; + return out; +} + +bool starts_with_punctuation(const std::string& s) { + return !s.empty() && std::string(".,!?;").find(s[0]) != std::string::npos; +} + +std::vector sizes_in_code_points(const std::vector& chunks) { + std::vector sizes; + std::transform(chunks.begin(), chunks.end(), std::back_inserter(sizes), code_points); + return sizes; +} + +bool all_bounded(const std::vector& chunks) { + return std::all_of(chunks.begin(), chunks.end(), [](const std::string& c) { return code_points(c) <= kLimit; }); +} + } // namespace TEST_CASE("long_first_segment_has_no_empty_chunk") { const std::string segment(600, 'a'); const std::vector chunks = split_phonemes(segment + ". bc.", kLimit); CAPTURE(chunks.size()); - CHECK(chunks == std::vector{segment, ". bc."}); + CHECK(chunks == std::vector{std::string(510, 'a'), std::string(90, 'a') + ". bc."}); } TEST_CASE("first_segment_at_the_limit_boundary") { @@ -47,13 +88,75 @@ TEST_CASE("chunks_are_nonempty_bounded_and_lossless") { REQUIRE(chunks.size() > 2); CHECK(std::none_of(chunks.begin(), chunks.end(), [](const std::string& c) { return c.empty(); })); - CHECK(std::all_of(chunks.begin() + 1, chunks.end(), [](const std::string& c) { return c.length() < kLimit; })); - - std::string joined; - for (const std::string& c : chunks) joined += c; - CHECK(strip_spaces(joined) == strip_spaces(input)); + CHECK(all_bounded(chunks)); + CHECK(strip_spaces(join(chunks)) == strip_spaces(input)); } TEST_CASE("empty_input_yields_no_chunks") { CHECK(split_phonemes("", kLimit).empty()); } + +TEST_CASE("boundary_units_around_the_limit") { + for (const std::string unit : {"a", "ʂ"}) { + CAPTURE(unit); + CHECK(split_phonemes(repeat(unit, 509), kLimit) == std::vector{repeat(unit, 509)}); + CHECK(split_phonemes(repeat(unit, 510), kLimit) == std::vector{repeat(unit, 510)}); + CHECK(split_phonemes(repeat(unit, 511), kLimit) == std::vector{repeat(unit, 510), unit}); + } + const std::string two_words = repeat("a", 254) + " " + repeat("a", 255); + CHECK(split_phonemes(two_words, kLimit) == std::vector{two_words}); + CHECK(split_phonemes(repeat("a", 254) + " " + repeat("a", 256), kLimit) == + std::vector{repeat("a", 254), repeat("a", 256)}); + CHECK(split_phonemes(repeat("a", 509) + ".", kLimit) == std::vector{repeat("a", 509) + "."}); + CHECK(split_phonemes(repeat("a", 510) + ".", kLimit) == std::vector{repeat("a", 509), "a."}); +} + +TEST_CASE("long_ascii_stretch_splits_at_word_boundaries") { + const std::string input = repeat("hello world ", 100); + const std::vector chunks = split_phonemes(input, kLimit); + CAPTURE(sizes_in_code_points(chunks)); + REQUIRE(chunks.size() > 1); + CHECK(all_bounded(chunks)); + for (const std::string& chunk : chunks) { + std::istringstream words(chunk); + for (std::string word; words >> word;) CHECK((word == "hello" || word == "world")); + } + CHECK(std::all_of(chunks.begin(), chunks.end() - 1, + [](const std::string& c) { return code_points(c) > kLimit - 6; })); + CHECK(strip_spaces(join(chunks)) == strip_spaces(input)); +} + +TEST_CASE("long_chinese_stretch_is_cut_after_a_tone_mark") { + // 16 characters, 28 bytes per unit. + const std::string input = repeat("ʈʂʊ→ŋkwo↗ni↘xau↓", 60); + const std::vector chunks = split_phonemes(input, kLimit); + CAPTURE(sizes_in_code_points(chunks)); + REQUIRE(chunks.size() == 2); + // The 510-character window holds 31 units plus ʈʂʊ→ŋkwo↗ni↘xa; its last tone mark ↘ is character 508. + CHECK(code_points(chunks[0]) == 508); + const std::string last_tone = "↘"; + CHECK(chunks[0].compare(chunks[0].size() - last_tone.size(), last_tone.size(), last_tone) == 0); + CHECK(chunks[0].size() > kLimit); + CHECK(std::all_of(chunks.begin(), chunks.end(), is_valid_utf8)); + CHECK(join(chunks) == input); +} + +TEST_CASE("space_free_word_over_the_limit_is_hard_split") { + const std::string input = repeat("ʈʂɕ", 400); + const std::vector chunks = split_phonemes(input, kLimit); + CHECK(sizes_in_code_points(chunks) == std::vector{510, 510, 180}); + CHECK(std::all_of(chunks.begin(), chunks.end(), is_valid_utf8)); + CHECK(join(chunks) == input); +} + +TEST_CASE("mixed_punctuation_and_long_stretches") { + const std::string input = "ni xau, " + repeat("wo ai ni ", 120) + "xau! " + repeat("ʈʂʊ→ŋkwo↗", 70) + "."; + const std::vector chunks = split_phonemes(input, kLimit); + CAPTURE(sizes_in_code_points(chunks)); + REQUIRE(chunks.size() > 2); + CHECK(std::none_of(chunks.begin(), chunks.end(), [](const std::string& c) { return c.empty(); })); + CHECK(all_bounded(chunks)); + CHECK(std::all_of(chunks.begin(), chunks.end(), is_valid_utf8)); + CHECK(std::none_of(chunks.begin(), chunks.end(), starts_with_punctuation)); + CHECK(strip_spaces(join(chunks)) == strip_spaces(input)); +} diff --git a/tests/test_synthesis.cpp b/tests/test_synthesis.cpp index 87d30f2..d7097ad 100644 --- a/tests/test_synthesis.cpp +++ b/tests/test_synthesis.cpp @@ -58,6 +58,21 @@ size_t synth_length(const char* text, const char* voice, unsigned flags) { return n; } +size_t code_points(const std::string& s) { + return static_cast( + std::count_if(s.begin(), s.end(), [](char c) { return (static_cast(c) & 0xC0) != 0x80; })); +} + +std::string join(const std::vector& parts, const std::string& separator) { + std::string out; + for (size_t i = 0; i < parts.size(); ++i) out += (i == 0 ? "" : separator) + parts[i]; + return out; +} + +double synth_seconds(const std::string& text, const char* voice) { + return static_cast(synth_length(text.c_str(), voice, 0)) / 24000.0; +} + // Minimal reader/writer for the voices file format (docs/adding-voices.md). using VoiceTable = std::map>; constexpr size_t kStyleDim = 256; @@ -281,6 +296,44 @@ TEST_CASE("style_row_follows_token_count") { kokoro_destroy(ctx); } +// Unpunctuated text longer than one chunk used to lose everything past byte 510 of the stretch. Commas add pauses, so +// the unpunctuated text is a bit shorter, but not by the truncated share. +TEST_CASE("unpunctuated_long_text_is_not_truncated") { + struct Variant { + const char* voice; + std::vector clauses; + std::string reference_separator; + std::vector separators; + }; + const std::vector english = { + "the old lighthouse keeper climbed the narrow stairs every single evening", + "he polished the great brass lamp until it shone across the dark water", + "the fishing boats followed its steady beam back to the quiet harbor", + "his daughter wrote down the weather in a small leather notebook", + "the storms of that winter were the worst anyone could remember", + "but the light never once failed the sailors who trusted it", + }; + std::vector english_twice = english; + english_twice.insert(english_twice.end(), english.begin(), english.end()); + const std::vector variants = { + {"af_maple", english_twice, ", and ", {" and ", ": "}}, + {kVoice, std::vector(8, "我们明天早上一起去公园散步然后回家吃饭"), "，", {""}}, + }; + + for (const Variant& v : variants) { + const double reference_s = synth_seconds(join(v.clauses, v.reference_separator), v.voice); + for (const std::string& sep : v.separators) { + const std::string text = join(v.clauses, sep); + CAPTURE(text); + REQUIRE(code_points(ph(text.c_str())) > 510); + const double plain_s = synth_seconds(text, v.voice); + MESSAGE(std::string(v.voice), " sep='", sep, "': ", plain_s, " s vs ", reference_s, " s punctuated"); + CHECK(plain_s > 0.85 * reference_s); + CHECK(plain_s < 1.15 * reference_s); + } + } +} + TEST_CASE("cpu_device_is_honored") { kokoro_options options = kokoro_default_options(); options.device = KOKORO_DEVICE_CPU; From 446399aa86a4fc5fa23b20b356555b85a32bcbeb Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 19 Oct 2026 15:25:40 -0700 Subject: [PATCH 2/2] Fix tone retention and punctuation-first phoneme chunking --- ARCHITECTURE.md | 6 +++-- src/PhonemeChunker.cpp | 58 ++++++++++++++++++++++++------------------ src/PhonemeChunker.h | 7 ++--- tests/test_chunker.cpp | 44 ++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 30 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 788acc7..049dec9 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -83,9 +83,11 @@ flowchart TD one UTF-8 phoneme string in Kokoro's symbol set. 4. **Chunk.** `split_phonemes` (`src/PhonemeChunker.cpp`) splits at `.,!?;`, keeping each mark with the text before it, trims the segments, and packs them into chunks of at most `MAX_PHONEME_LENGTH` = 510 UTF-8 characters, counted - as `PhonemeEncoder` splits them (so at most 510 tokens). A segment over the limit is split at spaces. A space-free + as `PhonemeEncoder` splits them (so at most 510 tokens). Punctuation groups are packed whole; an oversized group + starts a new chunk before splitting at spaces, so an earlier punctuation boundary takes precedence. A space-free word over the limit (unpunctuated Chinese) is cut after its last tone mark (`→ ↗ ↘ ↓`) within the limit, else at - exactly 510 characters. Nothing is dropped. + exactly 510 characters. A cut that would strand trailing punctuation retreats to the preceding tone boundary, or + reserves the final phoneme and its tone if no earlier tone boundary exists. Nothing is dropped. 5. **Encode.** `PhonemeEncoder::encode` maps each UTF-8 character to its id from `dict/vocab.txt` and **silently drops characters that are not in the vocab**. More than 510 tokens (unreachable after chunking) are capped at 510 with a `kokoro:` warning on stderr. diff --git a/src/PhonemeChunker.cpp b/src/PhonemeChunker.cpp index 13ae076..0f2f0d3 100644 --- a/src/PhonemeChunker.cpp +++ b/src/PhonemeChunker.cpp @@ -43,15 +43,14 @@ bool is_tone_mark(const std::string& word, std::size_t pos, std::size_t next) { return std::find(kToneMarks.begin(), kToneMarks.end(), character) != kToneMarks.end(); } -// Byte end of the piece of word starting at begin: the rest if it fits in max_length characters, else just past the -// last tone mark within the window, else exactly max_length characters. A cut that would leave only the trailing -// punctuation (which starts after last_core_start) moves back to last_core_start. +// Prefer the last tone boundary within the window that leaves a phoneme with trailing punctuation. If there is no +// such boundary, cut at the character limit, moving back to reserve the final phoneme (and its tone) when possible. std::size_t hard_cut(const std::string& word, std::size_t begin, std::size_t max_length, std::size_t last_core_start) { std::size_t end = begin; std::size_t after_tone = begin; for (std::size_t n = 0; n < max_length && end < word.size(); ++n) { const std::size_t next = utf8_next(word, end); - if (is_tone_mark(word, end, next)) after_tone = next; + if (next <= last_core_start && is_tone_mark(word, end, next)) after_tone = next; end = next; } if (end == word.size()) return end; @@ -60,12 +59,16 @@ std::size_t hard_cut(const std::string& word, std::size_t begin, std::size_t max return end; } -// Byte offset where the last non-punctuation character of word starts; npos if word is all punctuation. +// Byte offset of the final non-punctuation character; a final tone mark also reserves its preceding phoneme. std::size_t last_core_char_start(const std::string& word) { const std::size_t core_end = word.find_last_not_of(kPunctuation); if (core_end == std::string::npos) return std::string::npos; std::size_t start = core_end; while (start > 0 && (static_cast(word[start]) & 0xC0) == 0x80) --start; + if (start > 0 && is_tone_mark(word, start, core_end + 1)) { + --start; + while (start > 0 && (static_cast(word[start]) & 0xC0) == 0x80) --start; + } return start; } @@ -81,9 +84,8 @@ std::vector hard_split(const std::string& word, std::size_t max_len return pieces; } -// group if it fits; else its space-separated words (empty ones dropped), each word over the limit hard-split. +// Split an oversized group into space-separated words (empty ones dropped), hard-splitting words over the limit. std::vector pieces(const std::string& group, std::size_t max_length) { - if (utf8_length(group) <= max_length) return {group}; std::vector out; std::size_t begin = 0; while (begin <= group.size()) { @@ -100,33 +102,39 @@ std::vector pieces(const std::string& group, std::size_t max_length return out; } -// Joins pieces with single spaces, starting a new chunk when the next piece would push it past max_length. -std::vector pack(const std::vector& pieces, std::size_t max_length) { +// Append to the current chunk if it fits, else open a new chunk. A zero current_length forces a group boundary. +void append_piece(std::vector& chunks, std::size_t& current_length, const std::string& piece, + std::size_t piece_length, std::size_t max_length) { + if (current_length > 0 && current_length + 1 + piece_length <= max_length) { + chunks.back() += ' '; + chunks.back() += piece; + current_length += 1 + piece_length; + } else { + chunks.push_back(piece); + current_length = piece_length; + } +} + +// Pack punctuation groups whole when they fit. Start an oversized group in a new chunk before splitting its words. +std::vector pack(const std::vector& groups, std::size_t max_length) { std::vector chunks; - std::string current; std::size_t current_length = 0; - for (const std::string& piece : pieces) { - const std::size_t piece_length = utf8_length(piece); - if (!current.empty() && current_length + 1 + piece_length <= max_length) { - current += ' ' + piece; - current_length += 1 + piece_length; + for (const std::string& group : groups) { + const std::size_t group_length = utf8_length(group); + if (group_length <= max_length) { + append_piece(chunks, current_length, group, group_length, max_length); continue; } - if (!current.empty()) chunks.push_back(std::move(current)); - current = piece; - current_length = piece_length; + current_length = 0; + for (const std::string& piece : pieces(group, max_length)) { + append_piece(chunks, current_length, piece, utf8_length(piece), max_length); + } } - if (!current.empty()) chunks.push_back(std::move(current)); return chunks; } } // namespace std::vector split_phonemes(const std::string& phonemes, std::size_t max_length) { - std::vector all_pieces; - for (const std::string& group : attach_punctuation(segments(phonemes))) { - const std::vector group_pieces = pieces(group, max_length); - all_pieces.insert(all_pieces.end(), group_pieces.begin(), group_pieces.end()); - } - return pack(all_pieces, max_length); + return pack(attach_punctuation(segments(phonemes)), max_length); } diff --git a/src/PhonemeChunker.h b/src/PhonemeChunker.h index 35c3d6d..656cb88 100644 --- a/src/PhonemeChunker.h +++ b/src/PhonemeChunker.h @@ -5,7 +5,8 @@ #include // Splits a phoneme string into chunks of at most max_length UTF-8 characters (as PhonemeEncoder counts them, so at -// most max_length tokens) without dropping any phoneme. Splits at [.,!?;], keeping each mark with the text before it, -// then at spaces; a space-free word still too long is cut after its last tone mark (→ ↗ ↘ ↓) within the limit, else -// at exactly max_length characters. Segments are trimmed and joined with single spaces. +// most max_length tokens) without dropping any phoneme. Packs [.,!?;] groups whole, keeping each mark with the text +// before it. An oversized group starts a new chunk before splitting at spaces. A space-free word still too long is +// cut after its last tone mark (→ ↗ ↘ ↓) within the limit, else at max_length characters. Trailing punctuation +// reserves the final phoneme and its tone when possible. Segments are trimmed and joined with single spaces. std::vector split_phonemes(const std::string& phonemes, std::size_t max_length); diff --git a/tests/test_chunker.cpp b/tests/test_chunker.cpp index 28d194c..fe1316a 100644 --- a/tests/test_chunker.cpp +++ b/tests/test_chunker.cpp @@ -160,3 +160,47 @@ TEST_CASE("mixed_punctuation_and_long_stretches") { CHECK(std::none_of(chunks.begin(), chunks.end(), starts_with_punctuation)); CHECK(strip_spaces(join(chunks)) == strip_spaces(input)); } + +TEST_CASE("trailing_punctuation_keeps_the_final_toned_syllable") { + for (const std::string tone : {"→", "↗", "↘", "↓"}) { + for (const std::string punctuation : {".", "...", "!?"}) { + CAPTURE(tone); + CAPTURE(punctuation); + const std::string syllable = "ni" + tone; + const std::string input = repeat(syllable, 170) + punctuation; + const auto chunks = split_phonemes(input, kLimit); + CHECK(chunks == std::vector{repeat(syllable, 169), syllable + punctuation}); + CHECK(all_bounded(chunks)); + CHECK(join(chunks) == input); + CHECK(std::all_of(chunks.begin(), chunks.end(), is_valid_utf8)); + } + } +} + +TEST_CASE("trailing_tone_without_an_earlier_tone_keeps_its_phoneme") { + const std::string input = repeat("ʂ", 509) + "→."; + const auto chunks = split_phonemes(input, kLimit); + CHECK(chunks == std::vector{repeat("ʂ", 508), "ʂ→."}); + CHECK(join(chunks) == input); + CHECK(std::all_of(chunks.begin(), chunks.end(), is_valid_utf8)); +} + +TEST_CASE("punctuation_boundary_precedes_splitting_an_oversized_group") { + const std::string first_sentence = std::string(100, 'a') + "."; + const std::string word(199, 'b'); + const std::string input = first_sentence + " " + word + " " + word + " " + word + ". c."; + const auto chunks = split_phonemes(input, kLimit); + CHECK(chunks == std::vector{first_sentence, word + " " + word, word + ". c."}); + CHECK(strip_spaces(join(chunks)) == strip_spaces(input)); + CHECK(all_bounded(chunks)); +} + +TEST_CASE("punctuation_boundary_precedes_hard_splitting_a_word") { + const std::string first_sentence = std::string(100, 'a') + "."; + const std::string syllable = repeat("ʂ", 398) + "→"; + const std::string input = first_sentence + " " + repeat(syllable, 2) + "."; + const auto chunks = split_phonemes(input, kLimit); + CHECK(chunks == std::vector{first_sentence, syllable, syllable + "."}); + CHECK(strip_spaces(join(chunks)) == strip_spaces(input)); + CHECK(all_bounded(chunks)); +}