From 55f8a0bd2f078f36cf8ab2d87910ea820daf77b9 Mon Sep 17 00:00:00 2001 From: Rasaboun <40967731+Rasaboun@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:08:14 +0200 Subject: [PATCH] engine/bpe: map byte 0xAD like GPT-2 byte-level BPE (not identity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit init_byte_mappings treated bytes 161..255 as identity code points. GPT-2's bytes_to_unicode() excludes 173 (U+00AD, soft hyphen) from that range and assigns it the next free code point after the other non-printables (U+0143). Any UTF-8 sequence containing byte 0xAD -- e.g. 歯 (E6 AD AF) -- therefore tokenized to instead of the reference token, and decoded to U+FFFD. The fix must exclude 173 from BOTH the identity byte list and the identity branch of the assignment loop (the branch is selected by byte value, so moving the byte between lists alone has no effect). Verified: the resulting 256-entry mapping equals HF GPT2 bytes_to_unicode() byte-for-byte; cactus_tokenize("歯") -> [33687] as in the reference tokenizer. Refs #803 (second item). Signed-off-by: Rasaboun <40967731+Rasaboun@users.noreply.github.com> --- cactus-engine/src/bpe.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cactus-engine/src/bpe.cpp b/cactus-engine/src/bpe.cpp index 41d49c795..06e0b230e 100644 --- a/cactus-engine/src/bpe.cpp +++ b/cactus-engine/src/bpe.cpp @@ -267,6 +267,7 @@ void BPETokenizer::init_byte_mappings() const { for (int i = 161; i <= 255; ++i) { + if (i == 173) continue; // U+00AD is not an identity byte in GPT-2 byte-level BPE bytes.push_back(i); } @@ -274,6 +275,7 @@ void BPETokenizer::init_byte_mappings() const { for (int i = 0; i <= 32; ++i) remaining_bytes.push_back(i); remaining_bytes.push_back(127); for (int i = 128; i <= 160; ++i) remaining_bytes.push_back(i); + remaining_bytes.push_back(173); // -> U+0143, matching GPT-2 bytes_to_unicode() int unicode_start = 256; for (int byte : remaining_bytes) { @@ -287,7 +289,7 @@ void BPETokenizer::init_byte_mappings() const { std::string unicode_char(1, static_cast(byte)); byte_to_unicode_[byte] = unicode_char; unicode_to_byte_[unicode_char] = byte; - } else if (byte >= 161 && byte <= 255) { + } else if (byte >= 161 && byte <= 255 && byte != 173) { // 173 -> remaining branch std::string unicode_char; unicode_char += static_cast(0xC0 | (byte >> 6)); unicode_char += static_cast(0x80 | (byte & 0x3F));