From 17f731cfb2dd00cf8e0a58bab0e438e074cbbe97 Mon Sep 17 00:00:00 2001
From: gianni-cor <gianfranco.cordella@tether.io>
Date: Sat, 4 Jul 2026 01:20:20 +0200
Subject: [PATCH] fix: drop unused tokenizer vocab assets

---
 src/tokenizers/vocab/vocab.cpp | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/tokenizers/vocab/vocab.cpp b/src/tokenizers/vocab/vocab.cpp
index 4d2a9b84a..f651643f9 100644
--- a/src/tokenizers/vocab/vocab.cpp
+++ b/src/tokenizers/vocab/vocab.cpp
@@ -1,17 +1,15 @@
 #include "vocab.h"
 #include "clip_merges.hpp"
-#include "gemma2_merges.hpp"
-#include "gemma2_vocab.hpp"
 #include "gemma_merges.hpp"
 #include "gemma_vocab.hpp"
-#include "gpt_oss_merges.hpp"
-#include "gpt_oss_vocab.hpp"
 #include "mistral_merges.hpp"
 #include "mistral_vocab.hpp"
 #include "qwen_merges.hpp"
 #include "t5.hpp"
 #include "umt5.hpp"
 
+#include <stdexcept>
+
 std::string load_clip_merges() {
     std::string merges_utf8_str(reinterpret_cast<const char*>(clip_merges_utf8_c_str), sizeof(clip_merges_utf8_c_str));
     return merges_utf8_str;
@@ -53,21 +51,17 @@ std::string load_gemma_vocab_json() {
 }
 
 std::string load_gemma2_merges() {
-    std::string merges_utf8_str(reinterpret_cast<const char*>(gemma2_merges_utf8_c_str), sizeof(gemma2_merges_utf8_c_str));
-    return merges_utf8_str;
+    throw std::runtime_error("Gemma2 tokenizer vocab is disabled in this build");
 }
 
 std::string load_gemma2_vocab_json() {
-    std::string json_str(reinterpret_cast<const char*>(gemma2_vocab_json_utf8_c_str), sizeof(gemma2_vocab_json_utf8_c_str));
-    return json_str;
+    throw std::runtime_error("Gemma2 tokenizer vocab is disabled in this build");
 }
 
 std::string load_gpt_oss_merges() {
-    std::string merges_utf8_str(reinterpret_cast<const char*>(gpt_oss_merges_utf8_c_str), sizeof(gpt_oss_merges_utf8_c_str));
-    return merges_utf8_str;
+    throw std::runtime_error("GPT-OSS tokenizer vocab is disabled in this build");
 }
 
 std::string load_gpt_oss_vocab_json() {
-    std::string json_str(reinterpret_cast<const char*>(gpt_oss_vocab_json_utf8_c_str), sizeof(gpt_oss_vocab_json_utf8_c_str));
-    return json_str;
+    throw std::runtime_error("GPT-OSS tokenizer vocab is disabled in this build");
 }
\ No newline at end of file