Stable Diffusion 1.5

cactus-engine/CMakeLists.txt

@@ -27,6 +27,7 @@ set(ENGINE_SOURCES
     src/cloud.cpp
     src/init.cpp
     src/embed.cpp
+    src/generate_image.cpp
     src/complete.cpp
     src/transcribe.cpp
     src/stream.cpp

cactus-engine/cactus_engine.h

@@ -140,6 +140,18 @@ CACTUS_FFI_EXPORT int cactus_stream_transcribe_stop(
     size_t buffer_size
 );
 
+CACTUS_FFI_EXPORT int cactus_generate_image(
+    cactus_model_t model,
+    const char* prompt,
+    uint8_t* rgb_buffer,
+    size_t buffer_size,
+    unsigned int* image_width,
+    unsigned int* image_height,
+    int steps,
+    float guidance_scale,
+    unsigned long long seed
+);
+
 CACTUS_FFI_EXPORT int cactus_embed(
     cactus_model_t model,
     const char* text,

cactus-engine/src/bpe.cpp

@@ -3,6 +3,7 @@
 #include <sstream>
 #include <algorithm>
 #include <cstdio>
+#include <cstring>
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <fcntl.h>
@@ -455,6 +456,72 @@ std::vector<std::string> BPETokenizer::apply_bpe(const std::vector<std::string>&
     return current_tokens;
 }
 
+std::vector<std::string> BPETokenizer::clip_word_split(const std::string& text) const {
+    static const char* kContractions[] = {"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"};
+    auto is_letter = [](unsigned char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c >= 0x80; };
+    auto is_digit = [](unsigned char c) { return c >= '0' && c <= '9'; };
+    auto is_space = [](unsigned char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; };
+
+    std::vector<std::string> words;
+    size_t i = 0;
+    while (i < text.size()) {
+        const unsigned char c = static_cast<unsigned char>(text[i]);
+        if (is_space(c)) {
+            ++i;
+            continue;
+        }
+        if (c == '\'') {
+            bool matched = false;
+            for (const char* contraction : kContractions) {
+                const size_t len = std::strlen(contraction);
+                if (text.compare(i, len, contraction) == 0) {
+                    words.emplace_back(text.substr(i, len));
+                    i += len;
+                    matched = true;
+                    break;
+                }
+            }
+            if (matched) continue;
+        }
+        if (is_letter(c)) {
+            size_t start = i;
+            while (i < text.size() && is_letter(static_cast<unsigned char>(text[i]))) ++i;
+            words.emplace_back(text.substr(start, i - start));
+            continue;
+        }
+        if (is_digit(c)) {
+            words.emplace_back(text.substr(i, 1));
+            ++i;
+            continue;
+        }
+        size_t start = i;
+        while (i < text.size()) {
+            const unsigned char p = static_cast<unsigned char>(text[i]);
+            if (is_space(p) || is_letter(p) || is_digit(p) || p == '\'') break;
+            ++i;
+        }
+        if (i == start) ++i;
+        else words.emplace_back(text.substr(start, i - start));
+    }
+    return words;
+}
+
+void BPETokenizer::encode_clip_segment(const std::string& segment, std::vector<uint32_t>& token_ids) const {
+    std::string lowered = segment;
+    for (char& c : lowered) {
+        if (c >= 'A' && c <= 'Z') c += 'a' - 'A';
+    }
+    for (const auto& word : clip_word_split(lowered)) {
+        std::vector<std::string> chars = utf8_split(bytes_to_unicode(word));
+        if (chars.empty()) continue;
+        chars.back() += "</w>";
+        for (const auto& token : apply_bpe(chars)) {
+            auto it = token_to_id_.find(token);
+            token_ids.push_back(it != token_to_id_.end() ? it->second : unk_token_id_);
+        }
+    }
+}
+
 std::vector<uint32_t> BPETokenizer::encode(const std::string& text) const {
     if (text.empty()) return {};
 
@@ -468,6 +535,8 @@ std::vector<uint32_t> BPETokenizer::encode(const std::string& text) const {
         auto special_it = special_tokens_.find(segment);
         if (special_it != special_tokens_.end()) {
             token_ids.push_back(special_it->second);
+        } else if (runtime_config_.normalizer == TokenizerRuntimeConfig::Normalizer::CLIP) {
+            encode_clip_segment(segment, token_ids);
         } else {
             std::string normalized_segment = segment;
             std::vector<std::string> chars;
@@ -520,6 +589,20 @@ std::vector<uint32_t> BPETokenizer::encode(const std::string& text) const {
 }
 
 std::string BPETokenizer::decode(const std::vector<uint32_t>& tokens) const {
+    if (runtime_config_.decoder == TokenizerRuntimeConfig::Decoder::CLIP) {
+        std::string unicode_result;
+        unicode_result.reserve(tokens.size() * 6);
+        for (uint32_t token_id : tokens) {
+            if (token_id >= id_to_token_.size()) continue;
+            std::string tok = id_to_token_[token_id];
+            size_t pos = tok.find("</w>");
+            if (pos != std::string::npos) tok = tok.substr(0, pos) + " ";
+            unicode_result += tok;
+        }
+        std::string result = unicode_to_bytes(unicode_result);
+        while (!result.empty() && result.back() == ' ') result.pop_back();
+        return result;
+    }
     if (runtime_config_.decoder == TokenizerRuntimeConfig::Decoder::REPLACE_METASPACE) {
         std::string result;
         result.reserve(tokens.size() * 4);

cactus-engine/src/engine.h

@@ -149,7 +149,7 @@ struct Config {
     float rope_scaling_factor = 1.0f;
     float rope_mscale_all_dim = 0.0f;
 
-    enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, NEEDLE = 18, GENERIC = 19};
+    enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, NEEDLE = 18, GENERIC = 19, SD15 = 20};
     uint32_t predictor_hidden_dim = 0;
     uint32_t predictor_num_layers = 0;
     uint32_t tdt_joint_dim = 0;
@@ -287,8 +287,8 @@ struct ToolConstraintSpec {
 struct TokenizerRuntimeConfig {
     enum class TokenizerType { UNKNOWN, BPE, SENTENCEPIECE };
     enum class VocabFormat { UNKNOWN, ID_TAB_TOKEN, LINE_TOKEN };
-    enum class Normalizer { NONE, METASPACE, BYTE_LEVEL };
-    enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL };
+    enum class Normalizer { NONE, METASPACE, BYTE_LEVEL, CLIP };
+    enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL, CLIP };
 
     TokenizerType tokenizer_type = TokenizerType::UNKNOWN;
     VocabFormat vocab_format = VocabFormat::UNKNOWN;
@@ -298,6 +298,22 @@ struct TokenizerRuntimeConfig {
     bool has_chat_template = false;
 };
 
+struct DiffusionParams {
+    float beta_start = 0.00085f;
+    float beta_end = 0.012f;
+    std::string beta_schedule = "scaled_linear";
+    uint32_t num_train_timesteps = 1000;
+    uint32_t original_inference_steps = 50;
+    float timestep_scaling = 10.0f;
+    std::string prediction_type = "epsilon";
+};
+
+std::vector<float> diffusion_alphas_cumprod(const DiffusionParams& params);
+std::vector<uint32_t> diffusion_lcm_timesteps(const DiffusionParams& params, uint32_t steps);
+std::vector<float> diffusion_guidance_embedding(float guidance_embedding_scale, size_t dim);
+void write_typed_buffer(std::vector<uint8_t>& buf, Precision dst_prec,
+                        const void* src_data, size_t src_bytes, Precision src_prec);
+
 TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file);
 void load_special_tokens_map(const std::string& config_file, std::unordered_map<std::string, uint32_t>& special_tokens);
 std::vector<std::string> split_with_special_tokens(const std::string& text, const std::unordered_map<std::string, uint32_t>& special_tokens);
@@ -485,6 +501,8 @@ class BPETokenizer : public Tokenizer {
     std::string unicode_to_bytes(const std::string& text) const;
     std::vector<std::string> byte_level_split(const std::string& text) const;
     std::vector<std::string> utf8_split(const std::string& text) const;
+    std::vector<std::string> clip_word_split(const std::string& text) const;
+    void encode_clip_segment(const std::string& segment, std::vector<uint32_t>& token_ids) const;
 
     void cleanup_mmap();
 
@@ -686,6 +704,10 @@ class Model {
     uint32_t decode(const std::vector<uint32_t>& tokens, float temperature = -1.0f, float top_p = -1.0f,
                     size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr,
                     float min_p = 0.15f, float repetition_penalty = 1.1f);
+
+    int generate_image(const std::string& prompt, uint8_t* out_rgb, size_t out_capacity,
+                       uint32_t* out_width, uint32_t* out_height,
+                       int steps, float guidance_scale, uint64_t seed);
     void set_sample_seed(uint64_t seed) { sample_rng_.seed(static_cast<std::mt19937::result_type>(seed)); }
     bool prefill_and_sample_first_token(const std::vector<uint32_t>& tokens, uint32_t& out_token,
                                         float* out_uncertainty = nullptr);
@@ -1037,8 +1059,9 @@ class Model {
     Component* decoder_prefill_ = nullptr;
     Component* decoder_embed_ = nullptr;
     Component* prefill_encoder_ = nullptr;
-    enum class DecodeRoute { CACHED_STEP, DIRECT_DECODER_STEP, FULL_CONTEXT_TEXT, ENCODER_CROSS_KV_STEP };
+    enum class DecodeRoute { CACHED_STEP, DIRECT_DECODER_STEP, FULL_CONTEXT_TEXT, ENCODER_CROSS_KV_STEP, ITERATIVE_DENOISE };
     DecodeRoute decode_route_ = DecodeRoute::CACHED_STEP;
+    DiffusionParams diffusion_;
     Component* source_encoder_ = nullptr;
     Component* decoder_cross_kv_ = nullptr;
     Component* vision_encoder_ = nullptr;