diff --git a/backends/samsung/build.sh b/backends/samsung/build.sh index da8e211ee64..4c268a55ef9 100755 --- a/backends/samsung/build.sh +++ b/backends/samsung/build.sh @@ -7,6 +7,7 @@ X86_64_BUILD_DIR=${PROJECT_DIR}/build_samsung ANDROID_BUILD_DIR=${PROJECT_DIR}/build_samsung_android BUILD_ARCH=all CLEAN_BUILD_DIR=false +BUILD_VULKAN=OFF EXYNOS_AI_LITECORE_ROOT=${EXYNOS_AI_LITECORE_ROOT} ANDROID_NDK_ROOT=${ANDROID_NDK_ROOT} @@ -26,6 +27,9 @@ function usage() { --build, -b [x86_64, android, all] Default is all, x86_64 target to offline compilation, android target to online execution. + --vulkan Build the Vulkan backend into executor_runner (android, default is off). + Requires glslc from the Vulkan SDK (1.4.341.1+) in PATH, e.g. + export PATH=\$VULKAN_SDK/bin:\$PATH --clean, -c Clean the build cache. --help, -h Print the usage information. " @@ -89,36 +93,30 @@ function build_android() { -DCMAKE_BUILD_TYPE=Release \ -DEXECUTORCH_BUILD_ENN=ON \ -DEXYNOS_AI_LITECORE_ROOT=${EXYNOS_AI_LITECORE_ROOT} \ + -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH \ -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \ -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \ - -DEXECUTORCH_BUILD_EXTENSION_LLM=ON \ - -DEXECUTORCH_BUILD_EXTENSION_LLM_RUNNER=ON \ -DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \ -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \ + -DEXECUTORCH_BUILD_EXTENSION_LLM=ON \ + -DEXECUTORCH_BUILD_EXTENSION_LLM_RUNNER=ON \ -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \ -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \ + -DEXECUTORCH_BUILD_KERNELS_LLM=ON \ + -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \ + -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED_PORTABLE=ON \ -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \ + -DEXECUTORCH_BUILD_XNNPACK=ON \ + -DEXECUTORCH_BUILD_VULKAN=${BUILD_VULKAN} \ -DEXECUTORCH_ENABLE_LOGGING=1 \ -DEXECUTORCH_BUILD_DEVTOOLS=ON \ -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \ + -DBUILD_TESTING=OFF \ -S ${PROJECT_DIR} \ -B ${ANDROID_BUILD_DIR} - cmake --build ${ANDROID_BUILD_DIR} -j $(nproc) --target install - - LLAMA_EXAMPLE_ROOT=examples/models/llama - cmake ${PROJECT_DIR}/${LLAMA_EXAMPLE_ROOT} \ - -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_ROOT/build/cmake/android.toolchain.cmake \ - -DANDROID_ABI=${ANDROID_ABI} \ - -DCMAKE_INSTALL_PREFIX=${ANDROID_BUILD_DIR} \ - -DCMAKE_BUILD_TYPE=Release \ - -DANDROID_PLATFORM=${ANDROID_PLATFORM} \ - -DEXECUTORCH_ENABLE_LOGGING=ON \ - -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} \ - -B${ANDROID_BUILD_DIR}/${LLAMA_EXAMPLE_ROOT} - - cmake --build ${ANDROID_BUILD_DIR}/${LLAMA_EXAMPLE_ROOT} -j $(nproc) --config Release + cmake --build ${ANDROID_BUILD_DIR} -j $(nproc) --target install --config Release } # Main @@ -133,6 +131,9 @@ do let i++ ANDROID_NDK_ROOT="${!i}" ;; + "--vulkan") + BUILD_VULKAN=ON + ;; "--clean"|"-c") CLEAN_BUILD_DIR=true ;; diff --git a/backends/samsung/runtime/enn_api_implementation.cpp b/backends/samsung/runtime/enn_api_implementation.cpp index 0cdf810d5c4..560d6b6a1d4 100644 --- a/backends/samsung/runtime/enn_api_implementation.cpp +++ b/backends/samsung/runtime/enn_api_implementation.cpp @@ -7,9 +7,15 @@ * */ #include +#include + #include #include +#include +#include +#include + #define ENN_LOAD_API_FUNC(handle, name, enn_api_ptr) \ enn_api_ptr->name = \ reinterpret_cast(loadApiFunction(handle, #name, false)); \ @@ -38,6 +44,102 @@ EnnApi* EnnApi::getEnnApiInstance() { return &enn_api; } +namespace { + +constexpr uint32_t kPerfConfigIdDefault = 0; + +struct PerfConfig { + PerfModePreference mode; + uint32_t config_id; +}; + +// Primary source: ro.hardware. Falls back to ro.soc.model if unavailable. +std::string getHwName() { + char prop_value[PROP_VALUE_MAX] = {}; + if (__system_property_get("ro.hardware", prop_value) > 0) { + return std::string(prop_value); + } + return ""; +} + +std::string getSocName() { + char prop_value[PROP_VALUE_MAX] = {}; + if (__system_property_get("ro.soc.model", prop_value) > 0) { + return std::string(prop_value); + } + return ""; +} + +// Extracts trailing digits, e.g. "exynos9955" -> "9955", "s5e9965" -> "9965". +bool trimToTrailingDigits(std::string& value) { + const char* digits = "0123456789"; + auto last = value.find_last_of(digits); + if (last == std::string::npos) { + return false; + } + auto first = last; + while (first > 0 && std::strchr(digits, value[first - 1]) != nullptr) { + first--; + } + value = value.substr(first, last - first + 1); + return true; +} + +const std::unordered_map& getSocPerfConfigMap() { + static const std::unordered_map kSocPerfConfig = { + {"9955", {ENN_PREF_MODE_PERFORMANCE, kPerfConfigIdDefault}}, + {"9965", {ENN_PREF_MODE_CUSTOM, 100111}}, + }; + return kSocPerfConfig; +} + +PerfConfig getPerfConfigForSoc() { + std::string key = getHwName(); + if (key.empty()) { + key = getSocName(); + } + if (!key.empty()) { + trimToTrailingDigits(key); + } + + const auto& map = getSocPerfConfigMap(); + auto it = map.find(key); + if (it != map.end()) { + return it->second; + } + return {ENN_PREF_MODE_PERFORMANCE, kPerfConfigIdDefault}; +} + +// Mirrors LiteRT's litert::samsung::SetGenAiPerfConfigFromSoc: detect the +// SoC from Android system properties and apply its known-good perf mode, +// instead of leaving the ENN runtime on its own default. +void setPerfConfigFromSoc(const EnnApi* enn_api) { + PerfConfig config = getPerfConfigForSoc(); + if (enn_api->EnnSetPreferencePerfMode(static_cast(config.mode)) != + ENN_RET_SUCCESS) { + ENN_LOG_ERROR("Failed to set ENN preference perf mode."); + return; + } + if (config.config_id == kPerfConfigIdDefault) { + return; + } + if (enn_api->EnnSetPreferencePerfConfigId == nullptr || + enn_api->EnnSetPreferencePerfConfigId(config.config_id) != + ENN_RET_SUCCESS) { + ENN_LOG_WARN( + "Failed to set ENN perf config id %u, falling back to PERFORMANCE mode.", + config.config_id); + enn_api->EnnSetPreferencePerfMode( + static_cast(ENN_PREF_MODE_PERFORMANCE)); + } +} + +} // namespace + +void EnnApi::applyPerfConfigFromSoc() const { + setPerfConfigFromSoc(this); +} + EnnApi::EnnApi() { auto status = loadApiLib(); if (status != Error::Ok) { @@ -96,7 +198,13 @@ Error EnnApi::loadApiLib() { ENN_LOAD_API_FUNC( libenn_public_api_, EnnGetFileDescriptorFromEnnBuffer, this); ENN_LOAD_API_FUNC(libenn_public_api_, EnnOpenModelFromFd, this); - + // Optional: only newer ENN releases export these. + EnnSetPreferencePerfConfigId = + reinterpret_cast(loadApiFunction( + libenn_public_api_, "EnnSetPreferencePerfConfigId", true)); + EnnOpenModelFromFdWithWeight = + reinterpret_cast(loadApiFunction( + libenn_public_api_, "EnnOpenModelFromFdWithWeight", true)); return Error::Ok; } diff --git a/backends/samsung/runtime/enn_api_implementation.h b/backends/samsung/runtime/enn_api_implementation.h index 3e0bf1c5f2a..2c8e6577012 100644 --- a/backends/samsung/runtime/enn_api_implementation.h +++ b/backends/samsung/runtime/enn_api_implementation.h @@ -30,15 +30,29 @@ class EnnApi { static EnnApi* getEnnApiInstance(); bool isInitialized() const; + // Detects the SoC and applies its known-good ENN perf mode/config id. + // Must be called on the same thread that is about to open a model, and + // as the last preference-mutating call before that open, since + // EnnInitialize() resets the calling thread's preference on every call. + void applyPerfConfigFromSoc() const; + EnnReturn (*EnnInitialize)(void); EnnReturn (*EnnSetPreferencePerfMode)(const uint32_t val); EnnReturn (*EnnGetPreferencePerfMode)(uint32_t* val_ptr); + EnnReturn (*EnnSetPreferencePerfConfigId)(uint32_t val); EnnReturn (*EnnOpenModel)(const char* model_file, EnnModelId* model_id); EnnReturn (*EnnOpenModelFromMemory)( const char* va, const uint32_t size, EnnModelId* model_id); EnnReturn (*EnnOpenModelFromFd)(int _fd, EnnModelId* model_id); + EnnReturn (*EnnOpenModelFromFdWithWeight)( + const int fd, + const uint32_t size, + const uint32_t offset, + EnnBufferPtr* weights, + uint32_t n_weights, + EnnModelId* model_id); EnnReturn (*EnnSetFastIpc)(void); EnnReturn (*EnnUnsetFastIpc)(void); EnnReturn (*EnnExecuteModelFastIpc)( @@ -92,6 +106,7 @@ class EnnApi { typedef EnnReturn (*EnnInitialize_fn)(void); typedef EnnReturn (*EnnSetPreferencePerfMode_fn)(const uint32_t val); typedef EnnReturn (*EnnGetPreferencePerfMode_fn)(uint32_t* val_ptr); +typedef EnnReturn (*EnnSetPreferencePerfConfigId_fn)(uint32_t val); typedef EnnReturn ( *EnnOpenModel_fn)(const char* model_file, EnnModelId* model_id); typedef EnnReturn (*EnnOpenModelFromMemory_fn)( @@ -136,6 +151,13 @@ typedef EnnReturn (*EnnReleaseBuffer_fn)(EnnBufferPtr buf); typedef EnnReturn ( *EnnGetFileDescriptorFromEnnBuffer_fn)(EnnBufferPtr buffer, int32_t* fd); typedef EnnReturn (*EnnOpenModelFromFd_fn)(int _fd, EnnModelId* model_id); +typedef EnnReturn (*EnnOpenModelFromFdWithWeight_fn)( + const int fd, + const uint32_t size, + const uint32_t offset, + EnnBufferPtr* weights, + uint32_t n_weights, + EnnModelId* model_id); } // namespace enn } // namespace executor diff --git a/backends/samsung/runtime/enn_backend.cpp b/backends/samsung/runtime/enn_backend.cpp index 81484afc44c..0054926da7d 100644 --- a/backends/samsung/runtime/enn_backend.cpp +++ b/backends/samsung/runtime/enn_backend.cpp @@ -13,9 +13,16 @@ #include #include #include +#include #include +#include +#include #include +#include +#include +#include +#include #include #pragma clang diagnostic ignored "-Wglobal-constructors" @@ -23,6 +30,52 @@ namespace torch { namespace executor { +using executorch::runtime::NamedDataMap; + +const char ennWeightBufferKey[] = "enn_weight_buffer_hash_id"; + +namespace { +// Methods sharing weights (e.g. prefill and decode) reference the same buffer, +// so it is refcounted and released when the last delegate using it is +// destroyed. +struct SharedWeight { + FreeableBuffer buffer; + int32_t refs = 0; +}; + +std::mutex g_weight_cache_mutex; +std::unordered_map g_weight_cache; + +Result acquire_weight( + const NamedDataMap* named_data_map, + const std::string& key) { + std::lock_guard lock(g_weight_cache_mutex); + auto it = g_weight_cache.find(key); + if (it == g_weight_cache.end()) { + ET_CHECK_OR_RETURN_ERROR( + named_data_map != nullptr, + InvalidArgument, + "Weight-sharing delegate needs a NamedDataMap (.ptd) for key %s", + key.c_str()); + Result loaded = named_data_map->get_data(key.c_str()); + ET_CHECK_OK_OR_RETURN_ERROR( + loaded.error(), "Failed to load shared weight %s", key.c_str()); + it = g_weight_cache.emplace(key, SharedWeight{std::move(loaded.get()), 0}) + .first; + } + it->second.refs++; + return &it->second.buffer; +} + +void release_weight(const std::string& key) { + std::lock_guard lock(g_weight_cache_mutex); + auto it = g_weight_cache.find(key); + if (it != g_weight_cache.end() && --it->second.refs == 0) { + g_weight_cache.erase(it); + } +} +} // namespace + class EnnBackend final : public PyTorchBackendInterface { public: ~EnnBackend() = default; @@ -31,6 +84,11 @@ class EnnBackend final : public PyTorchBackendInterface { return true; } + std::string decode_spec_value(const CompileSpec& spec) const { + const char* char_ptr = static_cast(spec.value.buffer); + return std::string(char_ptr, spec.value.nbytes); + } + Result init( BackendInitContext& context, FreeableBuffer* processed, @@ -38,15 +96,53 @@ class EnnBackend final : public PyTorchBackendInterface { EXYNOS_ATRACE_NAME_LINE("backend_init"); MemoryAllocator* runtime_allocator = context.get_runtime_allocator(); auto executor = runtime_allocator->allocateInstance(); + ET_CHECK_OR_RETURN_ERROR( + executor != nullptr, + MemoryAllocationFailed, + "Failed to allocate EnnExecutor"); + new (executor) enn::EnnExecutor(); const char* binary_buf_addr = reinterpret_cast(processed->data()); size_t buf_size = processed->size(); - Error err = executor->initialize(binary_buf_addr, buf_size); + + auto it = std::find_if( + compile_specs.begin(), + compile_specs.end(), + [](const CompileSpec& spec) { + return std::strcmp(spec.key, ennWeightBufferKey) == 0; + }); + + Error err = Error::Ok; + if (it != compile_specs.end()) { + std::string weight_key = decode_spec_value(*it); + auto weight = acquire_weight(context.get_named_data_map(), weight_key); + if (!weight.ok()) { + executor->~EnnExecutor(); + return weight.error(); + } + executor->set_weight_key(weight_key); + ET_LOG( + Info, + "EnnBackend, [%s] shared weight %s size(%zu), addr(%p)", + context.get_method_name(), + weight_key.c_str(), + weight.get()->size(), + weight.get()->data()); + err = executor->initialize( + binary_buf_addr, + buf_size, + reinterpret_cast(weight.get()->data()), + weight.get()->size(), + true); + } else { + err = executor->initialize(binary_buf_addr, buf_size); + } if (err != Error::Ok) { ENN_LOG_ERROR("Exynos backend initialize failed."); - executor->~EnnExecutor(); + destroy(executor); return err; } + return executor; } @@ -85,7 +181,11 @@ class EnnBackend final : public PyTorchBackendInterface { EXYNOS_ATRACE_NAME_LINE("backend_destroy"); if (handle != nullptr) { auto executor = static_cast(handle); + std::string weight_key = executor->get_weight_key(); executor->~EnnExecutor(); + if (!weight_key.empty()) { + release_weight(weight_key); + } } } }; // namespace executor diff --git a/backends/samsung/runtime/enn_executor.cpp b/backends/samsung/runtime/enn_executor.cpp index 99a2ce11e5e..91318ada03b 100644 --- a/backends/samsung/runtime/enn_executor.cpp +++ b/backends/samsung/runtime/enn_executor.cpp @@ -20,28 +20,77 @@ namespace executor { namespace enn { Error EnnExecutor::initialize(const char* binary_buf_addr, size_t buf_size) { + return initialize(binary_buf_addr, buf_size, nullptr, 0, false); +} + +Error EnnExecutor::initialize( + const char* binary_buf_addr, + size_t buf_size, + const char* weight_buf_addr, + size_t weight_size, + bool use_weight_fd) { EXYNOS_ATRACE_FUNCTION_LINE(); auto sm_instance = executorch::backends::enn::shared_memory_manager:: SharedMemoryManager::getInstance(); const EnnApi* enn_api_inst = EnnApi::getEnnApiInstance(); ET_CHECK_OR_RETURN_ERROR( enn_api_inst->isInitialized(), Internal, "Enn initialize failed."); + // EnnInitialize() resets the calling thread's preference on every call, so + // re-apply it right before opening the model. + enn_api_inst->applyPerfConfigFromSoc(); + ET_CHECK_OR_RETURN_ERROR( + buf_size <= UINT32_MAX && weight_size <= UINT32_MAX, + InvalidArgument, + "Model (%zu) or weight (%zu) buffer exceeds the 4GiB ENN limit", + buf_size, + weight_size); EnnReturn ret = ENN_RET_SUCCESS; ET_LOG(Info, "Start to open model %p, %zu", binary_buf_addr, buf_size); - EnnBufferPtr shared_buffer = nullptr; - if (sm_instance->query(&shared_buffer, binary_buf_addr, buf_size)) { + EnnBufferPtr pte_buffer = nullptr; + if (sm_instance->query(&pte_buffer, binary_buf_addr, buf_size)) { int32_t fd; - if (shared_buffer->va == binary_buf_addr && - !enn_api_inst->EnnGetFileDescriptorFromEnnBuffer(shared_buffer, &fd)) { - ret = enn_api_inst->EnnOpenModelFromFd(fd, &model_id_); - if (ret == ENN_RET_SUCCESS) { - ET_LOG(Info, "Opened model from file descriptor, so fd is closed"); - sm_instance->free(shared_buffer->va); + if (pte_buffer->va == binary_buf_addr && + !enn_api_inst->EnnGetFileDescriptorFromEnnBuffer(pte_buffer, &fd)) { + if (use_weight_fd) { + ET_CHECK_OR_RETURN_ERROR( + enn_api_inst->EnnOpenModelFromFdWithWeight != nullptr, + NotSupported, + "This ENN runtime cannot open models with external weights."); + EnnBufferPtr ptd_buffer = nullptr; + ET_CHECK_OR_RETURN_ERROR( + sm_instance->query(&ptd_buffer, weight_buf_addr, weight_size), + Internal, + "PTD EnnBuffer not found."); + ET_LOG( + Info, + "Open FD & weights. fd: %d, model size %zu, offset: %u, weight buffer: %p", + fd, + buf_size, + pte_buffer->offset, + ptd_buffer->va); + ret = enn_api_inst->EnnOpenModelFromFdWithWeight( + fd, buf_size, pte_buffer->offset, &ptd_buffer, 1, &model_id_); + ET_CHECK_OR_RETURN_ERROR( + ret == ENN_RET_SUCCESS, + Internal, + "Open model with weights failed."); + ET_LOG(Info, "Opened model from file descriptor with weights"); + sm_instance->free(pte_buffer->va); + } else { + ret = enn_api_inst->EnnOpenModelFromFd(fd, &model_id_); + if (ret == ENN_RET_SUCCESS) { + ET_LOG(Info, "Opened model from file descriptor, so fd is closed"); + sm_instance->free(pte_buffer->va); + } } } } + ET_CHECK_OR_RETURN_ERROR( + !use_weight_fd || model_id_, + Internal, + "Failed to open the model with external weights from file descriptor."); if (!model_id_) { ET_LOG(Info, "Open model from memory"); ret = enn_api_inst->EnnOpenModelFromMemory( @@ -96,10 +145,13 @@ Error EnnExecutor::eval( } EXYNOS_ATRACE_END(); - ENN_LOG_DEBUG("Start to execute model."); + ENN_LOG_DEBUG("Starting model execution"); auto ret = enn_api_inst->EnnExecuteModel(model_id_); if (ret != ENN_RET_SUCCESS) { - ENN_LOG_ERROR("EnnExecuteModel Failed"); + ENN_LOG_ERROR( + "EnnExecuteModel failed, ret=%d (model_id=0x%" PRIX64 ")", + ret, + model_id_); return Error::Internal; } @@ -124,6 +176,11 @@ EnnExecutor::~EnnExecutor() { enn_api_inst->EnnReleaseBuffers(alloc_buffer_, num_of_buffers); } enn_api_inst->EnnCloseModel(model_id_); + + ET_LOG( + Info, + "EnnExecutor destroyed: model 0x%" PRIX64 " closed, resources released", + model_id_); } } // namespace enn diff --git a/backends/samsung/runtime/enn_executor.h b/backends/samsung/runtime/enn_executor.h index 68ee1f56847..cbfe5613e6b 100644 --- a/backends/samsung/runtime/enn_executor.h +++ b/backends/samsung/runtime/enn_executor.h @@ -11,6 +11,7 @@ #include #include +#include #include namespace torch { @@ -25,6 +26,12 @@ struct DataBuffer { class EnnExecutor { public: Error initialize(const char* binary_buf_addr, size_t buf_size); + Error initialize( + const char* binary_buf_addr, + size_t buf_size, + const char* weight_buf_addr, + size_t weight_size, + bool use_weight_fd); Error eval( const std::vector& inputs, @@ -37,6 +44,14 @@ class EnnExecutor { return num_of_outputs_; } + void set_weight_key(const std::string& key) { + weight_key_ = key; + } + + const std::string& get_weight_key() const { + return weight_key_; + } + ~EnnExecutor(); private: @@ -44,6 +59,7 @@ class EnnExecutor { EnnBufferPtr* alloc_buffer_ = nullptr; int32_t num_of_inputs_ = 0; int32_t num_of_outputs_ = 0; + std::string weight_key_; }; } // namespace enn diff --git a/backends/samsung/runtime/enn_type.h b/backends/samsung/runtime/enn_type.h index 4b91fe2d08a..f4ae534b020 100644 --- a/backends/samsung/runtime/enn_type.h +++ b/backends/samsung/runtime/enn_type.h @@ -53,3 +53,10 @@ typedef struct _NumberOfBuffersInfo { uint32_t n_in_buf; uint32_t n_out_buf; } NumberOfBuffersInfo; + +typedef enum _PerfModePreference { + ENN_PREF_MODE_LOW_POWER = 0, + ENN_PREF_MODE_BALANCED = 4, + ENN_PREF_MODE_CUSTOM = 5, + ENN_PREF_MODE_PERFORMANCE = 7, +} PerfModePreference; diff --git a/backends/samsung/runtime/logging.cpp b/backends/samsung/runtime/logging.cpp index 0a2a6e64282..d3f98111a9b 100644 --- a/backends/samsung/runtime/logging.cpp +++ b/backends/samsung/runtime/logging.cpp @@ -35,7 +35,11 @@ void EnnLogImpl(ENN_LOG_LEVEL log_level, const char* format, ...) { android_severity = ANDROID_LOG_UNKNOWN; break; } - __android_log_print(android_severity, "[Exynos ExecuTorch]", format, args); + va_list android_args; + va_copy(android_args, args); + __android_log_vprint( + android_severity, "[Exynos ExecuTorch]", format, android_args); + va_end(android_args); #endif const char* serverity_name; switch (log_level) { diff --git a/examples/samsung/CMakeLists.txt b/examples/samsung/CMakeLists.txt index 28a6dcc6660..d4fe9130eb9 100644 --- a/examples/samsung/CMakeLists.txt +++ b/examples/samsung/CMakeLists.txt @@ -4,7 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -cmake_minimum_required(VERSION 3.15) +cmake_minimum_required(VERSION 3.24) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) project(samsung_example) @@ -59,3 +59,182 @@ target_link_libraries( set_target_properties( enn_executor_runner PROPERTIES CXX_VISIBILITY_PRESET hidden ) + +option(EXYNOS_BUILD_LLM_RUNNER + "Build enn_executor_llm_runner and the Exynos LLM runner library" OFF +) +# C API in exynos_llm_wrapper.h. Always compiled into enn_executor_llm_runner; +# additionally shipped as libexynos_llm_wrapper.so when +# EXYNOS_BUILD_LLM_WRAPPER_SO is ON. +option(EXYNOS_BUILD_LLM_WRAPPER_SO + "Build libexynos_llm_wrapper.so and exynos_llm_wrapper_sample" OFF +) +if(EXYNOS_BUILD_LLM_WRAPPER_SO AND NOT EXYNOS_BUILD_LLM_RUNNER) + message( + FATAL_ERROR "EXYNOS_BUILD_LLM_WRAPPER_SO requires EXYNOS_BUILD_LLM_RUNNER" + ) +endif() +if(NOT EXYNOS_BUILD_LLM_RUNNER) + return() +endif() + +find_package(tokenizers REQUIRED) + +set(_llm_compile_options + -Wno-deprecated-declarations -Wno-unused-const-variable + -Wno-unused-variable -Wno-inconsistent-missing-override +) + +link_directories(${EXECUTORCH_ROOT}/build_samsung_android/lib) + +add_library(lib_portable_ops STATIC IMPORTED) +set_target_properties( + lib_portable_ops + PROPERTIES + IMPORTED_LOCATION + "${EXECUTORCH_ROOT}/build_samsung_android/lib/libportable_ops_lib.a" +) + +add_subdirectory(executor_runner/llm/runner) + +add_library(lib_tokenizer STATIC IMPORTED) +set_target_properties( + lib_tokenizer + PROPERTIES + IMPORTED_LOCATION + "${EXECUTORCH_ROOT}/build_samsung_android/extension/llm/tokenizers/libtokenizers.a" +) + +add_library(lib_extension_llm_runner STATIC IMPORTED) +set_target_properties( + lib_extension_llm_runner + PROPERTIES + IMPORTED_LOCATION + "${EXECUTORCH_ROOT}/build_samsung_android/extension/llm/runner/libextension_llm_runner.a" +) + +add_library(lib_enn_backend STATIC IMPORTED) +set_target_properties( + lib_enn_backend + PROPERTIES IMPORTED_LOCATION + "${EXECUTORCH_ROOT}/build_samsung_android/lib/libenn_backend.a" +) + +add_library(lib_executorch STATIC IMPORTED) +set_target_properties( + lib_executorch + PROPERTIES IMPORTED_LOCATION + "${EXECUTORCH_ROOT}/build_samsung_android/lib/libexecutorch.a" +) + +add_library(lib_custom_op STATIC IMPORTED) +set_target_properties( + lib_custom_op + PROPERTIES IMPORTED_LOCATION + "${EXECUTORCH_ROOT}/build_samsung_android/lib/libcustom_ops.a" +) + +set(enn_executor_llm_runner_deps + executorch_core extension_data_loader extension_module extension_tensor + extension_flat_tensor lib_extension_llm_runner +) + +set(xnnpack_backend_libs xnnpack_backend XNNPACK xnnpack-microkernels-prod + extension_threadpool pthreadpool kleidiai +) + +list( + APPEND + xnnpack_backend_libs + optimized_native_cpu_ops_lib + optimized_kernels + portable_kernels + cpublas + eigen_blas +) + +# Extra sources for cpuinfo +set(XNNPACK_ROOT ${EXECUTORCH_ROOT}/backends/xnnpack) +list(APPEND xnnpack_backend_libs cpuinfo) +list(APPEND _common_include_directories + ${XNNPACK_ROOT}/third-party/cpuinfo/include +) + +# quantized_ops_lib: Register quantized op kernels into the runtime +list(APPEND xnnpack_backend_libs quantized_kernels quantized_ops_lib) + +set(exynos_llm_link_libs + log + gflags + android + lib_portable_ops + ${enn_executor_llm_runner_deps} + ${xnnpack_backend_libs} + sentencepiece + re2 + lib_tokenizer + -Wl,--whole-archive + lib_executorch + lib_custom_op + lib_enn_backend + enn_logging + -Wl,--no-whole-archive +) + +add_library( + exynos_llm_wrapper_obj OBJECT + ${CMAKE_CURRENT_LIST_DIR}/executor_runner/exynos_llm_wrapper.cpp +) +target_compile_definitions(exynos_llm_wrapper_obj PRIVATE EXECUTORCH_VERSION=2) +target_compile_options(exynos_llm_wrapper_obj PRIVATE ${_llm_compile_options}) +target_link_libraries(exynos_llm_wrapper_obj PRIVATE enn_llm_runner) +set_target_properties( + exynos_llm_wrapper_obj PROPERTIES CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON +) + +add_executable( + enn_executor_llm_runner + ${CMAKE_CURRENT_LIST_DIR}/executor_runner/enn_executor_llm_runner.cpp +) + +target_include_directories( + enn_executor_llm_runner + PUBLIC ${_common_include_directories} + ${EXECUTORCH_ROOT}/backends/samsung/runtime + ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include +) + +target_compile_options(enn_executor_llm_runner PRIVATE ${_llm_compile_options}) +target_link_libraries( + enn_executor_llm_runner PRIVATE exynos_llm_wrapper_obj enn_llm_runner + ${exynos_llm_link_libs} +) + +if(EXYNOS_BUILD_LLM_WRAPPER_SO) + # Bundles all dependencies so only the .so + header need to be shipped. + add_library( + exynos_llm_wrapper SHARED $ + ) + set_target_properties(exynos_llm_wrapper PROPERTIES LINKER_LANGUAGE CXX) + target_include_directories( + exynos_llm_wrapper PUBLIC ${CMAKE_CURRENT_LIST_DIR}/executor_runner + ) + target_link_libraries( + exynos_llm_wrapper PRIVATE enn_llm_runner ${exynos_llm_link_libs} + -Wl,--exclude-libs,ALL + ) + + # Sample CLI consuming the public C API the way an app uses the .so. + add_executable( + exynos_llm_wrapper_sample + ${CMAKE_CURRENT_LIST_DIR}/executor_runner/exynos_llm_wrapper_sample_main.cpp + ) + target_compile_options( + exynos_llm_wrapper_sample PRIVATE ${_llm_compile_options} + ) + target_link_libraries(exynos_llm_wrapper_sample PRIVATE exynos_llm_wrapper) + set_target_properties( + exynos_llm_wrapper_sample PROPERTIES CXX_VISIBILITY_PRESET hidden + ) +endif() diff --git a/examples/samsung/build.sh b/examples/samsung/build.sh index 4568180777a..c5ed57ee8cc 100755 --- a/examples/samsung/build.sh +++ b/examples/samsung/build.sh @@ -19,8 +19,30 @@ echo ANDROID_NDK_ROOT=${ANDROID_NDK_ROOT} echo ANDROID_ABI=${ANDROID_ABI} echo ANDROID_PLATFORM=${ANDROID_PLATFORM} +usage() { + echo "Usage: $0 [--llm] [--llm-wrapper-so]" + echo " --llm Also build enn_executor_llm_runner." + echo " --llm-wrapper-so Also build libexynos_llm_wrapper.so and exynos_llm_wrapper_sample (implies --llm)." +} + main() { + local build_llm_runner=OFF + local build_llm_wrapper_so=OFF + for arg in "$@"; do + case "$arg" in + "--llm") build_llm_runner=ON ;; + "--llm-wrapper-so") build_llm_runner=ON; build_llm_wrapper_so=ON ;; + "--help"|"-h") usage; exit 0 ;; + *) echo "Unknown option: $arg"; usage; exit 1 ;; + esac + done + cd "$PROJECT_DIR" + + if [[ -z ${ANDROID_NDK} ]]; then + ANDROID_NDK=${ANDROID_NDK_ROOT} + fi + local build_dir_root="build_samsung_android" local example_root="examples/samsung" local build_dir_example="$PROJECT_DIR/${build_dir_root}/${example_root}" @@ -37,9 +59,16 @@ main() { -DANDROID_PLATFORM=$ANDROID_PLATFORM \ -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH \ -DCMAKE_BUILD_TYPE=Release \ + -DEXYNOS_BUILD_LLM_RUNNER=${build_llm_runner} \ + -DEXYNOS_BUILD_LLM_WRAPPER_SO=${build_llm_wrapper_so} \ -B"${build_dir_example}" \ "$PROJECT_DIR/${example_root}" - cmake --build build_samsung_android/examples/samsung/ --config Release + cmake --build ${build_dir_example} --config Release + + if [[ ${build_llm_wrapper_so} == OFF ]]; then + rm -f "${build_dir_example}/libexynos_llm_wrapper.so" \ + "${build_dir_example}/exynos_llm_wrapper_sample" + fi } main "$@" diff --git a/examples/samsung/executor_runner/README.md b/examples/samsung/executor_runner/README.md new file mode 100644 index 00000000000..19d5ac0f2c2 --- /dev/null +++ b/examples/samsung/executor_runner/README.md @@ -0,0 +1,61 @@ +# Exynos Executor Runner Configuration + +This directory contains the executor runner for LLMs on Exynos platforms. The runner requires a JSON configuration file to correctly set up the model architecture and runtime parameters. + +## Building + +The LLM runner is not part of the default example build. After building the backend with `./backends/samsung/build.sh`, build it with: + +```bash +./examples/samsung/build.sh --llm +``` + +## Using JSON Configuration Files + +The `enn_executor_llm_runner` binary accepts a path to a JSON configuration file via the `--config_path` argument. This file defines critical model dimensions and strategies used during execution. + +### Example Usage + +You can refer to `run_llama3_sample.sh` for a complete execution example. The key part of the command is: + +```bash +./enn_executor_llm_runner \ + --model_path="/data/local/tmp/llama3/llama3.pte" \ + --config_path="/data/local/tmp/exynos_llama3_1b.json" \ + --prompt="What is the capital of France?" \ + ... +``` + +For Gemma 4 multimodal inference, also pass the vision encoder and the image with `--vision_model_path`, `--image_path` and, optionally, `--prompt_before_image` for the text placed before the image. + +### Where to find config files + +Configuration files for supported models are located in `configs/`: +- Gemma 3 1B: `configs/exynos_gemma3_1b.json` +- Gemma 4 E2B: `configs/exynos_gemma4_e2b.json` +- Llama 3 1B: `configs/exynos_llama3_1b.json` + +Ensure these files are pushed to the target device (e.g., to `/data/local/tmp/`) before running the script. + +### Position embeddings + +`--pos_embedding_path` is optional: when the file is missing, the runner precomputes the RoPE tables from the config. For Gemma 4, generate the table that matches the exported model with: + +```bash +python -m executorch.examples.samsung.scripts.gemma4_pos_embedding \ + --config --max-seq-len 4096 --out pos_embedding.bin +``` + +## `exynos_llm_wrapper` library (app-porting reference) + +`libexynos_llm_wrapper.so` is a reference for porting the Exynos LLM runner into +an Android app: a small public C API (`exynos_llm_wrapper.h`) that hides the +runner internals behind an opaque handle. `exynos_llm_wrapper_sample` is a CLI +that exercises the same API on-device. + +`enn_executor_llm_runner` always drives the runner through this C API, but the +`.so` and the sample are only produced when requested: + +```bash +./examples/samsung/build.sh --llm-wrapper-so +``` diff --git a/examples/samsung/executor_runner/configs/exynos_gemma3_1b.json b/examples/samsung/executor_runner/configs/exynos_gemma3_1b.json new file mode 100644 index 00000000000..3adcedde9c8 --- /dev/null +++ b/examples/samsung/executor_runner/configs/exynos_gemma3_1b.json @@ -0,0 +1,21 @@ +{ + "model_id": "exynos_gemma3_1b", + "description": "Exynos Optimized Gemma3 1B Configuration", + "hidden_dim": 1152, + "n_heads": 4, + "n_kv_heads": 1, + "n_layers": 26, + "head_dim": 256, + "vocab_size": 262144, + "max_seq_len": 1024, + "max_context_len": 8192, + "embedding_strategy": "global_and_local", + "masking_strategy": "causal_local", + "model_version": "gemma3", + "tokenizer_type": "hf", + "prefill_seq_len": 128, + "decode_seq_len": 1, + "attn_mask_rows": 128, + "attn_mask_cols": 1024, + "sliding_window": 512 +} diff --git a/examples/samsung/executor_runner/configs/exynos_gemma4_e2b.json b/examples/samsung/executor_runner/configs/exynos_gemma4_e2b.json new file mode 100644 index 00000000000..aaa5d379b91 --- /dev/null +++ b/examples/samsung/executor_runner/configs/exynos_gemma4_e2b.json @@ -0,0 +1,69 @@ +{ + "model_id": "exynos_gemma4_e2b", + "description": "Exynos Optimized Gemma4 E2B Configuration", + "embedding_strategy": "global_and_local", + "masking_strategy": "causal_local", + "model_version": "gemma4", + "tokenizer_type": "hf", + "model_type": "gemma4", + "text_config": { + "hidden_size": 1536, + "intermediate_size": 6144, + "use_double_wide_mlp": true, + "num_hidden_layers": 35, + "num_attention_heads": 8, + "head_dim": 256, + "global_head_dim": 512, + "num_key_value_heads": 1, + "vocab_size": 262144, + "vocab_size_per_layer_input": 262144, + "max_seq_len": 4096, + "max_position_embeddings": 131072, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "rope_local_base_freq": 10000.0, + "partial_rotary_factor": 0.25, + "sliding_window": 512, + "hidden_activation": "gelu_pytorch_tanh", + "final_logit_softcapping": 30.0, + "hidden_size_per_layer_input": 256, + "num_kv_shared_layers": 20, + "tie_word_embeddings": true, + "layer_types": [ + "sliding_attention","sliding_attention","sliding_attention","sliding_attention","full_attention", + "sliding_attention","sliding_attention","sliding_attention","sliding_attention","full_attention", + "sliding_attention","sliding_attention","sliding_attention","sliding_attention","full_attention", + "sliding_attention","sliding_attention","sliding_attention","sliding_attention","full_attention", + "sliding_attention","sliding_attention","sliding_attention","sliding_attention","full_attention", + "sliding_attention","sliding_attention","sliding_attention","sliding_attention","full_attention", + "sliding_attention","sliding_attention","sliding_attention","sliding_attention","full_attention" + ] + }, + "audio_config": { + "model_type": "gemma4_audio", + "hidden_size": 1024, + "output_proj_dims": 1536, + "num_attention_heads": 8, + "num_hidden_layers": 12, + "rms_norm_eps": 1e-06, + "conv_kernel_size": 5, + "residual_weight": 0.5, + "attention_chunk_size": 12, + "attention_context_left": 13, + "attention_context_right": 0, + "attention_logit_cap": 50.0 + }, + "audio_seq_length": 750, + "audio_ms_per_token": 40, + "audio_sample_rate": 16000, + "max_audio_length_sec": 30, + "special_tokens": { + "audio_token_id": 258881, + "boa_token_id": 256000, + "eoa_token_id": 258883, + "boi_token_id": 255999, + "eoi_token_id": 258882, + "image_token_id": 258880, + "video_token_id": 258884 + } + } diff --git a/examples/samsung/executor_runner/configs/exynos_llama3_1b.json b/examples/samsung/executor_runner/configs/exynos_llama3_1b.json new file mode 100644 index 00000000000..b15807af27b --- /dev/null +++ b/examples/samsung/executor_runner/configs/exynos_llama3_1b.json @@ -0,0 +1,20 @@ +{ + "model_id": "exynos_llama3_1b", + "description": "Exynos Optimized Llama3 1B Configuration", + "hidden_dim": 2048, + "n_heads": 32, + "n_kv_heads": 8, + "n_layers": 16, + "head_dim": 64, + "vocab_size": 128256, + "max_seq_len": 1024, + "max_context_len": 8192, + "embedding_strategy": "global_only", + "masking_strategy": "causal", + "model_version": "llama3", + "tokenizer_type": "tiktoken", + "prefill_seq_len": 128, + "decode_seq_len": 1, + "attn_mask_rows": 128, + "attn_mask_cols": 1024 +} diff --git a/examples/samsung/executor_runner/enn_executor_llm_runner.cpp b/examples/samsung/executor_runner/enn_executor_llm_runner.cpp new file mode 100644 index 00000000000..475d531faac --- /dev/null +++ b/examples/samsung/executor_runner/enn_executor_llm_runner.cpp @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include +#include + +#include +#include +#include + +#if defined(ET_USE_THREADPOOL) +#include +#include +#endif + +DEFINE_string( + model_path, + "model.pte", + "Model serialized in flatbuffer format."); + +DEFINE_string(data_path, "", "Data file for the model."); + +DEFINE_string(pos_embedding_path, "", "rotary embedding stuff."); + +DEFINE_string(tokenizer_path, "tokenizer.bin", "Tokenizer stuff."); + +DEFINE_string(config_path, "", "Path to the model configuration JSON file."); + +DEFINE_string(vision_model_path, "", "Path to the Gemma 4 vision encoder PTE."); + +DEFINE_string(image_path, "", "Path to an input image (Gemma 4 multimodal)."); + +DEFINE_string( + prompt_before_image, + "", + "Text placed before the image in the user turn (Gemma 4 multimodal)."); + +DEFINE_string(prompt, "The answer to the ultimate question is", "Prompt."); + +DEFINE_string(system_prompt, "", "System prompt. Empty means none."); + +DEFINE_string( + assistant_prefill_prompt, + "", + "Assistant pre-fill appended verbatim to the very end of the rendered " + "prompt (after the assistant turn header). Ignored with --raw_prompt."); + +DEFINE_string( + prompt_file, + "", + "Read the prompt from this file instead of --prompt."); + +DEFINE_string( + system_prompt_file, + "", + "Read the system prompt from this file instead of --system_prompt."); + +// Raw mode: an already-formatted prompt (special tokens, system turn, tool +// declaration) fed verbatim, skipping chat-template wrapping. Supply it inline +// with --raw_prompt or from a file with --raw_prompt_file; either overrides +// --prompt / --system_prompt. +DEFINE_string(raw_prompt, "", "Inline raw (pre-formatted) prompt."); + +DEFINE_string( + raw_prompt_file, + "", + "Read the raw (pre-formatted) prompt from this file instead of --raw_prompt."); + +DEFINE_double( + temperature, + 0.8, + "Temperature; Default is 0.8. 0 = greedy argmax sampling (deterministic). Lower temperature = more deterministic"); + +DEFINE_int32( + seq_len, + 128, + "Total number of tokens to generate (prompt + output). If the number of input tokens + seq_len > max_seq_len, the output will be truncated to max_seq_len tokens."); + +DEFINE_int32( + cpu_threads, + -1, + "Number of CPU threads for inference. Defaults to -1, which implies we'll use a heuristic to derive the # of performant cores for a specific device."); + +DEFINE_int32( + num_bos, + 0, + "Number of BOS tokens to prepend to the prompt. Defaults to 0. If > 0, the prompt will be prepended with BOS tokens. This is useful for models that expect one or more BOS token at the start."); + +DEFINE_int32( + num_eos, + 0, + "Number of EOS tokens to append to the prompt. Defaults to 0. If > 0, the prompt will be appended with EOS tokens. This is useful for models that expect one or more EOS token at the end."); + +DEFINE_bool(is_decode_only, false, "Whether to run with decode method only."); + +DEFINE_int32( + max_token_limit, + 0, + "If > 0, print each token from the callback and call enn_llm_stop() right after the N-th generated token. Used to verify the stop API."); + +struct StopContext { + enn_runner* runner; + int32_t limit; + int32_t num_pieces = 0; + int32_t num_tokens = 0; +}; + +static void on_token_with_limit(const char* text, int32_t len, void* userdata) { + auto* ctx = static_cast(userdata); + fwrite(text, 1, static_cast(len), stdout); + fflush(stdout); + // The first callback is the echoed input prompt, not a generated token. + if (ctx->num_pieces++ == 0) { + return; + } + // The runner emits a trailing "\n" after generation ends; ignore it. + if (ctx->num_tokens >= ctx->limit) { + return; + } + if (++ctx->num_tokens == ctx->limit) { + enn_llm_stop(ctx->runner); + printf("\n[STOP] enn_llm_stop called at token %d\n", ctx->num_tokens); + fflush(stdout); + } +} + +static bool read_file(const std::string& path, std::string* out) { + std::ifstream file(path, std::ios::binary); + if (!file.is_open()) { + ET_LOG(Error, "Failed to open %s", path.c_str()); + return false; + } + std::stringstream ss; + ss << file.rdbuf(); + *out = ss.str(); + return true; +} + +int32_t main(int32_t argc, char** argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + + if ((!FLAGS_prompt_file.empty() && + !read_file(FLAGS_prompt_file, &FLAGS_prompt)) || + (!FLAGS_system_prompt_file.empty() && + !read_file(FLAGS_system_prompt_file, &FLAGS_system_prompt)) || + (!FLAGS_raw_prompt_file.empty() && + !read_file(FLAGS_raw_prompt_file, &FLAGS_raw_prompt))) { + return ENN_ERR_INVALID_ARG; + } + + // A non-empty raw prompt (inline or from file) is fed verbatim and overrides + // the templated --prompt / --system_prompt path. + const bool raw_mode = !FLAGS_raw_prompt.empty(); + const std::string& effective_prompt = + raw_mode ? FLAGS_raw_prompt : FLAGS_prompt; + + if (effective_prompt.empty()) { + ET_LOG(Error, "Input prompt cannot be empty."); + return ENN_ERR_INVALID_ARG; + } + if (FLAGS_cpu_threads < -1) { + ET_LOG(Error, "--cpu_threads must be -1 or a non-negative count."); + return ENN_ERR_INVALID_ARG; + } + +#if defined(ET_USE_THREADPOOL) + uint32_t num_performant_cores = FLAGS_cpu_threads == -1 + ? ::executorch::extension::cpuinfo::get_num_performant_cores() + : static_cast(FLAGS_cpu_threads); + ET_LOG( + Info, "Resetting threadpool with num threads = %d", num_performant_cores); + if (num_performant_cores > 0) { + ::executorch::extension::threadpool::get_threadpool() + ->_unsafe_reset_threadpool(num_performant_cores); + } +#endif + + enn_runner* runner = enn_llm_create( + FLAGS_model_path.c_str(), + FLAGS_data_path.c_str(), + FLAGS_tokenizer_path.c_str(), + FLAGS_pos_embedding_path.c_str(), + FLAGS_config_path.c_str(), + FLAGS_vision_model_path.c_str(), + FLAGS_is_decode_only); + if (runner == nullptr) { + ET_LOG(Error, "Failed to create LLM runner"); + return ENN_ERR_LOAD; + } + + const bool stop_test = FLAGS_max_token_limit > 0; + StopContext stop_ctx{runner, FLAGS_max_token_limit}; + int32_t error = enn_llm_generate( + runner, + effective_prompt.c_str(), + raw_mode ? "" : FLAGS_system_prompt.c_str(), + raw_mode ? "" : FLAGS_assistant_prefill_prompt.c_str(), + FLAGS_image_path.c_str(), + FLAGS_prompt_before_image.c_str(), + /*raw_prompt=*/raw_mode ? 1 : 0, + FLAGS_seq_len, + static_cast(FLAGS_temperature), + FLAGS_num_bos, + FLAGS_num_eos, + /*echo=*/1, + /*verbose=*/!stop_test, + stop_test ? on_token_with_limit : nullptr, + stop_test ? &stop_ctx : nullptr); + enn_llm_destroy(runner); + if (stop_test) { + printf( + "\n[STOP] limit=%d, generated=%d\n", + stop_ctx.limit, + stop_ctx.num_tokens); + } + if (error != ENN_OK) { + ET_LOG(Error, "Failed to generate token from LLM runner"); + } + return error; +} diff --git a/examples/samsung/executor_runner/exynos_llm_wrapper.cpp b/examples/samsung/executor_runner/exynos_llm_wrapper.cpp new file mode 100644 index 00000000000..3b9cbd221f5 --- /dev/null +++ b/examples/samsung/executor_runner/exynos_llm_wrapper.cpp @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * C API implementation over the Exynos LLM runner. Keeps the model(s) loaded + * across generate() calls so consecutive inferences do not re-prepare. + */ + +#include + +#include + +#include + +#include +#include +#include +#include + +namespace llm = ::executorch::extension::llm; + +struct enn_runner { + std::unique_ptr impl; +}; + +namespace { + +std::optional opt_or_null(const char* s) { + if (s == nullptr || *s == '\0') { + return std::nullopt; + } + return std::string(s); +} + +std::string str_or_empty(const char* s) { + return s == nullptr ? std::string() : std::string(s); +} + +} // namespace + +extern "C" { + +enn_runner* enn_llm_create( + const char* model_path, + const char* data_path, + const char* tokenizer_path, + const char* pos_embedding_path, + const char* config_path, + const char* vision_model_path, + int32_t is_decode_only) { + if (model_path == nullptr || tokenizer_path == nullptr || + config_path == nullptr) { + ET_LOG(Error, "enn_llm_create: model/tokenizer/config path required"); + return nullptr; + } + + example::RunnerPaths paths{ + .model_path = model_path, + .tokenizer_path = tokenizer_path, + .pos_embedding_path = opt_or_null(pos_embedding_path), + .data_path = opt_or_null(data_path), + .vision_model_path = opt_or_null(vision_model_path)}; + + example::RunnerConfig cfg{ + .config_path = config_path, .is_decode_only = is_decode_only != 0}; + + auto runner = example::create_llm_runner(paths, std::move(cfg)); + if (runner == nullptr) { + ET_LOG(Error, "enn_llm_create: failed to create LLM runner"); + return nullptr; + } + + return new enn_runner{std::move(runner)}; +} + +void enn_llm_destroy(enn_runner* runner) { + delete runner; +} + +void enn_llm_stop(enn_runner* runner) { + if (runner == nullptr) { + return; + } + runner->impl->stop(); +} + +int32_t enn_llm_generate( + enn_runner* runner, + const char* prompt, + const char* system_prompt, + const char* assistant_prefill_prompt, + const char* image_path, + const char* prompt_before_image, + int32_t raw_prompt, + int32_t seq_len, + float temperature, + int32_t num_bos, + int32_t num_eos, + int32_t echo, + int32_t verbose, + enn_token_cb on_token, + void* userdata) { + if (runner == nullptr || prompt == nullptr || *prompt == '\0') { + ET_LOG(Error, "enn_llm_generate: invalid argument"); + return ENN_ERR_INVALID_ARG; + } + + auto* lrunner = dynamic_cast(runner->impl.get()); + if (lrunner == nullptr) { + ET_LOG(Error, "enn_llm_generate: unexpected runner type"); + return ENN_ERR_INVALID_ARG; + } + + // Start each generation from a clean KV-cache. + lrunner->reset(); + + // System prompt and image inputs are threaded through the runner's config + // (read by generate). + lrunner->get_config().system_prompt = str_or_empty(system_prompt); + lrunner->get_config().assistant_prefill_prompt = + str_or_empty(assistant_prefill_prompt); + lrunner->get_config().image_path = str_or_empty(image_path); + lrunner->get_config().prompt_before_image = str_or_empty(prompt_before_image); + lrunner->get_config().raw_prompt = raw_prompt != 0; + lrunner->get_config().verbose = verbose != 0; + + llm::GenerationConfig gen_cfg{ + .echo = echo != 0, + .seq_len = seq_len >= 0 ? seq_len : -1, + .temperature = temperature, + .num_bos = num_bos, + .num_eos = num_eos}; + + auto err = runner->impl->generate( + prompt, + gen_cfg, + [on_token, userdata](const std::string& piece) { + if (on_token != nullptr) { + on_token(piece.data(), static_cast(piece.size()), userdata); + } + }, + {}); + if (err != executorch::runtime::Error::Ok) { + ET_LOG(Error, "enn_llm_generate: generation failed"); + return ENN_ERR_GENERATE; + } + return ENN_OK; +} + +} // extern "C" diff --git a/examples/samsung/executor_runner/exynos_llm_wrapper.h b/examples/samsung/executor_runner/exynos_llm_wrapper.h new file mode 100644 index 00000000000..94535c74167 --- /dev/null +++ b/examples/samsung/executor_runner/exynos_llm_wrapper.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * Public C API wrapping the Exynos LLM runner. + * + * Provided as a reference for porting the Exynos LLM runner into an Android + * app. + * + * Only these symbols are exported from libexynos_llm_wrapper.so. The underlying + * implementation (model loading, ENN backend, tokenizer, KV-cache) is kept + * internal to the library. + * + * Model preparation happens once in enn_llm_create. enn_llm_generate can then + * be called repeatedly (with or without an image) on the same handle without + * reloading or re-preparing the models. + */ + +#ifndef EXYNOS_LLM_WRAPPER_H_ +#define EXYNOS_LLM_WRAPPER_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define EXYNOS_LLM_WRAPPER_API __attribute__((visibility("default"))) +#else +#define EXYNOS_LLM_WRAPPER_API +#endif + +/** Values returned by the API functions. 0 = OK, non-zero = error. */ +enum { + ENN_OK = 0, + ENN_ERR_INVALID_ARG = 1, + ENN_ERR_LOAD = 2, + ENN_ERR_GENERATE = 3, +}; + +/** + * Callback invoked with each decoded text piece as tokens are generated. + * text points to len bytes of UTF-8 text (may not be NUL-terminated); copy it + * if you keep it beyond the call. Called on the same thread that calls + * enn_llm_generate; must not block. + */ +typedef void (*enn_token_cb)(const char* text, int32_t len, void* userdata); + +typedef struct enn_runner enn_runner; + +/** + * Load and prepare the model(s). Does all one-time work: decoder .pte/.ptd, + * tokenizer, positional embedding, optional vision encoder, and KV-cache + * allocation. Repeated calls are not allowed on the same handle. + * + * @param model_path Decoder .pte file (required). + * @param data_path Decoder .ptd weights, or NULL if none. + * @param tokenizer_path Tokenizer model file (required). + * @param pos_embedding_path Precomputed rotary embeddings, or NULL. + * @param config_path Runner config file (required). + * @param vision_model_path Vision encoder .pte, or NULL => text-only runner. + * @param is_decode_only Non-zero runs the prompt through the decode method only + * (no prefill method). + * @return A handle on success, NULL on failure. + */ +EXYNOS_LLM_WRAPPER_API enn_runner* enn_llm_create( + const char* model_path, + const char* data_path, + const char* tokenizer_path, + const char* pos_embedding_path, + const char* config_path, + const char* vision_model_path, + int32_t is_decode_only); + +/** + * Destroy the runner and release all resources. Safe to call with NULL. Must + * not run concurrently with enn_llm_generate or enn_llm_stop on the same + * handle. + */ +EXYNOS_LLM_WRAPPER_API void enn_llm_destroy(enn_runner* runner); + +/** + * Request the in-flight enn_llm_generate on runner to stop. Thread-safe; may + * be called from any thread, including from on_token. Generation returns + * ENN_OK before the next decode step. A no-op when no generation is running + * or runner is NULL. + */ +EXYNOS_LLM_WRAPPER_API void enn_llm_stop(enn_runner* runner); + +/** + * Generate a response for prompt (optionally with an image). The models stay + * prepared across calls; KV-cache is reset before each generation so every + * prompt starts fresh. Not reentrant: at most one generation may run on a + * handle at a time. + * + * @param runner Handle from enn_llm_create. + * @param prompt User prompt text (required, non-empty). When raw_prompt is + * non-zero, this is the already-formatted prompt (BOS, turn/tool special + * tokens, system turn) and is fed to the model verbatim. + * @param system_prompt System prompt text, or NULL/"" => none. Ignored when + * raw_prompt is non-zero. + * @param assistant_prefill_prompt Text appended verbatim to the very end of the + * rendered prompt, after the assistant turn header (a pre-filled start of the + * model's reply), or NULL/"" => none. Ignored when raw_prompt is non-zero. + * @param image_path Image file for this turn, or NULL => text-only. + * @param prompt_before_image Text placed before the image in the user turn + * (Gemma 4 multimodal); ignored when image_path is NULL. + * @param raw_prompt Non-zero => prompt is already fully formatted; skip the + * internal chat-template wrapping and system_prompt. + * @param seq_len Total token budget (prompt + generated), capped by the model's + * max sequence length; -1 => use the model's max sequence length. Must be + * -1 or larger than the prompt length. + * @param temperature Sampling temperature. + * @param num_bos BOS tokens to prepend to the prompt. + * @param num_eos EOS tokens to append to the prompt. + * @param echo Non-zero also passes the rendered input prompt to on_token + * before the generated tokens. + * @param verbose Non-zero also streams tokens to stdout. + * @param on_token Per-piece callback, or NULL to discard output. + * @param userdata Opaque pointer passed through to on_token. + * @return ENN_OK on success, an ENN_ERR_* code otherwise. + */ +EXYNOS_LLM_WRAPPER_API int32_t enn_llm_generate( + enn_runner* runner, + const char* prompt, + const char* system_prompt, + const char* assistant_prefill_prompt, + const char* image_path, + const char* prompt_before_image, + int32_t raw_prompt, + int32_t seq_len, + float temperature, + int32_t num_bos, + int32_t num_eos, + int32_t echo, + int32_t verbose, + enn_token_cb on_token, + void* userdata); + +#ifdef __cplusplus +} +#endif + +#endif // EXYNOS_LLM_WRAPPER_H_ diff --git a/examples/samsung/executor_runner/exynos_llm_wrapper_sample_main.cpp b/examples/samsung/executor_runner/exynos_llm_wrapper_sample_main.cpp new file mode 100644 index 00000000000..074114c589a --- /dev/null +++ b/examples/samsung/executor_runner/exynos_llm_wrapper_sample_main.cpp @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * Sample CLI consuming the public exynos_llm_wrapper C API + * (exynos_llm_wrapper.h). This mirrors how an Android app drives + * libexynos_llm_wrapper.so: create once, generate repeatedly, destroy at + * shutdown. Models stay prepared across generate() calls. + * + * Usage (paths are on-device): + * exynos_llm_wrapper_sample [vision] + * [--raw_prompt ] [--raw_prompt_file ] + * [--system_prompt ] [--system_prompt_file ] + * [--user_prompt ] [--user_prompt_file ] + * [--assistant_prefill_prompt ] [--assistant_prefill_prompt_file + * ] + * [--image_file ] + * [--max_output_tokens ] [--stop_after_sec ] [--echo] + * + * Each generation is one en-invocation: + * - `--raw_prompt ` / `--raw_prompt_file ` : feed verbatim + * (raw_prompt=1). + * - `--*_prompt ` : the next argument IS the text itself. + * - `--*_prompt_file `: read the file and use its contents. + * - `--image_file ` : the value IS the image path passed as image_path. + * The system/user/prefill texts are fed together (raw_prompt=0) as one + * generate call, along with any image. The user prompt is required; system + * and prefill may be omitted. A raw prompt always runs on its own, so a + * following system/user/prefill option starts the next generation. Repeating + * runs sequentially on the same handle, so the model stays prepared across + * calls. + * + * STREAMING=1 (default): the on_token callback prints each token as generated. + * STREAMING=0: the on_token callback appends to a std::string and the full + * answer is printed once when generation ends. Override with -DSTREAMING=0. + * + * Stop test (exercises enn_llm_stop(); both apply to every generation): + * --max_output_tokens N (> 0): call enn_llm_stop() right after the N-th + * output token of each generation and check that no more tokens arrive. + * Use prompts whose answer is longer than N. Prints PASS/FAIL per run. + * --stop_after_sec N (> 0): spawn a stopper thread that calls enn_llm_stop() + * N seconds into each generation, imitating a user interrupt. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "exynos_llm_wrapper.h" + +#ifndef STREAMING +#define STREAMING 1 +#endif + +// Option codes for the long-option table (start past ASCII so getopt_long +// distinguishes them from any single-char option). +enum { + OPT_RAW = 256, + OPT_RAW_FILE, + OPT_SYS, + OPT_SYS_FILE, + OPT_USR, + OPT_USR_FILE, + OPT_PRE, + OPT_PRE_FILE, + OPT_IMG_FILE, + OPT_MAX_TOKENS, + OPT_STOP_SEC, + OPT_ECHO, +}; + +struct GenContext { + enn_runner* runner; + int32_t max_output_tokens; + bool echo; + int32_t num_pieces = 0; + int32_t num_tokens = 0; + int32_t tokens_after_stop = 0; + std::string acc; +}; + +static void +count_for_stop_test(GenContext* ctx, const char* text, int32_t len) { + // When echo is on, the first callback is the echoed input prompt, not an + // output token, so skip it. With echo off every piece is an output token. + if (ctx->echo && ctx->num_pieces++ == 0) { + return; + } + // The runner emits a trailing "\n" after generation ends; ignore it. + if (ctx->num_tokens >= ctx->max_output_tokens) { + if (!(len == 1 && text[0] == '\n')) { + ++ctx->tokens_after_stop; + } + return; + } + if (++ctx->num_tokens == ctx->max_output_tokens) { + enn_llm_stop(ctx->runner); + printf("\n[STOP TEST] enn_llm_stop called at token %d\n", ctx->num_tokens); + fflush(stdout); + } +} + +static void on_token(const char* text, int32_t len, void* userdata) { + auto* ctx = static_cast(userdata); +#if STREAMING + fwrite(text, 1, (size_t)len, stdout); + fflush(stdout); +#else + ctx->acc.append(text, (size_t)len); +#endif + if (ctx->max_output_tokens > 0) { + count_for_stop_test(ctx, text, len); + } +} + +static bool read_file(const char* path, std::string* out) { + std::ifstream f(path); + if (!f.is_open()) { + fprintf(stderr, "failed to open %s\n", path); + return false; + } + std::stringstream ss; + ss << f.rdbuf(); + *out = ss.str(); + return true; +} + +static int run_one( + enn_runner* r, + const std::string& raw, + const std::string& sys, + const std::string& usr, + const std::string& pre, + const std::string& image, + int32_t max_output_tokens, + int32_t stop_after_sec, + bool echo) { + int32_t raw_prompt = !raw.empty(); + const char* prompt = raw_prompt ? raw.c_str() : usr.c_str(); + // raw prompts already contain the leading token; the templated path + // must add it so both modes feed the model the same prompt. + int32_t num_bos = raw_prompt ? 0 : 1; + + GenContext ctx{r, max_output_tokens, echo}; + + // Timer-based stopper thread imitating a user interrupt. Replaced by the + // token-count stop test when max_output_tokens is given. + std::mutex stop_mutex; + std::condition_variable stop_cv; + bool generation_done = false; + std::thread stopper; + if (stop_after_sec > 0) { + stopper = std::thread([&]() { + std::unique_lock lock(stop_mutex); + if (stop_cv.wait_for(lock, std::chrono::seconds(stop_after_sec), [&]() { + return generation_done; + })) { + return; + } + printf( + "\n[Stopper] triggering enn_llm_stop() after %d s\n", stop_after_sec); + fflush(stdout); + enn_llm_stop(r); + }); + } + + int rc = enn_llm_generate( + r, + prompt, + raw_prompt ? NULL : (sys.empty() ? NULL : sys.c_str()), + raw_prompt ? NULL : (pre.empty() ? NULL : pre.c_str()), + image.empty() ? NULL : image.c_str(), + NULL, + raw_prompt, + 3840, + 0.0f, + num_bos, + 0, + echo ? 1 : 0, + 0, + on_token, + &ctx); + + if (stopper.joinable()) { + { + std::lock_guard lock(stop_mutex); + generation_done = true; + } + stop_cv.notify_one(); + stopper.join(); + } + +#if !STREAMING + printf("%s\n", ctx.acc.c_str()); + fflush(stdout); +#endif + printf("\n[rc=%d]\n", rc); + + if (rc == ENN_OK && max_output_tokens > 0) { + bool pass = + ctx.num_tokens == max_output_tokens && ctx.tokens_after_stop == 0; + printf( + "[STOP TEST] %s: requested stop at %d, received %d, after stop %d\n", + pass ? "PASS" : "FAIL", + max_output_tokens, + ctx.num_tokens, + ctx.tokens_after_stop); + if (!pass) { + rc = ENN_ERR_GENERATE; + } + } + return rc; +} + +int main(int argc, char** argv) { + if (argc < 6) { + fprintf( + stderr, + "usage: %s model ptd tok pos cfg [vision] " + "[--raw_prompt STR | --raw_prompt_file F | " + "--system_prompt STR | --system_prompt_file F | " + "--user_prompt STR | --user_prompt_file F | " + "--assistant_prefill_prompt STR | --assistant_prefill_prompt_file F | " + "--image_file PATH]* " + "[--max_output_tokens N] [--stop_after_sec N] [--echo]\n", + argv[0]); + return 2; + } + const char* vision = + (argc > 6 && argv[6][0] && argv[6][0] != '-') ? argv[6] : NULL; + int idx = (argc > 6 && argv[6][0] && argv[6][0] != '-') ? 7 : 6; + + static const struct option long_opts[] = { + {"raw_prompt", required_argument, 0, OPT_RAW}, + {"raw_prompt_file", required_argument, 0, OPT_RAW_FILE}, + {"system_prompt", required_argument, 0, OPT_SYS}, + {"system_prompt_file", required_argument, 0, OPT_SYS_FILE}, + {"user_prompt", required_argument, 0, OPT_USR}, + {"user_prompt_file", required_argument, 0, OPT_USR_FILE}, + {"assistant_prefill_prompt", required_argument, 0, OPT_PRE}, + {"assistant_prefill_prompt_file", required_argument, 0, OPT_PRE_FILE}, + {"image_file", required_argument, 0, OPT_IMG_FILE}, + {"max_output_tokens", required_argument, 0, OPT_MAX_TOKENS}, + {"stop_after_sec", required_argument, 0, OPT_STOP_SEC}, + {"echo", no_argument, 0, OPT_ECHO}, + {0, 0, 0, 0}, + }; + + printf("[sample] creating runner...\n"); + enn_runner* r = + enn_llm_create(argv[1], argv[2], argv[3], argv[4], argv[5], vision, 0); + if (!r) { + fprintf(stderr, "enn_llm_create FAILED\n"); + return 1; + } + + // Options accumulate into one generation spec. A spec is either a raw + // (standalone) or the system/user/prefill triple combined into ONE generate + // call; each may come as a bare string or a file. Flush the accumulated spec + // only when a new raw begins (a raw group never shares a call with others). + // --max_output_tokens / --stop_after_sec / --echo are global and apply to + // every generation. + std::string raw, sys, usr, pre, image; + bool have = false; + int rc_total = 0; + int32_t max_output_tokens = 0; + int32_t stop_after_sec = 0; + bool echo = false; + + optind = idx; // resume scanning after the positional model arguments + int opt; + while ((opt = getopt_long(argc, argv, "", long_opts, NULL)) != -1) { + const bool is_file = opt == OPT_RAW_FILE || opt == OPT_SYS_FILE || + opt == OPT_USR_FILE || opt == OPT_PRE_FILE; + std::string content; + if (is_file) { + if (!read_file(optarg, &content)) { + rc_total = 1; + break; + } + } else if (opt != OPT_ECHO) { + content = optarg; + } + + const bool starts_templated = opt == OPT_SYS || opt == OPT_SYS_FILE || + opt == OPT_USR || opt == OPT_USR_FILE || opt == OPT_PRE || + opt == OPT_PRE_FILE; + const bool starts_raw = opt == OPT_RAW || opt == OPT_RAW_FILE; + if (have && (starts_raw || (starts_templated && !raw.empty()))) { + rc_total |= run_one( + r, + raw, + sys, + usr, + pre, + image, + max_output_tokens, + stop_after_sec, + echo); + raw.clear(); + sys.clear(); + usr.clear(); + pre.clear(); + image.clear(); + have = false; + } + + switch (opt) { + case OPT_RAW: + case OPT_RAW_FILE: + raw = content; + have = true; + break; + case OPT_SYS: + case OPT_SYS_FILE: + sys = content; + have = true; + break; + case OPT_USR: + case OPT_USR_FILE: + usr = content; + have = true; + break; + case OPT_PRE: + case OPT_PRE_FILE: + pre = content; + have = true; + break; + case OPT_IMG_FILE: + image = content; + have = true; + break; + case OPT_MAX_TOKENS: + max_output_tokens = atoi(content.c_str()); + break; + case OPT_STOP_SEC: + stop_after_sec = atoi(content.c_str()); + break; + case OPT_ECHO: + echo = true; + break; + default: + fprintf(stderr, "unknown argument: %s\n", argv[optind - 1]); + rc_total = 1; + break; + } + if (rc_total) + break; + } + if (have && !rc_total) { + rc_total |= run_one( + r, raw, sys, usr, pre, image, max_output_tokens, stop_after_sec, echo); + } + + printf("[sample] destroying runner...\n"); + enn_llm_destroy(r); + return rc_total ? 1 : 0; +} diff --git a/examples/samsung/executor_runner/llm/runner/CMakeLists.txt b/examples/samsung/executor_runner/llm/runner/CMakeLists.txt new file mode 100644 index 00000000000..842e9eab6f7 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/CMakeLists.txt @@ -0,0 +1,94 @@ +# Copyright (c) 2025 Samsung Electronics Co. LTD +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../../../tools/cmake/Utils.cmake) +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../../../tools/cmake/Codegen.cmake) + +# stb_image: same pinned revision as examples/models/gemma3. deprecated/ holds +# stb_image_resize.h. +include(FetchContent) +FetchContent_Declare( + stb + GIT_REPOSITORY https://github.com/nothings/stb.git + GIT_TAG f0569113c93ad095470c54bf34a17b36646bbbb5 +) +FetchContent_MakeAvailable(stb) + +set(_enn_llm_runner__srcs + ${CMAKE_CURRENT_LIST_DIR}/runner.cpp + ${CMAKE_CURRENT_LIST_DIR}/config.cpp + ${CMAKE_CURRENT_LIST_DIR}/decoder.cpp + ${CMAKE_CURRENT_LIST_DIR}/prefiller.cpp + ${CMAKE_CURRENT_LIST_DIR}/io_manager.cpp + ${CMAKE_CURRENT_LIST_DIR}/rotary_embedding.cpp + ${CMAKE_CURRENT_LIST_DIR}/vision_encoder.cpp + ${CMAKE_CURRENT_LIST_DIR}/stb_image_impl.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../examples/models/llama/tokenizer/llama_tiktoken.cpp +) + +target_include_directories( + extension_module INTERFACE ${_common_include_directories} +) + +add_library(lib_tokenizer STATIC IMPORTED) +set_target_properties( + lib_tokenizer + PROPERTIES + IMPORTED_LOCATION + "${CMAKE_CURRENT_BINARY_DIR}/../../../../../../build_samsung_android/lib/libtokenizers.a" +) + +add_library(lib_extension_llm_runner STATIC IMPORTED) +set_target_properties( + lib_extension_llm_runner + PROPERTIES + IMPORTED_LOCATION + "${CMAKE_CURRENT_BINARY_DIR}/../../../../../../build_samsung_android/extension/llm/runner/libextension_llm_runner.a" +) + +if(CMAKE_TOOLCHAIN_IOS + OR ANDROID + OR APPLE +) + # Building a share library on iOS requires code signing On Android we see + # duplicated registration when using shared lib + add_library(enn_llm_runner STATIC ${_enn_llm_runner__srcs}) +else() + add_library(enn_llm_runner SHARED ${_enn_llm_runner__srcs}) +endif() + +# For extension_llm_runner +if(NOT TARGET extension_llm_runner) + add_subdirectory( + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../extension/llm/runner + ${CMAKE_CURRENT_BINARY_DIR}/../../../../../extension/llm/runner + ) +endif() + +set(enn_llm_runner_deps + executorch_core extension_data_loader extension_module extension_tensor + extension_flat_tensor lib_extension_llm_runner +) + +target_compile_options(enn_llm_runner PRIVATE ${_llm_compile_options}) +target_link_libraries(enn_llm_runner PUBLIC ${enn_llm_runner_deps}) +target_link_libraries(enn_llm_runner PUBLIC tokenizers::tokenizers) +target_link_libraries(enn_llm_runner PUBLIC lib_tokenizer) + +target_include_directories( + enn_llm_runner + PUBLIC + ${EXECUTORCH_ROOT}/.. + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../extension/llm/tokenizers/include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../extension/llm/tokenizers/third-party/re2 + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../extension/llm/tokenizers/third-party/abseil-cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../extension/llm/tokenizers/third-party/json/include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../examples/models/llama/tokenizer + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../examples/samsung/executor_runner/llm/runner + ${stb_SOURCE_DIR} + ${stb_SOURCE_DIR}/deprecated + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../extension/llm/runner +) diff --git a/examples/samsung/executor_runner/llm/runner/config.cpp b/examples/samsung/executor_runner/llm/runner/config.cpp new file mode 100644 index 00000000000..0d9f464fd9f --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/config.cpp @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2025 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include + +#include +#include + +namespace example { + +using json = nlohmann::json; +using executorch::runtime::Error; + +Error RunnerConfig::load_from_json() { + ET_CHECK_OR_RETURN_ERROR( + !config_path.empty(), + InvalidArgument, + "A runner config path is required"); + + std::ifstream file(config_path); + ET_CHECK_OR_RETURN_ERROR( + file.is_open(), + InvalidArgument, + "Could not open config file: %s", + config_path.c_str()); + + json j = json::parse(file, nullptr, false); + ET_CHECK_OR_RETURN_ERROR( + !j.is_discarded(), + InvalidArgument, + "Invalid JSON format in config file: %s", + config_path.c_str()); + + if (j.contains("embedding_strategy") && j["embedding_strategy"].is_string()) { + std::string strategy = j["embedding_strategy"].get(); + if (strategy == "global_only") { + embedding_strategy = EmbeddingStrategy::kGlobalOnly; + } else if (strategy == "global_and_local") { + embedding_strategy = EmbeddingStrategy::kGlobalAndLocal; + } + } + + if (j.contains("masking_strategy") && j["masking_strategy"].is_string()) { + std::string strategy = j["masking_strategy"].get(); + if (strategy == "causal") { + masking_strategy = MaskingStrategy::kCausal; + } else if (strategy == "causal_local") { + masking_strategy = MaskingStrategy::kCausalLocal; + } else if (strategy == "hybrid_conv") { + masking_strategy = MaskingStrategy::kHybridConvAttention; + } + } + + if (j.contains("model_version") && j["model_version"].is_string()) { + std::string version_str = j["model_version"].get(); + if (version_str == "llama3") { + model_version = DecoderModelVersion::kLlama3; + } else if (version_str == "gemma3") { + model_version = DecoderModelVersion::kGemma3; + } else if (version_str == "gemma4") { + model_version = DecoderModelVersion::kGemma4; + } else if (version_str == "general") { + model_version = DecoderModelVersion::kGeneral; + } + } + + if (j.contains("tokenizer_type") && j["tokenizer_type"].is_string()) { + tokenizer_type = j["tokenizer_type"].get(); + } + + // Use text_config if present, otherwise fallback to root + const json* config_source = &j; + if (j.contains("text_config") && j["text_config"].is_object()) { + config_source = &j["text_config"]; + } + const auto& src = *config_source; + + switch (model_version) { + case DecoderModelVersion::kGemma4: { + auto args = std::make_unique(); + if (src.contains("hidden_size") && + src["hidden_size"].is_number_integer()) { + args->text_config.hidden_size = src["hidden_size"].get(); + } + if (src.contains("num_hidden_layers") && + src["num_hidden_layers"].is_number_integer()) { + args->text_config.num_hidden_layers = + src["num_hidden_layers"].get(); + } + if (src.contains("num_attention_heads") && + src["num_attention_heads"].is_number_integer()) { + args->text_config.num_attention_heads = + src["num_attention_heads"].get(); + } + if (src.contains("num_key_value_heads") && + src["num_key_value_heads"].is_number_integer()) { + args->text_config.num_key_value_heads = + src["num_key_value_heads"].get(); + } + if (src.contains("head_dim") && src["head_dim"].is_number_integer()) { + args->text_config.head_dim = src["head_dim"].get(); + } + if (src.contains("global_head_dim") && + src["global_head_dim"].is_number_integer()) { + args->text_config.global_head_dim = + src["global_head_dim"].get(); + } + if (src.contains("partial_rotary_factor") && + src["partial_rotary_factor"].is_number()) { + args->text_config.rope_percentage = + src["partial_rotary_factor"].get(); + } + if (src.contains("rope_theta") && src["rope_theta"].is_number()) { + args->text_config.base_freq_global = src["rope_theta"].get(); + } + if (src.contains("rope_local_base_freq") && + src["rope_local_base_freq"].is_number()) { + args->text_config.base_freq_local = + src["rope_local_base_freq"].get(); + } + if (src.contains("vocab_size") && src["vocab_size"].is_number_integer()) { + args->text_config.vocab_size = src["vocab_size"].get(); + } + if (src.contains("max_seq_len") && + src["max_seq_len"].is_number_integer()) { + args->text_config.max_seq_len = src["max_seq_len"].get(); + } + if (src.contains("max_position_embeddings") && + src["max_position_embeddings"].is_number_integer()) { + args->text_config.max_position_embeddings = + src["max_position_embeddings"].get(); + } + if (src.contains("sliding_window") && + src["sliding_window"].is_number_integer()) { + args->text_config.sliding_window = src["sliding_window"].get(); + } + if (src.contains("num_kv_shared_layers") && + src["num_kv_shared_layers"].is_number_integer()) { + args->num_kv_shared_layers = src["num_kv_shared_layers"].get(); + } + if (src.contains("sliding_window_pattern") && + src["sliding_window_pattern"].is_number_integer()) { + args->sliding_window_pattern = + src["sliding_window_pattern"].get(); + } + if (src.contains("layer_types") && src["layer_types"].is_array()) { + args->is_full_attention.clear(); + for (const auto& layer_type : src["layer_types"]) { + args->is_full_attention.push_back( + layer_type.is_string() && + layer_type.get() == "full_attention"); + } + } + model_args = std::move(args); + break; + } + case DecoderModelVersion::kLlama3: { + auto args = std::make_unique(); + if (src.contains("hidden_dim") && src["hidden_dim"].is_number_integer()) { + args->hidden_dim = src["hidden_dim"].get(); + } else if ( + src.contains("hidden_size") && + src["hidden_size"].is_number_integer()) { + args->hidden_dim = src["hidden_size"].get(); + } + if (src.contains("n_heads") && src["n_heads"].is_number_integer()) { + args->n_heads = src["n_heads"].get(); + } else if ( + src.contains("num_attention_heads") && + src["num_attention_heads"].is_number_integer()) { + args->n_heads = src["num_attention_heads"].get(); + } + if (src.contains("n_kv_heads") && src["n_kv_heads"].is_number_integer()) { + args->n_kv_heads = src["n_kv_heads"].get(); + } else if ( + src.contains("num_key_value_heads") && + src["num_key_value_heads"].is_number_integer()) { + args->n_kv_heads = src["num_key_value_heads"].get(); + } + if (src.contains("n_layers") && src["n_layers"].is_number_integer()) { + args->n_layers = src["n_layers"].get(); + } else if ( + src.contains("num_hidden_layers") && + src["num_hidden_layers"].is_number_integer()) { + args->n_layers = src["num_hidden_layers"].get(); + } + if (src.contains("head_dim") && src["head_dim"].is_number_integer()) { + args->head_dim = src["head_dim"].get(); + } + if (src.contains("vocab_size") && src["vocab_size"].is_number_integer()) { + args->vocab_size = src["vocab_size"].get(); + } + if (src.contains("max_seq_len") && + src["max_seq_len"].is_number_integer()) { + args->max_seq_len = src["max_seq_len"].get(); + } else if ( + src.contains("max_position_embeddings") && + src["max_position_embeddings"].is_number_integer()) { + args->max_seq_len = src["max_position_embeddings"].get(); + } + if (src.contains("max_context_len") && + src["max_context_len"].is_number_integer()) { + args->max_context_len = src["max_context_len"].get(); + } else if ( + src.contains("max_position_embeddings") && + src["max_position_embeddings"].is_number_integer()) { + args->max_context_len = src["max_position_embeddings"].get(); + } + if (src.contains("prefill_seq_len") && + src["prefill_seq_len"].is_number_integer()) { + args->prefill_seq_len = src["prefill_seq_len"].get(); + } + if (src.contains("decode_seq_len") && + src["decode_seq_len"].is_number_integer()) { + args->decode_seq_len = src["decode_seq_len"].get(); + } + if (src.contains("attn_mask_rows") && + src["attn_mask_rows"].is_number_integer()) { + args->attn_mask_rows = src["attn_mask_rows"].get(); + } + if (src.contains("attn_mask_cols") && + src["attn_mask_cols"].is_number_integer()) { + args->attn_mask_cols = src["attn_mask_cols"].get(); + } + if (src.contains("sliding_window") && + src["sliding_window"].is_number_integer()) { + args->sliding_window = src["sliding_window"].get(); + } else { + args->sliding_window = std::nullopt; + } + model_args = std::move(args); + break; + } + case DecoderModelVersion::kGemma3: { + auto args = std::make_unique(); + if (src.contains("hidden_dim") && src["hidden_dim"].is_number_integer()) { + args->hidden_dim = src["hidden_dim"].get(); + } else if ( + src.contains("hidden_size") && + src["hidden_size"].is_number_integer()) { + args->hidden_dim = src["hidden_size"].get(); + } + + if (src.contains("n_heads") && src["n_heads"].is_number_integer()) { + args->n_heads = src["n_heads"].get(); + } else if ( + src.contains("num_attention_heads") && + src["num_attention_heads"].is_number_integer()) { + args->n_heads = src["num_attention_heads"].get(); + } + + if (src.contains("n_kv_heads") && src["n_kv_heads"].is_number_integer()) { + args->n_kv_heads = src["n_kv_heads"].get(); + } else if ( + src.contains("num_key_value_heads") && + src["num_key_value_heads"].is_number_integer()) { + args->n_kv_heads = src["num_key_value_heads"].get(); + } + + if (src.contains("n_layers") && src["n_layers"].is_number_integer()) { + args->n_layers = src["n_layers"].get(); + } else if ( + src.contains("num_hidden_layers") && + src["num_hidden_layers"].is_number_integer()) { + args->n_layers = src["num_hidden_layers"].get(); + } + + if (src.contains("head_dim") && src["head_dim"].is_number_integer()) { + args->head_dim = src["head_dim"].get(); + } + + if (src.contains("vocab_size") && src["vocab_size"].is_number_integer()) { + args->vocab_size = src["vocab_size"].get(); + } + + if (src.contains("max_seq_len") && + src["max_seq_len"].is_number_integer()) { + args->max_seq_len = src["max_seq_len"].get(); + } else if ( + src.contains("max_position_embeddings") && + src["max_position_embeddings"].is_number_integer()) { + args->max_seq_len = src["max_position_embeddings"].get(); + } + + if (src.contains("max_context_len") && + src["max_context_len"].is_number_integer()) { + args->max_context_len = src["max_context_len"].get(); + } else if ( + src.contains("max_position_embeddings") && + src["max_position_embeddings"].is_number_integer()) { + args->max_context_len = src["max_position_embeddings"].get(); + } + + if (src.contains("prefill_seq_len") && + src["prefill_seq_len"].is_number_integer()) { + args->prefill_seq_len = src["prefill_seq_len"].get(); + } + if (src.contains("decode_seq_len") && + src["decode_seq_len"].is_number_integer()) { + args->decode_seq_len = src["decode_seq_len"].get(); + } + if (src.contains("attn_mask_rows") && + src["attn_mask_rows"].is_number_integer()) { + args->attn_mask_rows = src["attn_mask_rows"].get(); + } + if (src.contains("attn_mask_cols") && + src["attn_mask_cols"].is_number_integer()) { + args->attn_mask_cols = src["attn_mask_cols"].get(); + } + if (src.contains("sliding_window") && + src["sliding_window"].is_number_integer()) { + args->sliding_window = src["sliding_window"].get(); + } else { + args->sliding_window = std::nullopt; + } + model_args = std::move(args); + break; + } + case DecoderModelVersion::kGeneral: + default: + break; + } + return validate(); +} + +Error RunnerConfig::validate() const { + ET_CHECK_OR_RETURN_ERROR( + model_args != nullptr, + InvalidArgument, + "%s: model_version must be one of llama3, gemma3, gemma4", + config_path.c_str()); + const BaseModelArgs& args = *model_args; + ET_CHECK_OR_RETURN_ERROR( + args.get_n_layers() > 0 && args.get_n_kv_heads() > 0 && + args.get_hidden_dim() > 0, + InvalidArgument, + "%s: n_layers, n_kv_heads and hidden_dim must be positive", + config_path.c_str()); + ET_CHECK_OR_RETURN_ERROR( + args.get_head_dim() > 0 && args.get_head_dim() % 2 == 0 && + args.get_global_head_dim() > 0 && args.get_global_head_dim() % 2 == 0, + InvalidArgument, + "%s: head dims must be positive and even", + config_path.c_str()); + ET_CHECK_OR_RETURN_ERROR( + args.get_num_kv_shared_layers() >= 0 && + args.get_num_kv_shared_layers() < args.get_n_layers(), + InvalidArgument, + "%s: num_kv_shared_layers (%" PRId64 ") must be in [0, n_layers)", + config_path.c_str(), + args.get_num_kv_shared_layers()); + ET_CHECK_OR_RETURN_ERROR( + args.get_prefill_seq_len() >= 1 && + args.get_prefill_seq_len() <= args.get_max_seq_len(), + InvalidArgument, + "%s: prefill_seq_len (%" PRId64 ") must be in [1, max_seq_len (%" PRId64 + ")]", + config_path.c_str(), + args.get_prefill_seq_len(), + args.get_max_seq_len()); + ET_CHECK_OR_RETURN_ERROR( + args.get_decode_seq_len() == 1, + InvalidArgument, + "%s: only decode_seq_len == 1 is supported", + config_path.c_str()); + ET_CHECK_OR_RETURN_ERROR( + args.get_attn_mask_rows() == args.get_prefill_seq_len() && + args.get_attn_mask_cols() == args.get_max_seq_len(), + InvalidArgument, + "%s: attention mask must be [prefill_seq_len, max_seq_len]", + config_path.c_str()); + const bool needs_sliding_window = + masking_strategy == MaskingStrategy::kCausalLocal || + masking_strategy == MaskingStrategy::kHybridConvAttention; + ET_CHECK_OR_RETURN_ERROR( + !needs_sliding_window || args.get_sliding_window().value_or(0) > 0, + InvalidArgument, + "%s: the selected masking strategy needs a positive sliding_window", + config_path.c_str()); + return Error::Ok; +} +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/config.h b/examples/samsung/executor_runner/llm/runner/config.h new file mode 100644 index 00000000000..deec0db3afa --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/config.h @@ -0,0 +1,398 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace example { + +// Exynos Runtime metadata key constants +inline constexpr auto kPrefillArLen = "get_prefill_ar_len"; +inline constexpr auto kNumOfLayers = "get_n_layers"; +inline constexpr auto kNumOfHiddenDim = "get_hidden_dim"; +inline constexpr auto kNumOfHeads = "get_n_heads"; +inline constexpr auto kNumOfKVHeads = "get_n_kv_heads"; +inline constexpr auto kHeadDim = "get_head_dim"; + +inline constexpr auto default_method_name = "forward"; +inline constexpr auto default_method_name_prefill = "prefill_forward"; +inline constexpr auto default_method_name_decode = "decode_forward"; + +// Gemma 4 multimodal special tokens (matches the model's tokenizer). +inline constexpr int64_t kGemma4PadId = 0; +inline constexpr int64_t kGemma4BosId = 2; +inline constexpr int64_t kGemma4TurnStartId = 105; // <|turn| +inline constexpr int64_t kGemma4TurnEndId = 106; // +inline constexpr int64_t kGemma4ImageTokenId = 258880; +inline constexpr int64_t kGemma4BoiTokenId = 255999; // <|boi|> +inline constexpr int64_t kGemma4EoiTokenId = 258882; // <|eoi|> + +enum class DecoderModelVersion { + kLlama3, + kGemma3, + kGemma4, + kGeneral, +}; + +enum class EmbeddingStrategy { + kGlobalOnly, + kGlobalAndLocal, +}; + +enum class MaskingStrategy { + kCausal, + kCausalLocal, + kHybridConvAttention, // LIV-conv + GQA mask constraint +}; + +class RunnerPaths { + public: + std::string model_path; + std::string tokenizer_path; + std::optional pos_embedding_path = std::nullopt; + std::optional data_path = std::nullopt; + std::optional vision_model_path = std::nullopt; +}; + +class BaseModelArgs { + public: + virtual ~BaseModelArgs() = default; + virtual DecoderModelVersion get_version() const = 0; + virtual int64_t get_hidden_dim() const { + return 0; + } + virtual int64_t get_n_heads() const { + return 0; + } + virtual int64_t get_n_kv_heads() const { + return 0; + } + virtual int64_t get_n_layers() const { + return 0; + } + virtual int64_t get_head_dim() const { + return 0; + } + virtual int64_t get_global_head_dim() const { + return 0; + } + virtual int64_t get_vocab_size() const { + return 0; + } + virtual int64_t get_max_seq_len() const { + return 128; + } + virtual int64_t get_max_context_len() const { + return 128; + } + virtual int64_t get_prefill_seq_len() const { + return 128; + } + virtual int64_t get_decode_seq_len() const { + return 1; + } + virtual int64_t get_attn_mask_rows() const { + return 128; + } + virtual int64_t get_attn_mask_cols() const { + return 1024; + } + virtual std::optional get_sliding_window() const { + return std::nullopt; + } + // Number of trailing layers that reuse (share) KV cache from earlier layers + // and therefore do not maintain their own externally-managed cache. + virtual int64_t get_num_kv_shared_layers() const { + return 0; + } + // Per-layer head dimension. Defaults to a uniform head_dim; models with + // mixed local/global attention (e.g. Gemma 4) override this. + virtual int64_t get_head_dim_for_layer(int64_t layer_idx) const { + (void)layer_idx; + return get_head_dim(); + } + // Layers that actually produce/consume an externally-managed KV cache + // (the leading self-decoder layers). Shared layers are handled inside the + // delegate and are not cached by the runner. + int64_t get_num_cache_layers() const { + return get_n_layers() - get_num_kv_shared_layers(); + } +}; + +class Llama3Args : public BaseModelArgs { + public: + DecoderModelVersion get_version() const override { + return DecoderModelVersion::kLlama3; + } + int64_t hidden_dim = 0; + int64_t n_heads = 0; + int64_t n_kv_heads = 0; + int64_t n_layers = 0; + int64_t head_dim = 0; + int64_t vocab_size = 0; + int64_t max_seq_len = 128; + int64_t max_context_len = 128; + int64_t prefill_seq_len = 128; + int64_t decode_seq_len = 1; + int64_t attn_mask_rows = 128; + int64_t attn_mask_cols = 1024; + std::optional sliding_window; + + int64_t get_hidden_dim() const override { + return hidden_dim; + } + int64_t get_n_heads() const override { + return n_heads; + } + int64_t get_n_kv_heads() const override { + return n_kv_heads; + } + int64_t get_n_layers() const override { + return n_layers; + } + int64_t get_head_dim() const override { + return head_dim; + } + int64_t get_global_head_dim() const override { + return head_dim; + } + int64_t get_vocab_size() const override { + return vocab_size; + } + int64_t get_max_seq_len() const override { + return max_seq_len; + } + int64_t get_max_context_len() const override { + return max_context_len; + } + int64_t get_prefill_seq_len() const override { + return prefill_seq_len; + } + int64_t get_decode_seq_len() const override { + return decode_seq_len; + } + int64_t get_attn_mask_rows() const override { + return attn_mask_rows; + } + int64_t get_attn_mask_cols() const override { + return attn_mask_cols; + } + std::optional get_sliding_window() const override { + return sliding_window; + } +}; + +class Gemma3Args : public BaseModelArgs { + public: + DecoderModelVersion get_version() const override { + return DecoderModelVersion::kGemma3; + } + int64_t hidden_dim = 0; + int64_t n_heads = 0; + int64_t n_kv_heads = 0; + int64_t n_layers = 0; + int64_t head_dim = 0; + int64_t vocab_size = 0; + int64_t max_seq_len = 128; + int64_t max_context_len = 128; + int64_t prefill_seq_len = 128; + int64_t decode_seq_len = 1; + int64_t attn_mask_rows = 128; + int64_t attn_mask_cols = 1024; + std::optional sliding_window; + + int64_t get_hidden_dim() const override { + return hidden_dim; + } + int64_t get_n_heads() const override { + return n_heads; + } + int64_t get_n_kv_heads() const override { + return n_kv_heads; + } + int64_t get_n_layers() const override { + return n_layers; + } + int64_t get_head_dim() const override { + return head_dim; + } + int64_t get_global_head_dim() const override { + return head_dim; + } + int64_t get_vocab_size() const override { + return vocab_size; + } + int64_t get_max_seq_len() const override { + return max_seq_len; + } + int64_t get_max_context_len() const override { + return max_context_len; + } + int64_t get_prefill_seq_len() const override { + return prefill_seq_len; + } + int64_t get_decode_seq_len() const override { + return decode_seq_len; + } + int64_t get_attn_mask_rows() const override { + return attn_mask_rows; + } + int64_t get_attn_mask_cols() const override { + return attn_mask_cols; + } + std::optional get_sliding_window() const override { + return sliding_window; + } +}; + +class Gemma4Args : public BaseModelArgs { + public: + DecoderModelVersion get_version() const override { + return DecoderModelVersion::kGemma4; + } + + struct TextConfig { + int64_t hidden_size = 0; + int64_t intermediate_size = 0; + bool use_double_wide_mlp = false; + int64_t num_hidden_layers = 0; + int64_t num_attention_heads = 0; + int64_t head_dim = 0; + int64_t global_head_dim = 0; + int64_t num_key_value_heads = 0; + int64_t vocab_size = 0; + int64_t max_seq_len = 0; + int64_t max_position_embeddings = 0; + std::optional sliding_window; + + // RoPE specific configuration for Gemma 4 + float rope_percentage = 0.25f; + float base_freq_local = 10000.0f; + float base_freq_global = 1000000.0f; + } text_config; + + struct AudioConfig { + int64_t hidden_size = 0; + int64_t num_hidden_layers = 0; + int64_t num_attention_heads = 0; + } audio_config; + + int64_t audio_seq_length = 0; + int64_t attn_mask_rows = 128; + int64_t attn_mask_cols = 1024; + + // YOCO KV sharing: the last num_kv_shared_layers reuse earlier layers' KV. + int64_t num_kv_shared_layers = 0; + // Attention pattern period (every pattern-th layer is full_attention). + int64_t sliding_window_pattern = 5; + // Parsed from "layer_types"; true where the layer uses full (global) + // attention. Empty falls back to the sliding_window_pattern derivation. + std::vector is_full_attention; + + int64_t get_hidden_dim() const override { + return text_config.hidden_size; + } + int64_t get_n_heads() const override { + return text_config.num_attention_heads; + } + int64_t get_n_kv_heads() const override { + return text_config.num_key_value_heads; + } + int64_t get_n_layers() const override { + return text_config.num_hidden_layers; + } + int64_t get_head_dim() const override { + return text_config.head_dim; + } + int64_t get_global_head_dim() const override { + return text_config.global_head_dim; + } + int64_t get_vocab_size() const override { + return text_config.vocab_size; + } + int64_t get_max_seq_len() const override { + return text_config.max_seq_len; + } + int64_t get_max_context_len() const override { + return text_config.max_position_embeddings; + } + std::optional get_sliding_window() const override { + return text_config.sliding_window; + } + // Attention mask spans [prefill_seq_len, max_seq_len] (rows x cols). + int64_t get_attn_mask_rows() const override { + return get_prefill_seq_len(); + } + int64_t get_attn_mask_cols() const override { + return text_config.max_seq_len; + } + int64_t get_num_kv_shared_layers() const override { + return num_kv_shared_layers; + } + int64_t get_head_dim_for_layer(int64_t layer_idx) const override { + bool full; + if (layer_idx >= 0 && + layer_idx < static_cast(is_full_attention.size())) { + full = is_full_attention[layer_idx]; + } else { + full = ((layer_idx + 1) % sliding_window_pattern == 0); + } + return full ? text_config.global_head_dim : text_config.head_dim; + } +}; + +class RunnerConfig { + public: + std::string config_path; + bool is_decode_only = false; + float temperature = -1.0f; + + // Stream generated tokens to stdout (in addition to the token callback). + bool verbose = false; + + // Empty => no system prompt. + std::string system_prompt; + + // Text appended verbatim to the very end of the rendered prompt (after the + // assistant turn header), i.e. a pre-filled assistant response. Ignored in + // raw_prompt mode where the caller supplies the full prompt already. + std::string assistant_prefill_prompt; + + // Treat the prompt as already fully formatted: skip chat-template wrapping + // and system_prompt so a caller can feed raw special-token input verbatim. + bool raw_prompt = false; + + // Vision inputs (Gemma 4 multimodal). Empty image_path => text-only. + std::string image_path; + std::string prompt_before_image; + + // Strategies + EmbeddingStrategy embedding_strategy = EmbeddingStrategy::kGlobalOnly; + MaskingStrategy masking_strategy = MaskingStrategy::kCausal; + + std::string tokenizer_type; + + DecoderModelVersion model_version = DecoderModelVersion::kGeneral; + std::unique_ptr model_args; + + ET_NODISCARD executorch::runtime::Error load_from_json(); + + private: + ET_NODISCARD executorch::runtime::Error validate() const; +}; + +} // namespace example \ No newline at end of file diff --git a/examples/samsung/executor_runner/llm/runner/decoder.cpp b/examples/samsung/executor_runner/llm/runner/decoder.cpp new file mode 100644 index 00000000000..5d33f99b4f3 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/decoder.cpp @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2025 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Given inputs, run a text decoder and return logits. + +#include +#include +#include + +#include +#include +#include + +namespace example { + +namespace runtime = executorch::runtime; + +namespace { + +runtime::Error validate_inputs( + const runtime::MethodMeta& method_meta, + const std::vector& inputs) { + ET_CHECK_OR_RETURN_ERROR( + inputs.size() == method_meta.num_inputs(), + InvalidProgram, + "Method %s expects %zu inputs but the runner prepared %zu", + method_meta.name(), + method_meta.num_inputs(), + inputs.size()); + for (size_t i = 0; i < inputs.size(); ++i) { + if (!inputs[i].isTensor()) { + continue; + } + auto expected = ET_UNWRAP(method_meta.input_tensor_meta(i)); + const auto prepared_rank = inputs[i].toTensor().dim(); + ET_CHECK_OR_RETURN_ERROR( + static_cast(prepared_rank) == expected.sizes().size(), + InvalidProgram, + "Method %s input %zu: expected rank %zu, prepared rank %zu", + method_meta.name(), + i, + expected.sizes().size(), + static_cast(prepared_rank)); + } + return runtime::Error::Ok; +} + +} // namespace + +runtime::Result Decoder::slice_kth_token( + const runtime::EValue& output, + int64_t k) { + ET_CHECK_OR_RETURN_ERROR( + output.isTensor(), InvalidProgram, "Logits output is not a tensor"); + aten::Tensor logits = output.toTensor(); + + constexpr int64_t kExpectedNumDim = 3; + constexpr int64_t kSequenceDim = 1; + ET_CHECK_OR_RETURN_ERROR( + logits.dim() == kExpectedNumDim, + InvalidProgram, + "Logits rank %zu is not %" PRId64, + static_cast(logits.dim()), + kExpectedNumDim); + const int64_t seq_len = logits.size(kSequenceDim); + ET_CHECK_OR_RETURN_ERROR( + k >= 0 && k < seq_len, + InvalidArgument, + "Token index %" PRId64 " out of bounds for %" PRId64 " logits rows", + k, + seq_len); + + const auto sizes = logits.sizes(); + std::vector new_sizes = { + static_cast(sizes[0]), + 1, + static_cast(sizes[2])}; + uint8_t* row = logits.mutable_data_ptr() + + k * logits.strides()[kSequenceDim] * logits.element_size(); + last_logits_ = extension::from_blob(row, new_sizes, logits.scalar_type()); + return *last_logits_; +} + +runtime::Error Decoder::load() { + extract_llm_methods(); + + if (method_names_.size() > 1) { + auto ret_shared = create_shared_planned_memory(); + ET_CHECK_OR_RETURN_ERROR( + ret_shared == runtime::Error::Ok, + InvalidArgument, + "Failed to create share planned memory"); + } + + for (const auto& method : method_names_) { + if (method.empty() || module_->is_method_loaded(method)) { + continue; + } + + runtime::Error ret_load; + if (shared_method_memory_) { + ret_load = + module_->load_method(method, shared_method_memory_->allocator.get()); + } else { + ret_load = module_->load_method(method); + } + ET_CHECK_OR_RETURN_ERROR( + ret_load == runtime::Error::Ok, + InvalidArgument, + "Failed to load priority method %s", + method.c_str()); + ET_LOG(Info, "Loading Priority Method (%s)", method.c_str()); + } + + // Load remaining utility methods (e.g., get_genai_mode) + for (const auto& method : method_names_) { + if (module_->is_method_loaded(method)) { + continue; + } + + runtime::Error ret_load = module_->load_method(method); + ET_CHECK_OR_RETURN_ERROR( + ret_load == runtime::Error::Ok, + InvalidArgument, + "Failed to load utility method %s", + method.c_str()); + ET_LOG(Info, "Loading Utility Method (%s)", method.c_str()); + } + + auto ret = load_io_manager(); + ET_CHECK_OR_RETURN_ERROR( + ret == runtime::Error::Ok, InvalidArgument, "Failed to load io manager"); + + return ret; +} + +bool Decoder::is_method_loaded() { + return !prefill_method_.empty() && !decode_method_.empty() && + module_->is_method_loaded(prefill_method_) && + module_->is_method_loaded(decode_method_); +} + +runtime::Result Decoder::step( + extension::TensorPtr& tokens, + int64_t start_pos) { + const bool is_prefill = tokens->numel() > 1; + const std::string& method_stage = + is_prefill ? prefill_method_ : decode_method_; + + auto method_meta = ET_UNWRAP(module_->method_meta(method_stage.c_str())); + + if (method_meta.num_inputs() <= 1) { + auto outputs_res = module_->execute(method_stage.c_str(), tokens); + ET_CHECK_OK_OR_RETURN_ERROR(outputs_res.error()); + ET_CHECK_OR_RETURN_ERROR( + outputs_res.get().size() == 1 && outputs_res.get()[0].isTensor(), + InvalidProgram, + "Expected a single logits tensor from %s", + method_stage.c_str()); + return outputs_res.get()[0].toTensor(); + } + + std::vector start_pos_sizes = {1}; + auto start_pos_tensor = + extension::from_blob(&start_pos, start_pos_sizes, aten::ScalarType::Long); + auto inputs_res = is_prefill + ? io_manager_->prepare_prefill(tokens, start_pos_tensor, method_stage) + : io_manager_->prepare_decode(tokens, start_pos_tensor, method_stage); + ET_CHECK_OK_OR_RETURN_ERROR(inputs_res.error()); + std::vector inputs = std::move(inputs_res.get()); + ET_CHECK_OK_OR_RETURN_ERROR(validate_inputs(method_meta, inputs)); + + auto outputs_res = module_->execute(method_stage.c_str(), inputs); + ET_CHECK_OK_OR_RETURN_ERROR(outputs_res.error()); + ET_CHECK_OR_RETURN_ERROR( + !outputs_res.get().empty(), + InvalidProgram, + "%s returned no outputs", + method_stage.c_str()); + + if (is_prefill) { + ET_CHECK_OK_OR_RETURN_ERROR( + io_manager_->update_prefill(outputs_res.get(), method_stage), + "Failed to update prefill output by io_manager"); + } else { + ET_CHECK_OK_OR_RETURN_ERROR( + io_manager_->update_decode(outputs_res.get(), method_stage), + "Failed to update decode output by io_manager"); + } + return slice_kth_token(outputs_res.get()[0], tokens->numel() - 1); +} + +void Decoder::extract_llm_methods() { + if (method_names_.size() > 1) { + for (auto it = method_names_.begin(); it != method_names_.end(); ++it) { + if (it->find("prefill") != std::string::npos) { + prefill_method_ = *it; + } else if (it->find("decode") != std::string::npos) { + decode_method_ = *it; + } else { + ET_LOG( + Info, + "skipping io manager identify with %s method name.", + (*it).c_str()); + continue; + } + } + } else if (method_names_.size() == 1) { + const std::string& method = *method_names_.begin(); + prefill_method_ = method; + decode_method_ = method; + } +} + +runtime::Error Decoder::load_io_manager() { + if (method_names_.empty()) { + return runtime::Error::InvalidArgument; + } + return io_manager_->load(prefill_method_, decode_method_); +} + +runtime::Error Decoder::create_shared_planned_memory() { + if (shared_method_memory_ != nullptr) { + ET_LOG(Info, "Shared Planned memory is already created"); + return runtime::Error::Ok; + } + + auto prefill_meta = module_->method_meta(prefill_method_); + auto decode_meta = module_->method_meta(decode_method_); + if (!prefill_meta.ok() || !decode_meta.ok()) { + ET_LOG(Error, "Failed to get method metadata for shared memory setup"); + return runtime::Error::InvalidArgument; + } + + size_t prefill_bufs = prefill_meta->num_memory_planned_buffers(); + size_t decode_bufs = decode_meta->num_memory_planned_buffers(); + size_t num_buffers = std::max(prefill_bufs, decode_bufs); + + ET_LOG( + Info, + "Prefill_bufs(%zu), decode_bufs(%zu) num_buffers(%zu)", + prefill_bufs, + decode_bufs, + num_buffers); + + shared_method_memory_ = std::make_unique(); + shared_method_memory_->buffers.reserve(num_buffers); + shared_method_memory_->spans.reserve(num_buffers); + + for (size_t i = 0; i < num_buffers; ++i) { + int64_t prefill_size = 0; + if (i < prefill_bufs) { + prefill_size = ET_UNWRAP(prefill_meta->memory_planned_buffer_size(i)); + } + int64_t decode_size = 0; + if (i < decode_bufs) { + decode_size = ET_UNWRAP(decode_meta->memory_planned_buffer_size(i)); + } + size_t max_size = static_cast(std::max(prefill_size, decode_size)); + ET_LOG( + Info, + "Prefill_size(%" PRId64 "), decode_size(%" PRId64 ") max_size(%zu)", + prefill_size, + decode_size, + max_size); + + shared_method_memory_->buffers.emplace_back(max_size); + shared_method_memory_->spans.emplace_back( + shared_method_memory_->buffers.back().data(), max_size); + } + + shared_method_memory_->allocator = + std::make_unique( + runtime::Span>( + shared_method_memory_->spans.data(), + shared_method_memory_->spans.size())); + + return runtime::Error::Ok; +} + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/decoder.h b/examples/samsung/executor_runner/llm/runner/decoder.h new file mode 100644 index 00000000000..30e8f704113 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/decoder.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2025 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Given inputs, run a text decoder in LLM and return the output. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace example { + +namespace llm = ::executorch::extension::llm; +namespace runtime = ::executorch::runtime; +namespace aten = ::executorch::aten; +namespace extension = ::executorch::extension; + +class ET_EXPERIMENTAL Decoder : public llm::TextDecoderRunner { + public: + struct SharedMethodMemory { + std::vector> buffers; + std::vector> spans; + std::unique_ptr allocator; + }; + + explicit Decoder(extension::Module* module, llm::IOManager* io_manager) + : TextDecoderRunner(module, io_manager), shared_method_memory_(nullptr) {} + + runtime::Result step( + extension::TensorPtr& input, + int64_t start_pos) override; + + runtime::Error load() override; + + bool is_method_loaded() override; + + inline void set_method(std::string& method) { + method_names_.insert(method); + } + + private: + std::string prefill_method_; + std::string decode_method_; + std::unordered_set method_names_; + std::unique_ptr shared_method_memory_; + // Keeps the TensorImpl of the logits view returned by step() alive. + extension::TensorPtr last_logits_; + + runtime::Result slice_kth_token( + const runtime::EValue& output, + int64_t k); + void extract_llm_methods(); + runtime::Error load_io_manager(); + runtime::Error create_shared_planned_memory(); +}; + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/image_utils.h b/examples/samsung/executor_runner/llm/runner/image_utils.h new file mode 100644 index 00000000000..849df80b8a2 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/image_utils.h @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace example { + +using ::executorch::extension::TensorPtr; + +// Vision encoder constants (Gemma 4). Matches HF's image processor. +inline constexpr int32_t kVisionPatchSize = 16; +inline constexpr int32_t kVisionPatchDim = + 3 * kVisionPatchSize * kVisionPatchSize; // 768 +inline constexpr int32_t kVisionPoolingKernel = 3; +inline constexpr int32_t kVisionCellSize = + kVisionPatchSize * kVisionPoolingKernel; // 48 + +// The shipped gemma4_vision.pte accepts [1, 1260, 768] pixel_values and +// [1, 1260, 2] pixel_position_ids (STATIC shapes). 1260 = 140 pooled +// tokens * 9 (pooling kernel squared). +inline constexpr int32_t kVisionMaxSoftTokens = 140; +inline constexpr int32_t kVisionMaxPatches = + kVisionMaxSoftTokens * kVisionPoolingKernel * kVisionPoolingKernel; // 1260 +inline constexpr int32_t kVisionPooledTokens = + kVisionMaxPatches / (kVisionPoolingKernel * kVisionPoolingKernel); // 140 + +struct ImageData { + TensorPtr pixel_values; // (1, num_patches, 768) float32 + TensorPtr pixel_position_ids; // (1, num_patches, 2) int64, padding = -1 + int64_t num_valid_patches; + int32_t original_width; + int32_t original_height; +}; + +/// Load an image file (JPEG/PNG/etc.) into HWC interleaved RGB bytes. +inline std::vector +load_image_rgb(const std::string& path, int32_t* width, int32_t* height) { + int w = 0; + int h = 0; + int channels = 0; + unsigned char* data = stbi_load(path.c_str(), &w, &h, &channels, 3); + if (data == nullptr) { + throw std::runtime_error( + "Failed to load image: " + path + " (" + stbi_failure_reason() + ")"); + } + std::vector rgb(data, data + static_cast(w) * h * 3); + stbi_image_free(data); + *width = w; + *height = h; + return rgb; +} + +/// Compute target dimensions preserving aspect ratio. +/// Both dimensions will be divisible by kVisionCellSize (48). +inline std::pair get_aspect_ratio_preserving_size( + int32_t height, + int32_t width, + int32_t max_patches) { + double target_px = + static_cast(max_patches) * kVisionPatchSize * kVisionPatchSize; + double factor = std::sqrt(target_px / (static_cast(height) * width)); + double ideal_h = factor * height; + double ideal_w = factor * width; + + auto target_h = static_cast(std::floor(ideal_h / kVisionCellSize)) * + kVisionCellSize; + auto target_w = static_cast(std::floor(ideal_w / kVisionCellSize)) * + kVisionCellSize; + + int32_t max_side = + (max_patches / (kVisionPoolingKernel * kVisionPoolingKernel)) * + kVisionCellSize; + if (target_h == 0 && target_w == 0) { + throw std::runtime_error("Image too small for patch size"); + } + if (target_h == 0) { + target_h = kVisionCellSize; + target_w = std::min( + static_cast(std::floor(static_cast(width) / height)) * + kVisionCellSize, + max_side); + } else if (target_w == 0) { + target_w = kVisionCellSize; + target_h = std::min( + static_cast(std::floor(static_cast(height) / width)) * + kVisionCellSize, + max_side); + } + return {target_h, target_w}; +} + +/// Compute number of soft tokens after pooling. +inline int64_t compute_vision_num_tokens(int64_t num_patches) { + return num_patches / (kVisionPoolingKernel * kVisionPoolingKernel); +} + +/// Resize RGB image (HWC uint8) to target dimensions using bicubic +/// (Catmull-Rom) interpolation, matching Python's PIL.Image.BICUBIC. +/// PIL filters the stored sRGB values directly, so stb must not linearize +/// them (STBIR_COLORSPACE_SRGB would gamma-decode before filtering). +inline std::vector resize_rgb( + const uint8_t* src, + int32_t src_w, + int32_t src_h, + int32_t dst_w, + int32_t dst_h) { + std::vector dst(static_cast(dst_w) * dst_h * 3); + stbir_resize_uint8_generic( + src, + src_w, + src_h, + 0, + dst.data(), + dst_w, + dst_h, + 0, + 3, + STBIR_ALPHA_CHANNEL_NONE, + 0, + STBIR_EDGE_CLAMP, + STBIR_FILTER_CATMULLROM, + STBIR_COLORSPACE_LINEAR, + nullptr); + return dst; +} + +/// Patchify an RGB image (HWC uint8, already resized) into pixel_values +/// and position_ids tensors, padded to max_patches. +inline ImageData patchify_rgb_image( + const uint8_t* rgb, + int32_t width, + int32_t height, + int32_t max_soft_tokens) { + int32_t max_patches = + max_soft_tokens * kVisionPoolingKernel * kVisionPoolingKernel; + + // Resize if needed + auto [target_h, target_w] = + get_aspect_ratio_preserving_size(height, width, max_patches); + + std::vector resized; + const uint8_t* img_data = rgb; + if (target_h != height || target_w != width) { + resized = resize_rgb(rgb, width, height, target_w, target_h); + img_data = resized.data(); + height = target_h; + width = target_w; + } + + int32_t h_patches = height / kVisionPatchSize; + int32_t w_patches = width / kVisionPatchSize; + int32_t num_patches = h_patches * w_patches; + + // Allocate padded tensors + auto pixel_values = executorch::extension::zeros( + {1, max_patches, kVisionPatchDim}, executorch::aten::ScalarType::Float); + auto pixel_position_ids = executorch::extension::full( + {1, max_patches, 2}, -1, executorch::aten::ScalarType::Long); + + float* pv_data = pixel_values->mutable_data_ptr(); + int64_t* pos_data = pixel_position_ids->mutable_data_ptr(); + + // Extract patches in row-major order (y then x), matching HF's layout: + // patches[py * w_patches + px] = image[py*16:(py+1)*16, px*16:(px+1)*16] + for (int32_t py = 0; py < h_patches; ++py) { + for (int32_t px = 0; px < w_patches; ++px) { + int32_t patch_idx = py * w_patches + px; + float* patch_out = + pv_data + static_cast(patch_idx) * kVisionPatchDim; + + for (int32_t dy = 0; dy < kVisionPatchSize; ++dy) { + for (int32_t dx = 0; dx < kVisionPatchSize; ++dx) { + int32_t img_y = py * kVisionPatchSize + dy; + int32_t img_x = px * kVisionPatchSize + dx; + int32_t src_idx = (img_y * width + img_x) * 3; + int32_t dst_idx = (dy * kVisionPatchSize + dx) * 3; + for (int32_t c = 0; c < 3; ++c) { + patch_out[dst_idx + c] = img_data[src_idx + c] / 255.0f; + } + } + } + + // Position IDs: (x, y) patch coordinates + pos_data[patch_idx * 2] = px; + pos_data[patch_idx * 2 + 1] = py; + } + } + + return ImageData{ + std::move(pixel_values), + std::move(pixel_position_ids), + num_patches, + width, + height, + }; +} + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/io_manager.cpp b/examples/samsung/executor_runner/llm/runner/io_manager.cpp new file mode 100644 index 00000000000..b4c2fe05f1b --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/io_manager.cpp @@ -0,0 +1,731 @@ +/* + * Copyright (c) 2025 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include + +#include +#include + +namespace example { + +namespace llm = ::executorch::extension::llm; +namespace runtime = ::executorch::runtime; +namespace aten = ::executorch::aten; +namespace extension = ::executorch::extension; + +namespace { + +// Helper to fill a float region with a value, using memset when possible. +inline void fill_float_region(float* begin, float* end, float value) { + if (begin >= end) { + return; + } + if (value == 0.0f) { + std::memset(begin, 0, static_cast(end - begin) * sizeof(float)); + } else { + std::fill(begin, end, value); + } +} + +inline void +fill_float_region(float* base, size_t offset, size_t len, float value) { + fill_float_region(base + offset, base + offset + len, value); +} + +} // namespace + +LlmIOManager::LlmIOManager( + extension::Module& module, + const RunnerConfig& config) + : IOManager(module), config_(config) { + rotary_embedding_ = std::make_unique(config); + + // RunnerConfig::validate() guarantees 1 <= prefill_seq_len <= max_seq_len. + prefill_cache_len_ = config_.model_args->get_max_seq_len() - + config_.model_args->get_prefill_seq_len(); + decode_cache_len_ = config_.model_args->get_max_seq_len() - 1; + + allocate_buffers(); +} + +size_t LlmIOManager::get_attention_setup_tensor_count() const { + constexpr size_t kGlobalMaskCount = 1; + constexpr size_t kGlobalRopeCosCount = 1; + constexpr size_t kGlobalRopeSinCount = 1; + constexpr size_t kBaselineMetadataCount = + kGlobalMaskCount + kGlobalRopeCosCount + kGlobalRopeSinCount; + + size_t setup_tensor_count = kBaselineMetadataCount; + + const bool requires_local_mask = + (config_.masking_strategy == MaskingStrategy::kCausalLocal || + config_.masking_strategy == MaskingStrategy::kHybridConvAttention); + if (requires_local_mask) { + constexpr size_t kLocalMaskCount = 1; + setup_tensor_count += kLocalMaskCount; + } + + const bool requires_local_rope = + (config_.embedding_strategy == EmbeddingStrategy::kGlobalAndLocal); + if (requires_local_rope) { + constexpr size_t kLocalRopeCosCount = 1; + constexpr size_t kLocalRopeSinCount = 1; + setup_tensor_count += (kLocalRopeCosCount + kLocalRopeSinCount); + } + + return setup_tensor_count; +} + +ET_NODISCARD runtime::Error LlmIOManager::load( + const std::string& prefill_method, + const std::string& decode_method) { + (void)prefill_method; + (void)decode_method; + + if ((config_.masking_strategy == MaskingStrategy::kCausalLocal || + config_.masking_strategy == MaskingStrategy::kHybridConvAttention) && + !config_.model_args->get_sliding_window().has_value()) { + ET_LOG( + Error, + "Sliding window configuration is required for the selected masking strategy"); + return runtime::Error::InvalidArgument; + } + + ET_LOG( + Info, + "(%s) Model Config: layers(%zu), kv_heads(%zu), hidden_dim(%zu), head_dim(%zu), \ + vocab(%zu), max_seq(%zu), max_ctx(%zu), prefill_seq(%zu), decode_seq(%zu), \ + mask_rows(%zu), mask_cols(%zu), slide_win(%zu), emb_strat(%d), mask_strat(%d)", + ET_FUNCTION, + config_.model_args->get_n_layers(), + config_.model_args->get_n_kv_heads(), + config_.model_args->get_hidden_dim(), + config_.model_args->get_head_dim(), + config_.model_args->get_vocab_size(), + config_.model_args->get_max_seq_len(), + config_.model_args->get_max_context_len(), + config_.model_args->get_prefill_seq_len(), + config_.model_args->get_decode_seq_len(), + config_.model_args->get_attn_mask_rows(), + config_.model_args->get_attn_mask_cols(), + config_.model_args->get_sliding_window().value_or(0), + static_cast(config_.embedding_strategy), + static_cast(config_.masking_strategy)); + + return runtime::Error::Ok; +} + +ET_NODISCARD runtime::Error LlmIOManager::reset( + const std::string& prefill_method, + const std::string& decode_method) { + (void)prefill_method; + (void)decode_method; + + current_pos_ = 0; + prefill_done_ = false; + + for (size_t l = 0; l < config_.model_args->get_num_cache_layers(); l++) { + std::fill(prefill_k_input_[l].begin(), prefill_k_input_[l].end(), 0); + std::fill(prefill_v_input_[l].begin(), prefill_v_input_[l].end(), 0); + std::fill(decode_k_input_[l].begin(), decode_k_input_[l].end(), 0); + std::fill(decode_v_input_[l].begin(), decode_v_input_[l].end(), 0); + } + std::fill(attn_mask_.begin(), attn_mask_.end(), 0.0f); + + return runtime::Error::Ok; +} + +executorch::runtime::Error LlmIOManager::load_rotary_embedding( + std::optional pos_embedding_path) { + return rotary_embedding_->load(pos_embedding_path); +} + +executorch::runtime::Result> +LlmIOManager::prepare_prefill( + const executorch::extension::TensorPtr& input, + const executorch::extension::TensorPtr& start_pos, + const std::string& prefill_method) { + (void)prefill_method; + std::vector inputs; + inputs.reserve(get_input_reserve_size()); + const size_t prefill_seq_len = config_.model_args->get_prefill_seq_len(); + const size_t prompt_token_len = input->numel(); + + ET_CHECK_OR_RETURN_ERROR( + prompt_token_len <= prefill_seq_len, + InvalidArgument, + "Prompt chunk length %zu exceeds prefill sequence length %zu.", + prompt_token_len, + prefill_seq_len); + + const int64_t* prompt_tokens = input->const_data_ptr(); + input_tokens_.assign(prefill_seq_len, 0); + + std::copy( + prompt_tokens, prompt_tokens + prompt_token_len, input_tokens_.begin()); + + input_tokens_tensor_ = executorch::extension::from_blob( + input_tokens_.data(), + {1, static_cast(prefill_seq_len)}, + executorch::aten::ScalarType::Long); + inputs.emplace_back(input_tokens_tensor_); + + append_prelude_inputs(prefill_seq_len, inputs); + + const int64_t start_position = *start_pos->const_data_ptr(); + ET_CHECK_OR_RETURN_ERROR( + start_position >= 0 && + static_cast(start_position) + prompt_token_len <= + prefill_cache_len_, + InvalidArgument, + "Prompt does not fit: chunk at position %" PRId64 + " with %zu tokens exceeds the %zu-token prefill cache", + start_position, + prompt_token_len, + prefill_cache_len_); + current_pos_ = static_cast(start_position); + pos_ids_.resize(prefill_seq_len); + for (size_t i = 0; i < prefill_seq_len; ++i) { + pos_ids_[i] = current_pos_ + static_cast(i); + } + + // attention tensor + update_attn_mask(current_pos_, prompt_token_len); + attn_mask_tensor_ = executorch::extension::from_blob( + attn_mask_.data(), + {1, + 1, + static_cast( + config_.model_args->get_attn_mask_rows()), + static_cast( + config_.model_args->get_attn_mask_cols())}); + + inputs.emplace_back(attn_mask_tensor_); + + const auto masking_strategy = config_.masking_strategy; + switch (masking_strategy) { + case MaskingStrategy::kCausal: + break; + case MaskingStrategy::kCausalLocal: + case MaskingStrategy::kHybridConvAttention: + update_local_attn_mask(current_pos_, prompt_token_len); + const size_t mask_rows = config_.model_args->get_attn_mask_rows(); + const size_t mask_cols = config_.model_args->get_attn_mask_cols(); + local_attn_mask_tensor_ = executorch::extension::from_blob( + local_attn_mask_.data(), + {1, + 1, + static_cast(mask_rows), + static_cast(mask_cols)}); + inputs.emplace_back(local_attn_mask_tensor_); + break; + } + + ET_CHECK_OK_OR_RETURN_ERROR( + prepare_rotary_embedding_inputs(prefill_seq_len, inputs)); + + prefill_k_input_tensors_.clear(); + prefill_v_input_tensors_.clear(); + prefill_k_output_tensors_.clear(); + prefill_v_output_tensors_.clear(); + dummy_kv_tensors_.clear(); + append_kv_input_tensors( + prefill_k_input_, + prefill_v_input_, + prefill_k_input_tensors_, + prefill_v_input_tensors_, + executorch::aten::ScalarType::Float, + prefill_cache_len_, + prefill_dummy_kv_, + inputs); + return inputs; +} +executorch::runtime::Result> +LlmIOManager::prepare_decode( + const executorch::extension::TensorPtr& input, + const executorch::extension::TensorPtr& start_pos, + const std::string& decode_method) { + (void)decode_method; + const int64_t start_position = *start_pos->const_data_ptr(); + ET_CHECK_OR_RETURN_ERROR( + start_position >= 0 && + static_cast(start_position) <= decode_cache_len_, + InvalidArgument, + "Decode position %" PRId64 " exceeds max_seq_len %" PRId64, + start_position, + config_.model_args->get_max_seq_len()); + if (!prefill_done_) { + copy_prefill_to_decode_input(); + prefill_done_ = true; + } + current_pos_ = static_cast(start_position); + update_attn_mask(current_pos_, config_.model_args->get_decode_seq_len()); + std::vector inputs; + inputs.reserve(get_input_reserve_size()); + inputs.emplace_back(input); + + append_prelude_inputs(config_.model_args->get_decode_seq_len(), inputs); + + attn_mask_tensor_ = executorch::extension::from_blob( + attn_mask_.data(), + {1, + 1, + 1, + static_cast( + config_.model_args->get_attn_mask_cols())}); + inputs.emplace_back(attn_mask_tensor_); + switch (config_.masking_strategy) { + case MaskingStrategy::kCausal: + break; + case MaskingStrategy::kCausalLocal: + case MaskingStrategy::kHybridConvAttention: + update_local_attn_mask(current_pos_, 1); + local_attn_mask_tensor_ = executorch::extension::from_blob( + local_attn_mask_.data(), + {1, + 1, + 1, + static_cast( + config_.model_args->get_attn_mask_cols())}); + inputs.emplace_back(local_attn_mask_tensor_); + break; + } + + pos_ids_.clear(); + pos_ids_.resize(config_.model_args->get_decode_seq_len()); + pos_ids_[0] = current_pos_; + ET_CHECK_OK_OR_RETURN_ERROR(prepare_rotary_embedding_inputs( + config_.model_args->get_decode_seq_len(), inputs)); + + decode_k_input_tensors_.clear(); + decode_v_input_tensors_.clear(); + decode_k_output_tensors_.clear(); + decode_v_output_tensors_.clear(); + dummy_kv_tensors_.clear(); + append_kv_input_tensors( + decode_k_input_, + decode_v_input_, + decode_k_input_tensors_, + decode_v_input_tensors_, + executorch::aten::ScalarType::Float, + decode_cache_len_, + decode_dummy_kv_, + inputs); + return inputs; +} +executorch::runtime::Error LlmIOManager::update_prefill( + const std::vector& model_outputs, + const std::string& prefill_method) { + (void)prefill_method; + return update_kv_cache( + model_outputs, + prefill_k_input_, + prefill_v_input_, + prefill_cache_len_, + config_.model_args->get_prefill_seq_len()); +} +executorch::runtime::Error LlmIOManager::update_decode( + const std::vector& model_outputs, + const std::string& decode_method) { + (void)decode_method; + return update_kv_cache( + model_outputs, decode_k_input_, decode_v_input_, decode_cache_len_, 1); +} +void LlmIOManager::allocate_buffers() { + const size_t n_kv_heads = config_.model_args->get_n_kv_heads(); + const size_t num_cache_layers = config_.model_args->get_num_cache_layers(); + prefill_k_input_.resize(num_cache_layers); + prefill_v_input_.resize(num_cache_layers); + decode_k_input_.resize(num_cache_layers); + decode_v_input_.resize(num_cache_layers); + for (size_t l = 0; l < num_cache_layers; l++) { + const size_t head_dim = config_.model_args->get_head_dim_for_layer(l); + const size_t prefill_in_size = + n_kv_heads * prefill_cache_len_ * head_dim * sizeof(float); + const size_t decode_in_size = + n_kv_heads * decode_cache_len_ * head_dim * sizeof(float); + prefill_k_input_[l].resize(prefill_in_size, 0); + prefill_v_input_[l].resize(prefill_in_size, 0); + decode_k_input_[l].resize(decode_in_size, 0); + decode_v_input_[l].resize(decode_in_size, 0); + } + + // Dummy scratch for KV-shared layer slots, sized for the widest head_dim. + const size_t max_head_dim = std::max( + config_.model_args->get_head_dim(), + config_.model_args->get_global_head_dim()); + prefill_dummy_kv_.assign( + n_kv_heads * prefill_cache_len_ * max_head_dim * sizeof(float), 0); + decode_dummy_kv_.assign( + n_kv_heads * decode_cache_len_ * max_head_dim * sizeof(float), 0); + attn_mask_.resize( + config_.model_args->get_attn_mask_rows() * + config_.model_args->get_attn_mask_cols(), + -1.0f); + local_attn_mask_.resize( + config_.model_args->get_attn_mask_rows() * + config_.model_args->get_attn_mask_cols(), + -1.0f); +} +void LlmIOManager::copy_prefill_to_decode_input() { + const size_t n_kv_heads = config_.model_args->get_n_kv_heads(); + for (size_t l = 0; l < config_.model_args->get_num_cache_layers(); l++) { + const size_t head_dim = config_.model_args->get_head_dim_for_layer(l); + const size_t copy_size = prefill_cache_len_ * head_dim * sizeof(float); + for (size_t h = 0; h < n_kv_heads; h++) { + const char* k_src = prefill_k_input_[l].data() + + h * prefill_cache_len_ * head_dim * sizeof(float); + char* k_dst = decode_k_input_[l].data() + + h * decode_cache_len_ * head_dim * sizeof(float); + std::memcpy(k_dst, k_src, copy_size); + const char* v_src = prefill_v_input_[l].data() + + h * prefill_cache_len_ * head_dim * sizeof(float); + char* v_dst = decode_v_input_[l].data() + + h * decode_cache_len_ * head_dim * sizeof(float); + std::memcpy(v_dst, v_src, copy_size); + } + } +} + +void LlmIOManager::append_kv_input_tensors( + std::vector>& k_bufs, + std::vector>& v_bufs, + std::vector& k_tensors, + std::vector& v_tensors, + executorch::aten::ScalarType kv_scalar_type, + size_t cache_len, + std::vector& dummy_buf, + std::vector& inputs) const { + const size_t n_layers = config_.model_args->get_n_layers(); + const size_t num_cache_layers = config_.model_args->get_num_cache_layers(); + const auto n_kv_heads = static_cast( + config_.model_args->get_n_kv_heads()); + + // The model declares a KV input slot for every layer, in K[0..L-1] then + // V[0..L-1] order. The leading self-decoder layers carry the real cache; + // the trailing KV-shared (cross-decoder) layers are resolved inside the + // delegate and their slots are never read, so point them at zeroed scratch. + auto make_tensor = [&](std::vector>& bufs, + size_t layer) -> executorch::extension::TensorPtr { + char* data = + (layer < num_cache_layers) ? bufs[layer].data() : dummy_buf.data(); + return executorch::extension::from_blob( + data, + {1, + n_kv_heads, + static_cast(cache_len), + static_cast( + config_.model_args->get_head_dim_for_layer(layer))}, + kv_scalar_type); + }; + + for (size_t l = 0; l < n_layers; l++) { + auto k_in = make_tensor(k_bufs, l); + if (l < num_cache_layers) { + k_tensors.push_back(k_in); + } else { + dummy_kv_tensors_.push_back(k_in); + } + inputs.emplace_back(k_in); + } + for (size_t l = 0; l < n_layers; l++) { + auto v_in = make_tensor(v_bufs, l); + if (l < num_cache_layers) { + v_tensors.push_back(v_in); + } else { + dummy_kv_tensors_.push_back(v_in); + } + inputs.emplace_back(v_in); + } +} + +void LlmIOManager::update_attn_mask(size_t pos, size_t seq_len) { + const float maskFalse = -255.0f; + const float maskTrue = 0.0f; + + const size_t rows = config_.model_args->get_attn_mask_rows(); + const size_t cols = config_.model_args->get_attn_mask_cols(); + const size_t total_size = rows * cols; + + fill_float_region(attn_mask_.data(), 0, total_size, maskFalse); + + if (seq_len > 1) { + // Prefill: past positions [0, pos) and the causal block for the chunk, + // which occupies the last `rows` columns. + const size_t start_col = cols - rows; + for (size_t i = 0; i < seq_len; ++i) { + fill_float_region( + attn_mask_.data(), i * cols + start_col, i + 1, maskTrue); + } + for (size_t i = 0; i < rows; ++i) { + fill_float_region(attn_mask_.data(), i * cols, pos, maskTrue); + } + } else { + const size_t left_len = std::min(pos, cols); + if (left_len > 0) { + fill_float_region(attn_mask_.data(), 0, left_len, maskTrue); + } + + if (cols > 0) { + attn_mask_[cols - 1] = maskTrue; + } + } +} + +void LlmIOManager::update_local_attn_mask(size_t pos, size_t seq_len) { + const float maskFalse = -255.0f; + const float maskTrue = 0.0f; + + const size_t rows = config_.model_args->get_attn_mask_rows(); + const size_t cols = config_.model_args->get_attn_mask_cols(); + const size_t sliding_window = + config_.model_args->get_sliding_window().value(); + const size_t total_size = rows * cols; + + // Start with a copy of the full attention mask + std::memcpy( + local_attn_mask_.data(), attn_mask_.data(), total_size * sizeof(float)); + + if (seq_len > 1) { + // Prefill phase with sliding window: row i sits at position pos + i and + // sees the last sliding_window keys, so mask the oldest cached columns. + for (size_t i = 0; i < rows; ++i) { + const size_t query_pos = pos + i; + if (query_pos + 1 <= sliding_window) { + continue; + } + const size_t mask_len = std::min(query_pos + 1 - sliding_window, pos); + fill_float_region(local_attn_mask_.data(), i * cols, mask_len, maskFalse); + } + } else { + // Decode phase (single token) + // Set all positions up to pos to maskTrue + const size_t end = std::min(static_cast(pos), cols); + fill_float_region(local_attn_mask_.data(), 0, end, maskTrue); + + // If there are more than 512 True values, we need to keep only the last 512 + // Count True values efficiently by scanning from the end + size_t valid_count = 1; + for (size_t i = 0; i < cols; ++i) { + if (local_attn_mask_[i] == maskTrue) { + valid_count++; + } else { + break; + } + } + + if (valid_count > sliding_window) { + const size_t to_remove = valid_count - sliding_window; + // Set the oldest True values to maskFalse + for (size_t i = 0; i < to_remove; ++i) { + if (local_attn_mask_[i] == maskTrue) { + local_attn_mask_[i] = maskFalse; + } + } + } + + if (cols > 0) { + local_attn_mask_[cols - 1] = maskTrue; + } + } +} + +executorch::runtime::Error LlmIOManager::update_kv_cache( + const std::vector& model_outputs, + std::vector>& k_bufs, + std::vector>& v_bufs, + size_t cache_len, + size_t new_len) { + // model_outputs: [logits, k0..k_{C-1}, v0..v_{C-1}] where C is the number of + // cached (self-decoder) layers; KV-shared layers emit no new cache. + const size_t num_cache_layers = config_.model_args->get_num_cache_layers(); + ET_CHECK_OR_RETURN_ERROR( + model_outputs.size() == 1 + 2 * num_cache_layers, + InvalidProgram, + "Expected %zu outputs (logits + K/V for %zu layers), got %zu", + 1 + 2 * num_cache_layers, + num_cache_layers, + model_outputs.size()); + // The last decode step may run at pos == cache_len; its KV is never read. + if (current_pos_ >= cache_len) { + return executorch::runtime::Error::Ok; + } + const size_t rows = std::min(new_len, cache_len - current_pos_); + const size_t n_kv_heads = config_.model_args->get_n_kv_heads(); + for (size_t i = 0; i < 2 * num_cache_layers; i++) { + const size_t l = i % num_cache_layers; + const size_t head_dim = config_.model_args->get_head_dim_for_layer(l); + const runtime::EValue& out = model_outputs[1 + i]; + ET_CHECK_OR_RETURN_ERROR( + out.isTensor() && + out.toTensor().scalar_type() == aten::ScalarType::Float && + static_cast(out.toTensor().numel()) == + n_kv_heads * new_len * head_dim, + InvalidProgram, + "KV output %zu must be a float tensor of %zu elements", + 1 + i, + n_kv_heads * new_len * head_dim); + auto& bufs = i < num_cache_layers ? k_bufs : v_bufs; + copy_to_cache( + out.toTensor().const_data_ptr(), + new_len, + rows, + bufs[l].data(), + cache_len, + current_pos_, + head_dim); + } + return executorch::runtime::Error::Ok; +} + +executorch::runtime::Error LlmIOManager::prepare_rotary_embedding_inputs( + size_t seq_len, + std::vector& inputs) { + const int64_t head_dim = config_.model_args->get_head_dim(); + const int64_t global_head_dim = config_.model_args->get_global_head_dim(); + + const size_t global_element_size = seq_len * global_head_dim; + global_cos_data_.resize(global_element_size); + global_sin_data_.resize(global_element_size); + + switch (config_.embedding_strategy) { + case EmbeddingStrategy::kGlobalOnly: + break; + case EmbeddingStrategy::kGlobalAndLocal: { + const size_t local_element_size = seq_len * head_dim; + local_cos_data_.resize(local_element_size); + local_sin_data_.resize(local_element_size); + break; + } + } + + auto update_err = rotary_embedding_->update( + pos_ids_, + global_cos_data_.data(), + global_sin_data_.data(), + local_cos_data_.data(), + local_sin_data_.data()); + if (update_err != executorch::runtime::Error::Ok) { + ET_LOG(Error, "update pos_embedding failed !"); + return update_err; + } + + global_cos_tensor_ = executorch::extension::from_blob( + global_cos_data_.data(), + {1, + static_cast(seq_len), + static_cast(global_head_dim)}); + global_sin_tensor_ = executorch::extension::from_blob( + global_sin_data_.data(), + {1, + static_cast(seq_len), + static_cast(global_head_dim)}); + switch (config_.embedding_strategy) { + case EmbeddingStrategy::kGlobalOnly: + break; + case EmbeddingStrategy::kGlobalAndLocal: + local_cos_tensor_ = executorch::extension::from_blob( + local_cos_data_.data(), + {1, + static_cast(seq_len), + static_cast(head_dim)}); + local_sin_tensor_ = executorch::extension::from_blob( + local_sin_data_.data(), + {1, + static_cast(seq_len), + static_cast(head_dim)}); + break; + } + + inputs.emplace_back(global_cos_tensor_); + inputs.emplace_back(global_sin_tensor_); + switch (config_.embedding_strategy) { + case EmbeddingStrategy::kGlobalOnly: + break; + case EmbeddingStrategy::kGlobalAndLocal: + inputs.emplace_back(local_cos_tensor_); + inputs.emplace_back(local_sin_tensor_); + break; + } + + return executorch::runtime::Error::Ok; +} + +void LlmIOManager::set_vision_embeddings( + std::vector embeds, + int64_t num_tokens) { + vision_embeds_ = std::move(embeds); + num_vision_tokens_ = num_tokens; + vision_embed_idx_ = 0; + ET_LOG( + Info, + "Vision embeddings set: %lld tokens, %zu floats", + static_cast(num_vision_tokens_), + vision_embeds_.size()); +} + +void LlmIOManager::append_prelude_inputs( + size_t seq_len, + std::vector& inputs) { + if (config_.model_version != DecoderModelVersion::kGemma4) { + return; + } + const int64_t hidden_dim = config_.model_args->get_hidden_dim(); + + // inputs_embeds [1, seq_len, hidden_dim]: zeros for text-only inference. The + // decoder embeds token ids internally; this slot only matters at multimodal + // placeholder positions, which text-only prompts never contain. + inputs_embeds_data_.assign(seq_len * hidden_dim, 0.0f); + + // mm_mask [1, seq_len]: zeros select the internal text embeddings everywhere. + mm_mask_data_.assign(seq_len, 0.0f); + + // Inject vision embeddings at image-token positions (Gemma 4 multimodal). + // Only during prefill (seq_len > 1); decode tokens are never image tokens. + // Each position whose token is the image-token id consumes the next packed + // vision embedding and flips its mm_mask bit so the decoder picks + // inputs_embeds at that position. vision_embed_idx_ persists across + // micro-batch prefill chunks. + // The token id is also replaced with pad: the exported text decoder looks up + // per-layer embeddings (PLE) from input_ids directly, and HF / the export + // reference (Gemma4Model_ENN.forward) feed pad ids at multimodal positions. + // input_tokens_tensor_ aliases input_tokens_, so the in-place edit is seen + // by the model. + size_t vision_idx = vision_embed_idx_; + if (!vision_embeds_.empty() && seq_len > 1) { + for (size_t i = 0; i < seq_len; ++i) { + if (static_cast(input_tokens_[i]) == kGemma4ImageTokenId && + vision_idx < static_cast(num_vision_tokens_)) { + std::copy_n( + vision_embeds_.data() + vision_idx * hidden_dim, + hidden_dim, + inputs_embeds_data_.data() + i * hidden_dim); + mm_mask_data_[i] = 1.0f; + input_tokens_[i] = kGemma4PadId; + ++vision_idx; + } + } + } + vision_embed_idx_ = vision_idx; + + inputs_embeds_tensor_ = executorch::extension::from_blob( + inputs_embeds_data_.data(), + {1, + static_cast(seq_len), + static_cast(hidden_dim)}); + inputs.emplace_back(inputs_embeds_tensor_); + + mm_mask_tensor_ = executorch::extension::from_blob( + mm_mask_data_.data(), + {1, static_cast(seq_len)}); + inputs.emplace_back(mm_mask_tensor_); +} + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/io_manager.h b/examples/samsung/executor_runner/llm/runner/io_manager.h new file mode 100644 index 00000000000..493cee16980 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/io_manager.h @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2025 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace example { + +namespace llm = ::executorch::extension::llm; +namespace runtime = ::executorch::runtime; +namespace aten = ::executorch::aten; +namespace extension = ::executorch::extension; + +/** + * IO manager for models where the runner manages KV-cache externally. + * + * The model receives past KV-cache as input and produces new KV-cache entries + * as output. The runner is responsible for concatenating outputs into the + * cache and feeding the updated cache back on the next step. + * + * Supports micro-batch prefill: the runner may call prepare_prefill / + * update_prefill multiple times (each processing prefill_seq_len tokens) + * before switching to decode. The KV-cache accumulates in the prefill + * buffer across micro-batches and is copied to the decode buffer on the + * first prepare_decode call. + * + * Prefill (per micro-batch): + * Input: token[prefill_seq_len], start_pos, attn_mask[R,C], + * k_cache[1,H,past,D] x L, v_cache[1,H,past,D] x L + * Output: logits, k_cache[1,H,prefill_seq_len,D] x L, + * v_cache[1,H,prefill_seq_len,D] x L + * where past = max_seq_len - prefill_seq_len + * + * Decode: + * Input: token[1], start_pos, attn_mask[R,C], + * k_cache[1,H,max-1,D] x L, v_cache[1,H,max-1,D] x L + * Output: logits, k_cache[1,H,1,D] x L, v_cache[1,H,1,D] x L + */ +class ET_EXPERIMENTAL LlmIOManager : public llm::IOManager { + public: + LlmIOManager(extension::Module& module, const RunnerConfig& config); + ~LlmIOManager() override = default; + LlmIOManager(const LlmIOManager& other) = delete; + LlmIOManager& operator=(const LlmIOManager& rhs) = delete; + LlmIOManager(LlmIOManager&& other) = delete; + LlmIOManager& operator=(const LlmIOManager&& rhs) = delete; + + ET_NODISCARD runtime::Error load( + const std::string& prefill_method, + const std::string& decode_method) override; + + ET_NODISCARD runtime::Error reset( + const std::string& prefill_method, + const std::string& decode_method) override; + + ET_NODISCARD virtual runtime::Error load_rotary_embedding( + std::optional pos_embedding_path); + + virtual runtime::Result> prepare_prefill( + const extension::TensorPtr& input, + const extension::TensorPtr& start_pos, + const std::string& prefill_method) override; + + runtime::Result> prepare_prefill( + const extension::TensorPtr& input, + const extension::TensorPtr& start_pos) { + return prepare_prefill( + input, start_pos, example::default_method_name_prefill); + } + + runtime::Result> prepare_decode( + const extension::TensorPtr& input, + const extension::TensorPtr& start_pos, + const std::string& decode_method) override; + + runtime::Result> prepare_decode( + const extension::TensorPtr& input, + const extension::TensorPtr& start_pos) { + return prepare_decode( + input, start_pos, example::default_method_name_decode); + } + + ET_NODISCARD runtime::Error update_prefill( + const std::vector& model_outputs, + const std::string& prefill_method) override; + + ET_NODISCARD runtime::Error update_prefill( + const std::vector& model_outputs) { + return update_prefill(model_outputs, example::default_method_name_prefill); + } + + ET_NODISCARD runtime::Error update_decode( + const std::vector& model_outputs, + const std::string& decode_method) override; + + ET_NODISCARD runtime::Error update_decode( + const std::vector& model_outputs) { + return update_decode(model_outputs, example::default_method_name_decode); + } + + /** + * Supply pre-computed vision embeddings for Gemma 4 multimodal inference. + * + * @param embeds Packed valid vision embeddings (num_vision_tokens * hidden, + * contiguous float). The caller must have dropped padding. + * @param num_tokens Number of valid vision tokens (= embeds.size()/hidden). + */ + void set_vision_embeddings(std::vector embeds, int64_t num_tokens); + + size_t num_vision_embeds_consumed() const { + return vision_embed_idx_; + } + + const RunnerConfig& get_config() const { + return config_; + } + + size_t get_attention_setup_tensor_count() const; + + // [tokens, (Gemma 4: inputs_embeds, mm_mask), masks, RoPE cos/sin, K x L, + // V x L] + size_t get_input_reserve_size() const { + const size_t prelude = + config_.model_version == DecoderModelVersion::kGemma4 ? 2 : 0; + return 1 + prelude + get_attention_setup_tensor_count() + + 2 * static_cast(config_.model_args->get_n_layers()); + } + + size_t current_pos() const { + return current_pos_; + } + + private: + void allocate_buffers(); + + /** + * Copy prefill output KV cache (layout [1,H,prefill_cache_len,D]) into + * the decode input buffer (layout [1,H,decode_cache_len,D]). + * + * Since the head stride differs between the two layouts, we copy per-head. + */ + void copy_prefill_to_decode_input(); + + /** + * Copy model KV-cache outputs into the given buffers. + * + * @param model_outputs [logits, k0..k_{L-1}, v0..v_{L-1}] + * @param k_bufs per-layer K cache buffers (input/output) + * @param v_bufs per-layer V cache buffers (input/output) + * @param cache_len length of each cache buffer + * @param new_len number of new tokens to copy (prefill_seq_len or 1) + */ + ET_NODISCARD runtime::Error update_kv_cache( + const std::vector& model_outputs, + std::vector>& k_bufs, + std::vector>& v_bufs, + size_t cache_len, + size_t new_len); + + /** + * Copy the first `rows` of new cache entries from source data (layout + * [1,H,new_len,D]) into the cache buffer (layout [1,H,cache_len,D]) at pos. + */ + template + void copy_to_cache( + const T* src, + size_t new_len, + size_t rows, + char* cache, + size_t cache_len, + size_t pos, + size_t head_dim) { + for (size_t h = 0; h < config_.model_args->get_n_kv_heads(); h++) { + const T* src_head = src + h * new_len * head_dim; + char* dst_head = cache + h * cache_len * head_dim * sizeof(T) + + pos * head_dim * sizeof(T); + std::memcpy(dst_head, src_head, rows * head_dim * sizeof(T)); + } + } + + /** + * Append KV-cache input tensors for all layers. + * + * For each layer l, creates k and v tensors from the given buffers + * (shape [1, n_kv_heads, cache_len, head_dim]) and appends them + * to both the per-phase tensor wrappers and the model inputs vector. + */ + void append_kv_input_tensors( + std::vector>& k_bufs, + std::vector>& v_bufs, + std::vector& k_tensors, + std::vector& v_tensors, + executorch::aten::ScalarType kv_scalar_type, + size_t cache_len, + std::vector& dummy_buf, + std::vector& inputs) const; + + /** + * Append the Gemma 4 text-decoder prelude inputs that follow the token ids: + * inputs_embeds [1, seq_len, hidden_dim] and mm_mask [1, seq_len]. + * + * For text-only inference both are zero-filled: the decoder embeds the token + * ids internally and mm_mask == 0 selects those text embeddings everywhere, + * so inputs_embeds (the multimodal injection path) is ignored. + */ + void append_prelude_inputs( + size_t seq_len, + std::vector& inputs); + + /** + * Update the attention mask for a causal (lower-triangular) pattern. + * For each query row r in [0, seq_len), the mask allows attending to + * positions [0, pos + r] (value 1) and masks future positions (value 0). + */ + void update_attn_mask(size_t pos, size_t seq_len); + + /** + * @brief Prepare rotary embedding inputs for quantized models. + * + * This helper function computes and adds the rotary embedding tensors + * (global_cos, global_sin, local_cos, local_sin) to the inputs vector. + * It is used by both prepare_prefill and prepare_decode to avoid code + * duplication. + * + * @param seq_len The sequence length to use for the embedding computation. + * @param inputs Vector of EValue inputs to append the embedding tensors to. + */ + ET_NODISCARD runtime::Error prepare_rotary_embedding_inputs( + size_t seq_len, + std::vector& inputs); + + /** + * Update the local_attention mask for a causal (lower-triangular) pattern. + * For each query row r in [0, seq_len), the mask allows attending to + * positions [0, pos + r] (value 1) and masks future positions (value 0). + */ + void update_local_attn_mask(size_t pos, size_t seq_len); + const RunnerConfig& config_; + + size_t prefill_cache_len_; + size_t decode_cache_len_; + size_t current_pos_ = 0; + bool prefill_done_ = false; + + std::vector input_tokens_; + executorch::extension::TensorPtr input_tokens_tensor_; + std::vector pos_ids_; + + // Gemma 4 prelude inputs (zero-filled for text-only inference). + std::vector inputs_embeds_data_; + executorch::extension::TensorPtr inputs_embeds_tensor_; + std::vector mm_mask_data_; + executorch::extension::TensorPtr mm_mask_tensor_; + + // Gemma 4 vision embeddings injected at image-token positions. + // vision_embeds_ holds only the valid (non-padding) embeddings packed + // contiguously (num_vision_tokens * hidden). append_prelude_inputs copies + // them into inputs_embeds_data_ at the image-token slots and marks those + // slots in mm_mask_data_. + std::vector vision_embeds_; + int64_t num_vision_tokens_ = 0; + // Cursor into vision_embeds_ consumed by append_prelude_inputs. Persists + // across micro-batch prefill chunks so image tokens split across chunks map + // to the correct embeddings. Reset on set_vision_embeddings. + size_t vision_embed_idx_ = 0; + + // Zero-filled scratch feeding the KV input slots of KV-shared (cross-decoder) + // layers. These slots are declared model inputs but never read by the graph, + // so a single shared buffer (sized for the widest head_dim) backs them all. + std::vector prefill_dummy_kv_; + std::vector decode_dummy_kv_; + mutable std::vector dummy_kv_tensors_; + + // Per-layer input buffers: [1, n_kv_heads, cache_len, head_dim] flattened + // Use std::vector to support both float and half (float16) data types + std::vector> prefill_k_input_; + std::vector> prefill_v_input_; + std::vector> decode_k_input_; + std::vector> decode_v_input_; + + std::unique_ptr rotary_embedding_; + + std::vector global_cos_data_; + std::vector global_sin_data_; + std::vector local_cos_data_; + std::vector local_sin_data_; + + executorch::extension::TensorPtr global_cos_tensor_; + executorch::extension::TensorPtr global_sin_tensor_; + executorch::extension::TensorPtr local_cos_tensor_; + executorch::extension::TensorPtr local_sin_tensor_; + + // Attention mask buffer: [attn_mask_rows, attn_mask_cols] flattened + std::vector attn_mask_; + std::vector local_attn_mask_; + executorch::extension::TensorPtr attn_mask_tensor_; + executorch::extension::TensorPtr local_attn_mask_tensor_; + + // Tensor wrappers (kept alive between prepare and model execution) + std::vector prefill_k_input_tensors_; + std::vector prefill_v_input_tensors_; + std::vector prefill_k_output_tensors_; + std::vector prefill_v_output_tensors_; + std::vector decode_k_input_tensors_; + std::vector decode_v_input_tensors_; + std::vector decode_k_output_tensors_; + std::vector decode_v_output_tensors_; +}; + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/prefiller.cpp b/examples/samsung/executor_runner/llm/runner/prefiller.cpp new file mode 100644 index 00000000000..4f78405cd59 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/prefiller.cpp @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2025 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Given a text prompt, encode it using tokenizer and prefill the KV cache of a +// LLM. + +#include +#include + +namespace example { + +namespace llm = ::executorch::extension::llm; +namespace runtime = ::executorch::runtime; +namespace aten = ::executorch::aten; +namespace extension = ::executorch::extension; + +Prefiller::Prefiller( + llm::TextDecoderRunner* text_decoder_runner, + bool use_kv_cache, + bool enable_parallel_prefill, + int64_t max_seq_len) + : TextPrefiller( + text_decoder_runner, + use_kv_cache, + enable_parallel_prefill, + max_seq_len), + text_decoder_runner_(text_decoder_runner), + use_kv_cache_(use_kv_cache), + enable_parallel_prefill_(enable_parallel_prefill), + max_seq_len_(max_seq_len > 0 ? max_seq_len : 128) {} + +runtime::Result Prefiller::prefill( + std::vector& prompt_tokens, + int64_t& start_pos) { + ET_CHECK_OR_RETURN_ERROR( + !prompt_tokens.empty(), InvalidArgument, "Prompt cannot be empty"); + if (!text_decoder_runner_->is_method_loaded()) { + ET_CHECK_OK_OR_RETURN_ERROR(text_decoder_runner_->load()); + } + + // If prompt tokens exceed max_seq_len_, we need to chunk them + int32_t num_prompt_tokens = prompt_tokens.size(); + ET_LOG( + Info, + "Micro Batch: num_prompt(%d), prefill::max_seq_len(%ld), start_pos(%ld)", + num_prompt_tokens, + max_seq_len_, + start_pos); + + if (num_prompt_tokens > max_seq_len_) { + uint64_t cur_token = 0; + int num_tokens_to_process = 0; + + while (num_tokens_to_process < num_prompt_tokens) { + auto num_tokens_to_prefill_with = std::min( + num_prompt_tokens - num_tokens_to_process, max_seq_len_); + + std::vector prompt_tokens_to_process( + num_tokens_to_prefill_with); + std::copy( + prompt_tokens.begin() + num_tokens_to_process, + prompt_tokens.begin() + num_tokens_to_process + + num_tokens_to_prefill_with, + prompt_tokens_to_process.begin()); + + auto chunk_result = prefill_chunk(prompt_tokens_to_process, start_pos); + ET_CHECK_OK_OR_RETURN_ERROR(chunk_result.error()); + cur_token = chunk_result.get(); + + num_tokens_to_process += num_tokens_to_prefill_with; + } + + return cur_token; + } else { + // If prompt tokens don't exceed max_seq_len_, process them directly + return prefill_chunk(prompt_tokens, start_pos); + } +} + +runtime::Result Prefiller::prefill_chunk( + std::vector& prompt_tokens, + int64_t& start_pos) { + // enable_parallel_prefill_ maybe set even when not using kv cache + // When kv cache is not used, start pos is ignored + int32_t num_prompt_tokens = prompt_tokens.size(); + + // store the token + uint64_t cur_token; + if (enable_parallel_prefill_ || !use_kv_cache_) { + // initialize tensor wrappers + auto tokens = extension::from_blob( + prompt_tokens.data(), + {1, static_cast(prompt_tokens.size())}, + aten::ScalarType::Long); + + auto outputs_res = text_decoder_runner_->step(tokens, start_pos); + + ET_CHECK_OK_OR_RETURN_ERROR(outputs_res.error()); + ET_LOG( + Info, "Prefill token result numel(): %zu", outputs_res.get().numel()); + + start_pos += num_prompt_tokens; + cur_token = text_decoder_runner_->logits_to_token(outputs_res.get()); + } else { // sequential prefill + int64_t pos = 0; // position in the sequence + // NOLINTNEXTLINE(facebook-hte-ParameterUncheckedArrayBounds) + cur_token = prompt_tokens[0]; + + ET_LOG( + Info, + "num_prompt_token(%d), cur_token(%ld), start_pos(%ld)", + num_prompt_tokens, + cur_token, + start_pos); + + auto tokens = + extension::from_blob(&cur_token, {1, 1}, aten::ScalarType::Long); + + // run the first token and get back logits tensor. Assuming the first token + // is bos so don't callback. + auto logits_tensor = + ET_UNWRAP(text_decoder_runner_->step(tokens, start_pos)); + + pos += 1; // start the loop from index 1 + start_pos += 1; + + while (pos < num_prompt_tokens) { + // Run the model + // NOLINTNEXTLINE(facebook-hte-ParameterUncheckedArrayBounds) + cur_token = prompt_tokens[pos]; + + logits_tensor = ET_UNWRAP(text_decoder_runner_->step(tokens, start_pos)); + + pos++; + start_pos++; + } + + cur_token = text_decoder_runner_->logits_to_token(logits_tensor); + } + return cur_token; +} + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/prefiller.h b/examples/samsung/executor_runner/llm/runner/prefiller.h new file mode 100644 index 00000000000..fc98d284f1e --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/prefiller.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2025 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Given a text prompt, encode it using tokenizer and prefill the KV cache of a +// LLM. + +#pragma once + +#include +#include + +namespace example { + +namespace llm = ::executorch::extension::llm; +namespace runtime = ::executorch::runtime; +namespace extension = ::executorch::extension; + +class ET_EXPERIMENTAL Prefiller : public llm::TextPrefiller { + public: + Prefiller( + llm::TextDecoderRunner* text_decoder_runner, + bool use_kv_cache, + bool enable_parallel_prefill, + int64_t max_seq_len = 128); + + virtual ~Prefiller() = default; + + virtual runtime::Result prefill( + std::vector& prompt_tokens, + int64_t& start_pos) override; + + virtual runtime::Result prefill_chunk( + std::vector& prompt_tokens, + int64_t& start_pos) override; + + private: + /** + * Note: TextPrefiller does not own the TextDecoderRunner instance. + * The responsibility of managing the lifecycle of TextDecoderRunner + * lies with the outer class or entity (likely Runner) that creates + * and passes the TextDecoderRunner instance to TextPrefiller. + */ + llm::TextDecoderRunner* text_decoder_runner_; + bool use_kv_cache_; + bool enable_parallel_prefill_; + int64_t max_seq_len_; +}; + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/rotary_embedding.cpp b/examples/samsung/executor_runner/llm/runner/rotary_embedding.cpp new file mode 100644 index 00000000000..ae8755586fb --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/rotary_embedding.cpp @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include + +#include + +namespace example { + +namespace runtime = ::executorch::runtime; + +RotaryEmbedding::RotaryEmbedding(const RunnerConfig& config) + : config_(config), + loaded_(false), + seq_len_(config.model_args->get_max_seq_len()), + base_freq_global_(1000000.0f), + base_freq_local_(1000000.0f), + rope_percentage_(1.0f) { + switch (config_.model_version) { + case DecoderModelVersion::kGemma4: { + auto* args = static_cast(config_.model_args.get()); + normalized_global_head_dim_ = args->text_config.global_head_dim; + normalized_local_head_dim_ = args->text_config.head_dim; + base_freq_global_ = args->text_config.base_freq_global; + base_freq_local_ = args->text_config.base_freq_local; + rope_percentage_ = args->text_config.rope_percentage; + break; + } + case DecoderModelVersion::kLlama3: { + auto* args = static_cast(config_.model_args.get()); + normalized_global_head_dim_ = args->get_head_dim(); + normalized_local_head_dim_ = args->get_head_dim(); + base_freq_global_ = 500000.0f; + rope_percentage_ = 1.0f; + break; + } + case DecoderModelVersion::kGemma3: { + auto* args = static_cast(config_.model_args.get()); + normalized_global_head_dim_ = args->get_head_dim(); + normalized_local_head_dim_ = args->get_head_dim(); + base_freq_global_ = 1000000.0f; + base_freq_local_ = 10000.0f; + rope_percentage_ = 1.0f; + break; + } + default: { + break; + } + } + + global_bytes_ = config_.model_args->get_max_seq_len() * + normalized_global_head_dim_ * sizeof(float); + local_bytes_ = config_.model_args->get_max_seq_len() * + normalized_local_head_dim_ * sizeof(float); +} + +runtime::Error RotaryEmbedding::load( + std::optional pos_embedding_path) { + allocate_buffers(); + + if (pos_embedding_path.has_value() && !pos_embedding_path->empty()) { + runtime::Error err = load_from_file(pos_embedding_path.value()); + if (err == runtime::Error::Ok) { + return runtime::Error::Ok; + } + ET_LOG( + Info, + "Failed to load embeddings from file, falling back to precompute."); + } else { + ET_LOG( + Info, + "No valid position embedding path provided, falling back to precompute."); + } + + return load_by_precompute_rope_tables(); +} + +runtime::Error RotaryEmbedding::load_from_file(const std::string& path) { + std::ifstream file(path, std::ios::binary); + ET_CHECK_OR_RETURN_ERROR( + file.is_open(), + InvalidArgument, + "File I/O error: Cannot open file at '%s'.", + path.c_str()); + + file.seekg(0, std::ios::end); + size_t file_size = file.tellg(); + file.seekg(0, std::ios::beg); + + size_t expected_size = global_bytes_ * 2; + if (config_.embedding_strategy == EmbeddingStrategy::kGlobalAndLocal) { + expected_size += local_bytes_ * 2; + } + + ET_CHECK_OR_RETURN_ERROR( + file_size == expected_size, + InvalidArgument, + "Embedding file size mismatch. Expected %zu, got %zu.", + expected_size, + file_size); + + // ========================================================================= + // BINARY ASSET LAYOUT ASSUMPTION WARNING: + // The raw .bin file contains NO metadata, headers, or structural markers. + // This code relies entirely on a strict, sequential file-pointer progression. + // Any discrepancy in the packing sequence or byte padding during model + // export will cause silent memory corruption or misaligned frequencies. + // + // Expected File Byte Ordering Layout: + // +-----------------------------+-----------------------------+ + // | global_cos (global_bytes_) | global_sin (global_bytes_) | + // +-----------------------------+-----------------------------+ + // | local_cos (local_bytes_)* | local_sin (local_bytes_)* | + // +-----------------------------+-----------------------------+ + // (*Optional: Only if embedding_strategy == kGlobalAndLocal) + // ========================================================================= + + // Stream out the mandatory global attention layer matrices + file.read(global_cos_.get(), global_bytes_); + file.read(global_sin_.get(), global_bytes_); + if (config_.embedding_strategy == EmbeddingStrategy::kGlobalAndLocal) { + file.read(local_cos_.get(), local_bytes_); + file.read(local_sin_.get(), local_bytes_); + } + ET_CHECK_OR_RETURN_ERROR( + file.good(), InvalidArgument, "Failed to read '%s'.", path.c_str()); + + loaded_ = true; + return runtime::Error::Ok; +} + +runtime::Error RotaryEmbedding::load_by_precompute_rope_tables() { + float* global_cos_ptr = reinterpret_cast(global_cos_.get()); + float* global_sin_ptr = reinterpret_cast(global_sin_.get()); + + assert( + reinterpret_cast(global_cos_ptr) % alignof(float) == 0 && + "global_cos_ is misaligned for float access!"); + assert( + reinterpret_cast(global_sin_ptr) % alignof(float) == 0 && + "global_sin_ is misaligned for float access!"); + + ET_CHECK_OK_OR_RETURN_ERROR(compute_rope_table( + global_cos_ptr, + global_sin_ptr, + seq_len_, + normalized_global_head_dim_, + base_freq_global_, + rope_percentage_)); + + if (config_.embedding_strategy == EmbeddingStrategy::kGlobalAndLocal) { + float* local_cos_ptr = reinterpret_cast(local_cos_.get()); + float* local_sin_ptr = reinterpret_cast(local_sin_.get()); + + assert( + reinterpret_cast(local_cos_ptr) % alignof(float) == 0 && + "local_cos_ is misaligned for float access!"); + assert( + reinterpret_cast(local_sin_ptr) % alignof(float) == 0 && + "local_sin_ is misaligned for float access!"); + + // Sliding/local layers use full RoPE; the partial rotary factor applies + // only to the global (full-attention) layers. + ET_CHECK_OK_OR_RETURN_ERROR(compute_rope_table( + local_cos_ptr, + local_sin_ptr, + seq_len_, + normalized_local_head_dim_, + base_freq_local_, + /*rope_pct=*/1.0f)); + } + loaded_ = true; + ET_LOG(Info, "Rotary Embedding tables precomputed successfully."); + return runtime::Error::Ok; +} + +void RotaryEmbedding::allocate_buffers() { + global_cos_ = std::make_unique(global_bytes_); + global_sin_ = std::make_unique(global_bytes_); + + if (config_.embedding_strategy == EmbeddingStrategy::kGlobalAndLocal) { + local_cos_ = std::make_unique(local_bytes_); + local_sin_ = std::make_unique(local_bytes_); + } +} + +runtime::Error RotaryEmbedding::compute_rope_table( + float* cos_ptr, + float* sin_ptr, + size_t seq_len, + size_t dim, + float base, + float rope_pct) { + ET_CHECK_OR_RETURN_ERROR( + cos_ptr != nullptr, InvalidArgument, "cos_ptr cannot be nullptr."); + ET_CHECK_OR_RETURN_ERROR( + sin_ptr != nullptr, InvalidArgument, "sin_ptr cannot be nullptr."); + ET_CHECK_OR_RETURN_ERROR( + dim % 2 == 0, + InvalidArgument, + "The head dimension (dim) must be an even number for rotary pairs."); + ET_CHECK_OR_RETURN_ERROR( + base > 0.0f, + InvalidArgument, + "The RoPE base frequency must be greater than zero."); + ET_CHECK_OR_RETURN_ERROR( + rope_pct >= 0.0f && rope_pct <= 1.0f, + InvalidArgument, + "The rope_pct parameter must be a percentage value between 0.0 and 1.0 inclusive."); + + const size_t half_dim = dim / 2; + size_t rotary_dim = static_cast(static_cast(dim) * rope_pct); + rotary_dim = std::min(rotary_dim, dim); + rotary_dim &= ~size_t{1}; + + ET_CHECK_OR_RETURN_ERROR( + rotary_dim > 0, + InvalidArgument, + "rope_pct results in zero rotary dimensions."); + + const size_t rotary_half = rotary_dim / 2; + + // Partial RoPE uses the full head_dim as the inv_freq denominator (Gemma + // convention), so for full RoPE this reduces to the standard formula. + const float freq_denom = static_cast(dim); + + for (size_t pos = 0; pos < seq_len; ++pos) { + float* cos_row = cos_ptr + pos * dim; + float* sin_row = sin_ptr + pos * dim; + + // Non-rotary dimensions (partial RoPE) pass through unchanged: cos=1, + // sin=0. + for (size_t i = 0; i < dim; ++i) { + cos_row[i] = 1.0f; + sin_row[i] = 0.0f; + } + + for (size_t i = 0; i < rotary_half; ++i) { + // inv_freq_i = base^(-2i / head_dim) + const float exponent = -2.0f * static_cast(i) / freq_denom; + const float inv_freq = std::pow(base, exponent); + const float angle = static_cast(pos) * inv_freq; + + const float c = std::cos(angle); + const float s = std::sin(angle); + + // Split-half convention: + // + // first half: i + // second half: half_dim + i + // + // x = [x_0 ... x_{d/2-1} | x_{d/2} ... x_{d-1}] + // where each rotary pair is: (x_i, x_{i + d/2}) + // Both coordinates of the pair use the same cos/sin angle. + + cos_row[i] = c; + sin_row[i] = s; + + cos_row[half_dim + i] = c; + sin_row[half_dim + i] = s; + } + } + + return runtime::Error::Ok; +} + +runtime::Error RotaryEmbedding::update( + const std::vector& pos_ids, + float* global_cos, + float* global_sin, + float* local_cos, + float* local_sin) { + if (!loaded_) { + ET_LOG(Error, "RotaryEmbedding::update() called before load() succeeded."); + return runtime::Error::InvalidState; + } + if (pos_ids.empty()) { + return runtime::Error::InvalidArgument; + } + if (global_cos == nullptr || global_sin == nullptr) { + return runtime::Error::InvalidArgument; + } + if (config_.embedding_strategy == EmbeddingStrategy::kGlobalAndLocal && + (local_cos == nullptr || local_sin == nullptr)) { + return runtime::Error::InvalidArgument; + } + + ET_CHECK_OR_RETURN_ERROR( + pos_ids[0] >= 0 && + static_cast(pos_ids[0]) + pos_ids.size() <= seq_len_, + InvalidArgument, + "RoPE positions [%" PRId64 ", +%zu) exceed the %zu-entry table", + pos_ids[0], + pos_ids.size(), + seq_len_); + size_t pos = static_cast(pos_ids[0]); + size_t length = pos_ids.size(); + + // Global Copy + size_t g_head_dim = normalized_global_head_dim_; + size_t g_bytes_to_copy = length * g_head_dim * sizeof(float); + size_t g_element_offset = pos * g_head_dim * sizeof(float); + std::memcpy( + global_cos, global_cos_.get() + g_element_offset, g_bytes_to_copy); + std::memcpy( + global_sin, global_sin_.get() + g_element_offset, g_bytes_to_copy); + + // Local Copy + if (config_.embedding_strategy == EmbeddingStrategy::kGlobalAndLocal) { + size_t l_head_dim = normalized_local_head_dim_; + size_t l_bytes_to_copy = length * l_head_dim * sizeof(float); + size_t l_element_offset = pos * l_head_dim * sizeof(float); + std::memcpy( + local_cos, local_cos_.get() + l_element_offset, l_bytes_to_copy); + std::memcpy( + local_sin, local_sin_.get() + l_element_offset, l_bytes_to_copy); + } + + return runtime::Error::Ok; +} + +} // namespace example \ No newline at end of file diff --git a/examples/samsung/executor_runner/llm/runner/rotary_embedding.h b/examples/samsung/executor_runner/llm/runner/rotary_embedding.h new file mode 100644 index 00000000000..3f9b0882c49 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/rotary_embedding.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace example { + +namespace runtime = ::executorch::runtime; + +class RotaryEmbedding { + public: + RotaryEmbedding(const RunnerConfig& config); + ~RotaryEmbedding() = default; + + runtime::Error load(std::optional pos_embedding_path); + runtime::Error update( + const std::vector& pos_ids, + float* global_cos, + float* global_sin, + float* local_cos, + float* local_sin); + + private: + runtime::Error load_from_file(const std::string& path); + runtime::Error load_by_precompute_rope_tables(); + void allocate_buffers(); + runtime::Error compute_rope_table( + float* cos_ptr, + float* sin_ptr, + size_t seq_len, + size_t dim, + float base, + float rope_pct); + + const RunnerConfig& config_; + bool loaded_; + + size_t seq_len_; + size_t normalized_global_head_dim_; + size_t normalized_local_head_dim_; + size_t global_bytes_; + size_t local_bytes_; + float base_freq_global_; + float base_freq_local_; + float rope_percentage_; + + std::unique_ptr global_cos_; + std::unique_ptr global_sin_; + std::unique_ptr local_cos_; + std::unique_ptr local_sin_; +}; + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/runner.cpp b/examples/samsung/executor_runner/llm/runner/runner.cpp new file mode 100644 index 00000000000..8a0dccf9180 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/runner.cpp @@ -0,0 +1,803 @@ +/* + * Copyright (c) 2025 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// A simple LLM runner that includes preprocessing and post processing logic. +// The module takes in a string as input and emits a string as output. + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace example { + +namespace llm = ::executorch::extension::llm; +namespace runtime = ::executorch::runtime; +namespace aten = ::executorch::aten; +namespace extension = ::executorch::extension; + +using json = nlohmann::json; + +std::string LLMRunner::get_formatted_prompt( + const std::string& prompt, + const std::string& system_prompt, + const std::string& assistant_prefill_prompt, + DecoderModelVersion version) { + std::string formatted_prompt; + switch (version) { + case DecoderModelVersion::kLlama3: + if (!system_prompt.empty()) { + formatted_prompt.append( + "<|start_header_id|>system<|end_header_id|>\n\n"); + formatted_prompt.append(system_prompt); + formatted_prompt.append("<|eot_id|>"); + } + formatted_prompt.append("<|start_header_id|>user<|end_header_id|>\n\n"); + formatted_prompt.append(prompt); + formatted_prompt.append( + "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"); + // Pre-fill: text placed right after the assistant header as the start of + // the model's reply. + formatted_prompt.append(assistant_prefill_prompt); + break; + case DecoderModelVersion::kGemma3: + formatted_prompt.append("user\n"); + // Gemma 3 has no system role; the template prepends it to the user turn. + if (!system_prompt.empty()) { + formatted_prompt.append(system_prompt); + formatted_prompt.append("\n\n"); + } + formatted_prompt.append(prompt); + formatted_prompt.append("\n"); + formatted_prompt.append("model\n"); + // Pre-fill: text placed right after the model turn header. + formatted_prompt.append(assistant_prefill_prompt); + break; + case DecoderModelVersion::kGemma4: + if (!system_prompt.empty()) { + formatted_prompt.append("<|turn>system\n"); + formatted_prompt.append(system_prompt); + formatted_prompt.append("\n"); + } + formatted_prompt.append("<|turn>user\n"); + formatted_prompt.append(prompt); + formatted_prompt.append("\n"); + formatted_prompt.append("<|turn>model\n"); + // Pre-fill: text placed right after the model turn header. + formatted_prompt.append(assistant_prefill_prompt); + break; + case DecoderModelVersion::kGeneral: + formatted_prompt.append(prompt); + formatted_prompt.append(assistant_prefill_prompt); + break; + default: + formatted_prompt.append(prompt); + formatted_prompt.append(assistant_prefill_prompt); + break; + } + return formatted_prompt; +} + +/** + * This function creates and initializes a tokenizer from a file based on the + * specified type. Supported types are: "bpe", "tiktoken", "hf", and "sp". + */ +ET_EXPERIMENTAL std::unique_ptr load_tokenizer( + const std::string& type, + const std::string& tokenizer_path, + Version version); + +std::unique_ptr create_llm_runner( + const RunnerPaths& runner_paths, + RunnerConfig runner_config) { + ET_LOG( + Info, + "Creating LLM runner: model_path=%s, tokenizer_path=%s", + runner_paths.model_path.c_str(), + runner_paths.tokenizer_path.c_str()); + + if (runner_config.load_from_json() != runtime::Error::Ok) { + return nullptr; + } + + std::unique_ptr<::tokenizers::Tokenizer> tokenizer = load_tokenizer( + runner_config.tokenizer_type, + runner_paths.tokenizer_path, + Version::Default); + if (tokenizer == nullptr) { + ET_LOG( + Info, + "Failed to load tokenizer. Parsed tokenizer type is %s, located at %s.", + runner_config.tokenizer_type.c_str(), + runner_paths.tokenizer_path.c_str()); + return nullptr; + } + ET_LOG( + Info, + "[create_llm_runner] %s %s", + runner_paths.model_path.c_str(), + runner_paths.data_path.has_value() + ? runner_paths.data_path.value().c_str() + : "null"); + return create_llm_runner( + runner_paths, std::move(runner_config), std::move(tokenizer)); +} + +static const std::unordered_map kMetadataMap{ + {llm::kEnableDynamicShape, "enable_dynamic_shape"}, + {llm::kBosId, "get_bos_id"}, + {llm::kEosIds, "get_eos_ids"}, + {llm::kMaxSeqLen, "get_max_seq_len"}, + {llm::kMaxContextLen, "get_max_context_len"}, + {llm::kVocabSize, "get_vocab_size"}, + {llm::kUseKVCache, "use_kv_cache"}, + {llm::kUseSDPAWithKVCache, "use_sdpa_with_kv_cache"}, + {example::kHeadDim, "get_head_dim"}, + {example::kPrefillArLen, "get_prefill_ar_len"}, + {example::kNumOfHiddenDim, "get_hidden_dim"}, + {example::kNumOfHeads, "get_n_heads"}, + {example::kNumOfKVHeads, "get_n_kv_heads"}, + {example::kNumOfLayers, "get_n_layers"}, + {"get_sliding_window", "get_sliding_window"}}; + +LLMRunner::LLMRunner( + std::unordered_map metadata, + std::unique_ptr<::tokenizers::Tokenizer> tokenizer, + std::unique_ptr module, + std::unique_ptr text_decoder_runner, + std::unique_ptr text_prefiller, + std::unique_ptr text_token_generator, + std::unique_ptr stats, + std::unique_ptr io_manager, + RunnerConfig config, + DecoderModelVersion model_version, + float temperature) + : config_(std::move(config)), + tokenizer_(std::move(tokenizer)), + metadata_(std::move(metadata)), + module_(std::move(module)), + text_decoder_runner_(std::move(text_decoder_runner)), + text_prefiller_(std::move(text_prefiller)), + text_token_generator_(std::move(text_token_generator)), + io_manager_(std::move(io_manager)), + stats_(std::move(stats)), + temperature_(temperature), + model_version_(model_version) { + // Note: This constructor assumes that text_prefiller and text_token_generator + // already have references to the Module and TextDecoderRunner they need +} + +bool LLMRunner::is_loaded() const { + return text_prefiller_->is_loaded() && text_token_generator_->is_loaded(); +} + +runtime::Error LLMRunner::load() { + ET_CHECK_OK_OR_RETURN_ERROR(text_decoder_runner_->load()); + ET_CHECK_OK_OR_RETURN_ERROR(text_prefiller_->load()); + ET_CHECK_OK_OR_RETURN_ERROR(text_token_generator_->load()); + return runtime::Error::Ok; +} + +// Don't print with the same priority during warmup +#define RUNNER_ET_LOG(is_warmup, format, ...) \ + if (is_warmup) { \ + ET_LOG(Debug, format, __VA_ARGS__); \ + } else { \ + ET_LOG(Info, format, __VA_ARGS__); \ + } + +void replace_all_newlines(std::string& str) { + std::string from = "\\n"; + std::string to = "\n"; + + size_t start_pos = 0; + while ((start_pos = str.find(from, start_pos)) != std::string::npos) { + str.replace(start_pos, from.length(), to); + start_pos += to.length(); + } +} + +runtime::Error LLMRunner::generate_from_pos( + const std::string& prompt, + int64_t start_pos, + const llm::GenerationConfig& config, + std::function token_callback, + std::function stats_callback) { + // Prepare the inputs. + // Use ones-initialized inputs. + ET_CHECK_OR_RETURN_ERROR( + !prompt.empty(), InvalidArgument, "Prompt cannot be empty"); + if (!is_loaded()) { + stats_->model_load_start_ms = llm::time_in_ms(); + ET_CHECK_OK_OR_RETURN_ERROR(load()); + stats_->model_load_end_ms = llm::time_in_ms(); + } + + if (config.warming) { + ET_LOG(Info, "Doing a warmup run..."); + } + + RUNNER_ET_LOG( + config.warming, + "RSS after loading model: %f MiB (0 if unsupported)", + llm::get_rss_bytes() / 1024.0 / 1024.0); + + // Wrap the token_callback with print function + std::function wrapped_callback = + [this, token_callback, config](const std::string& piece) { + if (!config.warming && config_.verbose) { + llm::safe_printf(piece.c_str()); + fflush(stdout); + } + if (token_callback) { + token_callback(piece); + } + }; + + // First token time only measures the time it takes to encode the prompt and + // return a response token. + stats_->inference_start_ms = llm::time_in_ms(); + text_token_generator_->clear_stop(); + + // Raw prompts are already fully formatted (special tokens, system turn, tool + // declaration); feed them verbatim and skip every template transformation. + std::string input_prompt = config_.raw_prompt + ? prompt + : get_formatted_prompt( + prompt, + config_.system_prompt, + config_.assistant_prefill_prompt, + model_version_); + if (!config_.raw_prompt && model_version_ == DecoderModelVersion::kGemma3) { + replace_all_newlines(input_prompt); + } + + ::tokenizers::Result> encode_res = tokenizer_->encode( + input_prompt, + /*bos=*/config.num_bos, + /*eos=*/config.num_eos); + + ET_CHECK_TK_OK_OR_RETURN_ERROR( + encode_res.error(), "Failed to encode prompt %s", input_prompt.c_str()); + + // encode the (string) prompt into tokens sequence + std::vector prompt_tokens = encode_res.get(); + + // The Gemma 4 tokenizer's post-processor does not add BOS, so num_bos alone + // leaves the prompt without one. + if (model_version_ == DecoderModelVersion::kGemma4 && config.num_bos > 0 && + (prompt_tokens.empty() || + prompt_tokens.front() != tokenizer_->bos_tok())) { + prompt_tokens.insert(prompt_tokens.begin(), tokenizer_->bos_tok()); + } + + // Gemma 4 multimodal: run the vision encoder and inject the image token + // block. The user turn becomes: + // <|turn>user\n xN + // \nmodel\n + // holds the text AFTER the image; + // (config_.prompt_before_image) is the text placed BEFORE the image. + if (model_version_ == DecoderModelVersion::kGemma4 && + vision_encoder_ != nullptr && !config_.image_path.empty()) { + auto* llm_io = static_cast(io_manager_.get()); + + // Vision embeddings are injected only by the multi-token prefill method; + // single-token steps go through decode and would silently drop them. + ET_CHECK_OR_RETURN_ERROR( + !config_.is_decode_only && + config_.model_args->get_prefill_seq_len() > 1, + InvalidArgument, + "Image input requires a prefill method with seq_len > 1"); + + auto encode_err = vision_encoder_->encode(config_.image_path); + ET_CHECK_OK_OR_RETURN_ERROR(encode_err); + int64_t num_vision_tokens = vision_encoder_->num_valid_tokens(); + ET_CHECK_OR_RETURN_ERROR( + vision_encoder_->hidden_size() == + static_cast(config_.model_args->get_hidden_dim()), + InvalidProgram, + "Vision hidden size %lld != decoder hidden dim %lld", + static_cast(vision_encoder_->hidden_size()), + static_cast(config_.model_args->get_hidden_dim())); + + auto enc = [this]( + const std::string& s, std::vector& out) -> bool { + auto r = tokenizer_->encode(s, /*bos=*/0, /*eos=*/0); + if (r.error() != ::tokenizers::Error::Ok) { + ET_LOG(Error, "Failed to encode text segment"); + return false; + } + auto toks = std::move(r.get()); + out.insert(out.end(), toks.begin(), toks.end()); + return true; + }; + + std::vector vision_tokens; + vision_tokens.push_back(kGemma4BosId); + if (!enc("<|turn>user\n", vision_tokens) || + (!config_.prompt_before_image.empty() && + !enc(config_.prompt_before_image, vision_tokens))) { + return runtime::Error::InvalidArgument; + } + vision_tokens.push_back(kGemma4BoiTokenId); + for (int64_t i = 0; i < num_vision_tokens; ++i) { + vision_tokens.push_back(kGemma4ImageTokenId); + } + vision_tokens.push_back(kGemma4EoiTokenId); + if (!enc(prompt, vision_tokens) || + !enc("\n<|turn>model\n", vision_tokens)) { + return runtime::Error::InvalidArgument; + } + prompt_tokens = std::move(vision_tokens); + + llm_io->set_vision_embeddings( + vision_encoder_->vision_embeds(), num_vision_tokens); + } + + int num_prompt_tokens = prompt_tokens.size(); + + // The KV cache and RoPE tables only hold max_seq_len positions, whatever + // context length the model was trained with. + int64_t max_context_len = + std::min( + metadata_.at(llm::kMaxContextLen), metadata_.at(llm::kMaxSeqLen)) - + start_pos; + ET_CHECK_OR_RETURN_ERROR( + num_prompt_tokens >= 1, + InvalidArgument, + "Expected at least 1 prompt token"); + ET_CHECK_OR_RETURN_ERROR( + num_prompt_tokens < max_context_len, + InvalidArgument, + "num_prompt_tokens %d >= max_context_len %" PRId64 + ", Max seq length exceeded - please increase max seq len value in your export script", + num_prompt_tokens, + max_context_len); + + // Determine max_new_tokens using the GenerationConfig's resolve method, + // then subtract start_pos for max_new_tokens. + int max_new_tokens = + config.resolve_max_new_tokens(max_context_len, num_prompt_tokens); + + ET_LOG( + Info, + "Max new tokens resolved: %d, given start_pos %" PRId64 + ", num_prompt_tokens %zu, max_context_len %" PRId64, + max_new_tokens, + start_pos, + prompt_tokens.size(), + max_context_len); + ET_CHECK_OR_RETURN_ERROR( + max_new_tokens > 0, + InvalidArgument, + "Max new tokens %d is less than or equal to 0", + max_new_tokens); + // Prefill first + // Here feed all tokens to the model and get the next predicted token + // after the prompt. After that we will enter generate loop. + + // print prompts + if (config.echo) { + wrapped_callback(input_prompt); + } + int64_t pos = start_pos; + auto prefill_res = text_prefiller_->prefill(prompt_tokens, pos); + ET_CHECK_OK_OR_RETURN_ERROR(prefill_res.error()); + uint64_t cur_token = prefill_res.get(); + if (model_version_ == DecoderModelVersion::kGemma4 && + vision_encoder_ != nullptr && !config_.image_path.empty()) { + const auto consumed = static_cast(io_manager_.get()) + ->num_vision_embeds_consumed(); + ET_CHECK_OR_RETURN_ERROR( + consumed == static_cast(vision_encoder_->num_valid_tokens()), + InvalidState, + "Only %zu of %lld vision embeddings were injected during prefill", + consumed, + static_cast(vision_encoder_->num_valid_tokens())); + } + stats_->first_token_ms = llm::time_in_ms(); + stats_->prompt_eval_end_ms = llm::time_in_ms(); + + RUNNER_ET_LOG( + config.warming, + "\nRSS after prompt prefill: %f MiB (0 if unsupported)", + llm::get_rss_bytes() / 1024.0 / 1024.0); + + ::tokenizers::Result> newline_token = + tokenizer_->encode("\n"); + + ET_LOG( + Info, + "################ The start of token generation ################\n"); + + // Accumulate the full model answer (first token from prefill plus all + // decoded tokens) so it can be reprinted cleanly at the very end, free of + // interleaved logs. Warmed-up runs are skipped (they only measure speed). + std::string full_answer; + if (!config.warming) { + full_answer += + ET_UNWRAP_TOKENIZER(tokenizer_->decode(cur_token, cur_token)); + } + + // print the first token from prefill. No prev_token so use cur_token for it. + wrapped_callback( + ET_UNWRAP_TOKENIZER(tokenizer_->decode(cur_token, cur_token))); + + // start the main loop + prompt_tokens.push_back(cur_token); + + // Generate max_new_tokens - 1 because prefill already generated 1 token. + // Forward every decoded piece both to the stream and to the accumulator so + // the complete answer can be reprinted verbatim at the end. + std::function collect_callback = + [&full_answer, &wrapped_callback](const std::string& piece) { + full_answer += piece; + wrapped_callback(piece); + }; + int64_t num_generated_tokens = ET_UNWRAP(text_token_generator_->generate( + prompt_tokens, + pos, + max_new_tokens - 1, + temperature_ == -1.0f ? config.temperature : temperature_, + collect_callback)); + + // A temporary new line to separate stdout mixed with results. + wrapped_callback(ET_UNWRAP_TOKENIZER( + tokenizer_->decode(newline_token.get()[0], newline_token.get()[0]))); + + ET_LOG( + Info, "################ The end of token generation ################\n"); + + stats_->inference_end_ms = llm::time_in_ms(); + if (!config.warming) { + printf("\n"); + } + RUNNER_ET_LOG( + config.warming, + "RSS after finishing text generation: %f MiB (0 if unsupported)", + llm::get_rss_bytes() / 1024.0 / 1024.0); + + if (num_generated_tokens == max_new_tokens) { + RUNNER_ET_LOG(config.warming, "Max new tokens %i reached!", max_new_tokens); + } + + stats_->num_prompt_tokens = num_prompt_tokens; + stats_->num_generated_tokens = num_generated_tokens; + + if (config.warming) { + ET_LOG(Info, "Warmup run finished!"); + } else { + // Do not print report during warmup + print_report(*stats_); + } + if (stats_callback) { + stats_callback(*stats_); + } + + if (!config.warming && config_.verbose) { + // Reprint the complete answer on its own lines, after all logs and stats, + // so it is easy to identify and never overlaps other output. + printf("======== FULL ANSWER ========\n"); + printf("%s\n", full_answer.c_str()); + printf("=============================\n"); + fflush(stdout); + } + + return runtime::Error::Ok; +} + +runtime::Error LLMRunner::generate( + const std::string& prompt, + const llm::GenerationConfig& config, + std::function token_callback, + std::function stats_callback) { + return generate_from_pos(prompt, 0, config, token_callback, stats_callback); +} + +runtime::Error LLMRunner::warmup( + const std::string& prompt, + int32_t max_new_tokens) { + // Create a GenerationConfig for warmup + llm::GenerationConfig config{ + .echo = false, .max_new_tokens = max_new_tokens, .warming = true}; + + // Call generate with the warmup config + runtime::Error err = generate(prompt, config); + + // Reset stats after warmup, not resetting the std::unique_ptr! + stats_->reset(); + return err; +} + +void LLMRunner::stop() { + // Not gated on is_loaded(): it checks method "forward", absent here. + if (text_token_generator_ != nullptr) { + text_token_generator_->stop(); + } +} + +void LLMRunner::reset() { + auto* llm_io = static_cast(io_manager_.get()); + if (llm_io == nullptr) { + return; + } + if (llm_io->reset( + example::default_method_name_prefill, + example::default_method_name_decode) != runtime::Error::Ok) { + ET_LOG(Error, "Failed to reset the KV cache"); + } +} + +std::unique_ptr load_tokenizer( + const std::string& type, + const std::string& tokenizer_path, + Version version) { + ::executorch::runtime::runtime_init(); + + std::unique_ptr<::tokenizers::Tokenizer> tokenizer; + if (type == "bpe") { + tokenizer = std::make_unique<::tokenizers::Llama2cTokenizer>(); + } else if (type == "tiktoken") { + tokenizer = example::get_tiktoken_for_llama(version); + } else if (type == "hf") { + tokenizer = std::make_unique<::tokenizers::HFTokenizer>(); + } else if (type == "sp") { + tokenizer = std::make_unique<::tokenizers::SPTokenizer>(); + } else { + return nullptr; + } + + if (tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { + ET_LOG(Info, "Loaded %s tokenizer", type.c_str()); + return tokenizer; + } + + return nullptr; +} + +std::unordered_map get_llm_metadata( + tokenizers::Tokenizer* tokenizer, + extension::Module* module, + RunnerConfig& config) { + // Initialize metadata from JSON config values + std::unordered_map metadata({ + {kMetadataMap.at(llm::kEnableDynamicShape), false}, + {kMetadataMap.at(llm::kMaxSeqLen), + config.model_args ? config.model_args->get_max_seq_len() : 128}, + {kMetadataMap.at(llm::kMaxContextLen), + config.model_args ? config.model_args->get_max_context_len() : 128}, + {kMetadataMap.at(llm::kUseKVCache), true}, + {kMetadataMap.at(llm::kUseSDPAWithKVCache), false}, + {kMetadataMap.at(example::kPrefillArLen), 0}, + {kMetadataMap.at(example::kNumOfHiddenDim), + config.model_args ? config.model_args->get_hidden_dim() : 0}, + {kMetadataMap.at(example::kNumOfHeads), + config.model_args ? config.model_args->get_n_heads() : 0}, + {kMetadataMap.at(example::kNumOfKVHeads), + config.model_args ? config.model_args->get_n_kv_heads() : 0}, + {kMetadataMap.at(example::kNumOfLayers), + config.model_args ? config.model_args->get_n_layers() : 0}, + {kMetadataMap.at(example::kHeadDim), + config.model_args ? config.model_args->get_head_dim() : 0}, + }); + + auto method_names_result = module->method_names(); + if (method_names_result.error() != runtime::Error::Ok) { + ET_LOG(Error, "Failed reading method names"); + return metadata; + } + const auto method_names = method_names_result.get(); + + for (auto& pair : metadata) { + const auto& method_name = pair.first; + auto& value = pair.second; + + if (method_names.count(method_name)) { + auto get_result = module->get(method_name); + if (!get_result.ok()) { + ET_LOG(Error, "Failed to read metadata %s", method_name.c_str()); + continue; + } + value = get_result.get().toScalar().to(); + } else { + ET_LOG( + Debug, + "Method %s not found, using the default value %" PRId64, + method_name.c_str(), + value); + } + ET_LOG(Info, "Metadata: %s = %" PRId64, method_name.c_str(), value); + } + // Set tokenizer-related metadata + metadata[llm::kBosId] = tokenizer->bos_tok(); + metadata[llm::kVocabSize] = tokenizer->vocab_size(); + return metadata; +} + +std::unordered_set get_eos_ids( + tokenizers::Tokenizer* tokenizer, + extension::Module* module) { + std::unordered_set eos_ids = {tokenizer->eos_tok()}; + // Get EOS IDs if available + auto method_names_result = module->method_names(); + if (method_names_result.error() != runtime::Error::Ok) { + ET_LOG(Error, "Failed reading method names"); + return eos_ids; + } + const auto method_names = method_names_result.get(); + + if (method_names.count(llm::kEosIds)) { + eos_ids.clear(); + auto execute_result = module->execute(llm::kEosIds); + if (execute_result.error() != runtime::Error::Ok) { + ET_LOG(Error, "Failed to execute %s", llm::kEosIds); + return eos_ids; + } + for (const auto& eos_id : execute_result.get()) { + auto value = eos_id.toScalar().to(); + eos_ids.emplace(value); + ET_LOG(Info, "eos_id = %" PRId64, value); + } + } + return eos_ids; +} + +std::unique_ptr create_llm_runner( + const RunnerPaths& runner_paths, + RunnerConfig runner_config, + std::unique_ptr<::tokenizers::Tokenizer> tokenizer) { + if (!tokenizer || !tokenizer->is_loaded()) { + ET_LOG(Error, "Tokenizer is null or not loaded"); + return nullptr; + } + + DecoderModelVersion model_version = runner_config.model_version; + float temperature = runner_config.temperature; + bool is_decode_only = runner_config.is_decode_only; + + std::unique_ptr module; + if (runner_paths.data_path.has_value()) { + module = std::make_unique( + runner_paths.model_path, + runner_paths.data_path.value(), + extension::Module::LoadMode::File); + } else { + module = std::make_unique( + runner_paths.model_path, extension::Module::LoadMode::File); + } + + executorch::runtime::Result> methods_res = + module->method_names(); + if (methods_res.error() != executorch::runtime::Error::Ok) { + ET_LOG( + Error, + "Failed to get method names from %s", + runner_paths.model_path.c_str()); + return nullptr; + } + + auto methods = methods_res.get(); + std::vector register_methods; + if (is_decode_only) { + register_methods.emplace_back(default_method_name_decode); + } else { + for (auto method : methods) { + if (kMetadataMap.find(method) != kMetadataMap.end()) { + continue; + } + ET_LOG(Info, "Register method: \"%s\"", method.c_str()); + register_methods.emplace_back(method); + } + } + + ET_LOG(Info, "Reading metadata from model"); + auto metadata = + get_llm_metadata(tokenizer.get(), module.get(), runner_config); + + auto stats = std::make_unique(); + llm::Stats* stats_ptr = stats.get(); + + ET_LOG(Info, "Instantiate LLMRunner with null placeholders"); + std::unique_ptr runner = std::make_unique( + std::move(metadata), + std::move(tokenizer), + std::move(module), + nullptr, // text_decoder_runner placeholder + nullptr, // text_prefiller placeholder + nullptr, // text_token_generator placeholder + std::move(stats), + nullptr, // io_manager placeholder + std::move(runner_config), + model_version, + temperature); + + const auto& config = runner->get_config(); + + ET_LOG(Info, "Instantiate IO Manager"); + auto io_manager = + std::make_unique(*runner->get_module(), config); + + ET_LOG(Info, "Start loading Rotary Embedding"); + if (io_manager->load_rotary_embedding(runner_paths.pos_embedding_path) != + executorch::runtime::Error::Ok) { + ET_LOG(Error, "Failed to load or precompute pos_embedding."); + return nullptr; + } + + auto eos_ids = std::make_unique>( + get_eos_ids(runner->get_tokenizer(), runner->get_module())); + + auto text_decoder_runner = std::make_unique( + runner->get_module(), io_manager.get()); + + for (auto method : register_methods) { + text_decoder_runner->set_method(method); + } + + bool separated_prefill_decode = + !is_decode_only && config.model_args->get_prefill_seq_len() > 1; + ET_LOG( + Info, + "separated_prefill_decode(%s)", + (separated_prefill_decode) ? "True" : "False"); + + auto text_prefiller = std::make_unique( + text_decoder_runner.get(), + runner->get_metadata().at(llm::kUseKVCache), + runner->get_metadata().at(llm::kEnableDynamicShape) | + (separated_prefill_decode), + config.model_args->get_prefill_seq_len()); + + auto text_token_generator = std::make_unique( + runner->get_tokenizer(), + text_decoder_runner.get(), + runner->get_metadata().at(llm::kUseKVCache), + std::move(eos_ids), + stats_ptr); + + runner->set_io_manager(std::move(io_manager)); + runner->set_text_decoder_runner(std::move(text_decoder_runner)); + runner->set_text_prefiller(std::move(text_prefiller)); + runner->set_text_token_generator(std::move(text_token_generator)); + + // Optional Gemma 4 vision encoder (separate PTE). If no vision model path is + // configured, the runner stays text-only. + if (runner_paths.vision_model_path.has_value() && + !runner_paths.vision_model_path.value().empty()) { + auto vision_encoder = std::make_unique( + runner_paths.vision_model_path.value()); + auto verr = vision_encoder->load(); + if (verr != executorch::runtime::Error::Ok) { + ET_LOG(Error, "Failed to load vision encoder."); + return nullptr; + } + runner->set_vision_encoder(std::move(vision_encoder)); + } + + if (runner->load() != executorch::runtime::Error::Ok) { + ET_LOG(Error, "Failed to load the LLM runner."); + return nullptr; + } + + return runner; +} +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/runner.h b/examples/samsung/executor_runner/llm/runner/runner.h new file mode 100644 index 00000000000..3e006af868d --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/runner.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace example { + +namespace llm = ::executorch::extension::llm; +namespace runtime = ::executorch::runtime; +namespace extension = ::executorch::extension; + +class LLMRunner : public llm::IRunner { + public: + explicit LLMRunner( + std::unordered_map metadata, + std::unique_ptr<::tokenizers::Tokenizer> tokenizer, + std::unique_ptr module, + std::unique_ptr text_decoder_runner, + std::unique_ptr text_prefiller, + std::unique_ptr text_token_generator, + std::unique_ptr stats, + std::unique_ptr io_manager, + RunnerConfig config, + DecoderModelVersion model_version, + float temperature = -1.0f); + + bool is_loaded() const override; + + runtime::Error load() override; + + runtime::Error generate( + const std::string& prompt, + const llm::GenerationConfig& config, + std::function token_callback = {}, + std::function stats_callback = {}) override; + + runtime::Error generate_from_pos( + const std::string& prompt, + int64_t start_pos, + const llm::GenerationConfig& config, + std::function token_callback = {}, + std::function stats_callback = {}); + + runtime::Error warmup(const std::string& prompt, int32_t max_new_tokens); + + void stop() override; + + const RunnerConfig& get_config() const { + return config_; + } + RunnerConfig& get_config() { + return config_; + } + extension::Module* get_module() const { + return module_.get(); + } + ::tokenizers::Tokenizer* get_tokenizer() const { + return tokenizer_.get(); + } + const std::unordered_map& get_metadata() const { + return metadata_; + } + + void set_io_manager(std::unique_ptr io_manager) { + io_manager_ = std::move(io_manager); + } + void set_text_decoder_runner(std::unique_ptr runner) { + text_decoder_runner_ = std::move(runner); + } + void set_text_prefiller(std::unique_ptr prefiller) { + text_prefiller_ = std::move(prefiller); + } + void set_text_token_generator( + std::unique_ptr generator) { + text_token_generator_ = std::move(generator); + } + void set_vision_encoder(std::unique_ptr encoder) { + vision_encoder_ = std::move(encoder); + } + VisionEncoder* get_vision_encoder() const { + return vision_encoder_.get(); + } + + static std::string get_formatted_prompt( + const std::string& prompt, + const std::string& system_prompt, + const std::string& assistant_prefill_prompt, + DecoderModelVersion version); + + void reset() override; + + private: + RunnerConfig config_; + std::unique_ptr<::tokenizers::Tokenizer> tokenizer_; + std::unordered_map metadata_; + std::unique_ptr module_; + std::unique_ptr text_decoder_runner_; + std::unique_ptr text_prefiller_; + std::unique_ptr text_token_generator_; + std::unique_ptr io_manager_; + + std::unique_ptr vision_encoder_; + + std::unique_ptr stats_; + + float temperature_ = -1.0f; + DecoderModelVersion model_version_; +}; + +std::unique_ptr create_llm_runner( + const RunnerPaths& runner_paths, + RunnerConfig runner_config); + +std::unique_ptr create_llm_runner( + const RunnerPaths& runner_paths, + RunnerConfig runner_config, + std::unique_ptr<::tokenizers::Tokenizer> tokenizer); + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/stb_image_impl.cpp b/examples/samsung/executor_runner/llm/runner/stb_image_impl.cpp new file mode 100644 index 00000000000..02b01d4f407 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/stb_image_impl.cpp @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Single translation unit that instantiates the stb_image implementation. +// This must be included from exactly one .cpp in the runner. +#define STB_IMAGE_IMPLEMENTATION +#include + +#define STB_IMAGE_RESIZE_IMPLEMENTATION +#include diff --git a/examples/samsung/executor_runner/llm/runner/token_generator.h b/examples/samsung/executor_runner/llm/runner/token_generator.h new file mode 100644 index 00000000000..04af9e7f457 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/token_generator.h @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2025 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Generate tokens in a loop. +#pragma once + +#include +#include +#include +#include + +#include + +namespace example { + +namespace llm = ::executorch::extension::llm; +namespace runtime = ::executorch::runtime; +namespace aten = ::executorch::aten; +namespace extension = ::executorch::extension; + +class ET_EXPERIMENTAL TokenGenerator { + public: + TokenGenerator( + ::tokenizers::Tokenizer* tokenizer, + llm::TextDecoderRunner* text_decoder_runner, + bool use_kv_cache, + std::unique_ptr>&& eos_ids, + llm::Stats* stats) + : tokenizer_(tokenizer), + text_decoder_runner_(text_decoder_runner), + eos_ids_(std::move(eos_ids)), + use_kv_cache_(use_kv_cache), + stats_(stats) {} + + virtual ~TokenGenerator() = default; + + /** + * Token generation loop. + * @param tokens prompt tokens as well as the first token generated by + * prefill. + * @param start_pos the start position of the new tokens, based on how many + * prompt tokens is prefilled. + * @param max_new_tokens Maximum number of new tokens to generate. + * @param temperature controls the randomness of predictions by scaling the + * logits before applying softmax. A higher temperature results in more + * random predictions, while a lower temperature results in more deterministic + * predictions. + * @param token_callback what to do after a token is generated. + * @return how many tokens are generated. + */ + inline runtime::Result generate( + std::vector tokens, + int64_t start_pos, + int32_t max_new_tokens, + float temperature = 0.0f, + const std::function& token_callback = {}) { + ET_CHECK_OR_RETURN_ERROR( + !tokens.empty(), + InvalidArgument, + "Token generation loop shouldn't take empty tokens"); + int64_t pos = start_pos; // position in the sequence + + std::vector token_data; // allocate space for the tokens + std::vector token_shape; + + // Token after prefill + uint64_t cur_token = tokens.back(); + uint64_t prev_token; + + if (use_kv_cache_) { + // hard code these to size 1 as kv cache is locked to static size right + // now. + token_data = {cur_token}; + token_shape = {1, 1}; + } else { + token_data = tokens; + token_shape = {1, static_cast(tokens.size())}; + } + + // initialize tensor wrappers + auto tokens_managed = extension::from_blob( + token_data.data(), token_shape, aten::ScalarType::Long); + + // Generate our tokens + while (pos < start_pos + max_new_tokens) { + if (should_stop_.load()) { + break; + } + + // Run the model + auto logits_res = text_decoder_runner_->step(tokens_managed, pos); + + ET_CHECK_OK_OR_RETURN_ERROR(logits_res.error()); + executorch::aten::Tensor& logits_tensor = logits_res.get(); + + prev_token = cur_token; + + stats_->on_sampling_begin(); + cur_token = + text_decoder_runner_->logits_to_token(logits_tensor, temperature); + stats_->on_sampling_end(); + + pos++; + + if (use_kv_cache_) { + // update the token tensor. token_data will not be empty. + // NOLINTNEXTLINE(facebook-hte-LocalUncheckedArrayBounds) + token_data[0] = cur_token; + } else { + // push it to the back + token_data.push_back(cur_token); + ET_CHECK_OK_OR_RETURN_ERROR(extension::resize_tensor_ptr( + tokens_managed, {1, static_cast(token_data.size())})); + } + + // print the token as string, decode it with the Tokenizer object + token_callback( + ET_UNWRAP_TOKENIZER(tokenizer_->decode(prev_token, cur_token))); + + // data-dependent terminating condition: we have n_eos_ number of EOS + if (eos_ids_->find(cur_token) != eos_ids_->end()) { + // ET_LOG(Info, "Reached to the end of generation"); + break; + } + } + return pos - start_pos; + } + + /** + * Stop the generation loop. + */ + inline void stop() { + should_stop_.store(true); + } + + inline void clear_stop() { + should_stop_.store(false); + } + + /** + * Load the necessary resources for TokenGenerator. + * This method should be called before using the generate() method. + */ + runtime::Error load() { + // return text_decoder_runner_->load(); + return runtime::Error::Ok; + } + + /** + * Check if the TokenGenerator has been successfully loaded. + * @return True if the resources are loaded, false otherwise. + */ + bool inline is_loaded() const { + // Implementation to check if resources are loaded + return tokenizer_->is_loaded() && text_decoder_runner_->is_method_loaded(); + } + + private: + /** + * Note: TokenGenerator does not own the tokenizer_ and + * text_decoder_runner_. The lifecycle of these objects should be managed + * externally, likely in the Runner. This class assumes that the provided + * pointers remain valid for the duration of its use. + */ + ::tokenizers::Tokenizer* tokenizer_; + llm::TextDecoderRunner* text_decoder_runner_; + std::unique_ptr> eos_ids_; + bool use_kv_cache_; + + // state machine + std::atomic should_stop_{false}; + + // stats + llm::Stats* stats_; +}; + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/vision_encoder.cpp b/examples/samsung/executor_runner/llm/runner/vision_encoder.cpp new file mode 100644 index 00000000000..7cb0af6de2a --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/vision_encoder.cpp @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include +#include +#include +#include + +namespace example { + +namespace runtime = ::executorch::runtime; +namespace extension = ::executorch::extension; + +VisionEncoder::VisionEncoder(const std::string& model_path) + : model_path_(model_path) {} + +runtime::Error VisionEncoder::load() { + if (loaded_) { + return runtime::Error::Ok; + } + if (model_path_.empty()) { + ET_LOG(Error, "Vision model path is empty"); + return runtime::Error::InvalidArgument; + } + + module_ = std::make_unique( + model_path_, extension::Module::LoadMode::File); + + auto err = module_->load_method("vision_encoder"); + if (err != runtime::Error::Ok) { + ET_LOG( + Error, + "Failed to load vision_encoder method from %s", + model_path_.c_str()); + return err; + } + + auto method_meta = module_->method_meta("vision_encoder"); + ET_CHECK_OK_OR_RETURN_ERROR(method_meta.error()); + ET_CHECK_OR_RETURN_ERROR( + method_meta->num_inputs() >= 2 && method_meta->num_outputs() >= 2, + InvalidProgram, + "vision_encoder expects 2 inputs / 2 outputs, got %zu / %zu", + method_meta->num_inputs(), + method_meta->num_outputs()); + + auto pixel_meta = method_meta->input_tensor_meta(0); + ET_CHECK_OK_OR_RETURN_ERROR(pixel_meta.error()); + ET_CHECK_OR_RETURN_ERROR( + pixel_meta->scalar_type() == executorch::aten::ScalarType::Float, + InvalidProgram, + "vision_encoder pixel_values must be float32, got dtype %d", + static_cast(pixel_meta->scalar_type())); + const auto pixel_sizes = pixel_meta->sizes(); + ET_CHECK_OR_RETURN_ERROR( + pixel_sizes.size() == 3 && pixel_sizes[2] == kVisionPatchDim && + pixel_sizes[1] % (kVisionPoolingKernel * kVisionPoolingKernel) == 0, + InvalidProgram, + "Unexpected vision_encoder pixel_values shape"); + max_soft_tokens_ = + pixel_sizes[1] / (kVisionPoolingKernel * kVisionPoolingKernel); + + auto pos_meta = method_meta->input_tensor_meta(1); + ET_CHECK_OK_OR_RETURN_ERROR(pos_meta.error()); + position_ids_dtype_ = pos_meta->scalar_type(); + ET_CHECK_OR_RETURN_ERROR( + position_ids_dtype_ == executorch::aten::ScalarType::Long || + position_ids_dtype_ == executorch::aten::ScalarType::Int, + InvalidProgram, + "vision_encoder pixel_position_ids must be int64/int32, got dtype %d", + static_cast(position_ids_dtype_)); + + auto out_meta = method_meta->output_tensor_meta(0); + if (out_meta.ok() && out_meta->sizes().size() >= 3) { + hidden_size_ = out_meta->sizes()[2]; + } + ET_LOG( + Info, + "Vision encoder: max_soft_tokens %d, position_ids dtype %d, hidden %lld", + max_soft_tokens_, + static_cast(position_ids_dtype_), + static_cast(hidden_size_)); + + loaded_ = true; + ET_LOG(Info, "Vision encoder loaded: %s", model_path_.c_str()); + return runtime::Error::Ok; +} + +runtime::Error VisionEncoder::encode(const std::string& image_path) { + if (!loaded_) { + ET_LOG(Error, "Vision encoder is not loaded"); + return runtime::Error::InvalidState; + } + + // Load and patchify the image. + int32_t width = 0; + int32_t height = 0; + std::vector rgb; + try { + rgb = load_image_rgb(image_path, &width, &height); + } catch (const std::exception& e) { + ET_LOG(Error, "Failed to load image: %s", e.what()); + return runtime::Error::InvalidArgument; + } + + ImageData image = + patchify_rgb_image(rgb.data(), width, height, max_soft_tokens_); + + extension::TensorPtr position_ids = image.pixel_position_ids; + if (position_ids_dtype_ == executorch::aten::ScalarType::Int) { + position_ids = extension::empty( + {1, static_cast(position_ids->size(1)), 2}, + executorch::aten::ScalarType::Int); + std::copy_n( + image.pixel_position_ids->const_data_ptr(), + position_ids->numel(), + position_ids->mutable_data_ptr()); + } + + std::vector input_vec = {image.pixel_values, position_ids}; + + auto results = module_->execute("vision_encoder", input_vec); + ET_CHECK_OK_OR_RETURN_ERROR(results.error()); + + auto& outputs = results.get(); + if (outputs.size() < 2 || !outputs[0].isTensor() || !outputs[1].isTensor()) { + ET_LOG(Error, "Unexpected vision encoder outputs (%zu)", outputs.size()); + return runtime::Error::InvalidProgram; + } + + const auto& embeds = outputs[0].toTensor(); + const auto& mask = outputs[1].toTensor(); + + if (embeds.scalar_type() != executorch::aten::ScalarType::Float) { + ET_LOG(Error, "Vision embeddings are not float32"); + return runtime::Error::InvalidProgram; + } + + const float* embeds_data = embeds.const_data_ptr(); + const auto pooled = embeds.size(1); + const auto hidden = embeds.size(2); + hidden_size_ = hidden; + + ET_CHECK_OR_RETURN_ERROR( + mask.numel() == pooled, + InvalidProgram, + "Vision mask numel %lld != pooled tokens %lld", + static_cast(mask.numel()), + static_cast(pooled)); + + // The delegate may not preserve the BOOL dtype of the exported mask. + std::vector valid(pooled); + switch (mask.scalar_type()) { + case executorch::aten::ScalarType::Bool: + std::copy_n(mask.const_data_ptr(), pooled, valid.begin()); + break; + case executorch::aten::ScalarType::Byte: + std::copy_n(mask.const_data_ptr(), pooled, valid.begin()); + break; + case executorch::aten::ScalarType::Int: + std::copy_n(mask.const_data_ptr(), pooled, valid.begin()); + break; + case executorch::aten::ScalarType::Long: + std::copy_n(mask.const_data_ptr(), pooled, valid.begin()); + break; + case executorch::aten::ScalarType::Float: + std::copy_n(mask.const_data_ptr(), pooled, valid.begin()); + break; + default: + ET_LOG( + Error, + "Unsupported vision mask dtype %d", + static_cast(mask.scalar_type())); + return runtime::Error::InvalidProgram; + } + num_valid_tokens_ = std::count(valid.begin(), valid.end(), true); + + const int64_t expected_tokens = + compute_vision_num_tokens(image.num_valid_patches); + if (num_valid_tokens_ != expected_tokens) { + ET_LOG( + Error, + "Vision mask reports %lld valid tokens, expected %lld for %dx%d resized image", + static_cast(num_valid_tokens_), + static_cast(expected_tokens), + image.original_width, + image.original_height); + return runtime::Error::InvalidProgram; + } + + // Keep only valid tokens' embeddings, packed contiguously. + vision_embeds_.clear(); + vision_embeds_.reserve(static_cast(num_valid_tokens_) * hidden); + for (int64_t i = 0; i < pooled; ++i) { + if (valid[i]) { + vision_embeds_.insert( + vision_embeds_.end(), + embeds_data + i * hidden, + embeds_data + (i + 1) * hidden); + } + } + + ET_LOG( + Info, + "Vision encoded: %lld valid tokens out of %lld", + static_cast(num_valid_tokens_), + static_cast(pooled)); + + return runtime::Error::Ok; +} + +} // namespace example diff --git a/examples/samsung/executor_runner/llm/runner/vision_encoder.h b/examples/samsung/executor_runner/llm/runner/vision_encoder.h new file mode 100644 index 00000000000..7f2954320e9 --- /dev/null +++ b/examples/samsung/executor_runner/llm/runner/vision_encoder.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2026 Samsung Electronics Co. LTD + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace example { + +namespace runtime = ::executorch::runtime; +namespace extension = ::executorch::extension; + +/** + * Runs the Gemma 4 vision encoder (gemma4_vision.pte) on an input image and + * produces LM-ready vision embeddings plus the number of valid (non-padding) + * vision tokens. + * + * The vision encoder is a separate module from the text decoder. Its PTE has a + * single "vision_encoder" method accepting pre-patchified pixel values + * [1, max_patches, 768] and 2D patch positions [1, max_patches, 2], and + * returning embeddings [1, pooled, hidden] plus a BOOL validity mask [1, + * pooled]. + */ +class VisionEncoder { + public: + explicit VisionEncoder(const std::string& model_path); + + runtime::Error load(); + + /// Encode an image file and produce vision embeddings. + /// On success fills vision_embeds_ ([1, pooled, hidden], float) and sets + /// num_valid_tokens_ to the number of real (non-padding) vision tokens. + runtime::Error encode(const std::string& image_path); + + const std::vector& vision_embeds() const { + return vision_embeds_; + } + int64_t num_valid_tokens() const { + return num_valid_tokens_; + } + int64_t hidden_size() const { + return hidden_size_; + } + bool loaded() const { + return loaded_; + } + + private: + std::string model_path_; + std::unique_ptr module_; + bool loaded_ = false; + // Patch budget and position-id dtype the PTE was exported with; read from + // the method metadata in load(). + int32_t max_soft_tokens_ = kVisionMaxSoftTokens; + executorch::aten::ScalarType position_ids_dtype_ = + executorch::aten::ScalarType::Long; + int64_t hidden_size_ = 0; + int64_t num_valid_tokens_ = 0; + std::vector vision_embeds_; +}; + +} // namespace example diff --git a/examples/samsung/executor_runner/run_gemma3_sample.sh b/examples/samsung/executor_runner/run_gemma3_sample.sh new file mode 100755 index 00000000000..3a88c64a62b --- /dev/null +++ b/examples/samsung/executor_runner/run_gemma3_sample.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) 2026 Samsung Electronics Co. LTD +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +MODEL_PATH=/data/local/tmp/gemma3/gemma3.pte +WEIGHT_SHARED_MODEL_PATH=/data/local/tmp/gemma3/weights.ptd +TOKENIZER_PATH=/data/local/tmp/gemma3/tokenizer.json +POS_EMBEDDING_PATH=/data/local/tmp/gemma3/pos_embedding.bin +CONFIG_PATH="/data/local/tmp/exynos_gemma3_1b.json" + +SEQ_LEN=1024 +TEMPERATURE=0 +NUM_BOS=1 +INPUT_PROMPT="What is the capital of France?" + +chmod +x enn_executor_llm_runner + +./enn_executor_llm_runner \ + --model_path=$MODEL_PATH \ + --data_path=$WEIGHT_SHARED_MODEL_PATH \ + --seq_len=$SEQ_LEN \ + --temperature=$TEMPERATURE \ + --num_bos=$NUM_BOS \ + --pos_embedding_path=$POS_EMBEDDING_PATH \ + --tokenizer_path=$TOKENIZER_PATH \ + --prompt="$INPUT_PROMPT" \ + --config_path "$CONFIG_PATH" diff --git a/examples/samsung/executor_runner/run_llama3_sample.sh b/examples/samsung/executor_runner/run_llama3_sample.sh new file mode 100755 index 00000000000..d2d02a5bd04 --- /dev/null +++ b/examples/samsung/executor_runner/run_llama3_sample.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) 2026 Samsung Electronics Co. LTD +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +MODEL_PATH=/data/local/tmp/llama3/llama3.pte +WEIGHT_SHARED_MODEL_PATH=/data/local/tmp/llama3/weights.ptd +POS_EMBEDDING_PATH=/data/local/tmp/llama3/pos_embedding.bin +TOKENIZER_PATH=/data/local/tmp/llama3/tokenizer.model +CONFIG_PATH="/data/local/tmp/exynos_llama3_1b.json" + +SEQ_LEN=1024 +TEMPERATURE=0 +NUM_BOS=1 +INPUT_PROMPT="What is the capital of France?" + +chmod +x enn_executor_llm_runner + +./enn_executor_llm_runner \ + --model_path=$MODEL_PATH \ + --data_path=$WEIGHT_SHARED_MODEL_PATH \ + --pos_embedding_path=$POS_EMBEDDING_PATH \ + --tokenizer_path=$TOKENIZER_PATH \ + --config_path "$CONFIG_PATH" \ + --prompt="$INPUT_PROMPT" \ + --temperature=$TEMPERATURE \ + --seq_len=$SEQ_LEN \ + --num_bos=$NUM_BOS diff --git a/examples/samsung/scripts/gemma4_pos_embedding.py b/examples/samsung/scripts/gemma4_pos_embedding.py new file mode 100644 index 00000000000..7a8ab838256 --- /dev/null +++ b/examples/samsung/scripts/gemma4_pos_embedding.py @@ -0,0 +1,90 @@ +# Copyright (c) 2026 Samsung Electronics Co. LTD +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Precompute the Gemma4 RoPE position-embedding .bin for the Exynos LLM runner. + +Reproduces the RoPE convention from gemma4_attention.py: inv_freq denominator is +head_dim, full-attention (global) layers use partial rotary while sliding (local) +layers use full rotary, non-rotary dims zero-padded so split-half rotate_half +pairs correctly. Emits the layout RotaryEmbedding::load_from_file expects: + + [global_cos | global_sin | local_cos | local_sin] + +each table row-major [max_seq_len, head_dim] float32 (global head_dim 512, local +head_dim 256). +""" + +import argparse + +import torch + +from executorch.examples.models.gemma4.text_decoder.gemma4_config import Gemma4Config + + +def rope_table( + max_seq_len: int, + head_dim: int, + base: float, + partial_rotary_factor: float, + full_rope: bool, +): + rotary_dim = head_dim if full_rope else int(head_dim * partial_rotary_factor) + rope_angles = rotary_dim // 2 + inv_freq_rotated = 1.0 / ( + base ** (torch.arange(0, rotary_dim, 2).float() / head_dim) + ) + nope_angles = head_dim // 2 - rope_angles + if nope_angles > 0: + inv_freq = torch.cat([inv_freq_rotated, torch.zeros(nope_angles)]) + else: + inv_freq = inv_freq_rotated + + pos = torch.arange(max_seq_len).float() + freqs = torch.outer(pos, inv_freq) + emb = torch.cat((freqs, freqs), dim=-1) + return torch.cos(emb), torch.sin(emb) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--config", required=True, help="HF config.json path") + ap.add_argument("--max-seq-len", type=int, default=4096) + ap.add_argument("--out", required=True, help="output pos_embedding.bin path") + args = ap.parse_args() + + cfg = Gemma4Config.from_json(args.config) + + g_cos, g_sin = rope_table( + args.max_seq_len, + cfg.global_head_dim, + cfg.rope_theta, + cfg.partial_rotary_factor, + full_rope=False, + ) + l_cos, l_sin = rope_table( + args.max_seq_len, + cfg.head_dim, + cfg.rope_local_base_freq, + cfg.partial_rotary_factor, + full_rope=True, + ) + + with open(args.out, "wb") as f: + for t in (g_cos, g_sin, l_cos, l_sin): + f.write(t.contiguous().to(torch.float32).numpy().tobytes()) + + nbytes = args.max_seq_len * (cfg.global_head_dim + cfg.head_dim) * 2 * 4 + print( + f"wrote {args.out}: {nbytes} bytes " + f"(max_seq_len={args.max_seq_len}, global_head_dim={cfg.global_head_dim}, " + f"head_dim={cfg.head_dim}, rope_theta={cfg.rope_theta}, " + f"rope_local_base_freq={cfg.rope_local_base_freq}, " + f"partial_rotary_factor={cfg.partial_rotary_factor})" + ) + + +if __name__ == "__main__": + main()