From 51dc21fe6b79bfc18ad9934e3e53c65bd004084a Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Mon, 19 Oct 2026 02:33:58 +0800 Subject: [PATCH 1/5] [refactor](udf) Isolate Python process pools by function id ### What problem does this PR solve? Issue Number: None Problem Summary: Refactor Python UDF process scheduling to maintain an independent process pool for each function id. Each function owns a fixed-size pool, simplifying allocation, package isolation and lifecycle management for Python UDF, UDAF and UDTF. As part of the refactoring, add two function properties: pool_size controls the maximum process count per function on each BE and defaults to 3; auto_suspend specifies the idle timeout in seconds and defaults to -1 to keep pools resident. The existing 30-second monitor terminates eligible idle pools and subsequent invocations recreate them. Active clients, including clients being initialized, prevent reclamation. The new properties are persisted in function metadata and rejected for Java functions. Adapt DROP FUNCTION and permanent database removal to release the corresponding Python pools. Existing clients retain their processes and package files until release, and those processes remain included in memory accounting. Prevent deleted function ids from recreating package-cache entries that share files with active clients. Keep the existing one-shot cleanup notification behavior. ### Release note Python functions use independent process pools. The pool_size and auto_suspend properties control pool capacity and idle reclamation. ### Check List (For Author) - Test: Full BE/FE build; 83 BE unit tests; 46 FE unit tests; clang-tidy and header hygiene; cluster startup and basic query verification. Python regression suites were exercised during implementation. - Behavior changed: Yes, Python process allocation and lifecycle management are scoped by function id, with configurable pool size and idle reclamation. - Does this need documentation: Yes, English and Chinese documentation changes are prepared separately in doris-website. --- be/src/agent/task_worker_pool.cpp | 2 +- be/src/common/config.cpp | 3 - be/src/common/config.h | 2 - .../aggregate_function_python_udaf.cpp | 7 +- .../aggregate_function_python_udaf.h | 3 + be/src/exprs/function/function_python_udf.cpp | 9 +- .../table_function/python_udtf_function.cpp | 9 +- be/src/runtime/user_function_cache.cpp | 18 +- be/src/runtime/user_function_cache.h | 5 +- be/src/udf/python/python_client.cpp | 16 +- be/src/udf/python/python_client.h | 7 + be/src/udf/python/python_server.cpp | 752 ++++++++++-------- be/src/udf/python/python_server.h | 99 ++- be/src/udf/python/python_server.py | 250 +----- be/src/udf/python/python_udaf_client.cpp | 4 +- be/src/udf/python/python_udf_meta.h | 2 + be/src/udf/python/python_udf_runtime.h | 9 + be/test/runtime/user_function_cache_test.cpp | 34 +- be/test/udf/python/python_server_test.cpp | 504 +++++++++--- .../org/apache/doris/catalog/Function.java | 22 + .../doris/catalog/CatalogRecycleBin.java | 10 +- .../doris/catalog/FunctionToSqlConverter.java | 4 + .../catalog/FunctionToThriftConverter.java | 4 + .../apache/doris/catalog/FunctionUtil.java | 34 + .../doris/datasource/InternalCatalog.java | 8 + .../expressions/functions/udf/PythonUdaf.java | 16 +- .../expressions/functions/udf/PythonUdf.java | 16 +- .../expressions/functions/udf/PythonUdtf.java | 16 +- .../plans/commands/CreateFunctionCommand.java | 33 + .../doris/catalog/CreateFunctionTest.java | 103 +++ .../org/apache/doris/catalog/DropDbTest.java | 145 ++++ .../FunctionToThriftConverterTest.java | 50 ++ .../functions/udf/UdfVolatilityTest.java | 51 +- .../nereids/util/CacheExpressionTest.java | 2 +- gensrc/thrift/Types.thrift | 2 + .../test_pythonudaf_auto_suspend.out | 31 + .../test_pythonudf_auto_suspend.out | 41 + .../test_pythonudf_native_module_cache.out | 10 + .../test_pythonudtf_auto_suspend.out | 65 ++ .../pipeline/cloud_p0/conf/be_custom.conf | 1 - .../pipeline/cloud_p1/conf/be_custom.conf | 1 - .../pipeline/external/conf/be.conf | 1 - .../pipeline/nonConcurrent/conf/be.conf | 1 - regression-test/pipeline/p0/conf/be.conf | 1 - regression-test/pipeline/p1/conf/be.conf | 1 - .../pipeline/performance/conf/be_custom.conf | 1 - .../pipeline/vault_p0/conf/be_custom.conf | 1 - .../test_pythonudaf_auto_suspend.groovy | 66 ++ .../test_pythonudaf_pkg_isolation.groovy | 12 +- .../test_pythonudf_auto_suspend.groovy | 44 + .../test_pythonudf_inline_scalar.groovy | 13 +- .../test_pythonudf_native_module_cache.groovy | 68 ++ .../test_pythonudf_pkg_isolation.groovy | 12 +- .../udf_scripts/native_module_cache_helper.py | 20 + .../udf_scripts/native_module_cache_test.py | 31 + .../suites/pythonudf_p0/udf_scripts/pyudf.zip | Bin 15967 -> 17479 bytes .../test_pythonudtf_auto_suspend.groovy | 47 ++ .../test_pythonudtf_pkg_isolation.groovy | 12 +- 58 files changed, 1954 insertions(+), 777 deletions(-) create mode 100644 regression-test/data/pythonudaf_p0/test_pythonudaf_auto_suspend.out create mode 100644 regression-test/data/pythonudf_p0/test_pythonudf_auto_suspend.out create mode 100644 regression-test/data/pythonudf_p0/test_pythonudf_native_module_cache.out create mode 100644 regression-test/data/pythonudtf_p0/test_pythonudtf_auto_suspend.out create mode 100644 regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy create mode 100644 regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy create mode 100644 regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy create mode 100644 regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_helper.py create mode 100644 regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_test.py create mode 100644 regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index ab51e93e650da4..b7298e56378607 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -2602,7 +2602,7 @@ void clean_udf_cache_callback(const TAgentTaskRequest& req) { } if (drop_by_function_id) { UserFunctionCache::instance()->drop_function_cache(clean_req.function_id); - PythonServerManager::instance().clear_udaf_state_cache(clean_req.function_id); + PythonServerManager::instance().drop_function(clean_req.function_id); } LOG(INFO) << "clean udf cache callback finish: function_signature=" diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 4c9957dfdfa173..685f3c0a313228 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1198,9 +1198,6 @@ DEFINE_String(python_conda_root_path, ""); DEFINE_String(python_venv_root_path, "${DORIS_HOME}/lib/udf/python"); // python interpreter paths used by venv, e.g. /usr/bin/python3.7:/usr/bin/python3.6 DEFINE_String(python_venv_interpreter_paths, ""); -// max python processes in global shared pool, each version can have up to this many processes -// 0 means use CPU core count as default, otherwise use the specified value -DEFINE_mInt32(max_python_process_num, "0"); // Memory limit in bytes for all Python UDF processes; warning is logged when exceeded // default is 10GB DEFINE_mInt64(python_udf_processes_memory_limit_bytes, "10737418240"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 62becc90f28a1e..805eb41732affc 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1271,8 +1271,6 @@ DECLARE_String(python_conda_root_path); DECLARE_String(python_venv_root_path); // python interpreter paths used by venv, e.g. /usr/bin/python3.7:/usr/bin/python3.6 DECLARE_String(python_venv_interpreter_paths); -// max python processes in global shared pool, each version can have up to this many processes -DECLARE_mInt32(max_python_process_num); // Memory limit in bytes for all Python UDF processes; warning is logged when exceeded DECLARE_mInt64(python_udf_processes_memory_limit_bytes); diff --git a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp index ab6000783ad191..ae4bce41fb3fa6 100644 --- a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp +++ b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp @@ -191,6 +191,8 @@ Status AggregatePythonUDAFData::get(IColumn& to, const DataTypePtr& result_type, Status AggregatePythonUDAF::open() { // Build function metadata from TFunction _func_meta.id = _fn.id; + _func_meta.pool_size = _fn.__isset.pool_size ? _fn.pool_size : 3; + _func_meta.auto_suspend = _fn.__isset.auto_suspend ? _fn.auto_suspend : -1; _func_meta.name = _fn.name.function_name; // For UDAF, symbol is in aggregate_fn @@ -236,7 +238,8 @@ Status AggregatePythonUDAF::open() { if (_func_meta.type == PythonUDFLoadType::MODULE) { RETURN_IF_ERROR(UserFunctionCache::instance()->get_pypath( - _func_meta.id, _func_meta.location, _func_meta.checksum, &_func_meta.location)); + _func_meta.id, _func_meta.location, _func_meta.checksum, &_func_meta.location, + &_package_cache_entry)); } return Status::OK(); @@ -272,7 +275,7 @@ void AggregatePythonUDAF::create(AggregateDataPtr __restrict place) const { DCHECK(reinterpret_cast(place)) << "Place must not be null"; if (Status st = PythonServerManager::instance().get_client( - _func_meta, _python_version, &(this->data(place).client), _schema); + _func_meta, _python_version, &data(place).client, _schema, _package_cache_entry); UNLIKELY(!st.ok())) { this->data(place).~Data(); throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Failed to get Python UDAF client: {}", diff --git a/be/src/exprs/aggregate/aggregate_function_python_udaf.h b/be/src/exprs/aggregate/aggregate_function_python_udaf.h index 43a86aaa5900f7..d6744cb9432b7f 100644 --- a/be/src/exprs/aggregate/aggregate_function_python_udaf.h +++ b/be/src/exprs/aggregate/aggregate_function_python_udaf.h @@ -28,6 +28,8 @@ namespace doris { +struct UserFunctionCacheEntry; + /** * Aggregate state data for Python UDAF * @@ -172,6 +174,7 @@ class AggregatePythonUDAF final DataTypePtr _return_type; PythonUDFMeta _func_meta; PythonVersion _python_version; + std::shared_ptr _package_cache_entry; // Arrow Flight schema: [argument_types..., places: int64, binary_data: binary] // Used for all UDAF RPC operations // - places column is always present (NULL in single-place mode, actual place_id values in GROUP BY mode) diff --git a/be/src/exprs/function/function_python_udf.cpp b/be/src/exprs/function/function_python_udf.cpp index 9f7e40119eda5d..f6d266a7e439d7 100644 --- a/be/src/exprs/function/function_python_udf.cpp +++ b/be/src/exprs/function/function_python_udf.cpp @@ -53,6 +53,8 @@ Status PythonFunctionCall::open(FunctionContext* context, PythonVersion version; PythonUDFMeta func_meta; func_meta.id = _fn.id; + func_meta.pool_size = _fn.__isset.pool_size ? _fn.pool_size : 3; + func_meta.auto_suspend = _fn.__isset.auto_suspend ? _fn.auto_suspend : -1; func_meta.name = _fn.name.function_name; func_meta.symbol = _fn.scalar_fn.symbol; if (!_fn.function_code.empty()) { @@ -85,13 +87,16 @@ Status PythonFunctionCall::open(FunctionContext* context, LOG(INFO) << fmt::format("runtime_version: {}, func_meta: {}", version.to_string(), func_meta.to_string()); + std::shared_ptr package_cache_entry; if (func_meta.type == PythonUDFLoadType::MODULE) { RETURN_IF_ERROR(UserFunctionCache::instance()->get_pypath( - func_meta.id, func_meta.location, func_meta.checksum, &func_meta.location)); + func_meta.id, func_meta.location, func_meta.checksum, &func_meta.location, + &package_cache_entry)); } PythonUDFClientPtr client = nullptr; - RETURN_IF_ERROR(PythonServerManager::instance().get_client(func_meta, version, &client)); + RETURN_IF_ERROR(PythonServerManager::instance().get_client(func_meta, version, &client, nullptr, + std::move(package_cache_entry))); if (!client) { return Status::InternalError("Python UDF client is null"); diff --git a/be/src/exprs/table_function/python_udtf_function.cpp b/be/src/exprs/table_function/python_udtf_function.cpp index a45a0f95f15368..e87d40efff706c 100644 --- a/be/src/exprs/table_function/python_udtf_function.cpp +++ b/be/src/exprs/table_function/python_udtf_function.cpp @@ -61,6 +61,8 @@ PythonUDTFFunction::PythonUDTFFunction(const TFunction& t_fn) : TableFunction(), Status PythonUDTFFunction::open() { PythonUDFMeta python_udf_meta; python_udf_meta.id = _t_fn.id; + python_udf_meta.pool_size = _t_fn.__isset.pool_size ? _t_fn.pool_size : 3; + python_udf_meta.auto_suspend = _t_fn.__isset.auto_suspend ? _t_fn.auto_suspend : -1; python_udf_meta.name = _t_fn.name.function_name; python_udf_meta.symbol = _t_fn.scalar_fn.symbol; @@ -79,10 +81,11 @@ Status PythonUDTFFunction::open() { python_udf_meta.client_type = PythonClientType::UDTF; + std::shared_ptr package_cache_entry; if (python_udf_meta.type == PythonUDFLoadType::MODULE) { RETURN_IF_ERROR(UserFunctionCache::instance()->get_pypath( python_udf_meta.id, python_udf_meta.location, python_udf_meta.checksum, - &python_udf_meta.location)); + &python_udf_meta.location, &package_cache_entry)); } PythonVersion version; @@ -105,8 +108,8 @@ Status PythonUDTFFunction::open() { python_udf_meta.always_nullable = python_udf_meta.return_type->is_nullable(); RETURN_IF_ERROR(python_udf_meta.check()); - RETURN_IF_ERROR( - PythonServerManager::instance().get_client(python_udf_meta, version, &_udtf_client)); + RETURN_IF_ERROR(PythonServerManager::instance().get_client( + python_udf_meta, version, &_udtf_client, nullptr, std::move(package_cache_entry))); if (!_udtf_client) { return Status::InternalError("Failed to create Python UDTF client"); diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp index 7199cc7e1ff407..28a19d3a0ff026 100644 --- a/be/src/runtime/user_function_cache.cpp +++ b/be/src/runtime/user_function_cache.cpp @@ -498,10 +498,14 @@ Status UserFunctionCache::get_jarpath(int64_t fid, const std::string& url, } Status UserFunctionCache::get_pypath(int64_t fid, const std::string& url, - const std::string& checksum, std::string* libpath) { + const std::string& checksum, std::string* libpath, + std::shared_ptr* cache_entry) { std::shared_ptr entry = nullptr; RETURN_IF_ERROR(_get_cache_entry(fid, url, checksum, entry, LibType::PY_ZIP)); *libpath = entry->lib_file; + if (cache_entry != nullptr) { + *cache_entry = std::move(entry); + } return Status::OK(); } @@ -579,17 +583,11 @@ void UserFunctionCache::drop_function_cache(int64_t fid) { _entry_map.erase(it); } - // For Python UDF, clear module cache in Python server before deleting files - if (entry->type == LibType::PY_ZIP && !entry->lib_file.empty()) { - auto status = PythonServerManager::instance().clear_module_cache(entry->lib_file); - if (!status.ok()) [[unlikely]] { - LOG(WARNING) << "drop_function_cache: failed to clear Python module cache for " - << entry->lib_file << ": " << status.to_string(); - } - } - // Mark for deletion, destructor will delete the files entry->should_delete_library.store(true); + if (entry->type == LibType::PY_ZIP) { + PythonServerManager::instance().drop_function(fid); + } } } // namespace doris diff --git a/be/src/runtime/user_function_cache.h b/be/src/runtime/user_function_cache.h index e606899f969d4d..f71252a346380f 100644 --- a/be/src/runtime/user_function_cache.h +++ b/be/src/runtime/user_function_cache.h @@ -59,8 +59,11 @@ class UserFunctionCache { Status get_jarpath(int64_t fid, const std::string& url, const std::string& checksum, std::string* libpath); + // Optionally return a shared cache entry to pin package files: DROP only removes them + // after the last query/process releases this handle. The handle is for lifetime only. Status get_pypath(int64_t fid, const std::string& url, const std::string& checksum, - std::string* libpath); + std::string* libpath, + std::shared_ptr* cache_entry = nullptr); // Drop the cached function library by function id. void drop_function_cache(int64_t fid); diff --git a/be/src/udf/python/python_client.cpp b/be/src/udf/python/python_client.cpp index 716ae6f15f69ce..555b21a3118150 100644 --- a/be/src/udf/python/python_client.cpp +++ b/be/src/udf/python/python_client.cpp @@ -72,22 +72,20 @@ Status PythonClient::init(const PythonUDFMeta& func_meta, ProcessPtr process) { } Status PythonClient::close() { - if (!_inited || !_writer) { - return Status::OK(); - } - - auto writer_res = _writer->Close(); - if (!writer_res.ok()) { - // Don't propagate error from close, just log it - LOG(WARNING) << "Error closing Python client writer: " << writer_res.message(); + if (_inited && _writer) { + auto writer_res = _writer->Close(); + if (!writer_res.ok()) { + LOG(WARNING) << "Error closing Python client writer: " << writer_res.message(); + } } - + // Error handling may already have reset the writer; still release the pool lease. _inited = false; _begin = false; _arrow_client.reset(); _writer.reset(); _reader.reset(); _process.reset(); + _pool_lease.reset(); return Status::OK(); } diff --git a/be/src/udf/python/python_client.h b/be/src/udf/python/python_client.h index 62d576774e3c84..7a6363a77c99b5 100644 --- a/be/src/udf/python/python_client.h +++ b/be/src/udf/python/python_client.h @@ -28,6 +28,8 @@ namespace doris { +class PythonUDFPoolLease; + /** * Base class for Python UDF/UDAF/UDTF clients * @@ -102,6 +104,9 @@ class PythonClient { */ Status read_batch(std::shared_ptr* output); + // Released after the stream and process, so idle reclamation cannot interrupt cleanup. + std::shared_ptr _pool_lease; + // Common state bool _inited = false; bool _begin = false; // Track if Begin() has been called @@ -112,6 +117,8 @@ class PythonClient { ProcessPtr _process; private: + friend class PythonServerManager; + DISALLOW_COPY_AND_ASSIGN(PythonClient); }; diff --git a/be/src/udf/python/python_server.cpp b/be/src/udf/python/python_server.cpp index 2d06caa311067e..3761a1eb632362 100644 --- a/be/src/udf/python/python_server.cpp +++ b/be/src/udf/python/python_server.cpp @@ -38,104 +38,167 @@ #include "common/status.h" #include "cpp/sync_point.h" #include "runtime/thread_context.h" +#include "runtime/user_function_cache.h" #include "udf/python/python_udaf_client.h" #include "udf/python/python_udf_client.h" #include "udf/python/python_udtf_client.h" -#include "util/cpu_info.h" +#include "util/defer_op.h" namespace doris { -Result> -PythonServerManager::_get_or_create_process_pool(const PythonVersion& version) { - std::lock_guard lock(_pools_mutex); +// A client holds this from pool acquisition through stream/state cleanup. The pool may outlive +// its manager after DROP; releasing a lease must therefore only access the pool itself. +class PythonUDFPoolLease { +public: + explicit PythonUDFPoolLease(std::shared_ptr pool) + : _pool(std::move(pool)) {} + + ~PythonUDFPoolLease() { + std::scoped_lock lock(_pool->mutex); + DCHECK_GT(_pool->active_clients, 0); + if (--_pool->active_clients == 0) { + _pool->idle_since = std::chrono::steady_clock::now(); + } + } + +private: + std::shared_ptr _pool; +}; + +void PythonServerManager::ProcessRegistry::add(const ProcessPtr& process) { + std::scoped_lock lock(mutex); + processes.emplace_back(process); +} + +std::vector PythonServerManager::ProcessRegistry::snapshot() { + std::vector snapshot; + std::scoped_lock lock(mutex); + std::erase_if(processes, [&snapshot](const auto& weak_process) { + if (auto process = weak_process.lock()) { + snapshot.emplace_back(std::move(process)); + return false; + } + return true; + }); + return snapshot; +} + +Result> +PythonServerManager::_get_or_create_process_pool( + const PythonUDFMeta& meta, const PythonVersion& version, + std::shared_ptr package_cache_entry, + std::shared_ptr* lease) { + std::scoped_lock lock(_pools_mutex); // shutdown() owns the manager lifecycle. Once it starts, creating a new pool would let detached // init workers publish Python processes that the manager no longer tracks. if (_shutdown_flag.load(std::memory_order_acquire)) { return ResultError(Status::Error( "Python server manager is shutting down")); } - auto& pool = _process_pools[version]; + if (meta.pool_size <= 0) { + return ResultError( + Status::InvalidArgument("Python UDF pool_size must be a positive integer")); + } + auto& pool = _process_pools[meta.id]; if (!pool) { - pool = std::make_shared(); + pool = std::make_shared(); + pool->version = version; + pool->process_registry = _process_registry; + pool->function_id = meta.id; + pool->pool_size = meta.pool_size; + pool->auto_suspend = meta.auto_suspend; + pool->package_cache_entry = std::move(package_cache_entry); + // The first process can serve a client before the remaining slots finish initialization. + _start_health_check_thread(); + } + if (lease) { + auto acquired = std::make_shared(pool); + std::scoped_lock pool_lock(pool->mutex); + ++pool->active_clients; + *lease = std::move(acquired); } return pool; } -std::vector>> +std::vector> PythonServerManager::_snapshot_process_pools() { - std::lock_guard lock(_pools_mutex); - std::vector>> snapshot; + std::scoped_lock lock(_pools_mutex); + std::vector> snapshot; snapshot.reserve(_process_pools.size()); - for (const auto& [version, pool] : _process_pools) { - snapshot.emplace_back(version, pool); + for (const auto& [function_id, pool] : _process_pools) { + snapshot.emplace_back(pool); } return snapshot; } #ifdef BE_TEST -void PythonServerManager::set_process_pool_for_test(const PythonVersion& version, +void PythonServerManager::set_process_pool_for_test(const PythonUDFMeta& meta, + const PythonVersion& version, std::vector processes, bool initialized) { - auto versioned_pool = _get_or_create_process_pool(version).value(); - std::lock_guard lock(versioned_pool->mutex); - versioned_pool->processes = std::move(processes); - versioned_pool->state = initialized ? PoolState::INITIALIZED : PoolState::UNINITIALIZED; - versioned_pool->has_available_process = - std::any_of(versioned_pool->processes.begin(), versioned_pool->processes.end(), + auto function_pool = _get_or_create_process_pool(meta, version, nullptr).value(); + std::scoped_lock lock(function_pool->mutex); + for (const auto& process : processes) { + if (process) { + function_pool->process_registry->add(process); + } + } + function_pool->processes = std::move(processes); + function_pool->state = initialized ? PoolState::INITIALIZED : PoolState::UNINITIALIZED; + function_pool->has_available_process = + std::any_of(function_pool->processes.begin(), function_pool->processes.end(), [](const ProcessPtr& process) { return process && process->is_alive(); }); } -std::vector PythonServerManager::process_pool_snapshot_for_test( - const PythonVersion& version) { - auto versioned_pool = _get_or_create_process_pool(version).value(); - std::lock_guard lock(versioned_pool->mutex); - return versioned_pool->processes; +std::vector PythonServerManager::process_pool_snapshot_for_test(int64_t function_id) { + std::scoped_lock map_lock(_pools_mutex); + auto iter = _process_pools.find(function_id); + if (iter == _process_pools.end()) { + return {}; + } + std::scoped_lock lock(iter->second->mutex); + return iter->second->processes; } -bool PythonServerManager::process_pool_is_initialized_for_test(const PythonVersion& version) { - auto versioned_pool = _get_or_create_process_pool(version).value(); - std::lock_guard lock(versioned_pool->mutex); - return versioned_pool->state == PoolState::INITIALIZED; +bool PythonServerManager::process_pool_is_initialized_for_test(int64_t function_id) { + return wait_for_process_pool_initialized_for_test(function_id, std::chrono::milliseconds(0)); } bool PythonServerManager::wait_for_process_pool_initialized_for_test( - const PythonVersion& version, std::chrono::milliseconds timeout) { - auto versioned_pool_result = _get_or_create_process_pool(version); - if (!versioned_pool_result.has_value()) { - return false; + int64_t function_id, std::chrono::milliseconds timeout) { + std::shared_ptr pool; + { + std::scoped_lock map_lock(_pools_mutex); + pool = _process_pools.at(function_id); } - auto versioned_pool = versioned_pool_result.value(); - std::unique_lock lock(versioned_pool->mutex); - return versioned_pool->cv.wait_for(lock, timeout, [&versioned_pool]() { - return versioned_pool->state == PoolState::INITIALIZED; - }); + std::unique_lock lock(pool->mutex); + return pool->cv.wait_for(lock, timeout, + [&]() { return pool->state == PoolState::INITIALIZED; }); } bool PythonServerManager::wait_for_process_pool_initialization_finished_for_test( - const PythonVersion& version, std::chrono::milliseconds timeout) { - auto versioned_pool_result = _get_or_create_process_pool(version); - if (!versioned_pool_result.has_value()) { - return false; + int64_t function_id, std::chrono::milliseconds timeout) { + std::shared_ptr pool; + { + std::scoped_lock map_lock(_pools_mutex); + pool = _process_pools.at(function_id); } - auto versioned_pool = versioned_pool_result.value(); - std::unique_lock lock(versioned_pool->mutex); - return versioned_pool->cv.wait_for(lock, timeout, [&versioned_pool]() { - return versioned_pool->state != PoolState::INITIALIZING; - }); + std::unique_lock lock(pool->mutex); + return pool->cv.wait_for(lock, timeout, + [&]() { return pool->state != PoolState::INITIALIZING; }); } #endif bool PythonServerManager::_select_alive_process_from_pool(const std::vector& pool, ProcessPtr* process) { - auto alive_iter = std::min_element(pool.begin(), pool.end(), - [](const ProcessPtr& a, const ProcessPtr& b) { - const bool a_alive = a && a->is_alive(); - const bool b_alive = b && b->is_alive(); - if (a_alive != b_alive) { - return a_alive > b_alive; - } - return a.use_count() < b.use_count(); - }); + auto alive_iter = std::ranges::min_element(pool, [](const ProcessPtr& a, const ProcessPtr& b) { + const bool a_alive = a && a->is_alive(); + const bool b_alive = b && b->is_alive(); + if (a_alive != b_alive) { + return a_alive > b_alive; + } + return a.use_count() < b.use_count(); + }); if (alive_iter == pool.end() || !*alive_iter || !(*alive_iter)->is_alive()) { return false; } @@ -144,14 +207,16 @@ bool PythonServerManager::_select_alive_process_from_pool(const std::vector -Status PythonServerManager::get_client(const PythonUDFMeta& func_meta, const PythonVersion& version, - std::shared_ptr* client, - const std::shared_ptr& data_schema) { - std::shared_ptr versioned_pool = - DORIS_TRY(_ensure_pool_initialized(version)); +Status PythonServerManager::get_client( + const PythonUDFMeta& func_meta, const PythonVersion& version, + std::shared_ptr* client, const std::shared_ptr& data_schema, + std::shared_ptr package_cache_entry) { + std::shared_ptr lease; + std::shared_ptr function_pool = DORIS_TRY( + _ensure_pool_initialized(func_meta, version, std::move(package_cache_entry), &lease)); ProcessPtr process; - RETURN_IF_ERROR(_get_process(version, versioned_pool, &process)); + RETURN_IF_ERROR(_get_process(version, function_pool, &process)); if constexpr (std::is_same_v) { RETURN_IF_ERROR(ClientType::create(func_meta, std::move(process), data_schema, client)); @@ -159,117 +224,122 @@ Status PythonServerManager::get_client(const PythonUDFMeta& func_meta, const Pyt RETURN_IF_ERROR(ClientType::create(func_meta, std::move(process), client)); } + (*client)->_pool_lease = std::move(lease); return Status::OK(); } -Result> -PythonServerManager::_ensure_pool_initialized(const PythonVersion& version) { - auto versioned_pool_result = _get_or_create_process_pool(version); - if (!versioned_pool_result.has_value()) { - return ResultError(versioned_pool_result.error()); +void PythonServerManager::_start_pool_initialization( + const std::shared_ptr& function_pool, const PythonVersion& version, + std::chrono::milliseconds process_pool_init_timeout) { + // Called with the pool lock held; workers publish only after the caller starts waiting. + const int max_pool_size = function_pool->pool_size; + function_pool->state = PoolState::INITIALIZING; + function_pool->has_available_process = false; + function_pool->processes.resize(max_pool_size); + auto init_finished_count = std::make_shared>(0); + TEST_SYNC_POINT_CALLBACK("PythonServerManager::_ensure_pool_initialized:generation_started", + &version, init_finished_count.get()); + + LOG(INFO) << "Initializing Python process pool for function " << function_pool->function_id + << " with " << max_pool_size << " processes, runtime=" << version.to_string(); + + std::thread([function_pool, init_finished_count, max_pool_size, process_pool_init_timeout]() { + SCOPED_INIT_THREAD_CONTEXT(); + std::unique_lock lock(function_pool->mutex); + function_pool->cv.wait_for( + lock, process_pool_init_timeout, + [&function_pool, init_finished_count, max_pool_size]() { + return function_pool->state != PoolState::INITIALIZING || + init_finished_count->load(std::memory_order_acquire) >= max_pool_size; + }); + if (function_pool->state == PoolState::INITIALIZING) { + if (function_pool->has_available_process) { + function_pool->state = PoolState::INITIALIZED; + } else { + function_pool->state = PoolState::UNINITIALIZED; + } + } + function_pool->cv.notify_all(); + }).detach(); + + for (int i = 0; i < max_pool_size; ++i) { + std::thread([version, function_pool, i, max_pool_size, init_finished_count]() { + SCOPED_INIT_THREAD_CONTEXT(); + ProcessPtr process; + Status status = PythonServerManager::fork(version, &process); + const bool ok = status.ok() && process; + ProcessPtr process_to_shutdown; + { + std::scoped_lock lock(function_pool->mutex); + // shutdown() and repair can race with detached init workers after timeout. + // Late successful forks only fill slots that are still empty or dead. + if (ok && + (function_pool->state == PoolState::INITIALIZING || + function_pool->state == PoolState::INITIALIZED) && + i < function_pool->processes.size() && + (!function_pool->processes[i] || !function_pool->processes[i]->is_alive())) { + // Register only accepted processes; rejected forks shut down separately. + function_pool->process_registry->add(process); + process->set_package_cache_entry(function_pool->package_cache_entry); + function_pool->processes[i] = std::move(process); + function_pool->has_available_process = true; + } else if (ok) { + process_to_shutdown = std::move(process); + } else [[unlikely]] { + LOG(WARNING) << "Failed to create Python process " << (i + 1) << "/" + << max_pool_size << " for version " << version.to_string() << ": " + << status.to_string(); + } + } + init_finished_count->fetch_add(1, std::memory_order_acq_rel); + function_pool->cv.notify_all(); + if (process_to_shutdown) { + process_to_shutdown->shutdown(); + } + TEST_SYNC_POINT_CALLBACK( + "PythonServerManager::_ensure_pool_initialized:init_worker_finished", &version, + init_finished_count.get()); + }).detach(); } - auto versioned_pool = versioned_pool_result.value(); - const int max_pool_size = config::max_python_process_num > 0 ? config::max_python_process_num - : CpuInfo::num_cores(); +} + +Result> +PythonServerManager::_ensure_pool_initialized( + const PythonUDFMeta& meta, const PythonVersion& version, + std::shared_ptr package_cache_entry, + std::shared_ptr* lease) { + auto function_pool_result = + _get_or_create_process_pool(meta, version, std::move(package_cache_entry), lease); + if (!function_pool_result.has_value()) { + return ResultError(function_pool_result.error()); + } + auto function_pool = function_pool_result.value(); auto process_pool_init_timeout = PROCESS_POOL_INIT_TIMEOUT; TEST_SYNC_POINT_CALLBACK( "PythonServerManager::_ensure_pool_initialized:process_pool_init_timeout", &process_pool_init_timeout); - std::unique_lock lock(versioned_pool->mutex); - if (versioned_pool->state == PoolState::INITIALIZED) { - return versioned_pool; + std::unique_lock lock(function_pool->mutex); + if (function_pool->state == PoolState::INITIALIZED) { + return function_pool; } - if (versioned_pool->state != PoolState::STOPPED) { - if (versioned_pool->state != PoolState::INITIALIZING) { - versioned_pool->state = PoolState::INITIALIZING; - versioned_pool->has_available_process = false; - versioned_pool->processes.resize(max_pool_size); - auto init_finished_count = std::make_shared>(0); - TEST_SYNC_POINT_CALLBACK( - "PythonServerManager::_ensure_pool_initialized:generation_started", &version, - init_finished_count.get()); - - LOG(INFO) << "Initializing Python process pool for version " << version.to_string() - << " with " << max_pool_size << " processes (config::max_python_process_num=" - << config::max_python_process_num << ", CPU cores=" << CpuInfo::num_cores() - << ")"; - - std::thread([this, versioned_pool, init_finished_count, max_pool_size, - process_pool_init_timeout]() { - SCOPED_INIT_THREAD_CONTEXT(); - std::unique_lock lock(versioned_pool->mutex); - versioned_pool->cv.wait_for( - lock, process_pool_init_timeout, - [&versioned_pool, init_finished_count, max_pool_size]() { - return versioned_pool->state != PoolState::INITIALIZING || - init_finished_count->load(std::memory_order_acquire) >= - max_pool_size; - }); - if (versioned_pool->state == PoolState::INITIALIZING) { - if (versioned_pool->has_available_process) { - // Keep this under the pool lock. shutdown() must acquire the same lock - // before it can destroy manager-owned health-check state. - _start_health_check_thread(); - versioned_pool->state = PoolState::INITIALIZED; - } else { - versioned_pool->state = PoolState::UNINITIALIZED; - } - } - versioned_pool->cv.notify_all(); - }).detach(); - - for (int i = 0; i < max_pool_size; ++i) { - std::thread([version, versioned_pool, i, max_pool_size, init_finished_count]() { - SCOPED_INIT_THREAD_CONTEXT(); - ProcessPtr process; - Status status = PythonServerManager::fork(version, &process); - const bool ok = status.ok() && process; - ProcessPtr process_to_shutdown; - { - std::lock_guard lock(versioned_pool->mutex); - // shutdown() and repair can race with detached init workers after timeout. - // Late successful forks only fill slots that are still empty or dead. - if (ok && - (versioned_pool->state == PoolState::INITIALIZING || - versioned_pool->state == PoolState::INITIALIZED) && - i < versioned_pool->processes.size() && - (!versioned_pool->processes[i] || - !versioned_pool->processes[i]->is_alive())) { - versioned_pool->processes[i] = std::move(process); - versioned_pool->has_available_process = true; - } else if (ok) { - process_to_shutdown = std::move(process); - } else [[unlikely]] { - LOG(WARNING) << "Failed to create Python process " << (i + 1) << "/" - << max_pool_size << " for version " << version.to_string() - << ": " << status.to_string(); - } - } - init_finished_count->fetch_add(1, std::memory_order_acq_rel); - versioned_pool->cv.notify_all(); - if (process_to_shutdown) { - process_to_shutdown->shutdown(); - } - TEST_SYNC_POINT_CALLBACK( - "PythonServerManager::_ensure_pool_initialized:init_worker_finished", - &version, init_finished_count.get()); - }).detach(); - } + if (function_pool->state != PoolState::STOPPED) { + if (function_pool->state != PoolState::INITIALIZING) { + _start_pool_initialization(function_pool, version, process_pool_init_timeout); } // Wait only for the first usable process. INITIALIZED is set later by the last init worker // after every slot has attempted initialization. - versioned_pool->cv.wait_for(lock, process_pool_init_timeout, [&versioned_pool]() { - return versioned_pool->has_available_process || - versioned_pool->state == PoolState::STOPPED || - versioned_pool->state != PoolState::INITIALIZING; + function_pool->cv.wait_for(lock, process_pool_init_timeout, [&function_pool]() { + return function_pool->has_available_process || + function_pool->state == PoolState::STOPPED || + function_pool->state != PoolState::INITIALIZING; }); - if (versioned_pool->has_available_process) { - return versioned_pool; + if (function_pool->has_available_process) { + return function_pool; } - versioned_pool->cv.notify_all(); + function_pool->cv.notify_all(); } return ResultError(Status::Error( @@ -278,90 +348,95 @@ PythonServerManager::_ensure_pool_initialized(const PythonVersion& version) { version.to_string(), process_pool_init_timeout.count())); } -Status PythonServerManager::_get_process( - const PythonVersion& version, const std::shared_ptr& versioned_pool, - ProcessPtr* process) { +void PythonServerManager::_start_pool_repair( + const PythonVersion& version, const std::shared_ptr& function_pool) { + // Called with the pool lock held so concurrent requests cannot schedule the same repair. + function_pool->repairing = true; + // Repair is done in the background because fork can be slow. The current request still + // waits briefly below so a transient all-dead pool can recover without failing. + std::thread([version, function_pool]() { + SCOPED_INIT_THREAD_CONTEXT(); + int recreated = PythonServerManager::_repair_process_pool(version, function_pool); + { + std::scoped_lock lock(function_pool->mutex); + function_pool->repairing = false; + } + function_pool->cv.notify_all(); + if (recreated > 0) { + LOG(INFO) << "Repaired Python process pool for version " << version.to_string() + << ": recreated=" << recreated; + } + }).detach(); +} + +Status PythonServerManager::_get_process(const PythonVersion& version, + const std::shared_ptr& function_pool, + ProcessPtr* process) { { - std::unique_lock lock(versioned_pool->mutex); - std::vector& pool = versioned_pool->processes; + std::unique_lock lock(function_pool->mutex); + std::vector& pool = function_pool->processes; - if (versioned_pool->state == PoolState::STOPPED) { - versioned_pool->has_available_process = false; + if (function_pool->state == PoolState::STOPPED) { + function_pool->has_available_process = false; return Status::Error( "Python process pool has stopped for version {}", version.to_string()); } if (_select_alive_process_from_pool(pool, process)) [[likely]] { - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; return Status::OK(); } - versioned_pool->has_available_process = false; + function_pool->has_available_process = false; - if (versioned_pool->state == PoolState::INITIALIZING) { - versioned_pool->cv.wait_for(lock, PROCESS_REPAIR_WAIT_TIMEOUT, [&versioned_pool]() { - return std::any_of(versioned_pool->processes.begin(), - versioned_pool->processes.end(), + if (function_pool->state == PoolState::INITIALIZING) { + function_pool->cv.wait_for(lock, PROCESS_REPAIR_WAIT_TIMEOUT, [&function_pool]() { + return std::any_of(function_pool->processes.begin(), function_pool->processes.end(), [](const ProcessPtr& p) { return p && p->is_alive(); }) || - versioned_pool->state != PoolState::INITIALIZING; + function_pool->state != PoolState::INITIALIZING; }); - if (versioned_pool->state == PoolState::STOPPED) { + if (function_pool->state == PoolState::STOPPED) { return Status::Error( "Python process pool has stopped for version {}", version.to_string()); } if (_select_alive_process_from_pool(pool, process)) { - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; return Status::OK(); } - versioned_pool->has_available_process = false; + function_pool->has_available_process = false; return Status::Error( "Python process pool is initializing but has no available process for version " "{} after waiting {} ms", version.to_string(), PROCESS_REPAIR_WAIT_TIMEOUT.count()); } - if (versioned_pool->state != PoolState::INITIALIZED) { + if (function_pool->state != PoolState::INITIALIZED) { return Status::Error( "Python process pool is not initialized for version {}", version.to_string()); } - if (!versioned_pool->repairing) { - versioned_pool->repairing = true; - // Repair is done in the background because fork can be slow. The current request still - // waits briefly below so a transient all-dead pool can recover without failing. - std::thread([version, versioned_pool]() { - SCOPED_INIT_THREAD_CONTEXT(); - int recreated = PythonServerManager::_repair_process_pool(version, versioned_pool); - { - std::lock_guard lock(versioned_pool->mutex); - versioned_pool->repairing = false; - } - versioned_pool->cv.notify_all(); - if (recreated > 0) { - LOG(INFO) << "Repaired Python process pool for version " << version.to_string() - << ": recreated=" << recreated; - } - }).detach(); + if (!function_pool->repairing) { + _start_pool_repair(version, function_pool); } // Keep the request recoverable in the common case where the Python runtime can fork // normally and only the existing pool entries died. The wait is short so a wedged fork path // still returns SERVICE_UNAVAILABLE promptly. - versioned_pool->cv.wait_for(lock, PROCESS_REPAIR_WAIT_TIMEOUT, [&versioned_pool]() { - return std::any_of(versioned_pool->processes.begin(), versioned_pool->processes.end(), + function_pool->cv.wait_for(lock, PROCESS_REPAIR_WAIT_TIMEOUT, [&function_pool]() { + return std::any_of(function_pool->processes.begin(), function_pool->processes.end(), [](const ProcessPtr& p) { return p && p->is_alive(); }) || - versioned_pool->state == PoolState::STOPPED; + function_pool->state == PoolState::STOPPED; }); - if (versioned_pool->state == PoolState::STOPPED) { - versioned_pool->has_available_process = false; + if (function_pool->state == PoolState::STOPPED) { + function_pool->has_available_process = false; return Status::Error( "Python process pool has stopped for version {}", version.to_string()); } if (_select_alive_process_from_pool(pool, process)) { - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; return Status::OK(); } - versioned_pool->has_available_process = false; + function_pool->has_available_process = false; } return Status::Error( @@ -458,14 +533,47 @@ Status PythonServerManager::fork(const PythonVersion& version, ProcessPtr* proce return Status::OK(); } +void PythonServerManager::_reclaim_idle_pools(std::chrono::steady_clock::time_point now) { + std::vector> retired; + { + std::scoped_lock lock(_pools_mutex); + std::erase_if(_process_pools, [&](const auto& entry) { + const auto& pool = entry.second; + std::scoped_lock pool_lock(pool->mutex); + if (pool->auto_suspend < 0 || pool->active_clients != 0 || + std::chrono::duration_cast(now - pool->idle_since).count() < + pool->auto_suspend) { + return false; + } + // Stop late initialization/repair workers, but allow the next call to create a new pool. + pool->state = PoolState::STOPPED; + pool->has_available_process = false; + retired.emplace_back(pool); + return true; + }); + } + for (const auto& pool : retired) { + pool->cv.notify_all(); + std::vector processes; + { + std::scoped_lock lock(pool->mutex); + processes.swap(pool->processes); + } + for (const auto& process : processes) { + if (process) { + process->shutdown(); + } + } + } +} + void PythonServerManager::_start_health_check_thread() { - std::lock_guard lock(_health_check_mutex); + std::scoped_lock lock(_health_check_mutex); if (_health_check_thread || _shutdown_flag.load(std::memory_order_acquire)) return; - LOG(INFO) << "Starting Python process health check thread (interval: 30 seconds)"; + LOG(INFO) << "Starting Python process monitor"; _health_check_thread = std::make_unique([this]() { - // Health check loop while (!_shutdown_flag.load(std::memory_order_acquire)) { // Wait for interval or shutdown signal { @@ -477,6 +585,7 @@ void PythonServerManager::_start_health_check_thread() { if (_shutdown_flag.load(std::memory_order_acquire)) break; + _reclaim_idle_pools(std::chrono::steady_clock::now()); _check_and_recreate_processes(); _refresh_memory_stats(); } @@ -487,22 +596,22 @@ void PythonServerManager::_start_health_check_thread() { void PythonServerManager::_check_and_recreate_processes() { int total_recreated = 0; - for (auto& [version, versioned_pool] : _snapshot_process_pools()) { + for (const auto& function_pool : _snapshot_process_pools()) { { - std::lock_guard lock(versioned_pool->mutex); - if (versioned_pool->state != PoolState::INITIALIZED || versioned_pool->repairing) { + std::scoped_lock lock(function_pool->mutex); + if (function_pool->state != PoolState::INITIALIZED || function_pool->repairing) { continue; } // Share the same repair guard with foreground requests. Otherwise health check and // _get_process() can fork the same empty/dead slots at the same time under failures. - versioned_pool->repairing = true; + function_pool->repairing = true; } - int recreated = _repair_process_pool(version, versioned_pool); + int recreated = _repair_process_pool(function_pool->version, function_pool); { - std::lock_guard lock(versioned_pool->mutex); - versioned_pool->repairing = false; + std::scoped_lock lock(function_pool->mutex); + function_pool->repairing = false; } - versioned_pool->cv.notify_all(); + function_pool->cv.notify_all(); total_recreated += recreated; } @@ -512,17 +621,16 @@ void PythonServerManager::_check_and_recreate_processes() { } int PythonServerManager::_repair_process_pool( - const PythonVersion& version, const std::shared_ptr& versioned_pool) { - const int max_pool_size = config::max_python_process_num > 0 ? config::max_python_process_num - : CpuInfo::num_cores(); + const PythonVersion& version, const std::shared_ptr& function_pool) { + const int max_pool_size = function_pool->pool_size; std::vector died_process_indices; { - std::lock_guard lock(versioned_pool->mutex); - if (versioned_pool->state != PoolState::INITIALIZED) { + std::scoped_lock lock(function_pool->mutex); + if (function_pool->state != PoolState::INITIALIZED) { return 0; } - auto& pool = versioned_pool->processes; + auto& pool = function_pool->processes; died_process_indices.reserve(std::max(pool.size(), max_pool_size)); // Need to fix the following two cases // 1. Existing processes have died @@ -546,8 +654,8 @@ int PythonServerManager::_repair_process_pool( std::vector processes_to_shutdown; for (size_t index : died_process_indices) { { - std::lock_guard lock(versioned_pool->mutex); - if (versioned_pool->state != PoolState::INITIALIZED) { + std::scoped_lock lock(function_pool->mutex); + if (function_pool->state != PoolState::INITIALIZED) { break; } } @@ -557,22 +665,26 @@ int PythonServerManager::_repair_process_pool( if (status.ok() && new_process) { bool published = false; { - std::lock_guard lock(versioned_pool->mutex); - auto& pool = versioned_pool->processes; - if (versioned_pool->state != PoolState::INITIALIZED) { + std::scoped_lock lock(function_pool->mutex); + auto& pool = function_pool->processes; + if (function_pool->state != PoolState::INITIALIZED) { processes_to_shutdown.emplace_back(std::move(new_process)); } else if (index < pool.size()) { if (!pool[index] || !pool[index]->is_alive()) [[likely]] { + function_pool->process_registry->add(new_process); + new_process->set_package_cache_entry(function_pool->package_cache_entry); pool[index] = std::move(new_process); - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; recreated++; published = true; } else { processes_to_shutdown.emplace_back(std::move(new_process)); } } else if (pool.size() < static_cast(max_pool_size)) { + function_pool->process_registry->add(new_process); + new_process->set_package_cache_entry(function_pool->package_cache_entry); pool.emplace_back(std::move(new_process)); - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; recreated++; published = true; } else { @@ -580,7 +692,7 @@ int PythonServerManager::_repair_process_pool( } } if (published) { - versioned_pool->cv.notify_all(); + function_pool->cv.notify_all(); } } else { LOG(ERROR) << "Failed to recreate Python process for version " << version.to_string() @@ -589,21 +701,20 @@ int PythonServerManager::_repair_process_pool( } { - std::lock_guard lock(versioned_pool->mutex); - auto& pool = versioned_pool->processes; + std::scoped_lock lock(function_pool->mutex); + auto& pool = function_pool->processes; - if (versioned_pool->state != PoolState::INITIALIZED) { - versioned_pool->has_available_process = false; + if (function_pool->state != PoolState::INITIALIZED) { + function_pool->has_available_process = false; } else { // Keep empty/dead slots instead of shrinking the vector. Init workers are detached and // publish by original slot index; shrinking here would make a late successful init look // out-of-range and discard a usable process. - versioned_pool->has_available_process = std::any_of( - pool.begin(), pool.end(), - [](const ProcessPtr& process) { return process && process->is_alive(); }); + function_pool->has_available_process = std::ranges::any_of( + pool, [](const ProcessPtr& process) { return process && process->is_alive(); }); } } - versioned_pool->cv.notify_all(); + function_pool->cv.notify_all(); for (auto& process : processes_to_shutdown) { process->shutdown(); } @@ -611,31 +722,31 @@ int PythonServerManager::_repair_process_pool( } void PythonServerManager::shutdown() { - std::vector>> pools_to_shutdown; + std::vector> pools_to_shutdown; { - std::lock_guard lock(_pools_mutex); + std::scoped_lock lock(_pools_mutex); _shutdown_flag.store(true, std::memory_order_release); pools_to_shutdown.reserve(_process_pools.size()); - for (auto& [version, versioned_pool] : _process_pools) { - pools_to_shutdown.emplace_back(version, std::move(versioned_pool)); + for (auto& [function_id, function_pool] : _process_pools) { + pools_to_shutdown.emplace_back(std::move(function_pool)); } _process_pools.clear(); } - for (auto& [version, versioned_pool] : pools_to_shutdown) { - if (!versioned_pool) { + for (const auto& function_pool : pools_to_shutdown) { + if (!function_pool) { continue; } - std::lock_guard lock(versioned_pool->mutex); - versioned_pool->state = PoolState::STOPPED; - versioned_pool->has_available_process = false; - versioned_pool->repairing = false; - versioned_pool->cv.notify_all(); + std::scoped_lock lock(function_pool->mutex); + function_pool->state = PoolState::STOPPED; + function_pool->has_available_process = false; + function_pool->repairing = false; + function_pool->cv.notify_all(); } std::unique_ptr health_check_thread; { - std::lock_guard lock(_health_check_mutex); + std::scoped_lock lock(_health_check_mutex); health_check_thread = std::move(_health_check_thread); } _health_check_cv.notify_one(); @@ -643,23 +754,18 @@ void PythonServerManager::shutdown() { health_check_thread->join(); } - // Shutdown all processes - std::vector processes_to_shutdown; - for (auto& [version, versioned_pool] : pools_to_shutdown) { - if (!versioned_pool) { + // Include processes retained by clients after DROP. Snapshot ownership keeps pool clearing + // from destroying a process under the pool lock; all shutdowns happen outside locks. + auto processes_to_shutdown = _process_registry->snapshot(); + for (const auto& function_pool : pools_to_shutdown) { + if (!function_pool) { continue; } - std::lock_guard lock(versioned_pool->mutex); - auto& pool = versioned_pool->processes; - for (auto& process : pool) { - if (process) { - processes_to_shutdown.emplace_back(std::move(process)); - } - } - pool.clear(); - versioned_pool->cv.notify_all(); + std::scoped_lock lock(function_pool->mutex); + function_pool->processes.clear(); + function_pool->cv.notify_all(); } - for (auto& process : processes_to_shutdown) { + for (const auto& process : processes_to_shutdown) { process->shutdown(); } } @@ -692,21 +798,19 @@ Status PythonServerManager::_read_process_memory(pid_t pid, size_t* rss_bytes) { void PythonServerManager::_refresh_memory_stats() { int64_t total_rss = 0; - for (const auto& [version, versioned_pool] : _snapshot_process_pools()) { - std::lock_guard lock(versioned_pool->mutex); - const auto& pool = versioned_pool->processes; - for (const auto& process : pool) { - if (!process || !process->is_alive()) continue; - - size_t rss_bytes = 0; - Status s = _read_process_memory(process->get_child_pid(), &rss_bytes); + // DROP removes scheduling pools immediately, but active clients can retain their processes. + for (const auto& process : _process_registry->snapshot()) { + if (!process->is_alive()) { + continue; + } - if (s.ok()) { - total_rss += rss_bytes; - } else [[unlikely]] { - LOG(WARNING) << "Failed to read memory info for Python process (pid=" - << process->get_child_pid() << "): " << s.to_string(); - } + size_t rss_bytes = 0; + Status s = _read_process_memory(process->get_child_pid(), &rss_bytes); + if (s.ok()) { + total_rss += rss_bytes; + } else [[unlikely]] { + LOG(WARNING) << "Failed to read memory info for Python process (pid=" + << process->get_child_pid() << "): " << s.to_string(); } } _mem_tracker.set_consumption(total_rss); @@ -719,104 +823,46 @@ void PythonServerManager::_refresh_memory_stats() { } } -Status PythonServerManager::clear_module_cache(const std::string& location) { - if (location.empty()) { - return Status::InvalidArgument("Empty location for clear_module_cache"); - } - - std::string body = fmt::format(R"({{"location": "{}"}})", location); - return _broadcast_action_to_processes("clear_module_cache", body, - fmt::format("location={}", location)); -} - -void PythonServerManager::clear_udaf_state_cache(int64_t function_id) { - std::string body = fmt::format(R"({{"function_id": {}}})", function_id); - WARN_IF_ERROR(_broadcast_action_to_processes("clear_udaf_state_cache", body, - fmt::format("function_id={}", function_id)), - "failed to clear Python UDAF state cache"); -} - -Status PythonServerManager::_broadcast_action_to_processes(const std::string& action_type, - const std::string& body, - const std::string& log_name) { - int success_count = 0; - int fail_count = 0; - bool has_active_process = false; - - for (auto& [version, versioned_pool] : _snapshot_process_pools()) { - std::lock_guard lock(versioned_pool->mutex); - auto& pool = versioned_pool->processes; - for (auto& process : pool) { - if (!process || !process->is_alive()) { - continue; - } - has_active_process = true; - try { - auto loc_result = arrow::flight::Location::Parse(process->get_uri()); - if (!loc_result.ok()) [[unlikely]] { - fail_count++; - continue; - } - - auto client_result = arrow::flight::FlightClient::Connect(*loc_result); - if (!client_result.ok()) [[unlikely]] { - fail_count++; - continue; - } - auto client = std::move(*client_result); - - arrow::flight::Action action; - action.type = action_type; - action.body = arrow::Buffer::FromString(body); - - auto result_stream = client->DoAction(action); - if (!result_stream.ok()) { - fail_count++; - continue; - } - - auto result = (*result_stream)->Next(); - if (result.ok() && *result) { - success_count++; - } else { - fail_count++; - } - - } catch (...) { - fail_count++; - } +void PythonServerManager::drop_function(int64_t function_id) { + std::shared_ptr pool; + std::vector processes; + { + std::scoped_lock map_lock(_pools_mutex); + auto iter = _process_pools.find(function_id); + if (iter == _process_pools.end()) { + return; } + pool = std::move(iter->second); + // Stop publication before removing the pool, so concurrent shutdown cannot miss a late fork. + { + std::scoped_lock pool_lock(pool->mutex); + pool->state = PoolState::STOPPED; + pool->has_available_process = false; + processes.swap(pool->processes); + } + _process_pools.erase(iter); } - - if (!has_active_process) { - return Status::OK(); - } - - LOG(INFO) << action_type << " completed for " << log_name << ", success=" << success_count - << ", failed=" << fail_count; - - if (fail_count > 0) { - return Status::InternalError("{} failed for {}, success={}, failed={}", action_type, - log_name, success_count, fail_count); - } - - return Status::OK(); + pool->cv.notify_all(); + // Release pool ownership outside the mutex. Active Flight clients retain their process; + // the final client destroys it, shutting down Python before the zip is deleted. } // Explicit template instantiation for UDF, UDAF and UDTF clients template Status PythonServerManager::get_client( const PythonUDFMeta& func_meta, const PythonVersion& version, - std::shared_ptr* client, - const std::shared_ptr& data_schema); + std::shared_ptr* client, const std::shared_ptr& data_schema, + std::shared_ptr package_cache_entry); template Status PythonServerManager::get_client( const PythonUDFMeta& func_meta, const PythonVersion& version, std::shared_ptr* client, - const std::shared_ptr& data_schema); + const std::shared_ptr& data_schema, + std::shared_ptr package_cache_entry); template Status PythonServerManager::get_client( const PythonUDFMeta& func_meta, const PythonVersion& version, std::shared_ptr* client, - const std::shared_ptr& data_schema); + const std::shared_ptr& data_schema, + std::shared_ptr package_cache_entry); } // namespace doris diff --git a/be/src/udf/python/python_server.h b/be/src/udf/python/python_server.h index c72a2db43d1b94..48684741d27123 100644 --- a/be/src/udf/python/python_server.h +++ b/be/src/udf/python/python_server.h @@ -32,6 +32,8 @@ #include "udf/python/python_udf_runtime.h" namespace doris { +struct UserFunctionCacheEntry; +class PythonUDFPoolLease; class PythonServerManager { public: PythonServerManager() = default; @@ -46,15 +48,13 @@ class PythonServerManager { template Status get_client(const PythonUDFMeta& func_meta, const PythonVersion& version, std::shared_ptr* client, - const std::shared_ptr& data_schema = nullptr); + const std::shared_ptr& data_schema = nullptr, + std::shared_ptr package_cache_entry = nullptr); static Status fork(const PythonVersion& version, ProcessPtr* process); - // Clear Python module cache for a specific UDF location across all processes - Status clear_module_cache(const std::string& location); - - // Clear Python UDAF runtime state after DROP FUNCTION - void clear_udaf_state_cache(int64_t function_id); + // Remove this function's pool. Existing clients keep their processes and zip alive. + void drop_function(int64_t function_id); void shutdown(); @@ -62,40 +62,55 @@ class PythonServerManager { // For unit testing only. void check_and_recreate_processes_for_test() { _check_and_recreate_processes(); } - void set_process_pool_for_test(const PythonVersion& version, std::vector processes, - bool initialized = true); + void set_process_pool_for_test(const PythonUDFMeta& meta, const PythonVersion& version, + std::vector processes, bool initialized = true); - std::vector process_pool_snapshot_for_test(const PythonVersion& version); + std::vector process_pool_snapshot_for_test(int64_t function_id); - bool process_pool_is_initialized_for_test(const PythonVersion& version); + bool process_pool_is_initialized_for_test(int64_t function_id); - bool wait_for_process_pool_initialized_for_test(const PythonVersion& version, + bool wait_for_process_pool_initialized_for_test(int64_t function_id, std::chrono::milliseconds timeout); - bool wait_for_process_pool_initialization_finished_for_test(const PythonVersion& version, + bool wait_for_process_pool_initialization_finished_for_test(int64_t function_id, std::chrono::milliseconds timeout); - Status broadcast_action_to_processes_for_test(const std::string& action_type, - const std::string& body, - const std::string& log_name) { - return _broadcast_action_to_processes(action_type, body, log_name); - } #endif private: + friend class PythonUDFPoolLease; + enum class PoolState { UNINITIALIZED, INITIALIZING, // `Initialized` means: // 1. All process slots have attempted initialization. // 2. At least one live process is available for requests. - // 3. The health-check thread has been started. INITIALIZED, // Prevent the pool from being incorrectly reused after `shutdown()` STOPPED, }; - struct VersionedProcessPool { + // Includes processes held by clients after DROP, without extending their lifetime. + struct ProcessRegistry { + void add(const ProcessPtr& process); + std::vector snapshot(); + + std::mutex mutex; + std::vector> processes; + }; + + struct FunctionProcessPool { + // Interpreter information needed to replace dead processes in this function pool. + PythonVersion version; + // Detached fork/repair workers can outlive the manager. + std::shared_ptr process_registry; + int64_t function_id = 0; + int32_t pool_size = 3; + int64_t auto_suspend = -1; + size_t active_clients = 0; + std::chrono::steady_clock::time_point idle_since = std::chrono::steady_clock::now(); + std::shared_ptr package_cache_entry; std::mutex mutex; // Coordinates initialization and repair workers with foreground requests. std::condition_variable cv; @@ -107,25 +122,36 @@ class PythonServerManager { bool repairing = false; }; - /** - * Lazily initialize and return the process pool for specific Python version. + /** + * Lazily initialize and return the process pool for specific function. */ - Result> _ensure_pool_initialized( - const PythonVersion& version); + Result> _ensure_pool_initialized( + const PythonUDFMeta& meta, const PythonVersion& version, + std::shared_ptr package_cache_entry = nullptr, + std::shared_ptr* lease = nullptr); /** * Pick an available process from specific pool, recreating one on demand if needed. */ + static void _start_pool_initialization( + const std::shared_ptr& function_pool, const PythonVersion& version, + std::chrono::milliseconds process_pool_init_timeout); + + static void _start_pool_repair(const PythonVersion& version, + const std::shared_ptr& function_pool); + Status _get_process(const PythonVersion& version, - const std::shared_ptr& versioned_pool, + const std::shared_ptr& function_pool, ProcessPtr* process); /** - * Start health check background thread (called once by ensure_pool_initialized) + * Start health check background thread when the first function pool is created * Thread periodically checks process health and refreshes memory stats */ void _start_health_check_thread(); + void _reclaim_idle_pools(std::chrono::steady_clock::time_point now); + /** * Check process health and recreate dead processes */ @@ -145,7 +171,7 @@ class PythonServerManager { static constexpr std::chrono::milliseconds PROCESS_REPAIR_WAIT_TIMEOUT {1000}; #endif static int _repair_process_pool(const PythonVersion& version, - const std::shared_ptr& versioned_pool); + const std::shared_ptr& function_pool); /** * Read resident set size (RSS) for a single process from /proc/{pid}/statm @@ -157,18 +183,21 @@ class PythonServerManager { */ void _refresh_memory_stats(); - Result> _get_or_create_process_pool( - const PythonVersion& version); - std::vector>> - _snapshot_process_pools(); - Status _broadcast_action_to_processes(const std::string& action_type, const std::string& body, - const std::string& log_name); + Result> _get_or_create_process_pool( + const PythonUDFMeta& meta, const PythonVersion& version, + std::shared_ptr package_cache_entry, + std::shared_ptr* lease = nullptr); + std::vector> _snapshot_process_pools(); static bool _select_alive_process_from_pool(const std::vector& pool, ProcessPtr* process); - std::unordered_map> _process_pools; - // Protects the version -> pool handle map only. Per-version process operations are guarded - // by VersionedProcessPool::mutex. + // Weak references keep dropped-but-active processes visible without keeping them alive. + std::shared_ptr _process_registry = std::make_shared(); + + // function_id -> process_pool + std::unordered_map> _process_pools; + // Protects the function id -> pool map. DROP also stops the pool under this lock; + // when taking both locks, acquire this before FunctionProcessPool::mutex. std::mutex _pools_mutex; // Health check background thread std::unique_ptr _health_check_thread; diff --git a/be/src/udf/python/python_server.py b/be/src/udf/python/python_server.py index 14c28d6ef3c198..01b6c6801dc57d 100644 --- a/be/src/udf/python/python_server.py +++ b/be/src/udf/python/python_server.py @@ -31,7 +31,6 @@ import pickle import uuid from abc import ABC, abstractmethod -from contextlib import contextmanager from typing import Any, Callable, Optional, Tuple, get_origin, Dict from datetime import datetime from enum import Enum @@ -168,30 +167,6 @@ def monitor_parent_exit(): monitor_thread.start() -@contextmanager -def temporary_sys_path(path: str): - """ - Context manager to temporarily add a path to sys.path. - Ensures the path is removed after use to avoid pollution. - - Args: - path: Directory path to add to sys.path - - Yields: - None - """ - path_added = False - if path not in sys.path: - sys.path.insert(0, path) - path_added = True - - try: - yield - finally: - if path_added and path in sys.path: - sys.path.remove(path) - - def int32_to_uint32(value: int) -> int: """ Convert a signed int32 to unsigned uint32 representation. @@ -962,9 +937,8 @@ class ModuleUDFLoader(UDFLoader): """Loads a UDF from a Python module file (.py).""" # Module names that are forbidden for UDFs because they conflict with - # modules already imported by the server process. Loading a user module - # with one of these names would overwrite the entry in sys.modules and - # could break the server itself. + # modules already imported by the server process. A conflicting name could + # resolve to the server module instead of the requested UDF module. _FORBIDDEN_MODULE_NAMES: frozenset = frozenset({ "argparse", "base64", "gc", "importlib", "inspect", "ipaddress", "json", "sys", "os", "traceback", "logging", "time", "threading", @@ -973,48 +947,6 @@ class ModuleUDFLoader(UDFLoader): "logging.handlers", }) - # Class-level lock dictionary for thread-safe module imports - # Using RLock allows the same thread to acquire the lock multiple times - - # Key for _import_locks: module_name only (not location) - # sys.modules is a global dict keyed by module name. - # we need to ensure that imports with the same module name - # do not interfere with each other across different threads, - # even if they come from different file paths. - _import_locks: Dict[str, threading.Lock] = {} - _import_locks_lock = threading.Lock() - - # Key for _module_cache: location only - # since location already contains a unique function_id - _module_cache: Dict[str, Any] = {} - _module_cache_lock = threading.Lock() - - @classmethod - def _get_import_lock(cls, module_name: str) -> threading.Lock: - """ - Get or create an import lock for the given module namespace. - - Uses double-checked locking pattern for optimal performance: - - Fast path: return existing lock without acquiring global lock - - Slow path: create new lock under global lock protection - """ - # Lock by top-level package to avoid concurrent imports mutating shared - # parent entries in sys.modules. If we lock by full module name instead, - # pkg.mod.func1 and pkg.mod.func2 can import in parallel and race while - # initializing pkg/pkg.mod, causing flaky import failures (for example KeyError). - cache_key = module_name.split(".", 1)[0] - - # Fast path: check without lock (read-only, safe for most cases) - if cache_key in cls._import_locks: - return cls._import_locks[cache_key] - - # Slow path: create lock under protection - with cls._import_locks_lock: - # Double-check: another thread might have created it while we waited - if cache_key not in cls._import_locks: - cls._import_locks[cache_key] = threading.Lock() - return cls._import_locks[cache_key] - def load(self) -> AdaptivePythonUDF: """ Loads a UDF from a Python module file. @@ -1080,29 +1012,8 @@ def parse_symbol(self, symbol: str): return package_name, module_name, func_name - @staticmethod - def _clear_modules_from_sys(full_module_name: str) -> None: - """Remove a module and all its ancestor packages from sys.modules. - - To prevent the same module from being polluted by old caches - when loaded from different paths. - e.g., the pkg under path_a affecting the pkg.mdu_a under path_b, - the ancestor chain is cleared after each import. - - This ensures that subsequent imports always start from a fresh state. - """ - parts = full_module_name.split(".") - for i in range(len(parts)): - ancestor = ".".join(parts[: i + 1]) - sys.modules.pop(ancestor, None) - - def _get_or_import_module(self, location: str, full_module_name: str) -> Any: - """ - Get module from cache or import it (thread-safe). - - The cache is keyed by location alone, which already contains a unique - function_id assigned by the FE catalog. - """ + def _get_or_import_module(self, full_module_name: str) -> Any: + """Use Python's module cache and import locks within the function process.""" # Reject module names that would shadow server-critical modules top_level_name = full_module_name.split(".")[0] if top_level_name in ModuleUDFLoader._FORBIDDEN_MODULE_NAMES: @@ -1113,39 +1024,7 @@ def _get_or_import_module(self, location: str, full_module_name: str) -> Any: f"server-critical modules." ) - cache_key = location - - # Use a per-module lock to prevent race conditions during import - import_lock = ModuleUDFLoader._get_import_lock(full_module_name) - - with import_lock: - # Fast path: check cache first - if cache_key in ModuleUDFLoader._module_cache: - cached_module = ModuleUDFLoader._module_cache[cache_key] - if cached_module is not None and ( - hasattr(cached_module, "__file__") - or hasattr(cached_module, "__path__") - ): - return cached_module - else: - del ModuleUDFLoader._module_cache[cache_key] - - self._clear_modules_from_sys(full_module_name) - - with temporary_sys_path(location): - try: - module = importlib.import_module(full_module_name) - ModuleUDFLoader._module_cache[cache_key] = module - # Evict from sys.modules so future imports from a - # different location are not poisoned by this one. - self._clear_modules_from_sys(full_module_name) - return module - except Exception: - # Clean up any partially-imported modules - self._clear_modules_from_sys(full_module_name) - if cache_key in ModuleUDFLoader._module_cache: - del ModuleUDFLoader._module_cache[cache_key] - raise + return importlib.import_module(full_module_name) def _extract_function( self, module: Any, func_name: str, module_name: str @@ -1196,7 +1075,7 @@ def _load_single_file_udf( raise ImportError(f"Python file not found: {py_file}") try: - udf_module = self._get_or_import_module(location, module_name) + udf_module = self._get_or_import_module(module_name) return self._extract_function(udf_module, func_name, module_name) except (ImportError, AttributeError, TypeError) as e: raise ImportError( @@ -1244,7 +1123,7 @@ def _load_package_udf( try: full_module_name = self._build_full_module_name(package_name, module_name) - udf_module = self._get_or_import_module(location, full_module_name) + udf_module = self._get_or_import_module(full_module_name) return self._extract_function(udf_module, func_name, full_module_name) except (ImportError, AttributeError, TypeError) as e: raise ImportError( @@ -1737,6 +1616,9 @@ def __init__(self, location: str): # cannot reuse a class loaded from old inline code. self.udaf_state_managers: Dict[str, UDAFStateManager] = {} self.udaf_managers_lock = threading.Lock() + self._binding_lock = threading.Lock() + self._bound_runtime_version: Optional[str] = None + self._bound_function_id: Optional[int] = None def _get_udaf_state_manager( self, python_udaf_meta: PythonUDFMeta @@ -1855,6 +1737,29 @@ def parse_python_udf_meta( return python_udf_meta + def _validate_process_binding(self, python_udf_meta: PythonUDFMeta) -> None: + """Bind once and reject requests for a different function id.""" + with self._binding_lock: + if self._bound_function_id is None: + # One function owns this process. Keep its module root available + # for imports during evaluate and for pickle's module lookups. + if python_udf_meta.udf_load_type == 1: + sys.path.insert(0, python_udf_meta.location) + self._bound_runtime_version = python_udf_meta.runtime_version + self._bound_function_id = python_udf_meta.id + return + + if self._bound_runtime_version != python_udf_meta.runtime_version: + raise ValueError( + "Python process runtime mismatch: " + f"bound={self._bound_runtime_version}, requested={python_udf_meta.runtime_version}" + ) + if self._bound_function_id != python_udf_meta.id: + raise ValueError( + "Python process function id mismatch: " + f"bound={self._bound_function_id}, requested={python_udf_meta.id}" + ) + @staticmethod def check_schema( record_batch: pa.RecordBatch, expected_schema: pa.Schema @@ -2694,6 +2599,8 @@ def do_exchange( if not python_udf_meta: raise ValueError("Invalid or missing metadata in descriptor") + self._validate_process_binding(python_udf_meta) + if python_udf_meta.is_udf(): self._handle_exchange_udf(python_udf_meta, reader, writer) elif python_udf_meta.is_udaf(): @@ -2712,16 +2619,12 @@ def do_action( Handle Flight actions for cache management. Supported actions: - - "clear_module_cache": Clear Python module cache for a specific location - Body: JSON with "location" field (the UDF cache directory path) - "clear_udaf_state_cache": Clear UDAF runtime state for a dropped function id Body: JSON with "function_id" field """ action_type = action.type - if action_type == "clear_module_cache": - yield from self._handle_clear_module_cache(action.body.to_pybytes()) - elif action_type == "clear_udaf_state_cache": + if action_type == "clear_udaf_state_cache": yield from self._handle_clear_udaf_state_cache(action.body.to_pybytes()) else: raise flight.FlightUnavailableError(f"Unknown action: {action_type}") @@ -2750,87 +2653,6 @@ def _handle_clear_udaf_state_cache(self, body: bytes): "error": str(e) }).encode("utf-8")) - def _handle_clear_module_cache(self, body: bytes): - """ - Clear Python module cache for a specific UDF location. - - This removes modules from sys.modules that were loaded from the specified - location, allowing fresh imports when a new UDF with the same module name - is created. - """ - try: - params = json.loads(body.decode("utf-8")) - location = params.get("location", "") - - if not location: - yield flight.Result(b'{"success": false, "error": "empty location"}') - return - - cleared_modules = self._clear_modules_from_location(location) - - result = { - "success": True, - "cleared_modules": cleared_modules, - "location": location, - } - yield flight.Result(json.dumps(result).encode("utf-8")) - - except Exception as e: - logging.error("clear_module_cache failed: %s", e) - yield flight.Result(json.dumps({ - "success": False, - "error": str(e) - }).encode("utf-8")) - - def _clear_modules_from_location(self, location: str) -> list: - """ - Clear module cache for the given location. - - Acquires per-module import locks to ensure no concurrent import is - in progress for the modules being cleared, preventing race conditions - where sys.modules entries are removed mid-import. - - Returns list of cleared module names. - """ - cleared = [] - - with ModuleUDFLoader._module_cache_lock: - keys_to_remove = [ - key for key in ModuleUDFLoader._module_cache - if key[0] == location - ] - - # For each module, acquire its import lock before clearing. - # This ensures no concurrent _get_or_import_module is in progress - # for this (location, module_name) pair. - for key in keys_to_remove: - _, module_name = key - import_lock = ModuleUDFLoader._get_import_lock(module_name) - - with import_lock: - with ModuleUDFLoader._module_cache_lock: - if key in ModuleUDFLoader._module_cache: - del ModuleUDFLoader._module_cache[key] - - modules_to_remove = [ - name for name, mod in sys.modules.items() - if name == module_name or name.startswith(module_name + ".") - or ( - hasattr(mod, "__file__") and mod.__file__ is not None - and mod.__file__.startswith(location) - ) - ] - for mod_name in modules_to_remove: - del sys.modules[mod_name] - if mod_name not in cleared: - cleared.append(mod_name) - - if module_name not in cleared: - cleared.append(module_name) - - return cleared - - class UDAFOperationType(Enum): """Enum representing UDAF operation types.""" diff --git a/be/src/udf/python/python_udaf_client.cpp b/be/src/udf/python/python_udaf_client.cpp index 8b39524afa5696..a053da1e0197df 100644 --- a/be/src/udf/python/python_udaf_client.cpp +++ b/be/src/udf/python/python_udaf_client.cpp @@ -412,7 +412,9 @@ Status PythonUDAFClient::destroy(int64_t place_id) { } Status PythonUDAFClient::close() { - if (!_inited || !_writer) return Status::OK(); + if (!_inited || !_writer) { + return PythonClient::close(); + } // Destroy the place if it exists (cleanup on client destruction) if (_created_place_id.has_value()) { diff --git a/be/src/udf/python/python_udf_meta.h b/be/src/udf/python/python_udf_meta.h index 55c49abb30ad07..8a421d953b6f15 100644 --- a/be/src/udf/python/python_udf_meta.h +++ b/be/src/udf/python/python_udf_meta.h @@ -34,6 +34,8 @@ enum class PythonClientType : uint8_t { UDF = 0, UDAF = 1, UDTF = 2, UNKNOWN = 3 struct PythonUDFMeta { int64_t id = 0; + int32_t pool_size = 3; + int64_t auto_suspend = -1; std::string name; std::string symbol; std::string location; diff --git a/be/src/udf/python/python_udf_runtime.h b/be/src/udf/python/python_udf_runtime.h index 2d635dc11a1a21..9665a110ec131b 100644 --- a/be/src/udf/python/python_udf_runtime.h +++ b/be/src/udf/python/python_udf_runtime.h @@ -19,11 +19,14 @@ #include #include +#include #include "udf/python/python_env.h" namespace doris { +struct UserFunctionCacheEntry; + static const char* UNIX_SOCKET_PREFIX = "grpc+unix://"; // Use /tmp directory for Unix socket to avoid path length limit (max 107 chars) static const char* BASE_UNIX_SOCKET_PATH_TEMPLATE = "/tmp/doris_python_udf"; @@ -89,6 +92,10 @@ class PythonUDFProcess { pid_t get_child_pid() const { return _child_pid; } + void set_package_cache_entry(std::shared_ptr entry) { + _package_cache_entry = std::move(entry); + } + bool operator==(const PythonUDFProcess& other) const { return _child_pid == other._child_pid; } bool operator!=(const PythonUDFProcess& other) const { return !(*this == other); } @@ -104,6 +111,8 @@ class PythonUDFProcess { private: constexpr static size_t MAX_ACCUMULATED_LOG_SIZE = 65536; + // Keep package files available for lazy imports until this process is destroyed. + std::shared_ptr _package_cache_entry; bool _is_shutdown {false}; pid_t _child_pid; std::string _uri; diff --git a/be/test/runtime/user_function_cache_test.cpp b/be/test/runtime/user_function_cache_test.cpp index 57cd3149547e9a..de964ffc4afb8c 100644 --- a/be/test/runtime/user_function_cache_test.cpp +++ b/be/test/runtime/user_function_cache_test.cpp @@ -27,6 +27,7 @@ #include #include "common/status.h" +#include "util/md5.h" namespace doris { @@ -99,6 +100,16 @@ class UserFunctionCacheTest : public ::testing::Test { ofs.close(); } + std::string file_md5(const std::string& path) { + std::ifstream input(path, std::ios::binary); + std::string bytes((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + Md5Digest digest; + digest.update(bytes.data(), bytes.size()); + digest.digest(); + return digest.hex(); + } + private: const char* original_doris_home_ = nullptr; }; @@ -541,4 +552,25 @@ TEST_F(UserFunctionCacheTest, LoadEntryFromLibInvalidFileName) { EXPECT_FALSE(status.ok()); } -} // namespace doris \ No newline at end of file +TEST_F(UserFunctionCacheTest, PythonZipDeletionWaitsForRuntimeReference) { + const std::string cache_dir = test_dir_ + "/cache"; + ASSERT_TRUE(ufc.init(cache_dir).ok()); + + const std::string zip_path = test_dir_ + "/shared.zip"; + ASSERT_TRUE(create_zip_file(zip_path, {{"shared.py", "def evaluate(x): return x"}})); + const std::string checksum = file_md5(zip_path); + const std::string source_location = "file://" + zip_path; + + std::string local_path; + std::shared_ptr runtime_reference; + ASSERT_TRUE( + ufc.get_pypath(2001, source_location, checksum, &local_path, &runtime_reference).ok()); + + ufc.drop_function_cache(2001); + EXPECT_TRUE(std::filesystem::exists(local_path)); + runtime_reference.reset(); + EXPECT_FALSE(std::filesystem::exists(local_path)); + EXPECT_FALSE(std::filesystem::exists(local_path + ".zip")); +} + +} // namespace doris diff --git a/be/test/udf/python/python_server_test.cpp b/be/test/udf/python/python_server_test.cpp index 39e6143bc144db..9810f272fee63f 100644 --- a/be/test/udf/python/python_server_test.cpp +++ b/be/test/udf/python/python_server_test.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include "common/status.h" #include "cpp/sync_point.h" #include "udf/python/python_env.h" +#include "udf/python/python_udaf_client.h" #include "udf/python/python_udf_client.h" #include "udf/python/python_udf_meta.h" #include "util/defer_op.h" @@ -47,7 +49,7 @@ class PythonServerTest : public ::testing::Test { protected: std::string test_dir_; std::optional original_doris_home_; - int original_max_python_process_num_ = 0; + int test_pool_size_ = 1; void SetUp() override { test_dir_ = fs::temp_directory_path().string() + "/python_server_test_" + @@ -57,12 +59,10 @@ class PythonServerTest : public ::testing::Test { if (const char* doris_home = std::getenv("DORIS_HOME")) { original_doris_home_ = doris_home; } - original_max_python_process_num_ = config::max_python_process_num; } void TearDown() override { // Restore configuration - config::max_python_process_num = original_max_python_process_num_; if (!test_dir_.empty() && fs::exists(test_dir_)) { fs::remove_all(test_dir_); @@ -75,6 +75,49 @@ class PythonServerTest : public ::testing::Test { } } + void check_dropped_process_memory(bool repair) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + SCOPED_TRACE(repair ? "repaired process" : "initial process"); + PythonServerManager mgr; + auto meta = test_meta(version); + auto pool = mgr._ensure_pool_initialized(meta, version); + ASSERT_TRUE(pool.has_value()) << pool.error().to_string(); + ASSERT_TRUE( + mgr.wait_for_process_pool_initialized_for_test(meta.id, std::chrono::seconds(5))); + // Model the ProcessPtr retained by PythonClient during an active exchange. + ProcessPtr client_process; + ASSERT_TRUE(mgr._get_process(version, pool.value(), &client_process).ok()); + if (repair) { + client_process->shutdown(); + client_process.reset(); + mgr._check_and_recreate_processes(); + ASSERT_TRUE(mgr._get_process(version, pool.value(), &client_process).ok()); + } + mgr._refresh_memory_stats(); + ASSERT_GT(mgr._mem_tracker.consumption(), 0); + + std::weak_ptr process_lifetime = client_process; + mgr.drop_function(meta.id); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); + ASSERT_TRUE(client_process->is_alive()); + mgr._refresh_memory_stats(); + EXPECT_GT(mgr._mem_tracker.consumption(), 0); + + client_process.reset(); + EXPECT_TRUE(process_lifetime.expired()); + mgr._refresh_memory_stats(); + EXPECT_EQ(mgr._mem_tracker.consumption(), 0); + } + + PythonUDFMeta test_meta(const PythonVersion& version) const { + PythonUDFMeta meta; + meta.id = static_cast(std::hash()(version.to_string()) & INT64_MAX); + meta.pool_size = test_pool_size_; + return meta; + } + // Create a fake Python script that creates a socket file and keeps running // Arg: script prints "Python X.Y.Z" for version detection std::string create_fake_python_with_socket_creation(const std::string& version = "3.9.16") { @@ -339,11 +382,11 @@ TEST_F(PythonServerTest, ForkEnqueuesBackgroundReapWhenKilledStartFailureIsNotRe TEST_F(PythonServerTest, EnsurePoolInitializedWithInvalidVersionFails) { PythonServerManager mgr; - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonVersion invalid_version("3.99.99", "/non/existent/path", "/non/existent/python"); - auto result = mgr._ensure_pool_initialized(invalid_version); + auto result = mgr._ensure_pool_initialized(test_meta(invalid_version), invalid_version); // Verify: invalid version should cause initialization to fail EXPECT_FALSE(result.has_value()); @@ -355,7 +398,7 @@ TEST_F(PythonServerTest, EnsurePoolInitializedWithInvalidVersionFails) { TEST_F(PythonServerTest, EnsurePoolInitializedRetriesAfterRuntimeIsRepaired) { setup_doris_home(); - config::max_python_process_num = 1; + test_pool_size_ = 1; // The first executable starts but never publishes its Flight socket. This reproduces a real // worker-start failure and verifies that the pool does not remain stuck in INITIALIZING. @@ -407,7 +450,7 @@ TEST_F(PythonServerTest, EnsurePoolInitializedRetriesAfterRuntimeIsRepaired) { }); sync_point->enable_processing(); - auto failed_result = mgr._ensure_pool_initialized(version); + auto failed_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_FALSE(failed_result.has_value()); // Wait for both parts of the failed generation to finish. Otherwise an old detached worker can @@ -419,12 +462,12 @@ TEST_F(PythonServerTest, EnsurePoolInitializedRetriesAfterRuntimeIsRepaired) { })); } ASSERT_TRUE(mgr.wait_for_process_pool_initialization_finished_for_test( - version, std::chrono::seconds(5))); + test_meta(version).id, std::chrono::seconds(5))); // Repair the runtime in place and retry the same version key. Production can hit this when an // environment or server entry is fixed after a transient initialization failure. ASSERT_EQ(create_fake_python_with_socket_creation("3.9.16"), python_path); - auto recovered_result = mgr._ensure_pool_initialized(version); + auto recovered_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(recovered_result.has_value()) << recovered_result.error().to_string(); ProcessPtr process; @@ -443,7 +486,7 @@ TEST_F(PythonServerTest, EnsurePoolInitializedAfterShutdownReturnsServiceUnavail std::string python_path = create_fake_python_with_socket_creation("3.9.16"); PythonVersion version("3.9.16", test_dir_, python_path); - auto result = mgr._ensure_pool_initialized(version); + auto result = mgr._ensure_pool_initialized(test_meta(version), version); EXPECT_FALSE(result.has_value()); EXPECT_NE(result.error().to_string().find("shutting down"), std::string::npos); @@ -466,11 +509,11 @@ TEST_F(PythonServerTest, ShutdownCalledMultipleTimesDoesNotCrash) { TEST_F(PythonServerTest, ShutdownAfterFailedInitializationDoesNotCrash) { PythonServerManager mgr; - config::max_python_process_num = 1; + test_pool_size_ = 1; // Try initialization first (expected to fail) PythonVersion invalid_version("3.99.99", "/bad/path", "/bad/python"); - auto result = mgr._ensure_pool_initialized(invalid_version); + auto result = mgr._ensure_pool_initialized(test_meta(invalid_version), invalid_version); EXPECT_FALSE(result.has_value()); // Verify: calling shutdown after failed initialization does not crash @@ -481,11 +524,11 @@ TEST_F(PythonServerTest, GetProcessFromStoppedPoolReturnsUnavailable) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto pool_result = mgr._ensure_pool_initialized(version); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); mgr.shutdown(); @@ -498,46 +541,15 @@ TEST_F(PythonServerTest, GetProcessFromStoppedPoolReturnsUnavailable) { EXPECT_NE(status.to_string().find("stopped"), std::string::npos); } -TEST_F(PythonServerTest, ClearUdafStateCacheWithoutProcessesIsNoOp) { +TEST_F(PythonServerTest, DropUnknownFunctionIsNoOp) { PythonServerManager mgr; - EXPECT_NO_THROW(mgr.clear_udaf_state_cache(12345)); + EXPECT_NO_THROW(mgr.drop_function(12345)); } -TEST_F(PythonServerTest, ClearModuleCacheWithoutProcessesIsNoOp) { - PythonServerManager mgr; - - auto status = mgr.clear_module_cache("/tmp/python_udf_cache"); - EXPECT_TRUE(status.ok()) << status.to_string(); -} - -TEST_F(PythonServerTest, BroadcastActionWithInvalidProcessUriReturnsError) { - PythonServerManager mgr; - PythonVersion version("3.9.16", test_dir_, test_dir_ + "/bin/python3"); - ProcessPtr process = create_sleep_process(); - ASSERT_NE(process, nullptr); - ASSERT_TRUE(process->is_alive()); - process->set_uri_for_test("invalid-python-flight-uri"); - - mgr.set_process_pool_for_test(version, {process}); - auto status = mgr.broadcast_action_to_processes_for_test( - "clear_udaf_state_cache", R"({"function_id": 12345})", "function_id=12345"); - - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("clear_udaf_state_cache failed for function_id=12345"), - std::string::npos); - EXPECT_NE(status.to_string().find("success=0, failed=1"), std::string::npos); - - mgr.shutdown(); -} - -// ============================================================================ -// PythonServerManager::get_client() - client retrieval test -// ============================================================================ - TEST_F(PythonServerTest, GetClientWithInvalidVersionFails) { PythonServerManager mgr; - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonVersion invalid_version("3.9.16", "/invalid/path", "/invalid/python"); PythonUDFMeta meta; @@ -562,14 +574,14 @@ TEST_F(PythonServerTest, GetClientWithInvalidVersionFails) { TEST_F(PythonServerTest, DestructorCleansUpResources) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; ProcessPtr process; std::string socket_path; { PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto pool_result = mgr._ensure_pool_initialized(version); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); ASSERT_TRUE(mgr._get_process(version, pool_result.value(), &process).ok()); ASSERT_NE(process, nullptr); @@ -617,12 +629,12 @@ TEST_F(PythonServerTest, EnsurePoolInitializedSuccess) { std::string python_path = create_fake_python_with_socket_creation("3.9.16"); // Limit process pool to 1 to speed up the test - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto result = mgr._ensure_pool_initialized(version); + auto result = mgr._ensure_pool_initialized(test_meta(version), version); // Verify pool initialization succeeded EXPECT_TRUE(result.has_value()) << result.error().to_string(); @@ -636,12 +648,12 @@ TEST_F(PythonServerTest, EnsurePoolInitializedSucceedsWithOneStuckWorkerAndOneUs std::string python_path = create_fake_python_with_one_stuck_and_others_socket("python3.mixed", "3.9.16"); - config::max_python_process_num = 2; + test_pool_size_ = 2; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto result = mgr._ensure_pool_initialized(version); + auto result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(result.has_value()) << result.error().to_string(); @@ -652,9 +664,9 @@ TEST_F(PythonServerTest, EnsurePoolInitializedSucceedsWithOneStuckWorkerAndOneUs // Wait on the same condition variable used by the production coordinator instead of polling // or asserting on a racy intermediate state. - EXPECT_TRUE(mgr.wait_for_process_pool_initialized_for_test(version, + EXPECT_TRUE(mgr.wait_for_process_pool_initialized_for_test(test_meta(version).id, std::chrono::milliseconds(2000))); - EXPECT_TRUE(mgr.process_pool_is_initialized_for_test(version)); + EXPECT_TRUE(mgr.process_pool_is_initialized_for_test(test_meta(version).id)); mgr.shutdown(); } @@ -663,24 +675,24 @@ TEST_F(PythonServerTest, EnsurePoolInitializedIdempotent) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); // First initialization - auto result1 = mgr._ensure_pool_initialized(version); + auto result1 = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(result1.has_value()) << result1.error().to_string(); - auto first_snapshot = mgr.process_pool_snapshot_for_test(version); + auto first_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(first_snapshot.size(), 1); ASSERT_NE(first_snapshot[0], nullptr); pid_t first_pid = first_snapshot[0]->get_child_pid(); // Re-initialization must reuse both the versioned pool and its live process. - auto result2 = mgr._ensure_pool_initialized(version); + auto result2 = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(result2.has_value()) << result2.error().to_string(); EXPECT_EQ(result2.value(), result1.value()); - auto second_snapshot = mgr.process_pool_snapshot_for_test(version); + auto second_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(second_snapshot.size(), 1); EXPECT_EQ(second_snapshot[0], first_snapshot[0]); EXPECT_EQ(second_snapshot[0]->get_child_pid(), first_pid); @@ -692,13 +704,13 @@ TEST_F(PythonServerTest, GetProcessFromInitializedPool) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); // Initialize the pool first - auto init_result = mgr._ensure_pool_initialized(version); + auto init_result = mgr._ensure_pool_initialized(test_meta(version), version); EXPECT_TRUE(init_result.has_value()) << init_result.error().to_string(); // Get a process @@ -716,12 +728,12 @@ TEST_F(PythonServerTest, GetProcessRecreatesDeadProcessWhenNoAliveProcess) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto pool_result = mgr._ensure_pool_initialized(version); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); ProcessPtr first_process; @@ -748,7 +760,7 @@ TEST_F(PythonServerTest, GetProcessSkipsDeadProcessWhenAliveProcessExists) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 3; + test_pool_size_ = 3; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); @@ -765,8 +777,8 @@ TEST_F(PythonServerTest, GetProcessSkipsDeadProcessWhenAliveProcessExists) { dead_process->shutdown(); ASSERT_FALSE(dead_process->is_alive()); - mgr.set_process_pool_for_test(version, {alive_process, dead_process}); - auto pool_result = mgr._ensure_pool_initialized(version); + mgr.set_process_pool_for_test(test_meta(version), version, {alive_process, dead_process}); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); ProcessPtr selected; @@ -774,7 +786,7 @@ TEST_F(PythonServerTest, GetProcessSkipsDeadProcessWhenAliveProcessExists) { EXPECT_TRUE(status.ok()) << status.to_string(); EXPECT_EQ(selected, alive_process); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(pool_snapshot.size(), 2); EXPECT_FALSE(pool_snapshot[1]->is_alive()); EXPECT_EQ(pool_snapshot[1]->get_child_pid(), dead_pid); @@ -782,15 +794,30 @@ TEST_F(PythonServerTest, GetProcessSkipsDeadProcessWhenAliveProcessExists) { mgr.shutdown(); } -TEST_F(PythonServerTest, GetProcessSelectsLeastSharedProcess) { +TEST_F(PythonServerTest, RejectsInvalidPoolSizeBeforeStartingProcesses) { PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); - mgr.set_process_pool_for_test(version, {create_sleep_process(), create_sleep_process()}); + auto meta = test_meta(version); + for (const int pool_size : {0, -1}) { + meta.pool_size = pool_size; + auto result = mgr._ensure_pool_initialized(meta, version); + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().to_string().find("pool_size must be a positive integer"), + std::string::npos); + } + mgr.shutdown(); +} + +TEST_F(PythonServerTest, GetProcessSelectsLeastLoadedProcess) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + mgr.set_process_pool_for_test(test_meta(version), version, + {create_sleep_process(), create_sleep_process()}); - auto init_result = mgr._ensure_pool_initialized(version); + auto init_result = mgr._ensure_pool_initialized(test_meta(version), version); EXPECT_TRUE(init_result.has_value()) << init_result.error().to_string(); - // Holding p1 increases its shared ownership count, so the next client must select the other + // Keeping the first process shared_ptr raises its use_count, so select the other // live process. An implementation that always returns pool[0] fails this assertion. ProcessPtr p1, p2; ASSERT_TRUE(mgr._get_process(version, init_result.value(), &p1).ok()); @@ -806,13 +833,13 @@ TEST_F(PythonServerTest, ShutdownWithRunningProcesses) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 2; + test_pool_size_ = 2; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); // Initialize the pool - auto init_result = mgr._ensure_pool_initialized(version); + auto init_result = mgr._ensure_pool_initialized(test_meta(version), version); EXPECT_TRUE(init_result.has_value()) << init_result.error().to_string(); // Get a process reference @@ -859,15 +886,15 @@ TEST_F(PythonServerTest, MultipleVersionPools) { } fs::permissions(python310_path, fs::perms::owner_all); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version39("3.9.16", test_dir_, python39_path); PythonVersion version310("3.10.0", test_dir_, python310_path); // Initialize pools for two versions - auto pool39_result = mgr._ensure_pool_initialized(version39); - auto pool310_result = mgr._ensure_pool_initialized(version310); + auto pool39_result = mgr._ensure_pool_initialized(test_meta(version39), version39); + auto pool310_result = mgr._ensure_pool_initialized(test_meta(version310), version310); EXPECT_TRUE(pool39_result.has_value()) << pool39_result.error().to_string(); EXPECT_TRUE(pool310_result.has_value()) << pool310_result.error().to_string(); @@ -885,7 +912,7 @@ TEST_F(PythonServerTest, MultipleVersionPools) { TEST_F(PythonServerTest, EnsurePoolInitializedForDifferentVersionsDoesNotShareVersionLock) { setup_doris_home(); - config::max_python_process_num = 1; + test_pool_size_ = 1; std::string python39_path = create_fake_python_with_socket_creation("3.9.16"); std::string python310_path = test_dir_ + "/bin/python3.10"; @@ -940,10 +967,12 @@ TEST_F(PythonServerTest, EnsurePoolInitializedForDifferentVersionsDoesNotShareVe }); sync_point->enable_processing(); - auto future39 = std::async(std::launch::async, - [&]() { return mgr._ensure_pool_initialized(version39); }); - auto future310 = std::async(std::launch::async, - [&]() { return mgr._ensure_pool_initialized(version310); }); + auto future39 = std::async(std::launch::async, [&]() { + return mgr._ensure_pool_initialized(test_meta(version39), version39); + }); + auto future310 = std::async(std::launch::async, [&]() { + return mgr._ensure_pool_initialized(test_meta(version310), version310); + }); bool both_forks_entered = false; { @@ -965,15 +994,290 @@ TEST_F(PythonServerTest, EnsurePoolInitializedForDifferentVersionsDoesNotShareVe << "Both version pools must enter fork before either process starts"; } -// ============================================================================ -// PythonServerManager::_check_and_recreate_processes() - health-check recreation test -// ============================================================================ +TEST_F(PythonServerTest, FunctionsWithSameRuntimeHaveIndependentFixedPools) { + PythonServerManager mgr; + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonUDFMeta first; + first.id = 101; + first.pool_size = 1; + PythonUDFMeta second; + second.id = 102; + second.pool_size = 2; + auto first_result = mgr._ensure_pool_initialized(first, version); + ASSERT_TRUE(first_result.has_value()) << first_result.error().to_string(); + auto second_result = mgr._ensure_pool_initialized(second, version); + ASSERT_TRUE(second_result.has_value()) << second_result.error().to_string(); + ASSERT_TRUE(mgr.wait_for_process_pool_initialized_for_test(101, std::chrono::seconds(5))); + ASSERT_TRUE(mgr.wait_for_process_pool_initialized_for_test(102, std::chrono::seconds(5))); + auto first_pool = mgr.process_pool_snapshot_for_test(101); + auto second_pool = mgr.process_pool_snapshot_for_test(102); + ASSERT_EQ(first_pool.size(), 1); + ASSERT_EQ(second_pool.size(), 2); + EXPECT_NE(first_pool[0]->get_child_pid(), second_pool[0]->get_child_pid()); + EXPECT_NE(first_pool[0]->get_child_pid(), second_pool[1]->get_child_pid()); + ProcessPtr first_client, second_client; + ASSERT_TRUE(mgr._get_process(version, first_result.value(), &first_client).ok()); + ASSERT_TRUE(mgr._get_process(version, second_result.value(), &second_client).ok()); + EXPECT_NE(first_client, second_client); +} + +TEST_F(PythonServerTest, DropKeepsActiveProcessAliveAndDoesNotAffectOtherFunction) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + PythonUDFMeta first; + first.id = 101; + PythonUDFMeta second; + second.id = 102; + auto active = create_sleep_process(); + mgr.set_process_pool_for_test(first, version, {active}); + mgr.set_process_pool_for_test(second, version, {create_sleep_process()}); + mgr.drop_function(101); + EXPECT_TRUE(active->is_alive()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(101).empty()); + EXPECT_EQ(mgr.process_pool_snapshot_for_test(102).size(), 1); + active.reset(); +} + +TEST_F(PythonServerTest, DroppedClientProcessesRemainCountedUntilReleased) { + check_dropped_process_memory(false); +} + +TEST_F(PythonServerTest, RepairedDroppedClientProcessesRemainCountedUntilReleased) { + check_dropped_process_memory(true); +} + +TEST_F(PythonServerTest, DropDuringInitializationKeepsPeriodicMemoryAccounting) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonServerManager mgr; + struct ForkBarrier { + std::mutex mutex; + std::condition_variable cv; + int entries = 0; + bool released = false; + }; + auto barrier = std::make_shared(); + auto* sync_point = SyncPoint::get_instance(); + Defer clear_sync_point {[barrier, sync_point]() { + { + std::scoped_lock lock(barrier->mutex); + barrier->released = true; + } + barrier->cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_call_back("PythonServerManager::fork:before_process_start"); + }}; + sync_point->set_call_back("PythonServerManager::fork:before_process_start", [barrier](auto&&) { + std::unique_lock lock(barrier->mutex); + if (++barrier->entries == 2) { + barrier->cv.wait(lock, [&]() { return barrier->released; }); + } + }); + sync_point->enable_processing(); + auto meta = test_meta(version); + meta.pool_size = 2; + auto pool = mgr._ensure_pool_initialized(meta, version); + ASSERT_TRUE(pool.has_value()) << pool.error().to_string(); + ProcessPtr client_process; + ASSERT_TRUE(mgr._get_process(version, pool.value(), &client_process).ok()); + mgr.drop_function(meta.id); + ASSERT_TRUE(client_process->is_alive()); + + // Exercise the real periodic sampler: no explicit _refresh_memory_stats() call. + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(35); + while (mgr._mem_tracker.consumption() == 0 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + EXPECT_GT(mgr._mem_tracker.consumption(), 0); +} + +TEST_F(PythonServerTest, ShutdownStopsDroppedClientProcesses) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonServerManager mgr; + auto meta = test_meta(version); + auto pool = mgr._ensure_pool_initialized(meta, version); + ASSERT_TRUE(pool.has_value()) << pool.error().to_string(); + ASSERT_TRUE(mgr.wait_for_process_pool_initialized_for_test(meta.id, std::chrono::seconds(5))); + ProcessPtr client_process; + ASSERT_TRUE(mgr._get_process(version, pool.value(), &client_process).ok()); + mgr.drop_function(meta.id); + ASSERT_TRUE(client_process->is_alive()); + + mgr.shutdown(); + EXPECT_FALSE(client_process->is_alive()); +} + +TEST_F(PythonServerTest, AutoSuspendDestroysIdleProcessesAndAllowsRecreation) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonServerManager mgr; + auto meta = test_meta(version); + meta.auto_suspend = 0; + auto result = mgr._ensure_pool_initialized(meta, version); + ASSERT_TRUE(result.has_value()) << result.error().to_string(); + ASSERT_TRUE(mgr.wait_for_process_pool_initialized_for_test(meta.id, std::chrono::seconds(5))); + auto old_pool = result.value(); + auto process = mgr.process_pool_snapshot_for_test(meta.id).front(); + ASSERT_TRUE(process->is_alive()); + + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(process->is_alive()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); + auto reopened = mgr._ensure_pool_initialized(meta, version); + ASSERT_TRUE(reopened.has_value()) << reopened.error().to_string(); + EXPECT_NE(old_pool, reopened.value()); +} + +TEST_F(PythonServerTest, AutoSuspendWaitsForTheConfiguredIdleDuration) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + meta.auto_suspend = 3; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr).value(); + auto idle_since = pool->idle_since; + mgr._reclaim_idle_pools(idle_since + std::chrono::seconds(2)); + EXPECT_TRUE(process->is_alive()); + mgr._reclaim_idle_pools(idle_since + std::chrono::seconds(3)); + EXPECT_FALSE(process->is_alive()); +} + +TEST_F(PythonServerTest, NegativeAutoSuspendKeepsIdleProcessesAlive) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + for (int64_t timeout : {-1, -2}) { + auto meta = test_meta(version); + meta.id = 2000 - timeout; + meta.auto_suspend = timeout; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now() + std::chrono::hours(24)); + EXPECT_TRUE(process->is_alive()); + } +} + +TEST_F(PythonServerTest, AutoSuspendWaitsForLastClientAndStartsANewIdlePeriod) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + meta.auto_suspend = 3; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + std::shared_ptr first; + std::shared_ptr second; + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr, &first).value(); + ASSERT_TRUE(mgr._get_or_create_process_pool(meta, version, nullptr, &second).has_value()); + auto future = std::chrono::steady_clock::now() + std::chrono::hours(24); + mgr._reclaim_idle_pools(future); + EXPECT_TRUE(process->is_alive()); + first.reset(); + mgr._reclaim_idle_pools(future); + EXPECT_TRUE(process->is_alive()); + second.reset(); + EXPECT_EQ(0, pool->active_clients); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::seconds(2)); + EXPECT_TRUE(process->is_alive()); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::seconds(3)); + EXPECT_FALSE(process->is_alive()); +} + +TEST_F(PythonServerTest, AutoSuspendClientCloseReleasesLeaseWithoutAWriter) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + meta.auto_suspend = 0; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + PythonClient client; + auto pool = + mgr._get_or_create_process_pool(meta, version, nullptr, &client._pool_lease).value(); + client._process = process; + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_TRUE(process->is_alive()); + ASSERT_TRUE(client.close().ok()); + EXPECT_EQ(0, pool->active_clients); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(process->is_alive()); +} + +TEST_F(PythonServerTest, AutoSuspendInitializationFailureReleasesLease) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/missing_python"); + auto meta = test_meta(version); + meta.auto_suspend = 0; + std::shared_ptr client; + EXPECT_FALSE(mgr.get_client(meta, version, &client).ok()); + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr).value(); + EXPECT_EQ(0, pool->active_clients); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); +} + +TEST_F(PythonServerTest, AutoSuspendProtectsClientInitialization) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonServerManager mgr; + auto meta = test_meta(version); + meta.auto_suspend = 0; + std::promise started; + std::promise resume; + auto resumed = resume.get_future().share(); + auto* sync_point = SyncPoint::get_instance(); + sync_point->set_call_back("PythonServerManager::fork:before_process_start", [&](auto&&) { + started.set_value(); + resumed.wait(); + }); + sync_point->enable_processing(); + Defer cleanup {[&]() { + sync_point->disable_processing(); + sync_point->clear_call_back("PythonServerManager::fork:before_process_start"); + }}; + auto opening = std::async(std::launch::async, [&]() { + std::shared_ptr client; + return mgr.get_client(meta, version, &client); + }); + EXPECT_EQ(std::future_status::ready, started.get_future().wait_for(std::chrono::seconds(5))); + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr).value(); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now() + std::chrono::hours(24)); + EXPECT_EQ(1, pool->active_clients); + EXPECT_NE(PythonServerManager::PoolState::STOPPED, pool->state); + resume.set_value(); + // Either an initialization error or client destruction must release usage. + (void)opening.get(); + EXPECT_EQ(0, pool->active_clients); +} + +TEST_F(PythonServerTest, AutoSuspendMonitorTerminatesAnIdleProcess) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + meta.auto_suspend = 0; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + // A zero timeout becomes eligible immediately, but reclamation waits for the monitor scan. + std::this_thread::sleep_for(std::chrono::seconds(2)); + EXPECT_TRUE(process->is_alive()); + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(40); + while (process->is_alive() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + EXPECT_FALSE(process->is_alive()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); +} TEST_F(PythonServerTest, CheckAndRecreateProcessesRecreatesDeadProcess) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 3; + test_pool_size_ = 3; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); @@ -990,11 +1294,12 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesRecreatesDeadProcess) { dead_process->shutdown(); ASSERT_FALSE(dead_process->is_alive()); - mgr.set_process_pool_for_test(version, {alive_process, dead_process, nullptr}); + mgr.set_process_pool_for_test(test_meta(version), version, + {alive_process, dead_process, nullptr}); mgr.check_and_recreate_processes_for_test(); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(pool_snapshot.size(), 3); EXPECT_EQ(pool_snapshot[0], alive_process); @@ -1012,7 +1317,7 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsRepairingPool) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); @@ -1024,8 +1329,8 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsRepairingPool) { dead_process->shutdown(); ASSERT_FALSE(dead_process->is_alive()); - mgr.set_process_pool_for_test(version, {dead_process}); - auto pool_result = mgr._ensure_pool_initialized(version); + mgr.set_process_pool_for_test(test_meta(version), version, {dead_process}); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); { std::lock_guard lock(pool_result.value()->mutex); @@ -1034,7 +1339,7 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsRepairingPool) { mgr.check_and_recreate_processes_for_test(); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(pool_snapshot.size(), 1); ASSERT_NE(pool_snapshot[0], nullptr); EXPECT_FALSE(pool_snapshot[0]->is_alive()); @@ -1051,7 +1356,7 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsUninitializedPool) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); @@ -1063,11 +1368,11 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsUninitializedPool) { dead_process->shutdown(); ASSERT_FALSE(dead_process->is_alive()); - mgr.set_process_pool_for_test(version, {dead_process}, false); + mgr.set_process_pool_for_test(test_meta(version), version, {dead_process}, false); mgr.check_and_recreate_processes_for_test(); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(pool_snapshot.size(), 1); ASSERT_NE(pool_snapshot[0], nullptr); EXPECT_FALSE(pool_snapshot[0]->is_alive()); @@ -1080,7 +1385,7 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesKeepsDeadSlotsWhenRecreateFail setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 2; + test_pool_size_ = 2; PythonServerManager mgr; PythonVersion live_version("3.9.16", test_dir_, python_path); @@ -1098,11 +1403,12 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesKeepsDeadSlotsWhenRecreateFail ASSERT_FALSE(dead_process_2->is_alive()); PythonVersion invalid_version("3.9.16", test_dir_, test_dir_ + "/bin/nonexistent_python"); - mgr.set_process_pool_for_test(invalid_version, {dead_process_1, dead_process_2}); + mgr.set_process_pool_for_test(test_meta(invalid_version), invalid_version, + {dead_process_1, dead_process_2}); mgr.check_and_recreate_processes_for_test(); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(invalid_version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(invalid_version).id); ASSERT_EQ(pool_snapshot.size(), 2); EXPECT_FALSE(pool_snapshot[0]->is_alive()); EXPECT_FALSE(pool_snapshot[1]->is_alive()); diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java index 93783a47725d73..1c52f8e13925e3 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java @@ -112,6 +112,10 @@ public enum BinaryType { protected long expirationTime = 360; // default 6 hours; @SerializedName("rv") protected String runtimeVersion; + @SerializedName("ps") + protected int poolSize = 3; + @SerializedName("as") + protected long autoSuspend = -1; @SerializedName("fc") protected String functionCode; @SerializedName("vol") @@ -175,6 +179,8 @@ public Function(Function other) { this.isStaticLoad = other.isStaticLoad; this.expirationTime = other.expirationTime; this.runtimeVersion = other.runtimeVersion; + this.poolSize = other.poolSize; + this.autoSuspend = other.autoSuspend; this.functionCode = other.functionCode; this.volatility = other.getVolatility(); } @@ -288,6 +294,22 @@ public void setGlobal(boolean global) { isGlobal = global; } + public int getPoolSize() { + return poolSize; + } + + public void setPoolSize(int poolSize) { + this.poolSize = poolSize; + } + + public long getAutoSuspend() { + return autoSuspend; + } + + public void setAutoSuspend(long autoSuspend) { + this.autoSuspend = autoSuspend; + } + public String getRuntimeVersion() { return runtimeVersion; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index dcea067295c5d1..c3b4e47dd90d30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -322,13 +322,14 @@ private void eraseDatabase(long currentTimeMs, int keepNum) { // 2. erase each expired database one at a time for (Long dbId : expiredIds) { + Database db; writeLock(); try { RecycleDatabaseInfo dbInfo = idToDatabase.remove(dbId); if (dbInfo == null) { continue; } - Database db = dbInfo.getDb(); + db = dbInfo.getDb(); idToRecycleTime.remove(dbId); dbNameToIds.computeIfPresent(db.getFullName(), (k, v) -> { @@ -342,6 +343,7 @@ private void eraseDatabase(long currentTimeMs, int keepNum) { } finally { writeUnlock(); } + FunctionUtil.cleanUdfCache(db); } // 3. erase exceed number @@ -375,6 +377,7 @@ private void eraseDatabaseWithSameName(String dbName, long currentTimeMs, readUnlock(); } for (Long dbId : dbIdToErase) { + Database db; writeLock(); try { RecycleDatabaseInfo dbInfo = idToDatabase.get(dbId); @@ -384,6 +387,7 @@ private void eraseDatabaseWithSameName(String dbName, long currentTimeMs, if (!eraseAllTables(dbInfo)) { continue; } + db = dbInfo.getDb(); idToDatabase.remove(dbId); idToRecycleTime.remove(dbId); @@ -397,6 +401,7 @@ private void eraseDatabaseWithSameName(String dbName, long currentTimeMs, } finally { writeUnlock(); } + FunctionUtil.cleanUdfCache(db); } } @@ -1222,6 +1227,9 @@ public void eraseDatabaseInstantly(long dbId) throws DdlException { } finally { writeUnlock(); } + if (dbInfo != null) { + FunctionUtil.cleanUdfCache(dbInfo.getDb()); + } // 2. collect tables with same dbId List tableIdToErase = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java index 590a6582594e35..8db2644e331fa6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java @@ -84,6 +84,8 @@ public static String toSql(ScalarFunction fn, boolean ifNotExists) { boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE; sb.append(",\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\""); sb.append(",\n \"RUNTIME_VERSION\"=").append("\"" + Strings.nullToEmpty(fn.getRuntimeVersion()) + "\""); + sb.append(",\n \"POOL_SIZE\"=\"").append(fn.getPoolSize()).append("\""); + sb.append(",\n \"AUTO_SUSPEND\"=\"").append(fn.getAutoSuspend()).append("\""); appendExpirationTimeIfNeeded(sb, fn); sb.append(",\n \"VOLATILITY\"=").append("\"" + fn.getVolatility().toSql() + "\""); } else { @@ -173,6 +175,8 @@ public static String toSql(AggregateFunction fn, boolean ifNotExists) { sb.append("\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\","); sb.append("\n \"RUNTIME_VERSION\"=") .append("\"" + Strings.nullToEmpty(fn.getRuntimeVersion()) + "\","); + sb.append("\n \"POOL_SIZE\"=\"").append(fn.getPoolSize()).append("\","); + sb.append("\n \"AUTO_SUSPEND\"=\"").append(fn.getAutoSuspend()).append("\","); if (fn.getExpirationTime() != DEFAULT_EXPIRATION_TIME) { sb.append("\n \"EXPIRATION_TIME\"=").append("\"" + fn.getExpirationTime() + "\","); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java index 5c4ca6f8be9994..b0af83a6806ecd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java @@ -97,6 +97,8 @@ public static TFunction toThrift(ScalarFunction fn, Type realReturnType, Type[] tfn.setFunctionCode(fn.getFunctionCode()); } tfn.setRuntimeVersion(fn.getRuntimeVersion()); + tfn.setPoolSize(fn.getPoolSize()); + tfn.setAutoSuspend(fn.getAutoSuspend()); } if (fn.getDictFunction() != null) { tfn.setDictFunction(fn.getDictFunction()); @@ -144,6 +146,8 @@ public static TFunction toThrift(AggregateFunction fn, Type realReturnType, Type tfn.setFunctionCode(fn.getFunctionCode()); } tfn.setRuntimeVersion(fn.getRuntimeVersion()); + tfn.setPoolSize(fn.getPoolSize()); + tfn.setAutoSuspend(fn.getAutoSuspend()); } return tfn; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java index 76d454e9ed1b3a..99966425d56449 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java @@ -30,6 +30,9 @@ import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdf; import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdtf; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.task.AgentBatchTask; +import org.apache.doris.task.AgentTaskExecutor; +import org.apache.doris.task.CleanUDFCacheTask; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -48,6 +51,37 @@ public class FunctionUtil { private static final Logger LOG = LogManager.getLogger(FunctionUtil.class); + public static void cleanUdfCache(Map functionSignatures) { + if (functionSignatures.isEmpty()) { + // No function generation was removed. Never fall back to signature-based cleanup, + // which could delete a newly created function's cache when an old task arrives late. + return; + } + AgentBatchTask batchTask = new AgentBatchTask(); + for (long backendId : Env.getCurrentSystemInfo().getAllBackendIds(false)) { + for (Map.Entry function : functionSignatures.entrySet()) { + CleanUDFCacheTask task = new CleanUDFCacheTask(backendId, function.getValue(), function.getKey()); + batchTask.addTask(task); + LOG.info("clean udf cache in be {}, functionId {}", backendId, function.getKey()); + } + } + if (batchTask.getTaskNum() > 0) { + AgentTaskExecutor.submit(batchTask); + } + } + + public static void cleanUdfCache(Database db) { + Map functionSignatures; + db.readLock(); + try { + functionSignatures = db.getFunctions().stream() + .filter(function -> function.getBinaryType() == Function.BinaryType.PYTHON_UDF) + .collect(Collectors.toMap(Function::getId, Function::signatureString)); + } finally { + db.readUnlock(); + } + cleanUdfCache(functionSignatures); + } /** * @param function diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 1b879531b12acd..178c92e0fe14b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -478,6 +478,7 @@ public void replayCreateDb(Database db, String newDbName) { public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlException { LOG.info("begin drop database[{}], is force : {}", dbName, force); + Map functionSignatures = new HashMap<>(); // 1. check if database exists if (!tryLock(false)) { throw new DdlException("Failed to acquire catalog lock. Try again"); @@ -537,6 +538,11 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc Env.getCurrentRecycleBin().recycleDatabase(db, tableNames, tableIds, false, force, 0); recycleTime = Env.getCurrentRecycleBin().getRecycleTimeById(db.getId()); + if (force) { + functionSignatures = db.getFunctions().stream() + .filter(function -> function.getBinaryType() == Function.BinaryType.PYTHON_UDF) + .collect(Collectors.toMap(Function::getId, Function::signatureString)); + } } finally { db.writeUnlock(); } @@ -553,6 +559,8 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc unlock(); } + // Recoverable databases retain their function generations until the recycle bin erases them. + FunctionUtil.cleanUdfCache(functionSignatures); LOG.info("finish drop database[{}], is force : {}", dbName, force); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java index 39a27b72834d52..056d6ffa3092d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java @@ -66,6 +66,8 @@ public class PythonUdaf extends AggregateFunction implements ExplicitlyCastableS private final String checkSum; private final boolean isStaticLoad; private final long expirationTime; + private final int poolSize; + private final long autoSuspend; private final String runtimeVersion; private final String functionCode; @@ -80,7 +82,7 @@ public PythonUdaf(String name, long functionId, String dbName, Function.BinaryTy String initFn, String updateFn, String mergeFn, String serializeFn, String finalizeFn, String getValueFn, String removeFn, boolean isDistinct, String checkSum, boolean isStaticLoad, long expirationTime, - String runtimeVersion, String functionCode, Expression... args) { + String runtimeVersion, String functionCode, int poolSize, long autoSuspend, Expression... args) { super(name, isDistinct, args); this.dbName = dbName; this.functionId = functionId; @@ -102,6 +104,8 @@ public PythonUdaf(String name, long functionId, String dbName, Function.BinaryTy this.checkSum = checkSum; this.isStaticLoad = isStaticLoad; this.expirationTime = expirationTime; + this.poolSize = poolSize; + this.autoSuspend = autoSuspend; this.runtimeVersion = runtimeVersion; this.functionCode = functionCode; } @@ -135,7 +139,7 @@ public PythonUdaf withDistinctAndChildren(boolean isDistinct, List c return new PythonUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode, volatility, volatileIdentity, objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn, - isDistinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, + isDistinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @@ -150,7 +154,7 @@ public PythonUdaf withIgnoreUniqueId(boolean ignoreUniqueId) { return new PythonUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode, volatility, volatileIdentity.withIgnoreUniqueId(ignoreUniqueId), objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn, - distinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, + distinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @@ -162,7 +166,7 @@ public PythonUdaf withFreshVolatileIdentity() { return new PythonUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode, volatility, VolatileIdentity.newVolatileIdentity(), objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn, - distinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, + distinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @@ -223,7 +227,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca aggregate.isStaticLoad(), aggregate.getExpirationTime(), aggregate.getRuntimeVersion(), - aggregate.getFunctionCode(), + aggregate.getFunctionCode(), aggregate.getPoolSize(), aggregate.getAutoSuspend(), arguments); PythonUdafBuilder builder = new PythonUdafBuilder(udaf); @@ -261,6 +265,8 @@ public Function getCatalogFunction() { expr.setStaticLoad(isStaticLoad); expr.setExpirationTime(expirationTime); expr.setRuntimeVersion(runtimeVersion); + expr.setPoolSize(poolSize); + expr.setAutoSuspend(autoSuspend); expr.setFunctionCode(functionCode); expr.setVolatility(volatility); return expr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java index 8bf05a8a6afaea..3c939df341e121 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java @@ -60,6 +60,8 @@ public class PythonUdf extends ScalarFunction implements ExplicitlyCastableSigna private final String checkSum; private final boolean isStaticLoad; private final long expirationTime; + private final int poolSize; + private final long autoSuspend; private final String runtimeVersion; private final String functionCode; @@ -71,7 +73,7 @@ public PythonUdf(String name, long functionId, String dbName, Function.BinaryTyp NullableMode nullableMode, FunctionVolatility volatility, VolatileIdentity volatileIdentity, String objectFile, String symbol, String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, - String runtimeVersion, String functionCode, Expression... args) { + String runtimeVersion, String functionCode, int poolSize, long autoSuspend, Expression... args) { super(name, args); this.dbName = dbName; this.functionId = functionId; @@ -87,6 +89,8 @@ public PythonUdf(String name, long functionId, String dbName, Function.BinaryTyp this.checkSum = checkSum; this.isStaticLoad = isStaticLoad; this.expirationTime = expirationTime; + this.poolSize = poolSize; + this.autoSuspend = autoSuspend; this.runtimeVersion = runtimeVersion; this.functionCode = functionCode; } @@ -120,7 +124,7 @@ public PythonUdf withChildren(List children) { return new PythonUdf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, volatileIdentity, objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -134,7 +138,7 @@ public PythonUdf withIgnoreUniqueId(boolean ignoreUniqueId) { return new PythonUdf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, volatileIdentity.withIgnoreUniqueId(ignoreUniqueId), objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } /** Return a copy with a new per-call identity when this UDF is VOLATILE. */ @@ -146,7 +150,7 @@ public PythonUdf withFreshVolatileIdentity() { return new PythonUdf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, VolatileIdentity.newVolatileIdentity(), objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -190,7 +194,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca scalar.getCloseFnSymbol(), scalar.getChecksum(), scalar.isStaticLoad(), scalar.getExpirationTime(), scalar.getRuntimeVersion(), - scalar.getFunctionCode(), + scalar.getFunctionCode(), scalar.getPoolSize(), scalar.getAutoSuspend(), arguments); PythonUdfBuilder builder = new PythonUdfBuilder(udf); @@ -222,6 +226,8 @@ public Function getCatalogFunction() { expr.setStaticLoad(isStaticLoad); expr.setExpirationTime(expirationTime); expr.setRuntimeVersion(runtimeVersion); + expr.setPoolSize(poolSize); + expr.setAutoSuspend(autoSuspend); expr.setFunctionCode(functionCode); expr.setVolatility(volatility); return expr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java index 61aa694ff8ea60..1c275df3258250 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java @@ -60,6 +60,8 @@ public class PythonUdtf extends TableGeneratingFunction implements ExplicitlyCas private final String checkSum; private final boolean isStaticLoad; private final long expirationTime; + private final int poolSize; + private final long autoSuspend; private final String runtimeVersion; private final String functionCode; @@ -72,7 +74,7 @@ public PythonUdtf(String name, long functionId, String dbName, Function.BinaryTy String objectFile, String symbol, String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, - String runtimeVersion, String functionCode, Expression... args) { + String runtimeVersion, String functionCode, int poolSize, long autoSuspend, Expression... args) { super(name, args); this.dbName = dbName; this.functionId = functionId; @@ -88,6 +90,8 @@ public PythonUdtf(String name, long functionId, String dbName, Function.BinaryTy this.checkSum = checkSum; this.isStaticLoad = isStaticLoad; this.expirationTime = expirationTime; + this.poolSize = poolSize; + this.autoSuspend = autoSuspend; this.runtimeVersion = runtimeVersion; this.functionCode = functionCode; } @@ -101,7 +105,7 @@ public PythonUdtf withChildren(List children) { return new PythonUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, volatileIdentity, objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -115,7 +119,7 @@ public PythonUdtf withIgnoreUniqueId(boolean ignoreUniqueId) { return new PythonUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, volatileIdentity.withIgnoreUniqueId(ignoreUniqueId), objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -126,7 +130,7 @@ public PythonUdtf withFreshVolatileIdentity() { return new PythonUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, VolatileIdentity.newVolatileIdentity(), objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -179,6 +183,8 @@ public Function getCatalogFunction() { expr.setExpirationTime(expirationTime); expr.setUDTFunction(true); expr.setRuntimeVersion(runtimeVersion); + expr.setPoolSize(poolSize); + expr.setAutoSuspend(autoSuspend); expr.setFunctionCode(functionCode); expr.setVolatility(volatility); return expr; @@ -218,7 +224,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca scalar.isStaticLoad(), scalar.getExpirationTime(), scalar.getRuntimeVersion(), - scalar.getFunctionCode(), + scalar.getFunctionCode(), scalar.getPoolSize(), scalar.getAutoSuspend(), arguments); PythonUdtfBuilder builder = new PythonUdtfBuilder(udtf); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java index d04d0c7010f8c6..97b51a8cc45de6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java @@ -154,6 +154,8 @@ public class CreateFunctionCommand extends Command implements ForwardWithSync { public static final String IS_STATIC_LOAD = "static_load"; public static final String EXPIRATION_TIME = "expiration_time"; public static final String RUNTIME_VERSION = "runtime_version"; + public static final String POOL_SIZE = "pool_size"; + public static final String AUTO_SUSPEND = "auto_suspend"; public static final String VOLATILITY = "volatility"; private static final Pattern PYTHON_VERSION_PATTERN = Pattern.compile("^3\\.\\d{1,2}(?:\\.\\d{1,2})?$"); @@ -188,6 +190,8 @@ public class CreateFunctionCommand extends Command implements ForwardWithSync { // Keep IMMUTABLE as the UDAF/UDTF default for compatibility with previous behavior. private FunctionVolatility volatility = FunctionVolatility.IMMUTABLE; private String runtimeVersion; + private int poolSize = 3; + private long autoSuspend = -1; private String functionCode; /** @@ -343,6 +347,12 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { } } if (binaryType == Function.BinaryType.JAVA_UDF) { + if (properties.containsKey(POOL_SIZE)) { + throw new AnalysisException("pool_size property only supports PYTHON_UDF"); + } + if (properties.containsKey(AUTO_SUSPEND)) { + throw new AnalysisException("auto_suspend property only supports PYTHON_UDF"); + } FunctionUtil.checkEnableJavaUdf(); checkUdfSupportedTypes(); volatility = analyzeVolatility(defaultVolatility()); @@ -378,6 +388,23 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { + "'3.X.X' or '3.XX.XX' (e.g. '3.10.2').", runtimeVersionString)); } runtimeVersion = runtimeVersionString; + if (properties.containsKey(AUTO_SUSPEND)) { + try { + autoSuspend = Long.parseLong(properties.get(AUTO_SUSPEND)); + } catch (NumberFormatException e) { + throw new AnalysisException("Python UDF auto_suspend must be an integer number of seconds"); + } + } + if (properties.containsKey(POOL_SIZE)) { + try { + poolSize = Integer.parseInt(properties.get(POOL_SIZE)); + } catch (NumberFormatException e) { + throw new AnalysisException("Python UDF pool_size must be a positive integer"); + } + if (poolSize <= 0) { + throw new AnalysisException("Python UDF pool_size must be a positive integer"); + } + } } else if (properties.containsKey(VOLATILITY)) { throw new AnalysisException("volatility property only supports JAVA_UDF and PYTHON_UDF"); } @@ -551,6 +578,8 @@ private void analyzeUdtf() throws AnalysisException { function.setExpirationTime(expirationTime); function.setUDTFunction(true); function.setRuntimeVersion(runtimeVersion); + function.setPoolSize(poolSize); + function.setAutoSuspend(autoSuspend); function.setFunctionCode(functionCode); function.setVolatility(volatility); // Todo: maybe in create tables function, need register two function, one is @@ -626,6 +655,8 @@ private void analyzeUdaf() throws AnalysisException { function.setStaticLoad(isStaticLoad); function.setExpirationTime(expirationTime); function.setRuntimeVersion(runtimeVersion); + function.setPoolSize(poolSize); + function.setAutoSuspend(autoSuspend); function.setFunctionCode(functionCode); function.setVolatility(volatility); } @@ -664,6 +695,8 @@ private void analyzeUdf() throws AnalysisException { function.setStaticLoad(isStaticLoad); function.setExpirationTime(expirationTime); function.setRuntimeVersion(runtimeVersion); + function.setPoolSize(poolSize); + function.setAutoSuspend(autoSuspend); function.setFunctionCode(functionCode); function.setVolatility(volatility); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java index c7eeae3949e569..4b1705b7e92009 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java @@ -164,6 +164,7 @@ public void test() throws Exception { + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2');"; createFunction(defaultVolatileSql, ctx); Assertions.assertEquals(FunctionVolatility.VOLATILE, findFunction(db, "py_default").getVolatility()); + Assertions.assertEquals(3, findFunction(db, "py_default").getPoolSize()); String defaultImmutableUdafSql = "create aggregate function db1.py_agg_default(int) returns int " + "properties('type'='PYTHON_UDF', 'symbol'='Agg', 'runtime_version'='3.10.2');"; @@ -177,6 +178,108 @@ public void test() throws Exception { Assertions.assertEquals(FunctionVolatility.STABLE, findFunction(db, "py_table_stable").getVolatility()); } + @Test + public void testPoolSize() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database python_pool_db;"); + dorisAssert = new DorisAssert(ctx); + dorisAssert.useDatabase("python_pool_db"); + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("python_pool_db"); + for (String kind : new String[]{"", "aggregate ", "tables "}) { + String name = kind.isEmpty() ? "scalar_fn" : kind.trim() + "_fn"; + String ret = kind.equals("tables ") ? "array" : "int"; + createFunction("create " + kind + "function python_pool_db." + name + + "(int) returns " + ret + " properties('type'='PYTHON_UDF', " + + "'symbol'='evaluate', 'runtime_version'='3.10.2', 'pool_size'='3');", ctx); + Function fn = findFunction(db, name); + Assertions.assertEquals(3, fn.getPoolSize()); + Assertions.assertTrue(FunctionToSqlConverter.toSql(fn, false).contains("\"POOL_SIZE\"=\"3\"")); + } + for (String invalid : new String[]{"0", "-1", "abc", "1.5", "2147483648"}) { + assertCreateFunctionAnalysisException(ctx, + "create function python_pool_db.invalid_fn(int) returns int " + + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', " + + "'runtime_version'='3.10.2', 'pool_size'='" + invalid + "');", + "pool_size must be a positive integer"); + } + } + + @Test + public void testJavaUdfRejectsPoolSize() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database java_pool_db;"); + createFunction("create function java_pool_db.java_default(timestamp_ns) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='org.apache.doris.catalog.TimestampNsUdf');", ctx); + assertCreateFunctionAnalysisException(ctx, + "create function java_pool_db.java_pool(timestamp_ns) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='org.apache.doris.catalog.TimestampNsUdf', " + + "'pool_size'='3');", + "pool_size property only supports PYTHON_UDF"); + } + + @Test + public void testAutoSuspend() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database python_auto_suspend_db;"); + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("python_auto_suspend_db"); + for (String kind : new String[]{"", "aggregate ", "tables "}) { + String prefix = kind.isEmpty() ? "scalar" : kind.trim(); + String ret = kind.equals("tables ") ? "array" : "int"; + String definition = "(int) returns " + ret + " properties('type'='PYTHON_UDF', " + + "'symbol'='evaluate', 'runtime_version'='3.10.2'"; + createFunction("create " + kind + "function python_auto_suspend_db." + prefix + "_default" + + definition + ");", ctx); + assertAutoSuspendMetadata(findFunction(db, prefix + "_default"), -1); + if (kind.equals("tables ")) { + assertAutoSuspendMetadata(findFunction(db, prefix + "_default_outer"), -1); + } + long[] values = {Long.MIN_VALUE, -2, -1, 0, 30, Long.MAX_VALUE}; + for (int i = 0; i < values.length; i++) { + String name = prefix + "_explicit_" + i; + createFunction("create " + kind + "function python_auto_suspend_db." + name + + definition + ", 'auto_suspend'='" + values[i] + "');", ctx); + assertAutoSuspendMetadata(findFunction(db, name), values[i]); + if (kind.equals("tables ")) { + assertAutoSuspendMetadata(findFunction(db, name + "_outer"), values[i]); + } + } + } + } + + private void assertAutoSuspendMetadata(Function fn, long expected) { + Assertions.assertEquals(expected, fn.getAutoSuspend()); + Assertions.assertTrue(FunctionToSqlConverter.toSql(fn, false) + .contains("\"AUTO_SUSPEND\"=\"" + expected + "\"")); + Assertions.assertEquals(expected, FunctionToThriftConverter.toThrift( + fn, fn.getReturnType(), fn.getArgs(), new Boolean[]{true}).getAutoSuspend()); + } + + @Test + public void testJavaUdfRejectsAutoSuspend() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database java_auto_suspend_db;"); + createFunction("create function java_auto_suspend_db.java_default(timestamp_ns) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='org.apache.doris.catalog.TimestampNsUdf');", ctx); + assertCreateFunctionAnalysisException(ctx, + "create function java_auto_suspend_db.java_auto_suspend(timestamp_ns) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='org.apache.doris.catalog.TimestampNsUdf', " + + "'auto_suspend'='0');", + "auto_suspend property only supports PYTHON_UDF"); + } + + @Test + public void testAutoSuspendRejectsInvalidValues() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database invalid_auto_suspend_db;"); + for (String invalid : new String[]{"abc", "1.5", "9223372036854775808", "-9223372036854775809"}) { + assertCreateFunctionAnalysisException(ctx, + "create function invalid_auto_suspend_db.invalid_fn(int) returns int " + + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', " + + "'runtime_version'='3.10.2', 'auto_suspend'='" + invalid + "');", + "auto_suspend must be an integer number of seconds"); + } + } + @Test public void testCreatePythonFunctionRejectsObjectTypes() throws Exception { ConnectContext ctx = UtFrameUtils.createDefaultCtx(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java index 26df84fbbe3885..2a7f1d5cbd4fae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java @@ -19,14 +19,27 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.RecoverDatabaseCommand; +import org.apache.doris.task.AgentBatchTask; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskExecutor; +import org.apache.doris.task.CleanUDFCacheTask; import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; public class DropDbTest extends TestWithFeService { @@ -41,6 +54,138 @@ protected void runBeforeAll() throws Exception { + "distributed by hash(k2) buckets 1 " + "properties('replication_num' = '1');"); } + @Test + public void testDropDbPreservesFunctionsForRecovery() throws Exception { + Database db = createDatabaseWithPythonFunctions("drop_function_cache_db"); + Set functionIds = functionIds(db); + try (MockedStatic executor = Mockito.mockStatic(AgentTaskExecutor.class)) { + dropDatabaseWithSql("drop database drop_function_cache_db"); + RecoverDatabaseCommand recover = (RecoverDatabaseCommand) new NereidsParser() + .parseSingle("recover database drop_function_cache_db"); + recover.run(connectContext, null); + Database recovered = Env.getCurrentInternalCatalog().getDbOrMetaException("drop_function_cache_db"); + Assertions.assertEquals(functionIds, functionIds(recovered)); + executor.verifyNoInteractions(); + } + } + + @Test + public void testForceDropDbSubmitsFunctionCacheCleanup() throws Exception { + Database db = createDatabaseWithPythonFunctions("force_drop_function_cache_db"); + // Java database cleanup is unchanged; only the new Python pools need this notification. + db.getFunctions().stream() + .filter(function -> function.getFunctionName().getFunction().equals("py_scalar")) + .findFirst().orElseThrow().setBinaryType(Function.BinaryType.JAVA_UDF); + try (MockedStatic executor = Mockito.mockStatic(AgentTaskExecutor.class)) { + dropDatabaseWithSql("drop database force_drop_function_cache_db force"); + assertFunctionCacheCleanupSubmitted(executor, db); + } + Assertions.assertNull(Env.getCurrentInternalCatalog().getDbNullable(db.getFullName())); + } + + @Test + public void testExpiredDbSubmitsFunctionCacheCleanup() throws Exception { + assertErasedDbSubmitsFunctionCacheCleanup("expired"); + } + + @Test + public void testSameNameErasedDbSubmitsFunctionCacheCleanup() throws Exception { + assertErasedDbSubmitsFunctionCacheCleanup("same_name"); + } + + @Test + public void testInstantlyErasedDbSubmitsFunctionCacheCleanup() throws Exception { + assertErasedDbSubmitsFunctionCacheCleanup("instantly"); + } + + @Test + public void testDropDbWithoutFunctionsDoesNotSubmitCacheCleanup() throws Exception { + createDatabaseWithSql("create database drop_empty_function_cache_db"); + try (MockedStatic executor = Mockito.mockStatic(AgentTaskExecutor.class)) { + dropDatabaseWithSql("drop database drop_empty_function_cache_db force"); + executor.verifyNoInteractions(); + } + } + + private void assertErasedDbSubmitsFunctionCacheCleanup(String kind) throws Exception { + Database db = createDatabaseWithPythonFunctions("erase_" + kind + "_function_cache_db"); + dropDatabaseWithSql("drop database " + db.getFullName()); + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + try (MockedStatic executor = Mockito.mockStatic(AgentTaskExecutor.class)) { + switch (kind) { + case "expired": + recycleBin.setRecycleTimeByIdForReplay(db.getId(), 0L); + Deencapsulation.invoke(recycleBin, "eraseDatabase", System.currentTimeMillis(), -1); + break; + case "same_name": + recycleBin.setRecycleTimeByIdForReplay(db.getId(), 1L); + Deencapsulation.invoke(recycleBin, "eraseDatabaseWithSameName", db.getFullName(), + System.currentTimeMillis(), 0, new ArrayList<>(List.of(db.getId()))); + break; + case "instantly": + recycleBin.eraseDatabaseInstantly(db.getId()); + break; + default: + throw new AssertionError(kind); + } + Assertions.assertFalse(recycleBin.isRecycleDatabase(db.getId())); + assertFunctionCacheCleanupSubmitted(executor, db); + } + } + + private Database createDatabaseWithPythonFunctions(String dbName) throws Exception { + createDatabaseWithSql("create database " + dbName); + createFunction("create function " + dbName + ".py_scalar(int) returns int properties(" + + "'type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2')"); + createFunction("create aggregate function " + dbName + ".py_agg(int) returns int properties(" + + "'type'='PYTHON_UDF', 'symbol'='Agg', 'runtime_version'='3.10.2')"); + createFunction("create tables function " + dbName + ".py_table(int) returns array properties(" + + "'type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2')"); + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(dbName); + Assertions.assertEquals(4, functionIds(db).size()); + Assertions.assertTrue(db.getFunctions().stream() + .anyMatch(function -> function.getFunctionName().getFunction().equals("py_table_outer"))); + return db; + } + + private Set functionIds(Database db) { + return db.getFunctions().stream().map(Function::getId).collect(Collectors.toSet()); + } + + private void assertFunctionCacheCleanupSubmitted(MockedStatic executor, Database db) + throws Exception { + Map functions = db.getFunctions().stream() + .filter(function -> function.getBinaryType() == Function.BinaryType.PYTHON_UDF) + .collect(Collectors.toMap(Function::getId, Function::signatureString)); + Set databaseFunctionIds = functionIds(db); + Set backendIds = Env.getCurrentSystemInfo().getAllBackendsByAllCluster().keySet(); + Assertions.assertFalse(backendIds.isEmpty()); + ArgumentCaptor submitted = ArgumentCaptor.forClass(AgentBatchTask.class); + executor.verify(() -> AgentTaskExecutor.submit(submitted.capture()), Mockito.atLeastOnce()); + Map> cleanedFunctions = new HashMap<>(); + int taskCount = 0; + for (AgentBatchTask batch : submitted.getAllValues()) { + for (AgentTask task : batch.getAllTasks()) { + if (task instanceof CleanUDFCacheTask) { + CleanUDFCacheTask cleanup = (CleanUDFCacheTask) task; + long functionId = cleanup.toThrift().getFunctionId(); + // Recycle-bin scans may also erase other databases in the same test environment. + if (databaseFunctionIds.contains(functionId)) { + Assertions.assertTrue(functions.containsKey(functionId), "Only Python pools need cleanup"); + cleanedFunctions.computeIfAbsent(task.getBackendId(), ignored -> new HashMap<>()) + .put(functionId, cleanup.toThrift().getFunctionSignature()); + taskCount++; + } + } + } + } + Assertions.assertEquals(backendIds, cleanedFunctions.keySet()); + for (Map cleaned : cleanedFunctions.values()) { + Assertions.assertEquals(functions, cleaned); + } + Assertions.assertEquals(backendIds.size() * functions.size(), taskCount); + } + @Test public void testNormalDropDb() throws Exception { Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test1"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java index 2aafc93c8ea661..f80730c43ba4a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java @@ -18,17 +18,66 @@ package org.apache.doris.catalog; import org.apache.doris.catalog.Function.BinaryType; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TAggregateFunction; import org.apache.doris.thrift.TFunction; import org.apache.doris.thrift.TFunctionBinaryType; +import com.google.gson.JsonObject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class FunctionToThriftConverterTest { + @Test + void testPoolSizeSurvivesCopyAndThrift() { + ScalarFunction fn = ScalarFunction.createUdf(BinaryType.PYTHON_UDF, + new FunctionName("db1", "python_fn"), new Type[]{Type.INT}, Type.INT, + false, null, "evaluate", null, null); + Assertions.assertEquals(3, fn.getPoolSize()); + fn.setPoolSize(2); + ScalarFunction copy = new ScalarFunction(fn); + Assertions.assertEquals(2, copy.getPoolSize()); + ScalarFunction restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(fn), ScalarFunction.class); + Assertions.assertEquals(2, restored.getPoolSize()); + TFunction result = FunctionToThriftConverter.toThrift( + copy, Type.INT, new Type[]{Type.INT}, new Boolean[]{true}); + Assertions.assertEquals(2, result.getPoolSize()); + } + // ======================== ScalarFunction Tests ======================== + @Test + void testAutoSuspendSurvivesCopySerializationAndThrift() { + ScalarFunction scalar = ScalarFunction.createUdf(BinaryType.PYTHON_UDF, + new FunctionName("db1", "python_fn"), new Type[]{Type.INT}, Type.INT, + false, null, "evaluate", null, null); + AggregateFunction aggregate = new AggregateFunction( + new FunctionName("db1", "python_agg"), new Type[]{Type.INT}, Type.INT, false, + Type.INT, null, "init", "update", "merge", null, null, null, null); + aggregate.setBinaryType(BinaryType.PYTHON_UDF); + for (Function fn : new Function[]{scalar, aggregate}) { + Assertions.assertEquals(-1, fn.getAutoSuspend()); + JsonObject oldJson = GsonUtils.GSON.toJsonTree(fn).getAsJsonObject(); + oldJson.remove("as"); + Function oldRestored = GsonUtils.GSON.fromJson(oldJson, fn.getClass()); + Assertions.assertEquals(-1, oldRestored.getAutoSuspend()); + for (long value : new long[]{Long.MIN_VALUE, -2, 0, 30, Long.MAX_VALUE}) { + fn.setAutoSuspend(value); + Function copy = fn.clone(); + Assertions.assertEquals(value, copy.getAutoSuspend()); + JsonObject json = GsonUtils.GSON.toJsonTree(fn).getAsJsonObject(); + Assertions.assertEquals(value, json.get("as").getAsLong()); + Function restored = GsonUtils.GSON.fromJson(json, fn.getClass()); + Assertions.assertEquals(value, restored.getAutoSuspend()); + TFunction thrift = FunctionToThriftConverter.toThrift( + copy, Type.INT, new Type[]{Type.INT}, new Boolean[]{true}); + Assertions.assertTrue(thrift.isSetAutoSuspend()); + Assertions.assertEquals(value, thrift.getAutoSuspend()); + } + } + } + @Test void testScalarFunctionJavaUdf_symbolIsPopulated() { FunctionName name = new FunctionName("db1", "java_udf_fn"); @@ -41,6 +90,7 @@ void testScalarFunctionJavaUdf_symbolIsPopulated() { Assertions.assertTrue(result.isSetScalarFn()); Assertions.assertFalse(result.isSetAggregateFn()); + Assertions.assertFalse(result.isSetAutoSuspend()); Assertions.assertEquals("com.example.MyFn", result.getScalarFn().getSymbol()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java index 3c54a1acd18b5d..8a7e21911b28da 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java @@ -111,10 +111,47 @@ void testVolatilePythonUdafUsesUniqueIdentity() { Assertions.assertEquals(ignoredFirst, ignoredSecond); } + @Test + void testPoolSizeSurvivesExpressionCopies() { + PythonUdf udf = pythonUdf(FunctionVolatility.IMMUTABLE, VolatileIdentity.NON_VOLATILE); + PythonUdaf udaf = pythonUdaf(FunctionVolatility.IMMUTABLE); + PythonUdtf udtf = pythonUdtf(FunctionVolatility.IMMUTABLE); + Assertions.assertEquals(3, udf.withChildren(udf.children()).getCatalogFunction().getPoolSize()); + Assertions.assertEquals(3, udaf.withDistinctAndChildren(true, udaf.children()) + .getCatalogFunction().getPoolSize()); + Assertions.assertEquals(3, udtf.withChildren(udtf.children()).getCatalogFunction().getPoolSize()); + } + + @Test + void testAutoSuspendSurvivesExpressionCopies() { + for (long value : new long[]{-2, 0, 30}) { + PythonUdf udf = pythonUdf(FunctionVolatility.VOLATILE, + Udf.createVolatileIdentity(FunctionVolatility.VOLATILE), value); + PythonUdaf udaf = pythonUdaf(FunctionVolatility.VOLATILE, value); + PythonUdtf udtf = pythonUdtf(FunctionVolatility.VOLATILE, value); + for (Function fn : new Function[]{ + udf.withChildren(udf.children()).getCatalogFunction(), + udf.withIgnoreUniqueId(true).getCatalogFunction(), + udf.withFreshVolatileIdentity().getCatalogFunction(), + udaf.withDistinctAndChildren(true, udaf.children()).getCatalogFunction(), + udaf.withIgnoreUniqueId(true).getCatalogFunction(), + udaf.withFreshVolatileIdentity().getCatalogFunction(), + udtf.withChildren(udtf.children()).getCatalogFunction(), + udtf.withIgnoreUniqueId(true).getCatalogFunction(), + udtf.withFreshVolatileIdentity().getCatalogFunction()}) { + Assertions.assertEquals(value, fn.getAutoSuspend()); + } + } + } + private PythonUdf pythonUdf(FunctionVolatility volatility, VolatileIdentity volatileIdentity) { + return pythonUdf(volatility, volatileIdentity, -1); + } + + private PythonUdf pythonUdf(FunctionVolatility volatility, VolatileIdentity volatileIdentity, long autoSuspend) { return new PythonUdf("py_fn", 1, "db1", Function.BinaryType.PYTHON_UDF, signature(), NullableMode.ALWAYS_NULLABLE, volatility, volatileIdentity, - null, "evaluate", null, null, "", false, 360, "3.10.2", "", + null, "evaluate", null, null, "", false, 360, "3.10.2", "", 3, autoSuspend, new IntegerLiteral(1)); } @@ -125,16 +162,24 @@ private JavaUdf javaUdf(FunctionVolatility volatility, VolatileIdentity volatile } private PythonUdaf pythonUdaf(FunctionVolatility volatility) { + return pythonUdaf(volatility, -1); + } + + private PythonUdaf pythonUdaf(FunctionVolatility volatility, long autoSuspend) { return new PythonUdaf("py_agg", 1, "db1", Function.BinaryType.PYTHON_UDF, signature(), IntegerType.INSTANCE, NullableMode.ALWAYS_NULLABLE, volatility, Udf.createVolatileIdentity(volatility), null, "Agg", null, null, null, null, null, null, null, false, "", false, 360, - "3.10.2", "", new IntegerLiteral(1)); + "3.10.2", "", 3, autoSuspend, new IntegerLiteral(1)); } private PythonUdtf pythonUdtf(FunctionVolatility volatility) { + return pythonUdtf(volatility, -1); + } + + private PythonUdtf pythonUdtf(FunctionVolatility volatility, long autoSuspend) { return new PythonUdtf("py_table", 1, "db1", Function.BinaryType.PYTHON_UDF, signature(), NullableMode.ALWAYS_NULLABLE, volatility, Udf.createVolatileIdentity(volatility), - null, "evaluate", null, null, "", false, 360, "3.10.2", "", new IntegerLiteral(1)); + null, "evaluate", null, null, "", false, 360, "3.10.2", "", 3, autoSuspend, new IntegerLiteral(1)); } private FunctionSignature signature() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java index 11aeff8bad338e..eb6d590efbc18b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java @@ -79,7 +79,7 @@ void testAllUdfVolatilitiesRemainExcluded() { "file:///udf.jar", "F", "", "", "", false, 360, slot); Expression pythonUdf = new PythonUdf("f", 2, "db", Function.BinaryType.PYTHON_UDF, signature, Function.NullableMode.DEPEND_ON_ARGUMENT, volatility, identity, - "", "f", "", "", "", false, 360, "3.12", "", slot); + "", "f", "", "", "", false, 360, "3.12", "", 1, -1, slot); Assertions.assertEquals(volatility == FunctionVolatility.IMMUTABLE, javaUdf.isDeterministic()); Assertions.assertTrue(uncacheable(javaUdf)); Assertions.assertTrue(uncacheable(pythonUdf)); diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index e18c66ec89125a..bd7f2c55719760 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -420,6 +420,8 @@ struct TFunction { 17: optional TDictFunction dict_function 18: optional string runtime_version 19: optional string function_code + 20: optional i32 pool_size = 3 + 21: optional i64 auto_suspend = -1 // idle timeout in seconds; negative values disable suspension } enum TJdbcOperation { diff --git a/regression-test/data/pythonudaf_p0/test_pythonudaf_auto_suspend.out b/regression-test/data/pythonudaf_p0/test_pythonudaf_auto_suspend.out new file mode 100644 index 00000000000000..c8f9c9befe9f5e --- /dev/null +++ b/regression-test/data/pythonudaf_p0/test_pythonudaf_auto_suspend.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !result_0_0 -- +6 + +-- !result_1_0 -- +6 + +-- !result_0_1 -- +6 + +-- !result_1_1 -- +6 + +-- !result_0_2 -- +6 + +-- !result_1_2 -- +6 + +-- !result_0_3 -- +6 + +-- !result_1_3 -- +6 + +-- !result_2 -- +6 + +-- !result_3 -- +6 + diff --git a/regression-test/data/pythonudf_p0/test_pythonudf_auto_suspend.out b/regression-test/data/pythonudf_p0/test_pythonudf_auto_suspend.out new file mode 100644 index 00000000000000..c8671ef89d9213 --- /dev/null +++ b/regression-test/data/pythonudf_p0/test_pythonudf_auto_suspend.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !result_0_0 -- +11 +12 +13 + +-- !result_1_0 -- +11 +12 +13 + +-- !result_0_1 -- +11 +12 +13 + +-- !result_1_1 -- +11 +12 +13 + +-- !result_0_2 -- +11 +12 +13 + +-- !result_1_2 -- +11 +12 +13 + +-- !result_0_3 -- +11 +12 +13 + +-- !result_1_3 -- +11 +12 +13 + diff --git a/regression-test/data/pythonudf_p0/test_pythonudf_native_module_cache.out b/regression-test/data/pythonudf_p0/test_pythonudf_native_module_cache.out new file mode 100644 index 00000000000000..4000b9b000e55e --- /dev/null +++ b/regression-test/data/pythonudf_p0/test_pythonudf_native_module_cache.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !lazy_import -- +1107 + +-- !concurrent_import -- +12 1107 1107 + +-- !repeated_import -- +1107 \N + diff --git a/regression-test/data/pythonudtf_p0/test_pythonudtf_auto_suspend.out b/regression-test/data/pythonudtf_p0/test_pythonudtf_auto_suspend.out new file mode 100644 index 00000000000000..009a7d1b26408f --- /dev/null +++ b/regression-test/data/pythonudtf_p0/test_pythonudtf_auto_suspend.out @@ -0,0 +1,65 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !result_0_0 -- +11 +12 +13 + +-- !result_1_0 -- +\N + +-- !result_2_0 -- +11 +12 +13 + +-- !result_3_0 -- +\N + +-- !result_0_1 -- +11 +12 +13 + +-- !result_1_1 -- +\N + +-- !result_2_1 -- +11 +12 +13 + +-- !result_3_1 -- +\N + +-- !result_0_2 -- +11 +12 +13 + +-- !result_1_2 -- +\N + +-- !result_2_2 -- +11 +12 +13 + +-- !result_3_2 -- +\N + +-- !result_0_3 -- +11 +12 +13 + +-- !result_1_3 -- +\N + +-- !result_2_3 -- +11 +12 +13 + +-- !result_3_3 -- +\N + diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf b/regression-test/pipeline/cloud_p0/conf/be_custom.conf index 86b396dce3e824..1cdc9dc9c13592 100644 --- a/regression-test/pipeline/cloud_p0/conf/be_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/be_custom.conf @@ -65,7 +65,6 @@ fail_when_segment_rows_not_in_rowset_meta=true enable_python_udf_support=true python_env_mode=conda python_conda_root_path=/opt/miniforge3 -max_python_process_num=16 enable_cloud_make_rs_visible_on_be=true cloud_mow_sync_rowsets_when_load_txn_begin=false diff --git a/regression-test/pipeline/cloud_p1/conf/be_custom.conf b/regression-test/pipeline/cloud_p1/conf/be_custom.conf index 973b34376b6cfd..c70a36932d83cb 100644 --- a/regression-test/pipeline/cloud_p1/conf/be_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/be_custom.conf @@ -45,7 +45,6 @@ enable_segment_rows_check_core=true enable_python_udf_support=true python_env_mode=conda python_conda_root_path=/opt/miniconda3 -max_python_process_num=16 enable_cloud_make_rs_visible_on_be=true cloud_mow_sync_rowsets_when_load_txn_begin=false diff --git a/regression-test/pipeline/external/conf/be.conf b/regression-test/pipeline/external/conf/be.conf index 8300afb35939d5..b73e742feba87e 100644 --- a/regression-test/pipeline/external/conf/be.conf +++ b/regression-test/pipeline/external/conf/be.conf @@ -79,6 +79,5 @@ enable_graceful_exit_check=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/pipeline/nonConcurrent/conf/be.conf b/regression-test/pipeline/nonConcurrent/conf/be.conf index f0fc6ae7728f32..0c9549aaa0066f 100644 --- a/regression-test/pipeline/nonConcurrent/conf/be.conf +++ b/regression-test/pipeline/nonConcurrent/conf/be.conf @@ -99,6 +99,5 @@ enable_segment_rows_check_core=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf index d7145b68494773..b2adc781a0f4b8 100644 --- a/regression-test/pipeline/p0/conf/be.conf +++ b/regression-test/pipeline/p0/conf/be.conf @@ -102,6 +102,5 @@ fail_when_segment_rows_not_in_rowset_meta=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/pipeline/p1/conf/be.conf b/regression-test/pipeline/p1/conf/be.conf index 74bcff45aaf27e..02f7e183b27d54 100644 --- a/regression-test/pipeline/p1/conf/be.conf +++ b/regression-test/pipeline/p1/conf/be.conf @@ -85,6 +85,5 @@ enable_segment_rows_check_core=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/pipeline/performance/conf/be_custom.conf b/regression-test/pipeline/performance/conf/be_custom.conf index 3e2ba2aac4cc3f..e2565519772507 100644 --- a/regression-test/pipeline/performance/conf/be_custom.conf +++ b/regression-test/pipeline/performance/conf/be_custom.conf @@ -30,6 +30,5 @@ enable_graceful_exit_check=true enable_python_udf_support = true python_env_mode = venv python_venv_interpreter_paths = /usr/bin/python -max_python_process_num = 64 enable_cloud_random_segment_id=true \ No newline at end of file diff --git a/regression-test/pipeline/vault_p0/conf/be_custom.conf b/regression-test/pipeline/vault_p0/conf/be_custom.conf index ccfdac33c885cb..a5e682d04a7efe 100644 --- a/regression-test/pipeline/vault_p0/conf/be_custom.conf +++ b/regression-test/pipeline/vault_p0/conf/be_custom.conf @@ -45,6 +45,5 @@ enable_brpc_connection_check=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy new file mode 100644 index 00000000000000..d03cb5f30c0b97 --- /dev/null +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudaf_auto_suspend", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + sql "DROP TABLE IF EXISTS py_auto_suspend_input" + sql """CREATE TABLE py_auto_suspend_input(v INT) + DUPLICATE KEY(v) DISTRIBUTED BY HASH(v) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO py_auto_suspend_input VALUES (1), (2), (3)" + try { + // Check results before/after idling; BE unit tests separately verify process termination. + ["0", "1", "-2", null].eachWithIndex { timeout, index -> + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + def property = timeout == null ? "" : ", \"auto_suspend\"=\"${timeout}\"" + sql """CREATE AGGREGATE FUNCTION py_auto_suspend(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "symbol"="SumAgg", + "runtime_version"="${runtime}", "pool_size"="1" ${property}) + AS \$\$ +class SumAgg: + def __init__(self): + self.total = 0 + @property + def aggregate_state(self): + return self.total + def accumulate(self, v): + if v is not None: + self.total += v + def merge(self, other): + self.total += other + def finish(self): + return self.total +\$\$""" + "qt_result_0_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + Thread.sleep(2500) + "qt_result_1_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + } + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + def zip = "${context.file.parent}/udaf_scripts/python_udaf_drop_a/python_udaf_drop_test.zip" + scp_udf_file_to_all_be(zip) + scp_udf_file_to_all_fe(zip) + sql """CREATE AGGREGATE FUNCTION py_auto_suspend(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "symbol"="drop_udaf.SumAgg", + "file"="file://${zip}", "runtime_version"="${runtime}", + "pool_size"="1", "auto_suspend"="0")""" + qt_result_2("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + Thread.sleep(2500) + qt_result_3("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + } finally { + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + } +} diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy index 9523d2c76784f8..3dca47f21ee47b 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy @@ -45,7 +45,8 @@ suite('test_pythonudaf_pkg_isolation') { "type" = "PYTHON_UDF", "file" = "file://${zipA}", "symbol" = "mypkg.mod_x.SumAgg", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ sql """ @@ -53,7 +54,8 @@ suite('test_pythonudaf_pkg_isolation') { "type" = "PYTHON_UDF", "file" = "file://${zipB}", "symbol" = "mypkg.mod_x.SumAgg", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -66,7 +68,8 @@ suite('test_pythonudaf_pkg_isolation') { "type" = "PYTHON_UDF", "file" = "file://${zipA}", "symbol" = "mypkg.mod_y.SumAgg", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -79,7 +82,8 @@ suite('test_pythonudaf_pkg_isolation') { "type" = "PYTHON_UDF", "file" = "file://${zipB}", "symbol" = "mypkg.mod_y.SumAgg", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy new file mode 100644 index 00000000000000..f31e3b125e38e2 --- /dev/null +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudf_auto_suspend", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + sql "DROP TABLE IF EXISTS py_auto_suspend_input" + sql """CREATE TABLE py_auto_suspend_input(v INT) + DUPLICATE KEY(v) DISTRIBUTED BY HASH(v) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO py_auto_suspend_input VALUES (1), (2), (3)" + try { + // Check results before/after idling; BE unit tests separately verify process termination. + ["0", "1", "-2", null].eachWithIndex { timeout, index -> + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + def property = timeout == null ? "" : ", \"auto_suspend\"=\"${timeout}\"" + sql """CREATE FUNCTION py_auto_suspend(INT) RETURNS INT + PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + "runtime_version"="${runtime}", "pool_size"="1" ${property}) + AS \$\$ +def evaluate(v): + return v + 10 if v is not None else None +\$\$""" + "qt_result_0_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input ORDER BY v") + Thread.sleep(2500) + "qt_result_1_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input ORDER BY v") + } + } finally { + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + } +} diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy index 339ceb47cb7ce4..e85bba2d4a074f 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy @@ -19,6 +19,16 @@ suite("test_pythonudf_inline_basic") { // Test basic Python UDF using Inline mode def runtime_version = getPythonUdfRuntimeVersion() + for (def invalidPoolSize : ["0", "-1", "abc", "1.5", "2147483648"]) { + test { + sql """CREATE FUNCTION py_invalid_pool_size(INT) RETURNS INT PROPERTIES ( + "type" = "PYTHON_UDF", "symbol" = "evaluate", + "runtime_version" = "${runtime_version}", "pool_size" = "${invalidPoolSize}" + ) AS \$\$def evaluate(x): return x\$\$""" + exception "pool_size must be a positive integer" + } + } + try { // Test 1: Simple integer addition sql """ DROP FUNCTION IF EXISTS py_add(INT, INT); """ @@ -28,7 +38,8 @@ suite("test_pythonudf_inline_basic") { PROPERTIES ( "type" = "PYTHON_UDF", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "2" ) AS \$\$ def evaluate(a, b): diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy new file mode 100644 index 00000000000000..51710e240f6315 --- /dev/null +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.CountDownLatch + +suite("test_pythonudf_native_module_cache") { + def runtime_version = getPythonUdfRuntimeVersion() + def pyPath = """${context.file.parent}/udf_scripts/pyudf.zip""" + scp_udf_file_to_all_be(pyPath) + scp_udf_file_to_all_fe(pyPath) + + sql """DROP FUNCTION IF EXISTS py_native_module_cache(INT)""" + sql """ + CREATE FUNCTION py_native_module_cache(INT) RETURNS INT PROPERTIES ( + "type" = "PYTHON_UDF", + "file" = "file://${pyPath}", + "symbol" = "native_module_cache_test.evaluate", + "runtime_version" = "${runtime_version}", + "pool_size" = "1" + ) + """ + sql """DROP TABLE IF EXISTS test_pythonudf_native_module_cache_results""" + sql """ + CREATE TABLE test_pythonudf_native_module_cache_results (value INT) + DISTRIBUTED BY HASH(value) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + + try { + // The first call fails if the module directory is removed before evaluate. + qt_lazy_import """SELECT py_native_module_cache(7)""" + def ready = new CountDownLatch(4) + def start = new CountDownLatch(1) + def workers = (0..<4).collect { worker -> + thread("native-module-cache-${worker}") { + ready.countDown() + start.await() + 3.times { + sql """INSERT INTO test_pythonudf_native_module_cache_results + SELECT py_native_module_cache(7)""" + } + } + } + ready.await() + start.countDown() + workers.each { it.get() } + // Every client must observe exactly one execution of module-level code. + qt_concurrent_import """SELECT COUNT(*), MIN(value), MAX(value) + FROM test_pythonudf_native_module_cache_results""" + qt_repeated_import """SELECT py_native_module_cache(7), py_native_module_cache(NULL)""" + } finally { + try_sql("DROP FUNCTION IF EXISTS py_native_module_cache(INT)") + } +} diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy index 0107b4b582a89d..d883438119aaed 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy @@ -34,7 +34,8 @@ suite("test_pythonudf_pkg_isolation") { "type" = "PYTHON_UDF", "file" = "file://${zipA}", "symbol" = "mypkg.mod_x.evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ sql """ @@ -42,7 +43,8 @@ suite("test_pythonudf_pkg_isolation") { "type" = "PYTHON_UDF", "file" = "file://${zipB}", "symbol" = "mypkg.mod_x.evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -55,7 +57,8 @@ suite("test_pythonudf_pkg_isolation") { "type" = "PYTHON_UDF", "file" = "file://${zipA}", "symbol" = "mypkg.mod_y.evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -68,7 +71,8 @@ suite("test_pythonudf_pkg_isolation") { "type" = "PYTHON_UDF", "file" = "file://${zipB}", "symbol" = "mypkg.mod_y.evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ diff --git a/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_helper.py b/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_helper.py new file mode 100644 index 00000000000000..290343e4f801ab --- /dev/null +++ b/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_helper.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +def add_offset(value): + return value + 100 diff --git a/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_test.py b/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_test.py new file mode 100644 index 00000000000000..07b3b02f1ef93f --- /dev/null +++ b/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_test.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import builtins + + +# Keep the counter outside this module so an unintended reimport is observable. +builtins._doris_native_module_loads = getattr(builtins, "_doris_native_module_loads", 0) + 1 + + +def evaluate(value): + if value is None: + return None + # This dependency must remain importable after the loader returns. + from native_module_cache_helper import add_offset + return builtins._doris_native_module_loads * 1000 + add_offset(value) diff --git a/regression-test/suites/pythonudf_p0/udf_scripts/pyudf.zip b/regression-test/suites/pythonudf_p0/udf_scripts/pyudf.zip index 7a33bc3e20d22a932909d9567e87b59bcc390356..f5a576d25490161b66a9c0fc53d4fbfcbfd3045f 100644 GIT binary patch delta 1503 zcmca#bG(CbLxx#BGm8iV0|N)co0x!D@h_#JQhNHfSV^5th=zsb?5%@4lNf+gdC2?QQPNcC()MORX2VM7`Sj(@}MTOQ45+#1$1$zF7{; zwQbsoTr0JB^8D-F!ra|=h3C8p+%{v|XRA$TXLiXJ9A9{$^3q+6$4(zJjx_CKK7RVI z#QKajrY}Cn`X~Qfe&J4su;$wpoBy=={+U($`he(`S=**LJ>6Fs@~xVQ!>(WA*8`SQ zt`{74?a@(wxJdI>XWzN4*H5S3|GtqUCOLQ2)5trY%98QaO}tOedb64YMzOv-VJ zzZN-3_l4%o+Pik|US$6Lu)(kVWUSk5_FXrZZ9l1Zb6Lgh3s$vmkGHGrjTJbo$@A#? zwF=e+ntS_WVsb9bY>c0vD090=&%1u6^Q=`?frg>KT6;hJzPKrK$=AXt1Dy?my(Njg zI#&NLiMj{2OuS`bRkEo~Q%rrq@jGq1WVRpt`StyrsYfO17I*WVi|Q3TFl7ZpK_iQs z&XpIo!&+NOlXv3G8T5N&oY0qD=`u>ZxkGzoO_x>eU zNjO+zHZ_fjwkPb#!X+EDjqxGr~LGL z7q4b*ZF*!5Ap8ZDYsXG;xpSJ61Hr>dlj6GAF7+$9f4u4~a;vdZC3 zY|ol3_o{%L99`=TsrTHg_IPf*zE@bYCa%Q(<_ybuO*bwWf2-IXzr8vtkzvNsFVBy=wSYB$&~{*mb1v&13l$osh;`4obG$?TjFd%?HS zenPh8w#PBkjM_~@zb;Bz^+j6x>HEdWm%U!U-VzY;Kq>0wG2=}R|DI<{vp61Hx_?jn zE*;(7S7&Eb#ocFp!8mc(Jq@Y(FNAAvzHi>YInC@T`{WN6TyB`9q6DzCWMBZM8ipl} zX=V)g%Scv889DjBg>){+C~%U4r#NJz{4KE?g_QoFhM^U<0p6@^AQwph;c^ZJhAwLm F4*)Csvibl3 delta 34 kcmX@!!Fa!BLx$Pr9_z>KBCKo-Kp+N$kJ%X*(#$|S0LEzuF8}}l diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy new file mode 100644 index 00000000000000..29ceb439ddca0f --- /dev/null +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudtf_auto_suspend", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + sql "DROP TABLE IF EXISTS py_auto_suspend_input" + sql """CREATE TABLE py_auto_suspend_input(v INT) + DUPLICATE KEY(v) DISTRIBUTED BY HASH(v) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO py_auto_suspend_input VALUES (1), (2), (3)" + try { + // Check results before/after idling; BE unit tests separately verify process termination. + ["0", "1", "-2", null].eachWithIndex { timeout, index -> + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + def property = timeout == null ? "" : ", \"auto_suspend\"=\"${timeout}\"" + sql """CREATE TABLES FUNCTION py_auto_suspend(INT) RETURNS ARRAY + PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + "runtime_version"="${runtime}", "pool_size"="1" ${property}) + AS \$\$ +def evaluate(v): + if v is not None: + yield (v + 10,) +\$\$""" + "qt_result_0_${index}"("SELECT x FROM py_auto_suspend_input LATERAL VIEW py_auto_suspend(v) t AS x ORDER BY x") + "qt_result_1_${index}"("SELECT x FROM (SELECT CAST(NULL AS INT) v) s LATERAL VIEW py_auto_suspend_outer(v) t AS x") + Thread.sleep(2500) + "qt_result_2_${index}"("SELECT x FROM py_auto_suspend_input LATERAL VIEW py_auto_suspend(v) t AS x ORDER BY x") + "qt_result_3_${index}"("SELECT x FROM (SELECT CAST(NULL AS INT) v) s LATERAL VIEW py_auto_suspend_outer(v) t AS x") + } + } finally { + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + } +} diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy index e27030fd083e28..40e5777421092d 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy @@ -47,7 +47,8 @@ suite("test_pythonudtf_pkg_isolation") { "type" = "PYTHON_UDF", "file" = "file://${zipA}", "symbol" = "mypkg.mod_x.process", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ sql """ @@ -57,7 +58,8 @@ suite("test_pythonudtf_pkg_isolation") { "type" = "PYTHON_UDF", "file" = "file://${zipB}", "symbol" = "mypkg.mod_x.process", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -78,7 +80,8 @@ suite("test_pythonudtf_pkg_isolation") { "type" = "PYTHON_UDF", "file" = "file://${zipA}", "symbol" = "mypkg.mod_y.process", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -99,7 +102,8 @@ suite("test_pythonudtf_pkg_isolation") { "type" = "PYTHON_UDF", "file" = "file://${zipB}", "symbol" = "mypkg.mod_y.process", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ From 842ec3e6057abf490e8841d65fb7bdee7f1ce7c3 Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Mon, 19 Oct 2026 19:53:41 +0800 Subject: [PATCH 2/5] [fix](udf) Defer dropped Python pool cleanup to health checks ### What problem does this PR solve? Issue Number: N/A Problem Summary: Keep dropped function pools STOPPED in the manager while existing clients are alive, so a running UDAF cannot recreate the pool when it encounters another GROUP BY key. Reclaim stopped pools in the 30-second health check after the last client closes, regardless of auto_suspend. Document the distinction between DROP cleanup and idle suspension, and cover UDF/UDAF/UDTF client rejection and same-name function recreation. ### Release note None ### Check List (For Author) - Test: BE/FE build, 83 BE unit tests, 20 regression suites (including all 18 pythonudtf_p0 suites), clang-tidy, formatting and header hygiene passed for the lifecycle changes. The final equivalent guard consolidation was checked for boolean equivalence and formatting/header hygiene without rerunning the full suite. BE UT compilation used temporary adaptations to three unrelated baseline Variant test files; those adaptations were restored. - Behavior changed: Yes, DROP retains a STOPPED pool until the monitor observes no active clients and reclaims it. - Does this need documentation: No --- be/src/udf/python/python_server.cpp | 27 +++--- be/src/udf/python/python_server.h | 8 +- be/test/udf/python/python_server_test.cpp | 96 ++++++++++++++++++- .../test_pythonudaf_auto_suspend.groovy | 7 ++ .../test_pythonudf_auto_suspend.groovy | 7 ++ .../test_pythonudtf_auto_suspend.groovy | 7 ++ 6 files changed, 134 insertions(+), 18 deletions(-) diff --git a/be/src/udf/python/python_server.cpp b/be/src/udf/python/python_server.cpp index 3761a1eb632362..72209141e68c64 100644 --- a/be/src/udf/python/python_server.cpp +++ b/be/src/udf/python/python_server.cpp @@ -540,12 +540,19 @@ void PythonServerManager::_reclaim_idle_pools(std::chrono::steady_clock::time_po std::erase_if(_process_pools, [&](const auto& entry) { const auto& pool = entry.second; std::scoped_lock pool_lock(pool->mutex); - if (pool->auto_suspend < 0 || pool->active_clients != 0 || - std::chrono::duration_cast(now - pool->idle_since).count() < - pool->auto_suspend) { + // 1. Never reclaim a pool while any client still holds it. + // 2. A STOPPED pool is reclaimed on the next scan after its last client closes, + // regardless of auto_suspend. + // 3. Other pools require auto_suspend >= 0 and no clients for that many seconds. + // The last client closing resets idle_since, so each idle period starts afresh. + if (pool->active_clients != 0 || + (pool->state != PoolState::STOPPED && + (pool->auto_suspend < 0 || + std::chrono::duration_cast(now - pool->idle_since).count() < + pool->auto_suspend))) { return false; } - // Stop late initialization/repair workers, but allow the next call to create a new pool. + // Prevent detached initialization/repair workers from publishing into a retired pool. pool->state = PoolState::STOPPED; pool->has_available_process = false; retired.emplace_back(pool); @@ -825,26 +832,24 @@ void PythonServerManager::_refresh_memory_stats() { void PythonServerManager::drop_function(int64_t function_id) { std::shared_ptr pool; - std::vector processes; { std::scoped_lock map_lock(_pools_mutex); auto iter = _process_pools.find(function_id); if (iter == _process_pools.end()) { return; } - pool = std::move(iter->second); - // Stop publication before removing the pool, so concurrent shutdown cannot miss a late fork. + pool = iter->second; + // Keep the STOPPED pool in the map until the health check sees no active clients. + // For example, a GROUP BY UDAF may process group A, receive DROP, then encounter group B. + // B's create() must find this STOPPED pool and fail instead of starting a new pool while + // A's client is still alive. Existing clients may finish on their current processes. { std::scoped_lock pool_lock(pool->mutex); pool->state = PoolState::STOPPED; pool->has_available_process = false; - processes.swap(pool->processes); } - _process_pools.erase(iter); } pool->cv.notify_all(); - // Release pool ownership outside the mutex. Active Flight clients retain their process; - // the final client destroys it, shutting down Python before the zip is deleted. } // Explicit template instantiation for UDF, UDAF and UDTF clients diff --git a/be/src/udf/python/python_server.h b/be/src/udf/python/python_server.h index 48684741d27123..78ae34da9eee12 100644 --- a/be/src/udf/python/python_server.h +++ b/be/src/udf/python/python_server.h @@ -53,7 +53,7 @@ class PythonServerManager { static Status fork(const PythonVersion& version, ProcessPtr* process); - // Remove this function's pool. Existing clients keep their processes and zip alive. + // Stop accepting new clients. Health check removes the pool after its last client closes. void drop_function(int64_t function_id); void shutdown(); @@ -87,7 +87,7 @@ class PythonServerManager { // 1. All process slots have attempted initialization. // 2. At least one live process is available for requests. INITIALIZED, - // Prevent the pool from being incorrectly reused after `shutdown()` + // Reject new clients and late fork publication after DROP, shutdown or idle reclamation. STOPPED, }; @@ -146,10 +146,12 @@ class PythonServerManager { /** * Start health check background thread when the first function pool is created - * Thread periodically checks process health and refreshes memory stats + * Thread periodically reclaims pools, checks process health and refreshes memory stats. */ void _start_health_check_thread(); + // Only reclaim pools with no active clients: STOPPED pools on the next scan; other pools + // after auto_suspend seconds without a client (negative values disable idle reclamation). void _reclaim_idle_pools(std::chrono::steady_clock::time_point now); /** diff --git a/be/test/udf/python/python_server_test.cpp b/be/test/udf/python/python_server_test.cpp index 9810f272fee63f..43f47ba1c2f15d 100644 --- a/be/test/udf/python/python_server_test.cpp +++ b/be/test/udf/python/python_server_test.cpp @@ -38,6 +38,7 @@ #include "udf/python/python_udaf_client.h" #include "udf/python/python_udf_client.h" #include "udf/python/python_udf_meta.h" +#include "udf/python/python_udtf_client.h" #include "util/defer_op.h" namespace doris { @@ -82,7 +83,8 @@ class PythonServerTest : public ::testing::Test { SCOPED_TRACE(repair ? "repaired process" : "initial process"); PythonServerManager mgr; auto meta = test_meta(version); - auto pool = mgr._ensure_pool_initialized(meta, version); + std::shared_ptr lease; + auto pool = mgr._ensure_pool_initialized(meta, version, nullptr, &lease); ASSERT_TRUE(pool.has_value()) << pool.error().to_string(); ASSERT_TRUE( mgr.wait_for_process_pool_initialized_for_test(meta.id, std::chrono::seconds(5))); @@ -100,12 +102,15 @@ class PythonServerTest : public ::testing::Test { std::weak_ptr process_lifetime = client_process; mgr.drop_function(meta.id); - EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); + EXPECT_EQ(mgr.process_pool_snapshot_for_test(meta.id).size(), 1); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); ASSERT_TRUE(client_process->is_alive()); mgr._refresh_memory_stats(); EXPECT_GT(mgr._mem_tracker.consumption(), 0); client_process.reset(); + lease.reset(); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); EXPECT_TRUE(process_lifetime.expired()); mgr._refresh_memory_stats(); EXPECT_EQ(mgr._mem_tracker.consumption(), 0); @@ -1023,7 +1028,86 @@ TEST_F(PythonServerTest, FunctionsWithSameRuntimeHaveIndependentFixedPools) { EXPECT_NE(first_client, second_client); } -TEST_F(PythonServerTest, DropKeepsActiveProcessAliveAndDoesNotAffectOtherFunction) { +TEST_F(PythonServerTest, DropRejectsNewClientsUntilHealthCheckReclaimsThePool) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + std::shared_ptr lease; + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr, &lease).value(); + + mgr.drop_function(meta.id); + ASSERT_TRUE(mgr._process_pools.contains(meta.id)); + EXPECT_EQ(PythonServerManager::PoolState::STOPPED, pool->state); + EXPECT_TRUE(process->is_alive()); + + // UDAF can encounter a new GROUP BY key after DROP while an earlier group's client is alive. + // All three client factories must reject this pool instead of creating a replacement. + std::shared_ptr udf; + std::shared_ptr udaf; + std::shared_ptr udtf; + EXPECT_FALSE(mgr.get_client(meta, version, &udf).ok()); + EXPECT_FALSE(mgr.get_client(meta, version, &udaf).ok()); + EXPECT_FALSE(mgr.get_client(meta, version, &udtf).ok()); + EXPECT_EQ(pool, mgr._process_pools.at(meta.id)); + EXPECT_EQ(1, pool->active_clients); + + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_TRUE(mgr._process_pools.contains(meta.id)); + EXPECT_TRUE(process->is_alive()); + mgr.drop_function(meta.id); // Duplicate cleanup must keep the same STOPPED pool. + EXPECT_EQ(pool, mgr._process_pools.at(meta.id)); + + lease.reset(); + EXPECT_TRUE(mgr._process_pools.contains(meta.id)); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(mgr._process_pools.contains(meta.id)); + EXPECT_FALSE(process->is_alive()); +} + +TEST_F(PythonServerTest, HealthCheckReclaimsStoppedPoolsRegardlessOfAutoSuspend) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + for (int64_t timeout : {-1, 0, 3600}) { + auto meta = test_meta(version); + meta.id = 3000 + timeout; + meta.auto_suspend = timeout; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + mgr.drop_function(meta.id); + ASSERT_TRUE(mgr._process_pools.contains(meta.id)); + EXPECT_TRUE(process->is_alive()); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(mgr._process_pools.contains(meta.id)); + EXPECT_FALSE(process->is_alive()); + } +} + +TEST_F(PythonServerTest, MonitorReclaimsDroppedPoolAfterLastClientCloses) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + PythonClient client; + ASSERT_TRUE(mgr._get_or_create_process_pool(meta, version, nullptr, &client._pool_lease) + .has_value()); + client._process = process; + mgr.drop_function(meta.id); + std::this_thread::sleep_for(std::chrono::seconds(2)); + EXPECT_TRUE(process->is_alive()); + ASSERT_TRUE(client.close().ok()); + + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(40); + while (process->is_alive() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + EXPECT_FALSE(process->is_alive()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); +} + +TEST_F(PythonServerTest, DropDefersProcessCleanupAndDoesNotAffectOtherFunction) { PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); PythonUDFMeta first; @@ -1035,6 +1119,9 @@ TEST_F(PythonServerTest, DropKeepsActiveProcessAliveAndDoesNotAffectOtherFunctio mgr.set_process_pool_for_test(second, version, {create_sleep_process()}); mgr.drop_function(101); EXPECT_TRUE(active->is_alive()); + EXPECT_EQ(mgr.process_pool_snapshot_for_test(101).size(), 1); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(active->is_alive()); EXPECT_TRUE(mgr.process_pool_snapshot_for_test(101).empty()); EXPECT_EQ(mgr.process_pool_snapshot_for_test(102).size(), 1); active.reset(); @@ -1079,7 +1166,8 @@ TEST_F(PythonServerTest, DropDuringInitializationKeepsPeriodicMemoryAccounting) sync_point->enable_processing(); auto meta = test_meta(version); meta.pool_size = 2; - auto pool = mgr._ensure_pool_initialized(meta, version); + std::shared_ptr lease; + auto pool = mgr._ensure_pool_initialized(meta, version, nullptr, &lease); ASSERT_TRUE(pool.has_value()) << pool.error().to_string(); ProcessPtr client_process; ASSERT_TRUE(mgr._get_process(version, pool.value(), &client_process).ok()); diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy index d03cb5f30c0b97..a67597f8d1da55 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy @@ -48,6 +48,13 @@ class SumAgg: "qt_result_0_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") Thread.sleep(2500) "qt_result_1_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + // DROP now leaves a STOPPED pool until the monitor reclaims it. The next + // iteration recreates the same SQL name with a new function id and must still work. + sql "DROP FUNCTION py_auto_suspend(INT)" + test { + sql "SELECT py_auto_suspend(v) FROM py_auto_suspend_input" + exception "Can not found function" + } } sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" def zip = "${context.file.parent}/udaf_scripts/python_udaf_drop_a/python_udaf_drop_test.zip" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy index f31e3b125e38e2..cdf5ff19d9378d 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy @@ -37,6 +37,13 @@ def evaluate(v): "qt_result_0_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input ORDER BY v") Thread.sleep(2500) "qt_result_1_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input ORDER BY v") + // DROP now leaves a STOPPED pool until the monitor reclaims it. The next + // iteration recreates the same SQL name with a new function id and must still work. + sql "DROP FUNCTION py_auto_suspend(INT)" + test { + sql "SELECT py_auto_suspend(v) FROM py_auto_suspend_input" + exception "Can not found function" + } } } finally { sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy index 29ceb439ddca0f..e48314ebfab820 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy @@ -40,6 +40,13 @@ def evaluate(v): Thread.sleep(2500) "qt_result_2_${index}"("SELECT x FROM py_auto_suspend_input LATERAL VIEW py_auto_suspend(v) t AS x ORDER BY x") "qt_result_3_${index}"("SELECT x FROM (SELECT CAST(NULL AS INT) v) s LATERAL VIEW py_auto_suspend_outer(v) t AS x") + // DROP now leaves a STOPPED pool until the monitor reclaims it. The next + // iteration recreates the same SQL name with a new function id and must still work. + sql "DROP FUNCTION py_auto_suspend(INT)" + test { + sql "SELECT x FROM py_auto_suspend_input LATERAL VIEW py_auto_suspend(v) t AS x" + exception "Can not found function" + } } } finally { sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" From 73766e285858e09e155fe09d044fda1508840208 Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Tue, 20 Oct 2026 01:37:54 +0800 Subject: [PATCH 3/5] [fix](udf) Retry database UDF cache cleanup after backend recovery --- be/src/agent/task_worker_pool.cpp | 58 +++-- be/test/agent/task_worker_pool_test.cpp | 91 ++++++++ .../apache/doris/catalog/FunctionUtil.java | 9 +- .../org/apache/doris/master/MasterImpl.java | 3 + .../apache/doris/task/CleanUDFCacheTask.java | 8 +- .../doris/task/CleanUDFCacheTaskTest.java | 209 ++++++++++++++++++ ...test_pythonudaf_database_cleanup_retry.out | 13 ++ .../test_pythonudf_database_cleanup_retry.out | 13 ++ ...test_pythonudtf_database_cleanup_retry.out | 17 ++ ...t_pythonudaf_database_cleanup_retry.groovy | 114 ++++++++++ ...st_pythonudf_database_cleanup_retry.groovy | 114 ++++++++++ ...t_pythonudtf_database_cleanup_retry.groovy | 117 ++++++++++ 12 files changed, 743 insertions(+), 23 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java create mode 100644 regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out create mode 100644 regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out create mode 100644 regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out create mode 100644 regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy create mode 100644 regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy create mode 100644 regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index b7298e56378607..ef16da1b44c8eb 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -59,6 +59,7 @@ #include "common/metrics/doris_metrics.h" #include "common/status.h" #include "cpp/obj-client/obj_storage_client.h" +#include "cpp/sync_point.h" #include "exec/spill/spill_file_manager.h" #include "io/fs/file_system.h" #include "io/fs/hdfs_file_system.h" @@ -154,6 +155,7 @@ void remove_task_info(const TTaskType::type task_type, int64_t signature) { } void finish_task(const TFinishTaskRequest& finish_task_request) { + TEST_SYNC_POINT_RETURN_WITH_VOID("TaskWorkerPool::finish_task", &finish_task_request); // Return result to FE TMasterResult result; uint32_t try_time = 0; @@ -2579,34 +2581,50 @@ void clean_trash_callback(StorageEngine& engine, const TAgentTaskRequest& req) { } void clean_udf_cache_callback(const TAgentTaskRequest& req) { + Status status; const auto& clean_req = req.clean_udf_cache_req; if (clean_req.__isset.function_id && clean_req.function_id <= 0) { LOG(WARNING) << "skip clean udf cache request with invalid function_id=" << clean_req.function_id << ", function_signature=" << clean_req.function_signature; - return; - } - // Requests from old FEs do not set function_id and must keep signature-based cleanup. - const bool drop_by_function_id = clean_req.__isset.function_id; - - if (doris::config::enable_java_support) { - // The id, not just the signature: it is what the java-udf plugin keys its compiled - // classes by, because the signature carries no database and FE renders a variadic one - // differently here than on the requests that execute the function. - WARN_IF_ERROR( - Jni::PluginRegistry::clean_udf_cache( - drop_by_function_id ? clean_req.function_id : 0, - clean_req.function_signature), - fmt::format("failed to clean Java UDF cache, function_signature={}, function_id={}", + status = Status::InvalidArgument("Invalid function_id for UDF cache cleanup: {}", + clean_req.function_id); + } else { + // Requests from old FEs do not set function_id and must keep signature-based cleanup. + const bool drop_by_function_id = clean_req.__isset.function_id; + + if (doris::config::enable_java_support) { + // The id, not just the signature: it is what the java-udf plugin keys its compiled + // classes by, because the signature carries no database and FE renders a variadic one + // differently here than on the requests that execute the function. + WARN_IF_ERROR( + Jni::PluginRegistry::clean_udf_cache( + drop_by_function_id ? clean_req.function_id : 0, + clean_req.function_signature), + fmt::format( + "failed to clean Java UDF cache, function_signature={}, function_id={}", clean_req.function_signature, clean_req.function_id)); - } - if (drop_by_function_id) { - UserFunctionCache::instance()->drop_function_cache(clean_req.function_id); - PythonServerManager::instance().drop_function(clean_req.function_id); + } + if (drop_by_function_id) { + UserFunctionCache::instance()->drop_function_cache(clean_req.function_id); + PythonServerManager::instance().drop_function(clean_req.function_id); + } + + LOG(INFO) << "clean udf cache callback finish: function_signature=" + << clean_req.function_signature << ", function_id=" << clean_req.function_id; } - LOG(INFO) << "clean udf cache callback finish: function_signature=" - << clean_req.function_signature << ", function_id=" << clean_req.function_id; + // Report the result only for queued tasks; one-shot tasks use signature -1. + if (req.signature != -1) { + TFinishTaskRequest finish_task_request; + finish_task_request.__set_backend(BackendOptions::get_local_backend()); + finish_task_request.__set_task_type(req.task_type); + finish_task_request.__set_signature(req.signature); + finish_task_request.__set_task_status(status.to_thrift()); + finish_task(finish_task_request); + // Stop reporting this task as running so a failed result submission can trigger a retry. + remove_task_info(req.task_type, req.signature); + } } void report_index_policy_callback(const ClusterInfo* cluster_info) { diff --git a/be/test/agent/task_worker_pool_test.cpp b/be/test/agent/task_worker_pool_test.cpp index b8b2de0fc4b097..46893edd4daa15 100644 --- a/be/test/agent/task_worker_pool_test.cpp +++ b/be/test/agent/task_worker_pool_test.cpp @@ -19,16 +19,23 @@ #include #include +#include #include #include #include +#include +#include #include +#include #include "agent/agent_server.h" #include "cloud/cloud_storage_engine.h" +#include "common/config.h" +#include "cpp/sync_point.h" #include "runtime/cluster_info.h" #include "runtime/exec_env.h" +#include "runtime/user_function_cache.h" #include "storage/options.h" #include "storage/storage_engine.h" @@ -36,6 +43,90 @@ namespace doris { using namespace std::chrono_literals; +class CleanUDFCacheWorkerTest : public ::testing::Test { +protected: + UserFunctionCache cache; + UserFunctionCache* original_cache = nullptr; + bool original_java_support = false; + std::vector acknowledgements; + + void SetUp() override { + original_cache = ExecEnv::GetInstance()->_user_function_cache; + ExecEnv::GetInstance()->_user_function_cache = &cache; + original_java_support = config::enable_java_support; + config::enable_java_support = false; + auto* sync_point = SyncPoint::get_instance(); + sync_point->set_call_back("TaskWorkerPool::finish_task", [this](auto&& args) { + acknowledgements.push_back(*try_any_cast(args[0])); + *try_any_cast(args.back()) = true; + }); + sync_point->enable_processing(); + } + + void TearDown() override { + auto* sync_point = SyncPoint::get_instance(); + sync_point->disable_processing(); + sync_point->clear_call_back("TaskWorkerPool::finish_task"); + config::enable_java_support = original_java_support; + ExecEnv::GetInstance()->_user_function_cache = original_cache; + } + + TAgentTaskRequest request(int64_t signature, int64_t function_id = 938812) { + TAgentTaskRequest req; + req.__set_task_type(TTaskType::CLEAN_UDF_CACHE); + req.__set_signature(signature); + TCleanUDFCacheReq clean; + clean.__set_function_id(function_id); + clean.__set_function_signature("sum(INT)"); + req.__set_clean_udf_cache_req(clean); + return req; + } +}; + +TEST_F(CleanUDFCacheWorkerTest, OneShotCleanupDoesNotAcknowledge) { + clean_udf_cache_callback(request(-1)); + EXPECT_TRUE(acknowledgements.empty()); +} + +TEST_F(CleanUDFCacheWorkerTest, TrackedCleanupCanRunAgainAfterLostAcknowledgement) { + std::mutex mutex; + std::condition_variable cv; + int completed = 0; + TaskWorkerPool workers("test_clean_udf", 1, [&](const auto& req) { + clean_udf_cache_callback(req); + { + std::scoped_lock lock(mutex); + ++completed; + } + cv.notify_all(); + }); + auto req = request(938812); + ASSERT_TRUE(workers.submit_task(req).ok()); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, 5s, [&] { return completed == 1; })); + } + // A lost execution result must not prevent the same task from running again. + ASSERT_TRUE(workers.submit_task(req).ok()); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, 5s, [&] { return completed == 2; })); + } + ASSERT_EQ(2, acknowledgements.size()); + for (const auto& ack : acknowledgements) { + EXPECT_EQ(TTaskType::CLEAN_UDF_CACHE, ack.task_type); + EXPECT_EQ(938812, ack.signature); + EXPECT_EQ(TStatusCode::OK, ack.task_status.status_code); + } +} + +TEST_F(CleanUDFCacheWorkerTest, InvalidTrackedCleanupReportsFailure) { + clean_udf_cache_callback(request(938813, 0)); + ASSERT_EQ(1, acknowledgements.size()); + EXPECT_EQ(938813, acknowledgements.front().signature); + EXPECT_EQ(TStatusCode::INVALID_ARGUMENT, acknowledgements.front().task_status.status_code); +} + TEST(TaskWorkerPoolTest, TaskWorkerPool) { std::atomic_int count {0}; TaskWorkerPool workers("test", 2, [&](auto&& task) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java index 99966425d56449..0aa37d2228b51f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.types.DataType; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTaskExecutor; +import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.CleanUDFCacheTask; import com.google.common.collect.ImmutableList; @@ -60,8 +61,12 @@ public static void cleanUdfCache(Map functionSignatures) { AgentBatchTask batchTask = new AgentBatchTask(); for (long backendId : Env.getCurrentSystemInfo().getAllBackendIds(false)) { for (Map.Entry function : functionSignatures.entrySet()) { - CleanUDFCacheTask task = new CleanUDFCacheTask(backendId, function.getValue(), function.getKey()); - batchTask.addTask(task); + CleanUDFCacheTask task = new CleanUDFCacheTask(backendId, function.getValue(), function.getKey(), true); + // A temporarily unavailable backend can miss cleanup after the database is erased. + // Queue it for retry when the backend recovers; no database remains to trigger cleanup again. + if (AgentTaskQueue.addTask(task)) { + batchTask.addTask(task); + } LOG.info("clean udf cache in be {}, functionId {}", backendId, function.getKey()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java index 14bba10acc53e3..e72d88890787ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java @@ -199,6 +199,9 @@ public TMasterResult finishTaskInternal(TFinishTaskRequest request) { case DROP: finishDropReplica(task); break; + case CLEAN_UDF_CACHE: + AgentTaskQueue.removeTask(backendId, taskType, signature); + break; case SCHEMA_CHANGE: case ROLLUP: throw new RuntimeException("Schema change and rollup job is not used any more," diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CleanUDFCacheTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CleanUDFCacheTask.java index 85871ca202b012..d0a8c2c928bf90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CleanUDFCacheTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CleanUDFCacheTask.java @@ -30,7 +30,13 @@ public class CleanUDFCacheTask extends AgentTask { private long functionId; public CleanUDFCacheTask(long backendId, String signature, long functionId) { - super(null, backendId, TTaskType.CLEAN_UDF_CACHE, -1, -1, -1, -1, -1, -1, -1); + this(backendId, signature, functionId, false); + } + + public CleanUDFCacheTask(long backendId, String signature, long functionId, boolean retry) { + // Use functionId to track retries; -1 means no execution result is expected. + super(null, backendId, TTaskType.CLEAN_UDF_CACHE, -1, -1, -1, -1, -1, + retry ? functionId : -1, -1); this.functionSignature = signature; this.functionId = functionId; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java new file mode 100644 index 00000000000000..b292f006299aae --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.task; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.FunctionUtil; +import org.apache.doris.common.ClientPool; +import org.apache.doris.common.GenericPool; +import org.apache.doris.master.MasterImpl; +import org.apache.doris.master.ReportHandler; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.BackendService; +import org.apache.doris.thrift.TAgentTaskRequest; +import org.apache.doris.thrift.TBackend; +import org.apache.doris.thrift.TFinishTaskRequest; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TStatus; +import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TTaskType; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class CleanUDFCacheTaskTest { + private static final long BACKEND_ID = 10000L; + private static final long FUNCTION_ID = 20000L; + private static final String HOST = "127.0.0.1"; + private static final int BE_PORT = 9060; + + private final List submissions = new ArrayList<>(); + private Backend backend; + private MasterImpl master; + private MockedStatic envMock; + private MockedStatic executorMock; + private MockedConstruction reportHandlerMock; + + @BeforeEach + public void setUp() { + MetricRepo.init(); + AgentTaskQueue.clearAllTasks(); + SystemInfoService systemInfo = new SystemInfoService(); + backend = new Backend(BACKEND_ID, HOST, 9050); + backend.setBePort(BE_PORT); + systemInfo.addBackend(backend); + envMock = Mockito.mockStatic(Env.class); + envMock.when(Env::getCurrentSystemInfo).thenReturn(systemInfo); + executorMock = Mockito.mockStatic(AgentTaskExecutor.class); + executorMock.when(() -> AgentTaskExecutor.submit(Mockito.any(AgentBatchTask.class))) + .thenAnswer(invocation -> { + submissions.add(invocation.getArgument(0)); + return null; + }); + reportHandlerMock = Mockito.mockConstruction(ReportHandler.class, + (mock, context) -> Mockito.doNothing().when(mock).start()); + master = new MasterImpl(); + } + + @AfterEach + public void tearDown() { + reportHandlerMock.close(); + executorMock.close(); + envMock.close(); + AgentTaskQueue.clearAllTasks(); + } + + @Test + public void unavailableBackendCleanupIsRetriedByTaskReportAfterRecovery() throws Exception { + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)")); + AgentTask queued = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID); + Assertions.assertNotNull(queued); + Assertions.assertFalse(backend.isAlive()); + Assertions.assertEquals(1, submissions.size()); + submissions.get(0).run(); // AgentBatchTask skips the unavailable backend. + Assertions.assertSame(queued, + AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + Assertions.assertEquals(1, queued.getFailedTimes()); + + backend.setAlive(true); + reportTasks(); + Assertions.assertEquals(2, submissions.size()); + Assertions.assertEquals(List.of(queued), submissions.get(1).getAllTasks()); + TAgentTaskRequest request = submissions.get(1).toAgentTaskRequest(queued); + Assertions.assertEquals(FUNCTION_ID, request.getSignature()); + Assertions.assertEquals(FUNCTION_ID, request.getCleanUdfCacheReq().getFunctionId()); + + Assertions.assertEquals(TStatusCode.OK, finish(FUNCTION_ID, TStatusCode.OK)); + Assertions.assertNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + } + + @Test + @SuppressWarnings("unchecked") + public void rpcSubmissionFailureKeepsCleanupForTheNextReport() throws Exception { + backend.setAlive(true); + GenericPool originalPool = ClientPool.backendPool; + GenericPool failingPool = Mockito.mock(GenericPool.class); + Mockito.when(failingPool.borrowObject(Mockito.any(TNetworkAddress.class))) + .thenThrow(new Exception("simulated submit RPC failure")); + ClientPool.backendPool = failingPool; + try { + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)")); + AgentTask queued = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID); + Assertions.assertNotNull(queued); + submissions.get(0).run(); + Mockito.verify(failingPool).borrowObject(Mockito.any(TNetworkAddress.class)); + Assertions.assertSame(queued, + AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + reportTasks(); + Assertions.assertEquals(List.of(queued), submissions.get(1).getAllTasks()); + } finally { + ClientPool.backendPool = originalPool; + } + } + + @Test + public void duplicateCleanupKeepsPendingGenerationAndDistinctIdsDoNotCollide() { + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)", FUNCTION_ID + 1, "sum(INT)")); + AgentTask first = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID); + AgentTask second = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID + 1); + Assertions.assertNotNull(first); + Assertions.assertNotNull(second); + Assertions.assertNotSame(first, second); + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)")); + Assertions.assertEquals(1, submissions.size()); + Assertions.assertSame(first, + AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + } + + @Test + public void failedAcknowledgementKeepsTaskAndSuccessOnlyRemovesItsGeneration() { + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)", FUNCTION_ID + 1, "sum(INT)")); + AgentTask first = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID); + Assertions.assertNotNull(first); + finish(FUNCTION_ID, TStatusCode.INTERNAL_ERROR); + Assertions.assertSame(first, + AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + Assertions.assertEquals(1, first.getFailedTimes()); + Assertions.assertEquals(TStatusCode.OK, finish(FUNCTION_ID, TStatusCode.OK)); + Assertions.assertNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + Assertions.assertNotNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID + 1)); + // The task is already removed when the same execution result arrives again. + Assertions.assertEquals(TStatusCode.CANCELLED, finish(FUNCTION_ID, TStatusCode.OK)); + } + + @Test + public void recycledDatabaseSnapshotOnlyQueuesPythonFunctions() { + Database database = Mockito.mock(Database.class); + Function python = Mockito.mock(Function.class); + Mockito.when(python.getBinaryType()).thenReturn(Function.BinaryType.PYTHON_UDF); + Mockito.when(python.getId()).thenReturn(FUNCTION_ID); + Mockito.when(python.signatureString()).thenReturn("sum(INT)"); + Function java = Mockito.mock(Function.class); + Mockito.when(java.getBinaryType()).thenReturn(Function.BinaryType.JAVA_UDF); + Mockito.when(database.getFunctions()).thenReturn(List.of(python, java)); + FunctionUtil.cleanUdfCache(database); + Assertions.assertNotNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + Assertions.assertEquals(1, submissions.get(0).getTaskNum()); + Mockito.verify(database).readLock(); + Mockito.verify(database).readUnlock(); + } + + @Test + public void ordinaryDropKeepsTheOneShotTaskSignature() { + CleanUDFCacheTask task = new CleanUDFCacheTask(BACKEND_ID, "sum(INT)", FUNCTION_ID); + Assertions.assertEquals(-1L, task.getSignature()); + Assertions.assertEquals(FUNCTION_ID, task.toThrift().getFunctionId()); + Assertions.assertEquals(-1L, new AgentBatchTask().toAgentTaskRequest(task).getSignature()); + } + + private void reportTasks() throws Exception { + Method taskReport = ReportHandler.class.getDeclaredMethod("taskReport", long.class, Map.class, long.class); + taskReport.setAccessible(true); + taskReport.invoke(null, BACKEND_ID, Map.of(), 0L); + } + + private TStatusCode finish(long functionId, TStatusCode status) { + TFinishTaskRequest request = new TFinishTaskRequest(new TBackend(HOST, BE_PORT, 0), + TTaskType.CLEAN_UDF_CACHE, functionId, new TStatus(status)); + return master.finishTask(request).getStatus().getStatusCode(); + } +} diff --git a/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out b/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out new file mode 100644 index 00000000000000..f01937b2045607 --- /dev/null +++ b/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_0 -- +6 + +-- !after_0 -- +7 + +-- !before_1 -- +6 + +-- !after_1 -- +7 + diff --git a/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out b/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out new file mode 100644 index 00000000000000..da656270dbb80c --- /dev/null +++ b/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_0 -- +11 + +-- !after_0 -- +12 + +-- !before_1 -- +11 + +-- !after_1 -- +12 + diff --git a/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out b/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out new file mode 100644 index 00000000000000..6f720d35d5cc4b --- /dev/null +++ b/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_0 -- +10 +10 + +-- !after_0 -- +11 +11 + +-- !before_1 -- +10 +10 + +-- !after_1 -- +11 +11 + diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy new file mode 100644 index 00000000000000..fe86cb579d1b02 --- /dev/null +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudaf_database_cleanup_retry", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + def originalDb = sql("SELECT DATABASE()")[0][0] + def heartbeatPoint = "HeartbeatMgr.BackendHeartbeatHandler" + def zipA = "${context.file.parent}/udaf_scripts/python_udaf_drop_a/python_udaf_drop_test.zip" + def zipB = "${context.file.parent}/udaf_scripts/python_udaf_drop_b/python_udaf_drop_test.zip" + [zipA, zipB].each { zip -> + scp_udf_file_to_all_be(zip) + scp_udf_file_to_all_fe(zip) + } + def await = { String message, Closure ready -> + def deadline = System.currentTimeMillis() + 120000 + while (!ready()) { + if (System.currentTimeMillis() >= deadline) { + throw new IllegalStateException(message) + } + Thread.sleep(1000) + } + } + def cleanupTasks = { + sql_return_maparray("SHOW PROC '/tasks'").find { it.TaskType == "CLEAN_UDF_CACHE" } + } + def purgeDatabase = { db -> + sql "DROP DATABASE IF EXISTS ${db} FORCE" + // Remove leftover recycle-bin records before the next case. + def ids = sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .collect { it.DbId }.unique() + ids.each { id -> sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${id}" } + } + def createFunction = { zip -> + sql """CREATE AGGREGATE FUNCTION py_cleanup_retry(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "symbol"="drop_udaf.SumAgg", + "file"="file://${zip}", "runtime_version"="${runtime}", "pool_size"="1")""" + } + // Disable idle cleanup so this test verifies task retries. + ["force", "recycle"].eachWithIndex { mode, index -> + def db = "py_udaf_cleanup_retry_${mode}" + try { + purgeDatabase(db) + await("earlier database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + sql "CREATE DATABASE ${db}" + sql "USE ${db}" + def dbId = getDbId(db) + createFunction(zipA) + "qt_before_${index}"("SELECT py_cleanup_retry(v) FROM (SELECT 1 v UNION ALL SELECT 2 UNION ALL SELECT 3) s") + sql "USE ${originalDb}" + + def beIds = sql_return_maparray("SHOW BACKENDS").collect { it.BackendId.toString() } + // Simulate heartbeat failures while Python processes keep running. + GetDebugPoint().enableDebugPointForAllFEs(heartbeatPoint, + [deadBeIds: beIds.join(","), timeout: "180"]) + await("backends were not marked unavailable", { + sql_return_maparray("SHOW BACKENDS").every { !it.Alive.toBoolean() } + }) + if (mode == "recycle") { + sql "DROP DATABASE ${db}" + // Recoverable databases retain their function pools. + assertEquals(0, cleanupTasks().TotalNum.toInteger()) + sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${dbId}" + assertTrue(sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .every { it.DbId.toString() != dbId.toString() }) + } else { + sql "DROP DATABASE ${db} FORCE" + } + await("database cleanup was lost while BE was unavailable", { + def tasks = cleanupTasks() + tasks.TotalNum.toInteger() == beIds.size() && tasks.FailedNum.toInteger() == beIds.size() + }) + beIds.each { id -> + def tasks = sql_return_maparray("SHOW PROC '/tasks/CLEAN_UDF_CACHE/${id}'") + assertEquals(1, tasks.size()) + assertTrue(tasks[0].TaskSignature.toLong() > 0) + } + + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + await("backends did not recover", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + // The task leaves the queue only after cleanup succeeds and its result is reported. + await("BE recovery did not resend and acknowledge database cleanup", { + cleanupTasks().TotalNum.toInteger() == 0 + }) + sql "CREATE DATABASE ${db}" + sql "USE ${db}" + createFunction(zipB) + "qt_after_${index}"("SELECT py_cleanup_retry(v) FROM (SELECT 1 v UNION ALL SELECT 2 UNION ALL SELECT 3) s") + } finally { + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + sql "USE ${originalDb}" + purgeDatabase(db) + await("backends did not recover during cleanup", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + await("final database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + } + } +} diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy new file mode 100644 index 00000000000000..95603ac9dadfe6 --- /dev/null +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudf_database_cleanup_retry", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + def originalDb = sql("SELECT DATABASE()")[0][0] + def heartbeatPoint = "HeartbeatMgr.BackendHeartbeatHandler" + def zipA = "${context.file.parent}/udf_scripts/python_udf_drop_a/python_udf_drop_test.zip" + def zipB = "${context.file.parent}/udf_scripts/python_udf_drop_b/python_udf_drop_test.zip" + [zipA, zipB].each { zip -> + scp_udf_file_to_all_be(zip) + scp_udf_file_to_all_fe(zip) + } + def await = { String message, Closure ready -> + def deadline = System.currentTimeMillis() + 120000 + while (!ready()) { + if (System.currentTimeMillis() >= deadline) { + throw new IllegalStateException(message) + } + Thread.sleep(1000) + } + } + def cleanupTasks = { + sql_return_maparray("SHOW PROC '/tasks'").find { it.TaskType == "CLEAN_UDF_CACHE" } + } + def purgeDatabase = { db -> + sql "DROP DATABASE IF EXISTS ${db} FORCE" + // Remove leftover recycle-bin records before the next case. + def ids = sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .collect { it.DbId }.unique() + ids.each { id -> sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${id}" } + } + def createFunction = { zip -> + sql """CREATE FUNCTION py_cleanup_retry(INT) RETURNS INT + PROPERTIES("type"="PYTHON_UDF", "symbol"="drop_udf.evaluate", + "file"="file://${zip}", "runtime_version"="${runtime}", "pool_size"="1")""" + } + // Disable idle cleanup so this test verifies task retries. + ["force", "recycle"].eachWithIndex { mode, index -> + def db = "py_udf_cleanup_retry_${mode}" + try { + purgeDatabase(db) + await("earlier database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + sql "CREATE DATABASE ${db}" + sql "USE ${db}" + def dbId = getDbId(db) + createFunction(zipA) + "qt_before_${index}"("SELECT py_cleanup_retry(10)") + sql "USE ${originalDb}" + + def beIds = sql_return_maparray("SHOW BACKENDS").collect { it.BackendId.toString() } + // Simulate heartbeat failures while Python processes keep running. + GetDebugPoint().enableDebugPointForAllFEs(heartbeatPoint, + [deadBeIds: beIds.join(","), timeout: "180"]) + await("backends were not marked unavailable", { + sql_return_maparray("SHOW BACKENDS").every { !it.Alive.toBoolean() } + }) + if (mode == "recycle") { + sql "DROP DATABASE ${db}" + // Recoverable databases retain their function pools. + assertEquals(0, cleanupTasks().TotalNum.toInteger()) + sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${dbId}" + assertTrue(sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .every { it.DbId.toString() != dbId.toString() }) + } else { + sql "DROP DATABASE ${db} FORCE" + } + await("database cleanup was lost while BE was unavailable", { + def tasks = cleanupTasks() + tasks.TotalNum.toInteger() == beIds.size() && tasks.FailedNum.toInteger() == beIds.size() + }) + beIds.each { id -> + def tasks = sql_return_maparray("SHOW PROC '/tasks/CLEAN_UDF_CACHE/${id}'") + assertEquals(1, tasks.size()) + assertTrue(tasks[0].TaskSignature.toLong() > 0) + } + + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + await("backends did not recover", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + // The task leaves the queue only after cleanup succeeds and its result is reported. + await("BE recovery did not resend and acknowledge database cleanup", { + cleanupTasks().TotalNum.toInteger() == 0 + }) + sql "CREATE DATABASE ${db}" + sql "USE ${db}" + createFunction(zipB) + "qt_after_${index}"("SELECT py_cleanup_retry(10)") + } finally { + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + sql "USE ${originalDb}" + purgeDatabase(db) + await("backends did not recover during cleanup", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + await("final database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + } + } +} diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy new file mode 100644 index 00000000000000..026d6d5da119e3 --- /dev/null +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudtf_database_cleanup_retry", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + def originalDb = sql("SELECT DATABASE()")[0][0] + def heartbeatPoint = "HeartbeatMgr.BackendHeartbeatHandler" + def zipA = "${context.file.parent}/udtf_scripts/python_udtf_drop_a/python_udtf_drop_test.zip" + def zipB = "${context.file.parent}/udtf_scripts/python_udtf_drop_b/python_udtf_drop_test.zip" + [zipA, zipB].each { zip -> + scp_udf_file_to_all_be(zip) + scp_udf_file_to_all_fe(zip) + } + def await = { String message, Closure ready -> + def deadline = System.currentTimeMillis() + 120000 + while (!ready()) { + if (System.currentTimeMillis() >= deadline) { + throw new IllegalStateException(message) + } + Thread.sleep(1000) + } + } + def cleanupTasks = { + sql_return_maparray("SHOW PROC '/tasks'").find { it.TaskType == "CLEAN_UDF_CACHE" } + } + def purgeDatabase = { db -> + sql "DROP DATABASE IF EXISTS ${db} FORCE" + // Remove leftover recycle-bin records before the next case. + def ids = sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .collect { it.DbId }.unique() + ids.each { id -> sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${id}" } + } + def createFunction = { zip -> + sql """CREATE TABLES FUNCTION py_cleanup_retry(INT) RETURNS ARRAY + PROPERTIES("type"="PYTHON_UDF", "symbol"="drop_udtf.process", + "file"="file://${zip}", "runtime_version"="${runtime}", "pool_size"="1")""" + } + // Disable idle cleanup so this test verifies task retries. + ["force", "recycle"].eachWithIndex { mode, index -> + def db = "py_udtf_cleanup_retry_${mode}" + try { + purgeDatabase(db) + await("earlier database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + sql "CREATE DATABASE ${db}" + sql "USE ${db}" + def dbId = getDbId(db) + createFunction(zipA) + // Normal and outer functions each need a cleanup task on every backend. + def functionCount = sql("SHOW FUNCTIONS LIKE 'py_cleanup_retry%'").size() + "qt_before_${index}"("SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry(v) t AS x UNION ALL SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry_outer(v) t AS x ORDER BY x") + sql "USE ${originalDb}" + + def beIds = sql_return_maparray("SHOW BACKENDS").collect { it.BackendId.toString() } + // Simulate heartbeat failures while Python processes keep running. + GetDebugPoint().enableDebugPointForAllFEs(heartbeatPoint, + [deadBeIds: beIds.join(","), timeout: "180"]) + await("backends were not marked unavailable", { + sql_return_maparray("SHOW BACKENDS").every { !it.Alive.toBoolean() } + }) + if (mode == "recycle") { + sql "DROP DATABASE ${db}" + // Recoverable databases retain their function pools. + assertEquals(0, cleanupTasks().TotalNum.toInteger()) + sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${dbId}" + assertTrue(sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .every { it.DbId.toString() != dbId.toString() }) + } else { + sql "DROP DATABASE ${db} FORCE" + } + await("database cleanup was lost while BE was unavailable", { + def tasks = cleanupTasks() + tasks.TotalNum.toInteger() == beIds.size() * functionCount + && tasks.FailedNum.toInteger() == beIds.size() * functionCount + }) + beIds.each { id -> + def tasks = sql_return_maparray("SHOW PROC '/tasks/CLEAN_UDF_CACHE/${id}'") + assertEquals(functionCount, tasks.size()) + assertTrue(tasks.every { it.TaskSignature.toLong() > 0 }) + } + + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + await("backends did not recover", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + // The task leaves the queue only after cleanup succeeds and its result is reported. + await("BE recovery did not resend and acknowledge database cleanup", { + cleanupTasks().TotalNum.toInteger() == 0 + }) + sql "CREATE DATABASE ${db}" + sql "USE ${db}" + createFunction(zipB) + "qt_after_${index}"("SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry(v) t AS x UNION ALL SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry_outer(v) t AS x ORDER BY x") + } finally { + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + sql "USE ${originalDb}" + purgeDatabase(db) + await("backends did not recover during cleanup", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + await("final database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + } + } +} From e4fe4a695097426582d5cfb79e51ce03efd1f1fe Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Tue, 20 Oct 2026 19:14:59 +0800 Subject: [PATCH 4/5] [fix](udf) Retry dropped function cleanup and reclaim idle test pools Queue local and global DROP FUNCTION cleanup for retry after backend recovery. Set auto_suspend=0 in ordinary Python regression cases to release idle processes. Validation: full build, 17 FE unit tests, and 86 Python regression suites passed. --- .../apache/doris/catalog/FunctionUtil.java | 3 +- .../plans/commands/DropFunctionCommand.java | 32 +++------- .../doris/task/CleanUDFCacheTaskTest.java | 59 ++++++++++++++++++- ...test_pythonudaf_database_cleanup_retry.out | 12 ++++ .../test_pythonudf_database_cleanup_retry.out | 12 ++++ ...test_pythonudtf_database_cleanup_retry.out | 8 +++ ...test_python_raise_error_propagation.groovy | 16 +++++ .../test_pythonudaf_basic.groovy | 6 ++ ...thonudaf_complex_aggregation_inline.groovy | 14 +++++ ...thonudaf_complex_aggregation_module.groovy | 16 +++++ ...onudaf_complex_state_objects_inline.groovy | 12 ++++ ...onudaf_complex_state_objects_module.groovy | 16 +++++ .../test_pythonudaf_concurrent.groovy | 12 ++++ .../test_pythonudaf_data_types.groovy | 16 +++++ ...t_pythonudaf_database_cleanup_retry.groovy | 35 +++++++---- .../pythonudaf_p0/test_pythonudaf_drop.groovy | 12 ++++ .../test_pythonudaf_edge_cases.groovy | 16 +++++ .../test_pythonudaf_forbidden_module.groovy | 4 ++ .../test_pythonudaf_inline.groovy | 16 +++++ .../test_pythonudaf_inline_simple.groovy | 2 + .../test_pythonudaf_nested_query.groovy | 6 ++ .../test_pythonudaf_null_handling.groovy | 10 ++++ ...test_pythonudaf_object_types_inline.groovy | 12 ++++ .../test_pythonudaf_pkg_isolation.groovy | 4 ++ .../test_pythonudaf_simple.groovy | 2 + ...t_pythonudaf_window_advanced_inline.groovy | 12 ++++ ...t_pythonudaf_window_advanced_module.groovy | 14 +++++ .../test_pythonudaf_window_functions.groovy | 8 +++ .../test_pythonudf_udaf_mixed.groovy | 12 ++++ .../test_pythonudwf_comprehensive.groovy | 20 +++++++ .../test_python_udaf_complex.groovy | 24 ++++++++ .../test_python_udf_business_logic.groovy | 30 ++++++++++ .../test_python_udf_external_api.groovy | 12 ++-- .../test_python_udf_nlp_chinese.groovy | 20 +++++++ .../test_python_udtf_complex.groovy | 24 ++++++++ .../sanity/test_pythonudf_assertequal.groovy | 2 + .../test_pythonudf_assertlessthan.groovy | 2 + ...est_python_arrow_convertor_timezone.groovy | 6 +- .../test_pythonudf_aggregate.groovy | 4 ++ .../test_pythonudf_always_nullable.groovy | 8 +++ .../pythonudf_p0/test_pythonudf_array.groovy | 8 +++ .../test_pythonudf_base_data_type.groovy | 4 ++ .../test_pythonudf_boolean.groovy | 2 + .../test_pythonudf_complex_data_type.groovy | 12 ++++ .../test_pythonudf_concurrent.groovy | 19 +++++- .../test_pythonudf_data_types.groovy | 12 ++++ ...st_pythonudf_database_cleanup_retry.groovy | 35 +++++++---- .../pythonudf_p0/test_pythonudf_drop.groovy | 16 +++++ .../test_pythonudf_error_handling.groovy | 16 +++++ .../test_pythonudf_file_protocol.groovy | 10 ++++ .../pythonudf_p0/test_pythonudf_float.groovy | 4 ++ .../test_pythonudf_forbidden_module.groovy | 4 ++ .../test_pythonudf_global_function.groovy | 8 +++ .../test_pythonudf_inline_complex.groovy | 10 ++++ .../test_pythonudf_inline_scalar.groovy | 19 ++++-- .../test_pythonudf_inline_vector.groovy | 24 ++++++++ .../pythonudf_p0/test_pythonudf_int.groovy | 10 ++++ .../pythonudf_p0/test_pythonudf_ip.groovy | 20 +++++++ .../pythonudf_p0/test_pythonudf_map.groovy | 4 ++ .../test_pythonudf_mixed_params.groovy | 20 +++++++ .../pythonudf_p0/test_pythonudf_module.groovy | 2 + .../test_pythonudf_module_advanced.groovy | 2 + .../test_pythonudf_module_scalar.groovy | 56 ++++++++++++++++++ .../test_pythonudf_module_vector.groovy | 30 ++++++++++ .../test_pythonudf_multiline_inline.groovy | 8 +++ .../test_pythonudf_native_module_cache.groovy | 1 + .../test_pythonudf_nested_complex_type.groovy | 8 +++ .../test_pythonudf_no_input.groovy | 2 + .../test_pythonudf_object_types_inline.groovy | 10 ++++ .../test_pythonudf_pkg_isolation.groovy | 4 ++ .../test_pythonudf_ret_map.groovy | 8 +++ .../test_pythonudf_schema_check.groovy | 22 +++++++ .../pythonudf_p0/test_pythonudf_string.groovy | 2 + .../test_pythonudf_timestamp_ns.groovy | 8 +++ .../test_pythonudf_uuid_binary.groovy | 20 +++---- .../test_pythonudf_volatility.groovy | 8 +++ .../test_pythonudtf_basic_inline.groovy | 46 +++++++++++++++ .../test_pythonudtf_basic_module.groovy | 20 +++++++ .../test_pythonudtf_data_types_inline.groovy | 40 +++++++++++++ .../test_pythonudtf_data_types_module.groovy | 40 +++++++++++++ ...t_pythonudtf_database_cleanup_retry.groovy | 37 ++++++++---- .../pythonudtf_p0/test_pythonudtf_drop.groovy | 8 +++ .../test_pythonudtf_edge_cases_inline.groovy | 26 ++++++++ .../test_pythonudtf_edge_cases_module.groovy | 26 ++++++++ .../test_pythonudtf_exceptions_inline.groovy | 26 ++++++++ .../test_pythonudtf_exceptions_module.groovy | 26 ++++++++ .../test_pythonudtf_forbidden_module.groovy | 4 ++ .../test_pythonudtf_io_patterns_inline.groovy | 20 +++++++ .../test_pythonudtf_io_patterns_module.groovy | 20 +++++++ .../test_pythonudtf_no_input.groovy | 2 + ...test_pythonudtf_object_types_inline.groovy | 10 ++++ .../test_pythonudtf_pkg_isolation.groovy | 4 ++ ...t_pythonudtf_sql_integration_inline.groovy | 6 ++ ...t_pythonudtf_sql_integration_module.groovy | 6 ++ .../show/test_nereids_show_functions.groovy | 8 +++ 95 files changed, 1284 insertions(+), 90 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java index 0aa37d2228b51f..b20e69fd2e90e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java @@ -62,8 +62,7 @@ public static void cleanUdfCache(Map functionSignatures) { for (long backendId : Env.getCurrentSystemInfo().getAllBackendIds(false)) { for (Map.Entry function : functionSignatures.entrySet()) { CleanUDFCacheTask task = new CleanUDFCacheTask(backendId, function.getValue(), function.getKey(), true); - // A temporarily unavailable backend can miss cleanup after the database is erased. - // Queue it for retry when the backend recovers; no database remains to trigger cleanup again. + // DROP removes the metadata even if a backend is unavailable, so missed cleanup needs retries. if (AgentTaskQueue.addTask(task)) { batchTask.addTask(task); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFunctionCommand.java index 35296b9c29834f..2767c7b6fb98b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFunctionCommand.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSearchDesc; +import org.apache.doris.catalog.FunctionUtil; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -31,23 +32,17 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.system.Backend; -import org.apache.doris.task.AgentBatchTask; -import org.apache.doris.task.AgentTaskExecutor; -import org.apache.doris.task.CleanUDFCacheTask; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableMap; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * drop a alias or user defined function */ public class DropFunctionCommand extends Command implements ForwardWithSync { - private static final Logger LOG = LogManager.getLogger(DropFunctionCommand.class); private SetType setType; private final boolean ifExists; private final FunctionName functionName; @@ -89,25 +84,12 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { } functionIds = db.dropFunction(function, ifExists); } - if (functionIds.isEmpty()) { - // No function generation was removed. A signature-based cleanup task could arrive - // after a same-signature function is created and delete the new generation's cache. - return; - } - // BE will cache classload, when drop function, BE need clear cache - ImmutableMap backendsInfo = Env.getCurrentSystemInfo().getAllBackendsByAllCluster(); String functionSignature = getSignatureString(); - AgentBatchTask batchTask = new AgentBatchTask(); - for (Backend backend : backendsInfo.values()) { - for (long functionId : functionIds) { - CleanUDFCacheTask cleanUDFCacheTask = new CleanUDFCacheTask( - backend.getId(), functionSignature, functionId); - batchTask.addTask(cleanUDFCacheTask); - LOG.info("clean udf cache in be {}, beId {}, functionId {}", - backend.getHost(), backend.getId(), functionId); - } + Map functionSignatures = new HashMap<>(); + for (long functionId : functionIds) { + functionSignatures.put(functionId, functionSignature); } - AgentTaskExecutor.submit(batchTask); + FunctionUtil.cleanUdfCache(functionSignatures); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java index b292f006299aae..1d276b80a6bb5c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java @@ -17,15 +17,24 @@ package org.apache.doris.task; +import org.apache.doris.analysis.SetType; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.FunctionName; +import org.apache.doris.catalog.FunctionSearchDesc; import org.apache.doris.catalog.FunctionUtil; import org.apache.doris.common.ClientPool; import org.apache.doris.common.GenericPool; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.master.MasterImpl; import org.apache.doris.master.ReportHandler; import org.apache.doris.metric.MetricRepo; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.commands.DropFunctionCommand; +import org.apache.doris.nereids.trees.plans.commands.info.FunctionArgTypesInfo; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.BackendService; @@ -188,13 +197,61 @@ public void recycledDatabaseSnapshotOnlyQueuesPythonFunctions() { } @Test - public void ordinaryDropKeepsTheOneShotTaskSignature() { + public void legacyTaskKeepsTheOneShotTaskSignature() { CleanUDFCacheTask task = new CleanUDFCacheTask(BACKEND_ID, "sum(INT)", FUNCTION_ID); Assertions.assertEquals(-1L, task.getSignature()); Assertions.assertEquals(FUNCTION_ID, task.toThrift().getFunctionId()); Assertions.assertEquals(-1L, new AgentBatchTask().toAgentTaskRequest(task).getSignature()); } + @Test + public void localDropRetriesAfterBackendRecovery() throws Exception { + dropAndRetry(SetType.DEFAULT); + } + + @Test + public void globalDropRetriesAfterBackendRecovery() throws Exception { + dropAndRetry(SetType.GLOBAL); + } + + private void dropAndRetry(SetType scope) throws Exception { + Env env = Mockito.mock(Env.class, Mockito.RETURNS_DEEP_STUBS); + envMock.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getAccessManager().checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN))).thenReturn(true); + List functionIds = List.of(FUNCTION_ID, FUNCTION_ID + 1); + if (scope == SetType.GLOBAL) { + Mockito.when(env.getGlobalFunctionMgr().dropFunction( + Mockito.any(FunctionSearchDesc.class), Mockito.eq(false))).thenReturn(functionIds); + } else { + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + envMock.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + Database database = Mockito.mock(Database.class); + Mockito.when(catalog.getDbNullable("cleanup_db")).thenReturn(database); + Mockito.when(database.dropFunction(Mockito.any(FunctionSearchDesc.class), Mockito.eq(false))) + .thenReturn(functionIds); + } + DropFunctionCommand command = new DropFunctionCommand(scope, false, + new FunctionName("cleanup_db", "sum"), + new FunctionArgTypesInfo(List.of(IntegerType.INSTANCE), false)); + command.run(new ConnectContext(), null); + Assertions.assertEquals(1, submissions.size()); + submissions.get(0).run(); + for (long functionId : functionIds) { + AgentTask task = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, functionId); + Assertions.assertNotNull(task); + Assertions.assertEquals(1, task.getFailedTimes()); + Assertions.assertEquals(functionId, ((CleanUDFCacheTask) task).toThrift().getFunctionId()); + } + backend.setAlive(true); + reportTasks(); + Assertions.assertEquals(2, submissions.get(1).getTaskNum()); + for (long functionId : functionIds) { + Assertions.assertEquals(TStatusCode.OK, finish(functionId, TStatusCode.OK)); + Assertions.assertNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, functionId)); + } + } + private void reportTasks() throws Exception { Method taskReport = ReportHandler.class.getDeclaredMethod("taskReport", long.class, Map.class, long.class); taskReport.setAccessible(true); diff --git a/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out b/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out index f01937b2045607..115891b1272788 100644 --- a/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out +++ b/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out @@ -11,3 +11,15 @@ -- !after_1 -- 7 +-- !before_2 -- +6 + +-- !after_2 -- +7 + +-- !before_3 -- +6 + +-- !after_3 -- +7 + diff --git a/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out b/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out index da656270dbb80c..778262b74cbbc5 100644 --- a/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out +++ b/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out @@ -11,3 +11,15 @@ -- !after_1 -- 12 +-- !before_2 -- +11 + +-- !after_2 -- +12 + +-- !before_3 -- +11 + +-- !after_3 -- +12 + diff --git a/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out b/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out index 6f720d35d5cc4b..0870b357b7fc48 100644 --- a/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out +++ b/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out @@ -15,3 +15,11 @@ 11 11 +-- !before_2 -- +10 +10 + +-- !after_2 -- +11 +11 + diff --git a/regression-test/suites/pythonudaf_p0/test_python_raise_error_propagation.groovy b/regression-test/suites/pythonudaf_p0/test_python_raise_error_propagation.groovy index 4fadf5f4c2508f..589fa0d725782a 100644 --- a/regression-test/suites/pythonudaf_p0/test_python_raise_error_propagation.groovy +++ b/regression-test/suites/pythonudaf_p0/test_python_raise_error_propagation.groovy @@ -57,6 +57,8 @@ suite("test_python_raise_error_propagation") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -77,6 +79,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${udfPath}", "symbol" = "udf_errors.raise_in_module", "runtime_version" = "${runtime_version}", @@ -95,6 +99,8 @@ def evaluate(x): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineFinishErrorUDAF", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -130,6 +136,8 @@ class InlineFinishErrorUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineAccumulateErrorUDAF", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -166,6 +174,8 @@ class InlineAccumulateErrorUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineMergeErrorUDAF", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -202,6 +212,8 @@ class InlineMergeErrorUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${udafPath}", "symbol" = "udaf_errors.ModuleFinishErrorUDAF", "runtime_version" = "${runtime_version}", @@ -220,6 +232,8 @@ class InlineMergeErrorUDAF: RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "inline_raise_udtf", "runtime_version" = "${runtime_version}" ) AS \$\$ @@ -245,6 +259,8 @@ def inline_raise_udtf(x): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${udtfPath}", "symbol" = "pyudtf_module.exceptions_udtf.raise_in_module_udtf", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_basic.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_basic.groovy index 642fa2fabaa7f4..d07ac9970b0130 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_basic.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_basic.groovy @@ -60,6 +60,8 @@ suite("test_pythonudaf_basic") { "file"="file://${pyPath}", "symbol"="sum_int.SumInt", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -101,6 +103,8 @@ suite("test_pythonudaf_basic") { "file"="file://${pyPath}", "symbol"="sum_int.AvgDouble", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -148,6 +152,8 @@ suite("test_pythonudaf_basic") { "file"="file://${pyPath}", "symbol"="sum_int.SumInt", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_inline.groovy index 46c6f80ce4c91d..d026760619aec7 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_inline.groovy @@ -67,6 +67,8 @@ suite("test_pythonudaf_complex_aggregation_inline") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "VarianceUDAF", "runtime_version" = "${runtime_version}" ) @@ -115,6 +117,8 @@ class VarianceUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "StdDevUDAF", "runtime_version" = "${runtime_version}" ) @@ -165,6 +169,8 @@ class StdDevUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MedianUDAF", "runtime_version" = "${runtime_version}" ) @@ -210,6 +216,8 @@ class MedianUDAF: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CollectListUDAF", "runtime_version" = "${runtime_version}" ) @@ -250,6 +258,8 @@ class CollectListUDAF: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RangeUDAF", "runtime_version" = "${runtime_version}" ) @@ -299,6 +309,8 @@ class RangeUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "GeometricMeanUDAF", "runtime_version" = "${runtime_version}" ) @@ -344,6 +356,8 @@ class GeometricMeanUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WeightedAvgUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_module.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_module.groovy index 4d214c5b6c89b5..6e8d58ed516438 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_module.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_module.groovy @@ -73,6 +73,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.VarianceUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -93,6 +95,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.StdDevUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -113,6 +117,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.MedianUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -133,6 +139,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.CollectListUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -153,6 +161,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.RangeUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -173,6 +183,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.GeometricMeanUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -193,6 +205,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.WeightedAvgUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -304,6 +318,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.VarianceUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_inline.groovy index 9f974917df764d..ca643c32e34d19 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_inline.groovy @@ -81,6 +81,8 @@ suite("test_pythonudaf_complex_state_objects_inline") { RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "UserProfileUDAF", "runtime_version" = "${runtime_version}" ) @@ -157,6 +159,8 @@ class UserProfileUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "ProductStatsUDAF", "runtime_version" = "${runtime_version}" ) @@ -247,6 +251,8 @@ class ProductStatsUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "TransactionTimelineUDAF", "runtime_version" = "${runtime_version}" ) @@ -303,6 +309,8 @@ class TransactionTimelineUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "UniqueTrackerUDAF", "runtime_version" = "${runtime_version}" ) @@ -357,6 +365,8 @@ class UniqueTrackerUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CategorySummaryUDAF", "runtime_version" = "${runtime_version}" ) @@ -439,6 +449,8 @@ class CategorySummaryUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "HierarchicalAggUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_module.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_module.groovy index d0d1edbee052b2..609f08750867ea 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_module.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_module.groovy @@ -81,6 +81,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.UserProfileUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -97,6 +99,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.ProductStatsUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -113,6 +117,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.TransactionTimelineUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -129,6 +135,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.UniqueTrackerUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -145,6 +153,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.CategorySummaryUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -161,6 +171,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.HierarchicalAggUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -326,6 +338,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.UserProfileUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -347,6 +361,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.UserProfileUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_concurrent.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_concurrent.groovy index 961fa6984126cb..6ed48c47fcfaaa 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_concurrent.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_concurrent.groovy @@ -58,6 +58,8 @@ suite("test_pythonudaf_concurrent") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumUDAF", "runtime_version" = "${runtime_version}" ) @@ -89,6 +91,8 @@ class SumUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountUDAF", "runtime_version" = "${runtime_version}" ) @@ -120,6 +124,8 @@ class CountUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -157,6 +163,8 @@ class AvgUDAF: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MaxUDAF", "runtime_version" = "${runtime_version}" ) @@ -191,6 +199,8 @@ class MaxUDAF: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MinUDAF", "runtime_version" = "${runtime_version}" ) @@ -225,6 +235,8 @@ class MinUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumDoubleUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_data_types.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_data_types.groovy index 614446a6b91169..0c7cedfc69b6f0 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_data_types.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_data_types.groovy @@ -63,6 +63,8 @@ suite("test_pythonudaf_data_types") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumTinyIntUDAF", "runtime_version" = "${runtime_version}" ) @@ -98,6 +100,8 @@ class SumTinyIntUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumSmallIntUDAF", "runtime_version" = "${runtime_version}" ) @@ -133,6 +137,8 @@ class SumSmallIntUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumBigIntUDAF", "runtime_version" = "${runtime_version}" ) @@ -168,6 +174,8 @@ class SumBigIntUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumFloatUDAF", "runtime_version" = "${runtime_version}" ) @@ -203,6 +211,8 @@ class SumFloatUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumDecimalUDAF", "runtime_version" = "${runtime_version}" ) @@ -238,6 +248,8 @@ class SumDecimalUDAF: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "ConcatStrUDAF", "runtime_version" = "${runtime_version}" ) @@ -276,6 +288,8 @@ class ConcatStrUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountTrueUDAF", "runtime_version" = "${runtime_version}" ) @@ -326,6 +340,8 @@ class CountTrueUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "IntToDoubleSumUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy index fe86cb579d1b02..9cc4b7d60d06b8 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy @@ -44,23 +44,25 @@ suite("test_pythonudaf_database_cleanup_retry", "nonConcurrent") { .collect { it.DbId }.unique() ids.each { id -> sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${id}" } } - def createFunction = { zip -> - sql """CREATE AGGREGATE FUNCTION py_cleanup_retry(INT) RETURNS BIGINT - PROPERTIES("type"="PYTHON_UDF", "symbol"="drop_udaf.SumAgg", + def createFunction = { zip, scope -> + sql """CREATE ${scope} AGGREGATE FUNCTION py_cleanup_retry(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "symbol"="drop_udaf.SumAgg", "file"="file://${zip}", "runtime_version"="${runtime}", "pool_size"="1")""" } - // Disable idle cleanup so this test verifies task retries. - ["force", "recycle"].eachWithIndex { mode, index -> + ["force", "recycle", "function", "global"].eachWithIndex { mode, index -> def db = "py_udaf_cleanup_retry_${mode}" + def scope = mode == "global" ? "GLOBAL" : "" try { + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } purgeDatabase(db) await("earlier database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) sql "CREATE DATABASE ${db}" sql "USE ${db}" def dbId = getDbId(db) - createFunction(zipA) + createFunction(zipA, scope) "qt_before_${index}"("SELECT py_cleanup_retry(v) FROM (SELECT 1 v UNION ALL SELECT 2 UNION ALL SELECT 3) s") - sql "USE ${originalDb}" def beIds = sql_return_maparray("SHOW BACKENDS").collect { it.BackendId.toString() } // Simulate heartbeat failures while Python processes keep running. @@ -69,7 +71,10 @@ suite("test_pythonudaf_database_cleanup_retry", "nonConcurrent") { await("backends were not marked unavailable", { sql_return_maparray("SHOW BACKENDS").every { !it.Alive.toBoolean() } }) - if (mode == "recycle") { + if (mode in ["function", "global"]) { + sql "DROP ${scope} FUNCTION py_cleanup_retry(INT)" + } else if (mode == "recycle") { + sql "USE ${originalDb}" sql "DROP DATABASE ${db}" // Recoverable databases retain their function pools. assertEquals(0, cleanupTasks().TotalNum.toInteger()) @@ -77,9 +82,10 @@ suite("test_pythonudaf_database_cleanup_retry", "nonConcurrent") { assertTrue(sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") .every { it.DbId.toString() != dbId.toString() }) } else { + sql "USE ${originalDb}" sql "DROP DATABASE ${db} FORCE" } - await("database cleanup was lost while BE was unavailable", { + await("function cleanup was lost while backend was unavailable", { def tasks = cleanupTasks() tasks.TotalNum.toInteger() == beIds.size() && tasks.FailedNum.toInteger() == beIds.size() }) @@ -94,16 +100,21 @@ suite("test_pythonudaf_database_cleanup_retry", "nonConcurrent") { sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } }) // The task leaves the queue only after cleanup succeeds and its result is reported. - await("BE recovery did not resend and acknowledge database cleanup", { + await("backend recovery did not complete function cleanup", { cleanupTasks().TotalNum.toInteger() == 0 }) - sql "CREATE DATABASE ${db}" + if (mode in ["force", "recycle"]) { + sql "CREATE DATABASE ${db}" + } sql "USE ${db}" - createFunction(zipB) + createFunction(zipB, scope) "qt_after_${index}"("SELECT py_cleanup_retry(v) FROM (SELECT 1 v UNION ALL SELECT 2 UNION ALL SELECT 3) s") } finally { GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) sql "USE ${originalDb}" + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } purgeDatabase(db) await("backends did not recover during cleanup", { sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_drop.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_drop.groovy index 1745a14864d46c..9d67b8fb57f0c6 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_drop.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_drop.groovy @@ -55,6 +55,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { sql """ CREATE AGGREGATE FUNCTION py_drop_sum_once(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udaf.SumAgg", "runtime_version" = "${runtime_version}" @@ -74,6 +76,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { sql """ CREATE AGGREGATE FUNCTION py_drop_sum_a(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udaf.SumAgg", "runtime_version" = "${runtime_version}" @@ -82,6 +86,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { sql """ CREATE AGGREGATE FUNCTION py_drop_sum_b(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipB}", "symbol" = "drop_udaf.SumAgg", "runtime_version" = "${runtime_version}" @@ -109,6 +115,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { sql """ CREATE AGGREGATE FUNCTION py_drop_sum_reconnect(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udaf.SumAgg", "runtime_version" = "${runtime_version}" @@ -136,6 +144,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineDropRecreateUdaf", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -165,6 +175,8 @@ class InlineDropRecreateUdaf: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineDropRecreateUdaf", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_edge_cases.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_edge_cases.groovy index 9bd7ed3890a79f..9ce752e3084e35 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_edge_cases.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_edge_cases.groovy @@ -64,6 +64,8 @@ suite("test_pythonudaf_edge_cases") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumLargeUDAF", "runtime_version" = "${runtime_version}" ) @@ -102,6 +104,8 @@ class SumLargeUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MinValUDAF", "runtime_version" = "${runtime_version}" ) @@ -140,6 +144,8 @@ class MinValUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MaxValUDAF", "runtime_version" = "${runtime_version}" ) @@ -178,6 +184,8 @@ class MaxValUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountDistinctUDAF", "runtime_version" = "${runtime_version}" ) @@ -217,6 +225,8 @@ class CountDistinctUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "ProductUDAF", "runtime_version" = "${runtime_version}" ) @@ -260,6 +270,8 @@ class ProductUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AbsSumUDAF", "runtime_version" = "${runtime_version}" ) @@ -298,6 +310,8 @@ class AbsSumUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SafeAvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -342,6 +356,8 @@ class SafeAvgUDAF: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SignSummaryUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_forbidden_module.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_forbidden_module.groovy index 4892a249f61056..7d9d08b55a02a6 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_forbidden_module.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_forbidden_module.groovy @@ -50,6 +50,8 @@ suite("test_pythonudaf_forbidden_module") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "${forbiddenCase.symbol}", "runtime_version" = "${runtime_version}" @@ -68,6 +70,8 @@ suite("test_pythonudaf_forbidden_module") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "safepkg_udaf.pathlib.SafePathlibUDAF", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline.groovy index 4c6955ca99a937..99ca41a8785bff 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline.groovy @@ -54,6 +54,8 @@ suite("test_pythonudaf_inline") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumUDAF", "runtime_version" = "${runtime_version}" ) @@ -100,6 +102,8 @@ class SumUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -150,6 +154,8 @@ class AvgUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountUDAF", "runtime_version" = "${runtime_version}" ) @@ -194,6 +200,8 @@ class CountUDAF: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MaxUDAF", "runtime_version" = "${runtime_version}" ) @@ -288,6 +296,8 @@ class MaxUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumUDAF", "runtime_version" = "${runtime_version}" ) @@ -361,6 +371,8 @@ class SumUDAF: RETURNS LARGEINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumLargeIntUDAF", "runtime_version" = "${runtime_version}" ) @@ -404,6 +416,8 @@ class SumLargeIntUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountPrivateIPv4UDAF", "runtime_version" = "${runtime_version}" ) @@ -447,6 +461,8 @@ class CountPrivateIPv4UDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountLoopbackIPv6UDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline_simple.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline_simple.groovy index 90e216f2704deb..0199c4ceb70288 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline_simple.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline_simple.groovy @@ -49,6 +49,8 @@ suite("test_pythonudaf_inline_simple") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MySum", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_nested_query.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_nested_query.groovy index bdedf6e810c3fc..98c3b9dc8ae9b9 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_nested_query.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_nested_query.groovy @@ -86,6 +86,8 @@ suite("test_pythonudaf_nested_query") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "TotalRevenueUDAF", "runtime_version" = "${runtime_version}" ) @@ -117,6 +119,8 @@ class TotalRevenueUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "OrderCountUDAF", "runtime_version" = "${runtime_version}" ) @@ -148,6 +152,8 @@ class OrderCountUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgOrderValueUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_null_handling.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_null_handling.groovy index 36107e2b5c7513..8f8d19360f006d 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_null_handling.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_null_handling.groovy @@ -62,6 +62,8 @@ suite("test_pythonudaf_null_handling") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountNonNullUDAF", "runtime_version" = "${runtime_version}" ) @@ -99,6 +101,8 @@ class CountNonNullUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumNullUDAF", "runtime_version" = "${runtime_version}" ) @@ -140,6 +144,8 @@ class SumNullUDAF: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "FirstNonNullUDAF", "runtime_version" = "${runtime_version}" ) @@ -178,6 +184,8 @@ class FirstNonNullUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountNullUDAF", "runtime_version" = "${runtime_version}" ) @@ -215,6 +223,8 @@ class CountNullUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgCoalesceUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_object_types_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_object_types_inline.groovy index d37f5d33074a4a..8e49c74ad84292 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_object_types_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_object_types_inline.groovy @@ -24,6 +24,8 @@ suite("test_pythonudaf_object_types_inline") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -51,6 +53,8 @@ class Agg: RETURNS HLL PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -79,6 +83,8 @@ class Agg: INTERMEDIATE BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -107,6 +113,8 @@ class Agg: INTERMEDIATE BITMAP PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -134,6 +142,8 @@ class Agg: RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -161,6 +171,8 @@ class Agg: RETURNS STRUCT>> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy index 3dca47f21ee47b..446af2fdf2fce9 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy @@ -43,6 +43,7 @@ suite('test_pythonudaf_pkg_isolation') { sql """ CREATE AGGREGATE FUNCTION py_pkg_a_sum_x(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_x.SumAgg", "runtime_version" = "${runtime_version}", @@ -52,6 +53,7 @@ suite('test_pythonudaf_pkg_isolation') { sql """ CREATE AGGREGATE FUNCTION py_pkg_b_sum_x(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_x.SumAgg", "runtime_version" = "${runtime_version}", @@ -66,6 +68,7 @@ suite('test_pythonudaf_pkg_isolation') { sql """ CREATE AGGREGATE FUNCTION py_pkg_a_sum_y(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_y.SumAgg", "runtime_version" = "${runtime_version}", @@ -80,6 +83,7 @@ suite('test_pythonudaf_pkg_isolation') { sql """ CREATE AGGREGATE FUNCTION py_pkg_b_sum_y(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_y.SumAgg", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_simple.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_simple.groovy index ab97a86dcafb6e..204433ccf64ef9 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_simple.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_simple.groovy @@ -53,6 +53,8 @@ suite("test_pythonudaf_simple") { "file"="file://${pyPath}", "symbol"="sum_int.SumInt", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_inline.groovy index 3a329299850682..dc1deef0b0c73c 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_inline.groovy @@ -67,6 +67,8 @@ suite("test_pythonudaf_window_advanced_inline") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MovingAvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -103,6 +105,8 @@ class MovingAvgUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WindowStdDevUDAF", "runtime_version" = "${runtime_version}" ) @@ -143,6 +147,8 @@ class WindowStdDevUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "LastValueUDAF", "runtime_version" = "${runtime_version}" ) @@ -177,6 +183,8 @@ class LastValueUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WindowMinUDAF", "runtime_version" = "${runtime_version}" ) @@ -210,6 +218,8 @@ class WindowMinUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WindowMaxUDAF", "runtime_version" = "${runtime_version}" ) @@ -429,6 +439,8 @@ class WindowMaxUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Percentile50UDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_module.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_module.groovy index 90894a11af9044..8a9deb8daae432 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_module.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_module.groovy @@ -73,6 +73,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.MovingAvgUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -89,6 +91,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.WindowStdDevUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -105,6 +109,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.LastValueUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -121,6 +127,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.WindowMinUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -134,6 +142,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.WindowMaxUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -333,6 +343,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.Percentile50UDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -385,6 +397,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.MovingAvgUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_functions.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_functions.groovy index 1159f906b0c8d4..9af547eca94eaa 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_functions.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_functions.groovy @@ -71,6 +71,8 @@ suite("test_pythonudaf_window_functions") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningSumUDAF", "runtime_version" = "${runtime_version}" ) @@ -104,6 +106,8 @@ class RunningSumUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningCountUDAF", "runtime_version" = "${runtime_version}" ) @@ -137,6 +141,8 @@ class RunningCountUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningAvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -407,6 +413,8 @@ class RunningAvgUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WindowFirstUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudf_udaf_mixed.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudf_udaf_mixed.groovy index 10d9370d4cdf3c..8d20bc3fd15d6a 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudf_udaf_mixed.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudf_udaf_mixed.groovy @@ -58,6 +58,8 @@ suite("test_pythonudf_udaf_mixed") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "final_price", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -77,6 +79,8 @@ def final_price(price, discount): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "double_value", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -96,6 +100,8 @@ def double_value(x): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "get_prefix", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -117,6 +123,8 @@ def get_prefix(s): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumUDAF", "runtime_version" = "${runtime_version}" ) @@ -148,6 +156,8 @@ class SumUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountUDAF", "runtime_version" = "${runtime_version}" ) @@ -179,6 +189,8 @@ class CountUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgDoubleUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudwf_comprehensive.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudwf_comprehensive.groovy index 970b7517574bbb..8ece5850564429 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudwf_comprehensive.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudwf_comprehensive.groovy @@ -151,6 +151,8 @@ suite("test_pythonudwf_comprehensive") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningSumUDWF", "runtime_version" = "${runtime_version}" ) @@ -182,6 +184,8 @@ class RunningSumUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningAvgUDWF", "runtime_version" = "${runtime_version}" ) @@ -219,6 +223,8 @@ class RunningAvgUDWF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningCountUDWF", "runtime_version" = "${runtime_version}" ) @@ -250,6 +256,8 @@ class RunningCountUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MovingAvgUDWF", "runtime_version" = "${runtime_version}" ) @@ -284,6 +292,8 @@ class MovingAvgUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "StdDevUDWF", "runtime_version" = "${runtime_version}" ) @@ -322,6 +332,8 @@ class StdDevUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MinUDWF", "runtime_version" = "${runtime_version}" ) @@ -356,6 +368,8 @@ class MinUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MaxUDWF", "runtime_version" = "${runtime_version}" ) @@ -390,6 +404,8 @@ class MaxUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "FirstValueUDWF", "runtime_version" = "${runtime_version}" ) @@ -426,6 +442,8 @@ class FirstValueUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "LastValueUDWF", "runtime_version" = "${runtime_version}" ) @@ -458,6 +476,8 @@ class LastValueUDWF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RankUDWF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udaf_complex.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udaf_complex.groovy index c7c3c87c3e512e..539c20a0d4506e 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udaf_complex.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udaf_complex.groovy @@ -70,6 +70,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.WeightedAverageUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -92,6 +94,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.SkewnessUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -114,6 +118,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.KurtosisUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -136,6 +142,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.CorrelationUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -158,6 +166,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.CovarianceUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -180,6 +190,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.RetentionRateUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -228,6 +240,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.ConversionFunnelUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -278,6 +292,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.GiniCoefficientUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -324,6 +340,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.HerfindahlIndexUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -372,6 +390,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.TrendSlopeUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -425,6 +445,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.VolatilityUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -473,6 +495,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.JsonArrayAggUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udf_business_logic.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udf_business_logic.groovy index e63e2014bfe8fc..171fd214ed8204 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udf_business_logic.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udf_business_logic.groovy @@ -37,6 +37,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_moving_average", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -56,6 +58,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_ewma", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -75,6 +79,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.detect_anomalies_zscore", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -94,6 +100,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_trend", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -134,6 +142,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_percentiles", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -153,6 +163,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_statistics", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -172,6 +184,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_correlation", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -194,6 +208,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_rfm_score", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -241,6 +257,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_ltv", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -262,6 +280,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_churn_probability", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -309,6 +329,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_npv", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -330,6 +352,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_irr", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -349,6 +373,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_sharpe_ratio", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -368,6 +394,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.normalize_values", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -389,6 +417,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.bin_values", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udf_external_api.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udf_external_api.groovy index 931d52b99e1d8e..de94234d0c33b8 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udf_external_api.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udf_external_api.groovy @@ -30,28 +30,28 @@ suite("test_python_udf_external_api") { sql """ DROP FUNCTION IF EXISTS py_fetch_post_title(INT); """ sql """ CREATE FUNCTION py_fetch_post_title(INT) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_post_title", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_post_title", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ qt_fetch_post_title """ SELECT py_fetch_post_title(1); """ sql """ DROP FUNCTION IF EXISTS py_fetch_user_email(INT); """ sql """ CREATE FUNCTION py_fetch_user_email(INT) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_user_email", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_user_email", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ qt_fetch_user_email """ SELECT py_fetch_user_email(1); """ sql """ DROP FUNCTION IF EXISTS py_fetch_user_name(INT); """ sql """ CREATE FUNCTION py_fetch_user_name(INT) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_user_name", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_user_name", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ qt_fetch_user_name """ SELECT py_fetch_user_name(1); """ sql """ DROP FUNCTION IF EXISTS py_fetch_comments_count(INT); """ sql """ CREATE FUNCTION py_fetch_comments_count(INT) RETURNS INT - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_comments_count", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_comments_count", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ qt_fetch_comments_count """ SELECT py_fetch_comments_count(1); """ @@ -59,7 +59,7 @@ suite("test_python_udf_external_api") { sql """ DROP FUNCTION IF EXISTS py_get_ip_location(STRING); """ sql """ CREATE FUNCTION py_get_ip_location(STRING) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.get_ip_location", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.get_ip_location", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ def ip_result = sql """ SELECT py_get_ip_location('8.8.8.8') """ assertTrue(ip_result.size() == 1) @@ -68,7 +68,7 @@ suite("test_python_udf_external_api") { sql """ DROP FUNCTION IF EXISTS py_get_weather(DOUBLE, DOUBLE); """ sql """ CREATE FUNCTION py_get_weather(DOUBLE, DOUBLE) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.get_weather", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.get_weather", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ def weather_result = sql """ SELECT py_get_weather(39.9042, 116.4074) """ assertTrue(weather_result.size() == 1) diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy index d2f6a804edc7d3..bb60f301df14a7 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy @@ -37,6 +37,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.chinese_word_segment", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -78,6 +80,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.chinese_word_segment_json", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -100,6 +104,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.chinese_word_count", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -121,6 +127,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.extract_keywords_tfidf", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -159,6 +167,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.pos_tagging", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -181,6 +191,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.extract_nouns", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -203,6 +215,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.extract_verbs", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -225,6 +239,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.extract_all_entities", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -264,6 +280,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.chinese_text_similarity", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -305,6 +323,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.split_sentences", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udtf_complex.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udtf_complex.groovy index deeeb6ca64b28b..bd0e1b7bf7bef7 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udtf_complex.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udtf_complex.groovy @@ -37,6 +37,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_date_range", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -77,6 +79,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_time_slots", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -99,6 +103,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_json_array", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -139,6 +145,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_json_object_keys", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -178,6 +186,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.flatten_nested_json", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -216,6 +226,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_histogram_bins", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -238,6 +250,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_quartile_distribution", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -260,6 +274,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_ngrams", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -298,6 +314,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_sentences", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -320,6 +338,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_order_items", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -359,6 +379,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_user_events", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -397,6 +419,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_bbox_to_tiles", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertequal.groovy b/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertequal.groovy index 44cbaa9a6716fa..fa7fd1eaebe76c 100644 --- a/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertequal.groovy +++ b/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertequal.groovy @@ -43,6 +43,8 @@ suite("test_pythonudf_assertequal") { "file"="file://${pyPath}", "symbol"="assert_equal_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertlessthan.groovy b/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertlessthan.groovy index 19b85f25c0dc9c..1191f535bde45a 100644 --- a/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertlessthan.groovy +++ b/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertlessthan.groovy @@ -44,6 +44,8 @@ suite("test_pythonudf_assertlessthan") { "file"="file://${pyPath}", "symbol"="assert_lessthan_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_python_arrow_convertor_timezone.groovy b/regression-test/suites/pythonudf_p0/test_python_arrow_convertor_timezone.groovy index 0112e61ebb5a33..840bb9171801df 100644 --- a/regression-test/suites/pythonudf_p0/test_python_arrow_convertor_timezone.groovy +++ b/regression-test/suites/pythonudf_p0/test_python_arrow_convertor_timezone.groovy @@ -32,7 +32,7 @@ suite("test_python_arrow_convertor_timezone") { sql "DROP FUNCTION IF EXISTS ${name}(DATETIME(6))" } sql """CREATE FUNCTION ${scalar}(DATETIME(6)) RETURNS STRING - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(value): return None if value is None else value.strftime('%Y-%m-%d %H:%M:%S.%f') @@ -40,13 +40,13 @@ def evaluate(value): // Declare the yielded string as the array element; ARRAY> emits a struct. sql """CREATE TABLES FUNCTION ${tableFunction}(DATETIME(6)) RETURNS ARRAY - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(value): yield (None if value is None else value.strftime('%Y-%m-%d %H:%M:%S.%f'),) \$\$""" sql """CREATE AGGREGATE FUNCTION ${aggregate}(DATETIME(6)) RETURNS DATETIME(6) - PROPERTIES("type"="PYTHON_UDF", "symbol"="Maximum", + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="Maximum", "runtime_version"="${runtimeVersion}") AS \$\$ class Maximum: def __init__(self): diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_aggregate.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_aggregate.groovy index 59940e4ed417f9..e214085ac24596 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_aggregate.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_aggregate.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_aggregate") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "volatility" = "immutable" @@ -120,6 +122,8 @@ def evaluate(score): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "volatility" = "immutable" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_always_nullable.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_always_nullable.groovy index 21589f0c53c0fc..0a48ea89b4cc89 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_always_nullable.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_always_nullable.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_always_nullable") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -52,6 +54,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "false", "runtime_version" = "${runtime_version}" @@ -75,6 +79,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "false", "runtime_version" = "${runtime_version}" @@ -152,6 +158,8 @@ def evaluate(x): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_array.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_array.groovy index c5b71db5e7fc50..d3edb4ab88f94b 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_array.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_array.groovy @@ -56,6 +56,8 @@ suite("test_pythonudf_array") { "file"="file://${pyPath}", "symbol"="array_int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -68,6 +70,8 @@ suite("test_pythonudf_array") { "file"="file://${pyPath}", "symbol"="array_return_array_int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -81,6 +85,8 @@ suite("test_pythonudf_array") { "file"="file://${pyPath}", "symbol"="array_return_array_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -93,6 +99,8 @@ suite("test_pythonudf_array") { "file"="file://${pyPath}", "symbol"="array_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_base_data_type.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_base_data_type.groovy index 32114872eabbb9..ff90d560ea6486 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_base_data_type.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_base_data_type.groovy @@ -66,6 +66,8 @@ CREATE FUNCTION row_to_csv_all( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "row_to_csv_all_impl", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -224,6 +226,8 @@ def row_to_csv_all_impl( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_data_type.row_to_csv_all_impl", "always_nullable" = "true", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_boolean.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_boolean.groovy index f3f14532dd53c6..19705a2fd7450a 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_boolean.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_boolean.groovy @@ -48,6 +48,8 @@ suite("test_pythonudf_boolean") { "file"="file://${pyPath}", "symbol"="boolean_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_complex_data_type.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_complex_data_type.groovy index 76206717135e97..33d79cf62a4631 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_complex_data_type.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_complex_data_type.groovy @@ -40,6 +40,8 @@ CREATE FUNCTION array_to_csv( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "array_to_csv_impl", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -117,6 +119,8 @@ def array_to_csv_impl(int_arr, str_arr, nested_arr): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file"="file://${pyPath}", "symbol" = "python_udf_array_type.array_to_csv_impl", "always_nullable" = "true", @@ -172,6 +176,8 @@ CREATE FUNCTION map_to_csv( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "map_to_csv_impl", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -237,6 +243,8 @@ def map_to_csv_impl(map1, map2): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file"="file://${pyPath}", "symbol" = "python_udf_map_type.map_to_csv_impl", "always_nullable" = "true", @@ -289,6 +297,8 @@ CREATE FUNCTION struct_to_csv( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "struct_to_csv_impl", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -371,6 +381,8 @@ def struct_to_csv_impl(person, point): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file"="file://${pyPath}", "symbol" = "python_udf_struct_type.struct_to_csv_impl", "always_nullable" = "true", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_concurrent.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_concurrent.groovy index 2de5e2a9249662..8b1b97b08558f5 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_concurrent.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_concurrent.groovy @@ -56,6 +56,8 @@ suite("test_pythonudf_concurrent") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "add_int", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -75,6 +77,8 @@ def add_int(a, b): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "multiply_int", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -94,6 +98,8 @@ def multiply_int(a, b): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "divide_double", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -113,6 +119,8 @@ def divide_double(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "concat_str", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -132,6 +140,8 @@ def concat_str(s1, s2): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "str_len", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -186,6 +196,8 @@ def str_len(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "vec_add", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -204,6 +216,8 @@ def vec_add(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "vec_sub", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -222,6 +236,8 @@ def vec_sub(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "vec_max", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -287,6 +303,8 @@ def vec_max(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "int_to_str", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -342,4 +360,3 @@ def int_to_str(n): } } - diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_data_types.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_data_types.groovy index 6b15d9382bd953..0100f72186e1d8 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_data_types.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_data_types.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_data_types") { RETURNS TINYINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -47,6 +49,8 @@ def evaluate(x): RETURNS SMALLINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -67,6 +71,8 @@ def evaluate(x): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -87,6 +93,8 @@ def evaluate(x): RETURNS DECIMAL(10,2) PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -108,6 +116,8 @@ def evaluate(x): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -128,6 +138,8 @@ def evaluate(d): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy index 95603ac9dadfe6..837f04393a6e72 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy @@ -44,23 +44,25 @@ suite("test_pythonudf_database_cleanup_retry", "nonConcurrent") { .collect { it.DbId }.unique() ids.each { id -> sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${id}" } } - def createFunction = { zip -> - sql """CREATE FUNCTION py_cleanup_retry(INT) RETURNS INT - PROPERTIES("type"="PYTHON_UDF", "symbol"="drop_udf.evaluate", + def createFunction = { zip, scope -> + sql """CREATE ${scope} FUNCTION py_cleanup_retry(INT) RETURNS INT + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "symbol"="drop_udf.evaluate", "file"="file://${zip}", "runtime_version"="${runtime}", "pool_size"="1")""" } - // Disable idle cleanup so this test verifies task retries. - ["force", "recycle"].eachWithIndex { mode, index -> + ["force", "recycle", "function", "global"].eachWithIndex { mode, index -> def db = "py_udf_cleanup_retry_${mode}" + def scope = mode == "global" ? "GLOBAL" : "" try { + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } purgeDatabase(db) await("earlier database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) sql "CREATE DATABASE ${db}" sql "USE ${db}" def dbId = getDbId(db) - createFunction(zipA) + createFunction(zipA, scope) "qt_before_${index}"("SELECT py_cleanup_retry(10)") - sql "USE ${originalDb}" def beIds = sql_return_maparray("SHOW BACKENDS").collect { it.BackendId.toString() } // Simulate heartbeat failures while Python processes keep running. @@ -69,7 +71,10 @@ suite("test_pythonudf_database_cleanup_retry", "nonConcurrent") { await("backends were not marked unavailable", { sql_return_maparray("SHOW BACKENDS").every { !it.Alive.toBoolean() } }) - if (mode == "recycle") { + if (mode in ["function", "global"]) { + sql "DROP ${scope} FUNCTION py_cleanup_retry(INT)" + } else if (mode == "recycle") { + sql "USE ${originalDb}" sql "DROP DATABASE ${db}" // Recoverable databases retain their function pools. assertEquals(0, cleanupTasks().TotalNum.toInteger()) @@ -77,9 +82,10 @@ suite("test_pythonudf_database_cleanup_retry", "nonConcurrent") { assertTrue(sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") .every { it.DbId.toString() != dbId.toString() }) } else { + sql "USE ${originalDb}" sql "DROP DATABASE ${db} FORCE" } - await("database cleanup was lost while BE was unavailable", { + await("function cleanup was lost while backend was unavailable", { def tasks = cleanupTasks() tasks.TotalNum.toInteger() == beIds.size() && tasks.FailedNum.toInteger() == beIds.size() }) @@ -94,16 +100,21 @@ suite("test_pythonudf_database_cleanup_retry", "nonConcurrent") { sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } }) // The task leaves the queue only after cleanup succeeds and its result is reported. - await("BE recovery did not resend and acknowledge database cleanup", { + await("backend recovery did not complete function cleanup", { cleanupTasks().TotalNum.toInteger() == 0 }) - sql "CREATE DATABASE ${db}" + if (mode in ["force", "recycle"]) { + sql "CREATE DATABASE ${db}" + } sql "USE ${db}" - createFunction(zipB) + createFunction(zipB, scope) "qt_after_${index}"("SELECT py_cleanup_retry(10)") } finally { GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) sql "USE ${originalDb}" + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } purgeDatabase(db) await("backends did not recover during cleanup", { sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_drop.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_drop.groovy index 8596855ebdb50b..c196a2d2bee8e4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_drop.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_drop.groovy @@ -55,6 +55,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { sql """ CREATE FUNCTION py_drop_once(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udf.evaluate", "runtime_version" = "${runtime_version}" @@ -74,6 +76,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { sql """ CREATE FUNCTION py_drop_a(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udf.evaluate", "runtime_version" = "${runtime_version}" @@ -82,6 +86,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { sql """ CREATE FUNCTION py_drop_b(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipB}", "symbol" = "drop_udf.evaluate", "runtime_version" = "${runtime_version}" @@ -109,6 +115,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { sql """ CREATE FUNCTION py_drop_reconnect(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udf.evaluate", "runtime_version" = "${runtime_version}" @@ -134,6 +142,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -154,6 +164,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -184,6 +196,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -213,6 +227,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_error_handling.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_error_handling.groovy index 9b4def35e9c478..5a1049ecc327b7 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_error_handling.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_error_handling.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_error_handling") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -52,6 +54,8 @@ def evaluate(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -77,6 +81,8 @@ def evaluate(s, index): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -103,6 +109,8 @@ def evaluate(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -163,6 +171,8 @@ def evaluate(arr, index): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -186,6 +196,8 @@ def evaluate(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "module_only", "runtime_version" = "${runtime_version}" ) @@ -204,6 +216,8 @@ def evaluate(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = ".evaluate", "runtime_version" = "${runtime_version}" ) @@ -224,6 +238,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "nonexistent_func", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_file_protocol.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_file_protocol.groovy index 004737662a8ba2..b767e404e7fa55 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_file_protocol.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_file_protocol.groovy @@ -32,6 +32,8 @@ suite("test_pythonudf_file_protocol") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "int_test.evaluate", "runtime_version" = "${runtime_version}" @@ -47,6 +49,8 @@ suite("test_pythonudf_file_protocol") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "string_test.evaluate", "runtime_version" = "${runtime_version}" @@ -62,6 +66,8 @@ suite("test_pythonudf_file_protocol") { RETURNS FLOAT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "float_test.evaluate", "runtime_version" = "${runtime_version}", @@ -78,6 +84,8 @@ suite("test_pythonudf_file_protocol") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "boolean_test.evaluate", "runtime_version" = "${runtime_version}" @@ -127,6 +135,8 @@ suite("test_pythonudf_file_protocol") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${context.file.parent}/udf_scripts/missing_inline_precedence.zip", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_float.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_float.groovy index 2c61719cb57e51..6d91279b375cc4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_float.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_float.groovy @@ -53,6 +53,8 @@ suite("test_pythonudf_float") { "file"="file://${pyPath}", "symbol"="float_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}", "always_nullable" = "true", "volatility" = "immutable" @@ -73,6 +75,8 @@ suite("test_pythonudf_float") { "file"="file://${pyPath}", "symbol"="double_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}", "always_nullable" = "true" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_forbidden_module.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_forbidden_module.groovy index 2cc93b24ca0f88..065c049fd19512 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_forbidden_module.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_forbidden_module.groovy @@ -39,6 +39,8 @@ suite("test_pythonudf_forbidden_module") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "${forbiddenCase.symbol}", "runtime_version" = "${runtime_version}", @@ -58,6 +60,8 @@ suite("test_pythonudf_forbidden_module") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "safepkg_udf.logging.evaluate", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_global_function.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_global_function.groovy index ae57a0afcb6f3d..e924f4ef019bf4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_global_function.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_global_function.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_global_function") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -47,6 +49,8 @@ def evaluate(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -67,6 +71,8 @@ def evaluate(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -122,6 +128,8 @@ def evaluate(a, b): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_complex.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_complex.groovy index 8a84223221d80b..e68d0cf7932112 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_complex.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_complex.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_inline_complex") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -47,6 +49,8 @@ def evaluate(arr): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -67,6 +71,8 @@ def evaluate(s): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -90,6 +96,8 @@ def evaluate(val1, weight1, val2, weight2): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -110,6 +118,8 @@ def evaluate(first_name, last_name): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy index e85bba2d4a074f..41a5d74cde2209 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy @@ -22,7 +22,7 @@ suite("test_pythonudf_inline_basic") { for (def invalidPoolSize : ["0", "-1", "abc", "1.5", "2147483648"]) { test { sql """CREATE FUNCTION py_invalid_pool_size(INT) RETURNS INT PROPERTIES ( - "type" = "PYTHON_UDF", "symbol" = "evaluate", + "type" = "PYTHON_UDF", "auto_suspend"="0", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "pool_size" = "${invalidPoolSize}" ) AS \$\$def evaluate(x): return x\$\$""" exception "pool_size must be a positive integer" @@ -37,6 +37,7 @@ suite("test_pythonudf_inline_basic") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "pool_size" = "2" @@ -56,8 +57,10 @@ def evaluate(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) AS \$\$ def evaluate(s1, s2): @@ -77,8 +80,10 @@ def evaluate(s1, s2): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) AS \$\$ def evaluate(x): @@ -98,8 +103,10 @@ def evaluate(x): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) AS \$\$ def evaluate(num): @@ -120,8 +127,10 @@ def evaluate(num): RETURNS LARGEINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) AS \$\$ def evaluate(val): diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_vector.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_vector.groovy index c5398227262aa5..1818e03fdba694 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_vector.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_vector.groovy @@ -54,6 +54,8 @@ suite("test_pythonudf_inline_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "add", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -83,6 +85,8 @@ def add(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "multiply", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -112,6 +116,8 @@ def multiply(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "concat", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -141,6 +147,8 @@ def concat(s1: pd.Series, s2: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "get_max", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -171,6 +179,8 @@ def get_max(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "sqrt", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -200,6 +210,8 @@ def sqrt(x: pd.Series) -> pd.Series: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "to_upper", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -228,6 +240,8 @@ def to_upper(s: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "weighted_sum", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -257,6 +271,8 @@ def weighted_sum(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "negate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -285,6 +301,8 @@ def negate(b: pd.Series) -> pd.Series: RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "greater", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -314,6 +332,8 @@ def greater(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "str_len", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -342,6 +362,8 @@ def str_len(s: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "fill_null", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -370,6 +392,8 @@ def fill_null(x: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "cumsum", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_int.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_int.groovy index 126f4e2bfa97fb..c9d20bd8dc6465 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_int.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_int.groovy @@ -58,6 +58,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -70,6 +72,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -82,6 +86,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -94,6 +100,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -105,6 +113,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_ip.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_ip.groovy index 7c1221c6f1f9e3..b821c4b427a83e 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_ip.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_ip.groovy @@ -50,6 +50,8 @@ suite("test_pythonudf_ip") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -75,6 +77,8 @@ def evaluate(ip): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -101,6 +105,8 @@ def evaluate(ip): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -133,6 +139,8 @@ def evaluate(ip, cidr_str): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -164,6 +172,8 @@ def evaluate(cidr_str): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -195,6 +205,8 @@ def evaluate(cidr_str): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -222,6 +234,8 @@ def evaluate(ip): RETURNS IPv4 PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -250,6 +264,8 @@ def evaluate(ip_int): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -275,6 +291,8 @@ def evaluate(ip): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -301,6 +319,8 @@ def evaluate(ip): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_map.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_map.groovy index effa9ccecd326f..fd1d983ed3c0ab 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_map.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_map.groovy @@ -47,6 +47,8 @@ suite("test_pythonudf_map") { "file"="file://${pyPath}", "symbol"="map_int_int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -72,6 +74,8 @@ suite("test_pythonudf_map") { "file"="file://${pyPath}", "symbol"="map_string_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_mixed_params.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_mixed_params.groovy index ae849c90b2788c..ab2e58d84ff1d4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_mixed_params.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_mixed_params.groovy @@ -68,6 +68,8 @@ suite("test_pythonudf_mixed_params") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_multiply_constant", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -100,6 +102,8 @@ def py_vec_multiply_constant(values: pd.Series, multiplier: float) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_calc_total", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -136,6 +140,8 @@ def py_vec_calc_total(price: pd.Series, quantity: pd.Series, tax_rate: float) -> RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_apply_discount", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -170,6 +176,8 @@ def py_vec_apply_discount(price: pd.Series, discount_rate: pd.Series) -> pd.Seri RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_complex_calc", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -211,6 +219,8 @@ def py_vec_complex_calc(price: pd.Series, quantity: pd.Series, discount_rate: pd RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_add_prefix", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -244,6 +254,8 @@ def py_vec_add_prefix(categories: pd.Series, prefix: str) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_add_int_inline", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -277,6 +289,8 @@ def py_vec_add_int_inline(quantities: pd.Series, bonus: int) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_conditional_discount", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -317,6 +331,8 @@ def py_vec_conditional_discount(price: pd.Series, discount_rate: pd.Series, thre RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_scale_and_add", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -352,6 +368,8 @@ def py_vec_scale_and_add(scale_factor: float, prices: pd.Series, quantities: pd. RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_alternating", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -394,6 +412,8 @@ def py_vec_alternating(prices: pd.Series, markup: float, quantities: pd.Series, RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_multi_scalar", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_module.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_module.groovy index 14835ebe17421c..c2088e9a6f4de0 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_module.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_module.groovy @@ -28,6 +28,8 @@ suite("test_pythonudf_module") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file"="file://${pyPath}", "symbol" = "python_udf_module_test.main.safe_ltv", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_module_advanced.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_module_advanced.groovy index 19d386e801a10a..4f0e6ce05d0794 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_module_advanced.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_module_advanced.groovy @@ -32,6 +32,8 @@ suite("test_pythonudf_module_advanced") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "python_udf_module_test.main.safe_ltv", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_module_scalar.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_module_scalar.groovy index 51fc8dfe299a3a..566048b951e587 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_module_scalar.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_module_scalar.groovy @@ -66,6 +66,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.add_three_numbers", "runtime_version" = "${runtime_version}", @@ -89,6 +91,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.safe_divide_with_precision", "runtime_version" = "${runtime_version}", @@ -112,6 +116,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.calculate_discount_price", "runtime_version" = "${runtime_version}", @@ -136,6 +142,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.compound_interest", "runtime_version" = "${runtime_version}", @@ -159,6 +167,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.calculate_bmi", "runtime_version" = "${runtime_version}", @@ -182,6 +192,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.fibonacci", "runtime_version" = "${runtime_version}", @@ -206,6 +218,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.is_prime", "runtime_version" = "${runtime_version}", @@ -231,6 +245,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.gcd", "runtime_version" = "${runtime_version}", @@ -254,6 +270,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.lcm", "runtime_version" = "${runtime_version}", @@ -279,6 +297,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.reverse_string", "runtime_version" = "${runtime_version}", @@ -302,6 +322,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.count_vowels", "runtime_version" = "${runtime_version}", @@ -325,6 +347,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.count_words", "runtime_version" = "${runtime_version}", @@ -348,6 +372,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.capitalize_words", "runtime_version" = "${runtime_version}", @@ -371,6 +397,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.is_palindrome", "runtime_version" = "${runtime_version}", @@ -394,6 +422,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.string_similarity", "runtime_version" = "${runtime_version}", @@ -417,6 +447,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.mask_email", "runtime_version" = "${runtime_version}", @@ -440,6 +472,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.extract_domain", "runtime_version" = "${runtime_version}", @@ -463,6 +497,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.levenshtein_distance", "runtime_version" = "${runtime_version}", @@ -488,6 +524,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.days_between_dates", "runtime_version" = "${runtime_version}", @@ -511,6 +549,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.is_weekend", "runtime_version" = "${runtime_version}", @@ -534,6 +574,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.get_quarter", "runtime_version" = "${runtime_version}", @@ -557,6 +599,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.age_in_years", "runtime_version" = "${runtime_version}", @@ -581,6 +625,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.is_in_range", "runtime_version" = "${runtime_version}", @@ -604,6 +650,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.xor_operation", "runtime_version" = "${runtime_version}", @@ -629,6 +677,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.calculate_grade", "runtime_version" = "${runtime_version}", @@ -652,6 +702,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.categorize_age", "runtime_version" = "${runtime_version}", @@ -675,6 +727,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.calculate_tax", "runtime_version" = "${runtime_version}", @@ -698,6 +752,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.truncate_string", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_module_vector.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_module_vector.groovy index 231c18243e0b50..438da1fd43b63c 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_module_vector.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_module_vector.groovy @@ -61,6 +61,8 @@ suite("test_pythonudf_module_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_add_with_constant", "runtime_version" = "${runtime_version}", @@ -85,6 +87,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_multiply_and_round", "runtime_version" = "${runtime_version}", @@ -109,6 +113,8 @@ suite("test_pythonudf_module_vector") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_string_concat_with_separator", "runtime_version" = "${runtime_version}", @@ -133,6 +139,8 @@ suite("test_pythonudf_module_vector") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_string_title_case", "runtime_version" = "${runtime_version}", @@ -156,6 +164,8 @@ suite("test_pythonudf_module_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_conditional_value", "runtime_version" = "${runtime_version}", @@ -180,6 +190,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_percentage_calculation", "runtime_version" = "${runtime_version}", @@ -204,6 +216,8 @@ suite("test_pythonudf_module_vector") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_is_in_range", "runtime_version" = "${runtime_version}", @@ -227,6 +241,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_safe_divide", "runtime_version" = "${runtime_version}", @@ -251,6 +267,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_exponential_decay", "runtime_version" = "${runtime_version}", @@ -275,6 +293,8 @@ suite("test_pythonudf_module_vector") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_string_extract_first_word", "runtime_version" = "${runtime_version}", @@ -298,6 +318,8 @@ suite("test_pythonudf_module_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_abs_difference", "runtime_version" = "${runtime_version}", @@ -322,6 +344,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_power", "runtime_version" = "${runtime_version}", @@ -345,6 +369,8 @@ suite("test_pythonudf_module_vector") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_boolean_and", "runtime_version" = "${runtime_version}", @@ -369,6 +395,8 @@ suite("test_pythonudf_module_vector") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_boolean_or", "runtime_version" = "${runtime_version}", @@ -393,6 +421,8 @@ suite("test_pythonudf_module_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_clip_values", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_multiline_inline.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_multiline_inline.groovy index cc59150367f891..dcc1977602b63d 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_multiline_inline.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_multiline_inline.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_multiline_inline") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -51,6 +53,8 @@ def evaluate(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -99,6 +103,8 @@ def evaluate(customer_type, amount, quantity): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -132,6 +138,8 @@ def evaluate(text): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy index 51710e240f6315..5deb0594f773f2 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy @@ -27,6 +27,7 @@ suite("test_pythonudf_native_module_cache") { sql """ CREATE FUNCTION py_native_module_cache(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${pyPath}", "symbol" = "native_module_cache_test.evaluate", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_nested_complex_type.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_nested_complex_type.groovy index ca62629237518b..3bc36906a7e322 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_nested_complex_type.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_nested_complex_type.groovy @@ -82,6 +82,8 @@ suite("test_pythonudf_nested_complex_type") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -198,6 +200,8 @@ def evaluate(array_map, map_array_map, struct_nested, map_struct_nested): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -296,6 +300,8 @@ def evaluate(array_maps: list, map_array_maps: list, struct_nesteds: list, map_s RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -394,6 +400,8 @@ def evaluate(array_maps: pd.Series, map_array_maps: pd.Series, struct_nesteds: p RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_no_input.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_no_input.groovy index a0d78f4629dff0..0de051774c29c4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_no_input.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_no_input.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_no_input") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_object_types_inline.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_object_types_inline.groovy index b141e6d503ea26..e02e0d11925993 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_object_types_inline.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_object_types_inline.groovy @@ -24,6 +24,8 @@ suite("test_pythonudf_object_types_inline") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -41,6 +43,8 @@ def evaluate(v): RETURNS HLL PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -58,6 +62,8 @@ def evaluate(v): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -75,6 +81,8 @@ def evaluate(v): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -92,6 +100,8 @@ def evaluate(v): RETURNS STRUCT>> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy index d883438119aaed..93903d38170906 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy @@ -32,6 +32,7 @@ suite("test_pythonudf_pkg_isolation") { sql """ CREATE FUNCTION py_pkg_a_mod_x(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_x.evaluate", "runtime_version" = "${runtime_version}", @@ -41,6 +42,7 @@ suite("test_pythonudf_pkg_isolation") { sql """ CREATE FUNCTION py_pkg_b_mod_x(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_x.evaluate", "runtime_version" = "${runtime_version}", @@ -55,6 +57,7 @@ suite("test_pythonudf_pkg_isolation") { sql """ CREATE FUNCTION py_pkg_a_mod_y(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_y.evaluate", "runtime_version" = "${runtime_version}", @@ -69,6 +72,7 @@ suite("test_pythonudf_pkg_isolation") { sql """ CREATE FUNCTION py_pkg_b_mod_y(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_y.evaluate", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_ret_map.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_ret_map.groovy index 7ed7317210744e..b7db5ac57ddbca 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_ret_map.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_ret_map.groovy @@ -67,6 +67,8 @@ suite("test_pythonudf_ret_map") { "file"="file://${pyPath}", "symbol"="map_ret_int_int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -79,6 +81,8 @@ suite("test_pythonudf_ret_map") { "file"="file://${pyPath}", "symbol"="map_ret_string_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -92,6 +96,8 @@ suite("test_pythonudf_ret_map") { "file"="file://${pyPath}", "symbol"="map_ret_int_double_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -104,6 +110,8 @@ suite("test_pythonudf_ret_map") { "file"="file://${pyPath}", "symbol"="map_int_double_ret_string_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_schema_check.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_schema_check.groovy index 15e8620bd06a91..4da7f5795e9330 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_schema_check.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_schema_check.groovy @@ -62,6 +62,8 @@ suite("test_pythonudf_schema_check") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_add_int_sc", "runtime_version" = "${runtime_version}" ) @@ -106,6 +108,8 @@ def py_add_int_sc(a, b): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_add_bigint", "runtime_version" = "${runtime_version}" ) @@ -134,6 +138,8 @@ def py_add_bigint(a, b): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_add_double", "runtime_version" = "${runtime_version}" ) @@ -162,6 +168,8 @@ def py_add_double(a, b): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_sum_three", "runtime_version" = "${runtime_version}" ) @@ -191,6 +199,8 @@ def py_sum_three(a, b, c): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_multiply", "runtime_version" = "${runtime_version}", "vectorized" = "true" @@ -223,6 +233,8 @@ def py_vec_multiply(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_divide", "runtime_version" = "${runtime_version}", "vectorized" = "true" @@ -255,6 +267,8 @@ def py_vec_divide(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_calc", "runtime_version" = "${runtime_version}", "vectorized" = "true" @@ -287,6 +301,8 @@ def py_vec_calc(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_string_upper", "runtime_version" = "${runtime_version}" ) @@ -314,6 +330,8 @@ def py_string_upper(s): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_bool_not", "runtime_version" = "${runtime_version}" ) @@ -354,6 +372,8 @@ def py_bool_not(b): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_scale", "runtime_version" = "${runtime_version}", "vectorized" = "true" @@ -411,6 +431,8 @@ def py_vec_scale(values: pd.Series, factor: float) -> pd.Series: RETURNS FLOAT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_add_float", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_string.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_string.groovy index 4a0498ecd85309..39f5b725d0625c 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_string.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_string.groovy @@ -61,6 +61,8 @@ suite("test_pythonudf_string") { "file"="file://${pyPath}", "symbol"="string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_timestamp_ns.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_timestamp_ns.groovy index 64d4f8fcd06b03..8a6ce75d4b841c 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_timestamp_ns.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_timestamp_ns.groovy @@ -67,6 +67,8 @@ suite("test_pythonudf_timestamp_ns") { RETURNS TIMESTAMP_NS PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "always_nullable" = "true" @@ -82,6 +84,8 @@ def evaluate(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "always_nullable" = "true" @@ -97,6 +101,8 @@ def evaluate(value): RETURNS MAP PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "always_nullable" = "true" @@ -112,6 +118,8 @@ def evaluate(value): RETURNS STRUCT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_uuid_binary.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_uuid_binary.groovy index f6b6d48e27d575..19d9e5c36c8d16 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_uuid_binary.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_uuid_binary.groovy @@ -44,7 +44,7 @@ suite("test_pythonudf_uuid_binary") { sql "DROP FUNCTION IF EXISTS ${definition[0]}(${definition[1]})" sql """CREATE FUNCTION ${definition[0]}(${definition[1]}) RETURNS ${definition[1]} - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def validate(value): @@ -72,7 +72,7 @@ def evaluate(value): sql "DROP FUNCTION IF EXISTS py_uuid_binary_list(UUID)" sql """CREATE FUNCTION py_uuid_binary_list(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def evaluate(values: list): @@ -83,7 +83,7 @@ def evaluate(values: list): \$\$""" sql "DROP FUNCTION IF EXISTS py_uuid_binary_series(UUID)" sql """CREATE FUNCTION py_uuid_binary_series(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import pandas as pd import uuid @@ -95,7 +95,7 @@ def evaluate(values: pd.Series): \$\$""" sql "DROP FUNCTION IF EXISTS py_uuid_binary_vector_array(ARRAY)" sql """CREATE FUNCTION py_uuid_binary_vector_array(ARRAY) RETURNS ARRAY - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def evaluate(rows: list): @@ -110,7 +110,7 @@ def evaluate(rows: list): py_uuid_binary_vector_array(a) FROM python_uuid_binary ORDER BY id""" sql "DROP FUNCTION IF EXISTS py_uuid_binary_broadcast(UUID)" sql """CREATE FUNCTION py_uuid_binary_broadcast(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def evaluate(values: list): @@ -119,7 +119,7 @@ def evaluate(values: list): qt_broadcast "SELECT id, py_uuid_binary_broadcast(u) FROM python_uuid_binary ORDER BY id" sql "DROP FUNCTION IF EXISTS py_uuid_binary_string(STRING)" sql """CREATE FUNCTION py_uuid_binary_string(STRING) RETURNS STRING - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(value): if value is not None and not isinstance(value, str): @@ -141,7 +141,7 @@ def evaluate(value): ["value", "values: list"].eachWithIndex { parameter, index -> sql "DROP FUNCTION IF EXISTS py_uuid_binary_not_nullable_${index}(UUID)" sql """CREATE FUNCTION py_uuid_binary_not_nullable_${index}(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "always_nullable"="false", + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "always_nullable"="false", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(${parameter}): return None @@ -155,7 +155,7 @@ def evaluate(${parameter}): def functionName = "py_uuid_binary_invalid_${index}" sql "DROP FUNCTION IF EXISTS ${functionName}(UUID)" sql """CREATE FUNCTION ${functionName}(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(value): return ${expression} @@ -167,7 +167,7 @@ def evaluate(value): } sql "DROP FUNCTION IF EXISTS py_uuid_binary_min(UUID)" sql """CREATE AGGREGATE FUNCTION py_uuid_binary_min(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="UuidMin", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="UuidMin", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid class UuidMin: @@ -192,7 +192,7 @@ class UuidMin: qt_udaf_null "SELECT py_uuid_binary_min(u) FROM python_uuid_binary WHERE id = 5" sql "DROP FUNCTION IF EXISTS py_uuid_binary_rows(UUID)" sql """CREATE TABLES FUNCTION py_uuid_binary_rows(UUID) RETURNS ARRAY - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def evaluate(value): diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_volatility.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_volatility.groovy index 691e21efedb6c8..df7c118271aa3a 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_volatility.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_volatility.groovy @@ -54,6 +54,8 @@ suite("test_pythonudf_volatility") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "volatility" = "immutable", @@ -72,6 +74,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "volatility" = "stable" @@ -89,6 +93,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "volatility" = "volatile" @@ -106,6 +112,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_inline.groovy index b99c52dfacd459..3e9815e1e31d67 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_inline.groovy @@ -33,6 +33,8 @@ suite("test_pythonudtf_basic_inline") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "split_string_udtf", "runtime_version" = "${runtime_version}" ) @@ -78,6 +80,8 @@ def split_string_udtf(input_str): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "generate_series_udtf", "runtime_version" = "${runtime_version}" ) @@ -130,6 +134,8 @@ def generate_series_udtf(start, end): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "running_sum_udtf", "runtime_version" = "${runtime_version}" ) @@ -179,6 +185,8 @@ def running_sum_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "explode_json_udtf", "runtime_version" = "${runtime_version}" ) @@ -229,6 +237,8 @@ def explode_json_udtf(json_str): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "top_n_udtf", "runtime_version" = "${runtime_version}" ) @@ -281,6 +291,8 @@ def top_n_udtf(value, n): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "duplicate_udtf", "runtime_version" = "${runtime_version}" ) @@ -325,6 +337,8 @@ def duplicate_udtf(text, n): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "filter_positive_udtf", "runtime_version" = "${runtime_version}" ) @@ -369,6 +383,8 @@ def filter_positive_udtf(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "cartesian_udtf", "runtime_version" = "${runtime_version}" ) @@ -418,6 +434,8 @@ def cartesian_udtf(list1, list2): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "filter_negative_udtf", "runtime_version" = "${runtime_version}" ) @@ -512,6 +530,8 @@ def filter_negative_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "nullable_processor_udtf", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -561,6 +581,8 @@ def nullable_processor_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "non_nullable_processor_udtf", "runtime_version" = "${runtime_version}", "always_nullable" = "false" @@ -643,6 +665,8 @@ def non_nullable_processor_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "default_nullable_udtf", "runtime_version" = "${runtime_version}" ) @@ -689,6 +713,8 @@ def default_nullable_udtf(text): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "nullable_explode_udtf", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -746,6 +772,8 @@ def nullable_explode_udtf(csv_string): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "scalar_int_udtf", "runtime_version" = "${runtime_version}" ) @@ -772,6 +800,8 @@ def scalar_int_udtf(start, end): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "scalar_string_udtf", "runtime_version" = "${runtime_version}" ) @@ -798,6 +828,8 @@ def scalar_string_udtf(text): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "mixed_style_udtf", "runtime_version" = "${runtime_version}" ) @@ -828,6 +860,8 @@ def mixed_style_udtf(n): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "return_scalar_udtf", "runtime_version" = "${runtime_version}" ) @@ -852,6 +886,8 @@ def return_scalar_udtf(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "multi_field_udtf", "runtime_version" = "${runtime_version}" ) @@ -883,6 +919,8 @@ def multi_field_udtf(n): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_value_udtf", "runtime_version" = "${runtime_version}" ) @@ -938,6 +976,8 @@ def process_value_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "split_words_udtf", "runtime_version" = "${runtime_version}" ) @@ -995,6 +1035,8 @@ def split_words_udtf(text): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "expand_range_udtf", "runtime_version" = "${runtime_version}" ) @@ -1051,6 +1093,8 @@ def expand_range_udtf(n): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "parse_csv_udtf", "runtime_version" = "${runtime_version}" ) @@ -1211,6 +1255,8 @@ def parse_csv_udtf(csv_line): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "largeint_expand_udtf", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_module.groovy index 44c121cfdd724d..0904e31257babd 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_module.groovy @@ -38,6 +38,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.split_string_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -76,6 +78,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.generate_series_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -122,6 +126,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.running_sum_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -164,6 +170,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.explode_json_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -201,6 +209,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.top_n_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -247,6 +257,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.duplicate_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -285,6 +297,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.filter_positive_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -323,6 +337,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.cartesian_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -362,6 +378,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.filter_negative_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -450,6 +468,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.nullable_processor_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}", "always_nullable" = "true" ); diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_inline.groovy index d3e5b6a3bbc162..cc8739bf72c932 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_inline.groovy @@ -31,6 +31,8 @@ suite("test_pythonudtf_data_types_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_tinyint", "runtime_version" = "${runtime_version}" ) @@ -73,6 +75,8 @@ def process_tinyint(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_smallint", "runtime_version" = "${runtime_version}" ) @@ -115,6 +119,8 @@ def process_smallint(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_bigint", "runtime_version" = "${runtime_version}" ) @@ -157,6 +163,8 @@ def process_bigint(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_float", "runtime_version" = "${runtime_version}" ) @@ -199,6 +207,8 @@ def process_float(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_double", "runtime_version" = "${runtime_version}" ) @@ -243,6 +253,8 @@ def process_double(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_boolean", "runtime_version" = "${runtime_version}" ) @@ -285,6 +297,8 @@ def process_boolean(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_string", "runtime_version" = "${runtime_version}" ) @@ -327,6 +341,8 @@ def process_string(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_date", "runtime_version" = "${runtime_version}" ) @@ -370,6 +386,8 @@ def process_date(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_datetime", "runtime_version" = "${runtime_version}" ) @@ -416,6 +434,8 @@ def process_datetime(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_array_int", "runtime_version" = "${runtime_version}" ) @@ -463,6 +483,8 @@ def process_array_int(arr): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_array_string", "runtime_version" = "${runtime_version}" ) @@ -509,6 +531,8 @@ def process_array_string(arr): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_struct", "runtime_version" = "${runtime_version}" ) @@ -559,6 +583,8 @@ def process_struct(person): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_multi_types", "runtime_version" = "${runtime_version}" ) @@ -602,6 +628,8 @@ def process_multi_types(num, text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_decimal", "runtime_version" = "${runtime_version}" ) @@ -650,6 +678,8 @@ def process_decimal(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_map_string", "runtime_version" = "${runtime_version}" ) @@ -699,6 +729,8 @@ def process_map_string(map_str): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_nested_array", "runtime_version" = "${runtime_version}" ) @@ -751,6 +783,8 @@ def process_nested_array(nested_str): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_array_structs", "runtime_version" = "${runtime_version}" ) @@ -800,6 +834,8 @@ def process_array_structs(data): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_struct_array", "runtime_version" = "${runtime_version}" ) @@ -844,6 +880,8 @@ def process_struct_array(data): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "extract_json_fields", "runtime_version" = "${runtime_version}" ) @@ -894,6 +932,8 @@ def extract_json_fields(json_str): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_complex_struct", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_module.groovy index e055dbb47e8eba..b1d079c661e425 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_module.groovy @@ -37,6 +37,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_tinyint", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -74,6 +76,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_smallint", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -111,6 +115,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_bigint", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -148,6 +154,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_float", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -185,6 +193,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_double", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -222,6 +232,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_boolean", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -259,6 +271,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -296,6 +310,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_date", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -333,6 +349,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_datetime", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -373,6 +391,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_array_int", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -413,6 +433,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_array_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -452,6 +474,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_struct", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -492,6 +516,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_multi_types", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -530,6 +556,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_decimal", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -570,6 +598,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_map_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -607,6 +637,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_nested_array", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -644,6 +676,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_array_structs", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -681,6 +715,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_struct_array", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -718,6 +754,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.extract_json_fields", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -755,6 +793,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_complex_struct", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy index 026d6d5da119e3..134e52f229157d 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy @@ -44,25 +44,27 @@ suite("test_pythonudtf_database_cleanup_retry", "nonConcurrent") { .collect { it.DbId }.unique() ids.each { id -> sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${id}" } } - def createFunction = { zip -> - sql """CREATE TABLES FUNCTION py_cleanup_retry(INT) RETURNS ARRAY - PROPERTIES("type"="PYTHON_UDF", "symbol"="drop_udtf.process", + def createFunction = { zip, scope -> + sql """CREATE ${scope} TABLES FUNCTION py_cleanup_retry(INT) RETURNS ARRAY + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "symbol"="drop_udtf.process", "file"="file://${zip}", "runtime_version"="${runtime}", "pool_size"="1")""" } - // Disable idle cleanup so this test verifies task retries. - ["force", "recycle"].eachWithIndex { mode, index -> + ["force", "recycle", "function"].eachWithIndex { mode, index -> def db = "py_udtf_cleanup_retry_${mode}" + def scope = mode == "global" ? "GLOBAL" : "" try { + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } purgeDatabase(db) await("earlier database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) sql "CREATE DATABASE ${db}" sql "USE ${db}" def dbId = getDbId(db) - createFunction(zipA) + createFunction(zipA, scope) // Normal and outer functions each need a cleanup task on every backend. - def functionCount = sql("SHOW FUNCTIONS LIKE 'py_cleanup_retry%'").size() + def functionCount = sql("SHOW ${scope} FUNCTIONS LIKE 'py_cleanup_retry%'").size() "qt_before_${index}"("SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry(v) t AS x UNION ALL SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry_outer(v) t AS x ORDER BY x") - sql "USE ${originalDb}" def beIds = sql_return_maparray("SHOW BACKENDS").collect { it.BackendId.toString() } // Simulate heartbeat failures while Python processes keep running. @@ -71,7 +73,10 @@ suite("test_pythonudtf_database_cleanup_retry", "nonConcurrent") { await("backends were not marked unavailable", { sql_return_maparray("SHOW BACKENDS").every { !it.Alive.toBoolean() } }) - if (mode == "recycle") { + if (mode in ["function", "global"]) { + sql "DROP ${scope} FUNCTION py_cleanup_retry(INT)" + } else if (mode == "recycle") { + sql "USE ${originalDb}" sql "DROP DATABASE ${db}" // Recoverable databases retain their function pools. assertEquals(0, cleanupTasks().TotalNum.toInteger()) @@ -79,9 +84,10 @@ suite("test_pythonudtf_database_cleanup_retry", "nonConcurrent") { assertTrue(sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") .every { it.DbId.toString() != dbId.toString() }) } else { + sql "USE ${originalDb}" sql "DROP DATABASE ${db} FORCE" } - await("database cleanup was lost while BE was unavailable", { + await("function cleanup was lost while backend was unavailable", { def tasks = cleanupTasks() tasks.TotalNum.toInteger() == beIds.size() * functionCount && tasks.FailedNum.toInteger() == beIds.size() * functionCount @@ -97,16 +103,21 @@ suite("test_pythonudtf_database_cleanup_retry", "nonConcurrent") { sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } }) // The task leaves the queue only after cleanup succeeds and its result is reported. - await("BE recovery did not resend and acknowledge database cleanup", { + await("backend recovery did not complete function cleanup", { cleanupTasks().TotalNum.toInteger() == 0 }) - sql "CREATE DATABASE ${db}" + if (mode in ["force", "recycle"]) { + sql "CREATE DATABASE ${db}" + } sql "USE ${db}" - createFunction(zipB) + createFunction(zipB, scope) "qt_after_${index}"("SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry(v) t AS x UNION ALL SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry_outer(v) t AS x ORDER BY x") } finally { GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) sql "USE ${originalDb}" + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } purgeDatabase(db) await("backends did not recover during cleanup", { sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_drop.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_drop.groovy index 7eeb9de281a43f..25290aa34be90f 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_drop.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_drop.groovy @@ -57,6 +57,8 @@ suite("test_pythonudtf_drop", "nonConcurrent") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udtf.process", "runtime_version" = "${runtime_version}" @@ -87,6 +89,8 @@ suite("test_pythonudtf_drop", "nonConcurrent") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udtf.process", "runtime_version" = "${runtime_version}" @@ -97,6 +101,8 @@ suite("test_pythonudtf_drop", "nonConcurrent") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipB}", "symbol" = "drop_udtf.process", "runtime_version" = "${runtime_version}" @@ -145,6 +151,8 @@ suite("test_pythonudtf_drop", "nonConcurrent") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udtf.process", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_inline.groovy index cbdd84b86f143b..bbdbc7b3538f6a 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_inline.groovy @@ -33,6 +33,8 @@ suite("test_pythonudtf_edge_cases_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "handle_null_int", "runtime_version" = "${runtime_version}" ) @@ -75,6 +77,8 @@ def handle_null_int(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "handle_null_string", "runtime_version" = "${runtime_version}" ) @@ -119,6 +123,8 @@ def handle_null_string(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "handle_empty_array", "runtime_version" = "${runtime_version}" ) @@ -166,6 +172,8 @@ def handle_empty_array(arr): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "handle_null_struct", "runtime_version" = "${runtime_version}" ) @@ -230,6 +238,8 @@ def handle_null_struct(person): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_empty_table", "runtime_version" = "${runtime_version}" ) @@ -267,6 +277,8 @@ def process_empty_table(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_single_row", "runtime_version" = "${runtime_version}" ) @@ -308,6 +320,8 @@ def process_single_row(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_long_string", "runtime_version" = "${runtime_version}" ) @@ -353,6 +367,8 @@ def process_long_string(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_large_array", "runtime_version" = "${runtime_version}" ) @@ -399,6 +415,8 @@ def process_large_array(arr): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "output_explosion", "runtime_version" = "${runtime_version}" ) @@ -445,6 +463,8 @@ def output_explosion(n): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_special_numbers", "runtime_version" = "${runtime_version}" ) @@ -503,6 +523,8 @@ def process_special_numbers(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_special_doubles", "runtime_version" = "${runtime_version}" ) @@ -565,6 +587,8 @@ def process_special_doubles(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_special_strings", "runtime_version" = "${runtime_version}" ) @@ -626,6 +650,8 @@ def process_special_strings(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_boundary_dates", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_module.groovy index 8940a6983ffd16..dfb1becba5937c 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_module.groovy @@ -39,6 +39,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.handle_null_int", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -74,6 +76,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.handle_null_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -109,6 +113,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.handle_empty_array", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -147,6 +153,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.handle_null_struct", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -190,6 +198,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_empty_table", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -222,6 +232,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_single_row", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -257,6 +269,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_long_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -294,6 +308,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_large_array", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -331,6 +347,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.output_explosion", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -371,6 +389,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_special_numbers", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -412,6 +432,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_special_doubles", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -452,6 +474,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_special_strings", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -492,6 +516,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_boundary_dates", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_inline.groovy index ee8d8bafc47baf..765b485c4b1db4 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_inline.groovy @@ -33,6 +33,8 @@ suite("test_pythonudtf_exceptions_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "safe_divide", "runtime_version" = "${runtime_version}" ) @@ -84,6 +86,8 @@ def safe_divide(a, b): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "check_overflow", "runtime_version" = "${runtime_version}" ) @@ -144,6 +148,8 @@ def check_overflow(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "parse_number", "runtime_version" = "${runtime_version}" ) @@ -196,6 +202,8 @@ def parse_number(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "check_type", "runtime_version" = "${runtime_version}" ) @@ -251,6 +259,8 @@ def check_type(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "safe_array_access", "runtime_version" = "${runtime_version}" ) @@ -303,6 +313,8 @@ def safe_array_access(arr, position): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "compute_stats", "runtime_version" = "${runtime_version}" ) @@ -358,6 +370,8 @@ def compute_stats(arr): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "access_struct_fields", "runtime_version" = "${runtime_version}" ) @@ -415,6 +429,8 @@ def access_struct_fields(person): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "slice_string", "runtime_version" = "${runtime_version}" ) @@ -476,6 +492,8 @@ def slice_string(text, start, end): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "check_text_encoding", "runtime_version" = "${runtime_version}" ) @@ -531,6 +549,8 @@ def check_text_encoding(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_conditional", "runtime_version" = "${runtime_version}" ) @@ -588,6 +608,8 @@ def process_conditional(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "conditional_yield", "runtime_version" = "${runtime_version}" ) @@ -639,6 +661,8 @@ def conditional_yield(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "classify_number_range", "runtime_version" = "${runtime_version}" ) @@ -701,6 +725,8 @@ def classify_number_range(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "validate_date", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_module.groovy index 29443b8214b0b1..25b44c60a10a9b 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_module.groovy @@ -39,6 +39,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.safe_divide", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -79,6 +81,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.check_overflow", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -122,6 +126,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.parse_number", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -163,6 +169,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.check_type", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -206,6 +214,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.safe_array_access", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -247,6 +257,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.compute_stats", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -290,6 +302,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.access_struct_fields", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -333,6 +347,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.slice_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -375,6 +391,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.check_text_encoding", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -419,6 +437,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.process_conditional", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -459,6 +479,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.conditional_yield", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -504,6 +526,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.classify_number_range", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -545,6 +569,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.validate_date", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_forbidden_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_forbidden_module.groovy index a25852fc69b507..4a5ccd0f203f74 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_forbidden_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_forbidden_module.groovy @@ -53,6 +53,8 @@ suite("test_pythonudtf_forbidden_module") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "${forbiddenCase.symbol}", "runtime_version" = "${runtime_version}" @@ -76,6 +78,8 @@ suite("test_pythonudtf_forbidden_module") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "safepkg_udtf.inspect.safe_udtf", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_inline.groovy index b5a1bd5479173e..bc7d07bab69f94 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_inline.groovy @@ -32,6 +32,8 @@ suite("test_pythonudtf_io_patterns_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "one_to_one", "runtime_version" = "${runtime_version}" ) @@ -76,6 +78,8 @@ def one_to_one(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "one_to_many", "runtime_version" = "${runtime_version}" ) @@ -121,6 +125,8 @@ def one_to_many(n): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "one_to_zero", "runtime_version" = "${runtime_version}" ) @@ -166,6 +172,8 @@ def one_to_zero(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "one_to_variable", "runtime_version" = "${runtime_version}" ) @@ -222,6 +230,8 @@ def one_to_variable(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "aggregate_pattern", "runtime_version" = "${runtime_version}" ) @@ -274,6 +284,8 @@ def aggregate_pattern(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "explosive", "runtime_version" = "${runtime_version}" ) @@ -321,6 +333,8 @@ def explosive(all_rows, all_cols): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "conditional", "runtime_version" = "${runtime_version}" ) @@ -375,6 +389,8 @@ def conditional(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "all_or_nothing", "runtime_version" = "${runtime_version}" ) @@ -428,6 +444,8 @@ def all_or_nothing(text, min_length): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "empty_input", "runtime_version" = "${runtime_version}" ) @@ -468,6 +486,8 @@ def empty_input(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "batch_process", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_module.groovy index d84725f13ea0b3..367a2b5067bf67 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_module.groovy @@ -38,6 +38,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.one_to_one", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -76,6 +78,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.one_to_many", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -114,6 +118,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.one_to_zero", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -152,6 +158,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.one_to_variable", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -195,6 +203,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.aggregate_pattern", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -235,6 +245,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.explosive", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -274,6 +286,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.conditional", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -312,6 +326,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.all_or_nothing", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -354,6 +370,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.empty_input", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -389,6 +407,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.batch_process", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_no_input.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_no_input.groovy index e5b90a6ca9ffc7..76aa26e3c611cd 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_no_input.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_no_input.groovy @@ -27,6 +27,8 @@ suite("test_pythonudtf_no_input") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "emit_values", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_object_types_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_object_types_inline.groovy index 0d4259a6e6e02d..70b73ab4f3b858 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_object_types_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_object_types_inline.groovy @@ -24,6 +24,8 @@ suite("test_pythonudtf_object_types_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -41,6 +43,8 @@ def evaluate(v): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -58,6 +62,8 @@ def evaluate(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -75,6 +81,8 @@ def evaluate(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -92,6 +100,8 @@ def evaluate(v): RETURNS ARRAY>>> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy index 40e5777421092d..359e63c7934978 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy @@ -45,6 +45,7 @@ suite("test_pythonudtf_pkg_isolation") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_x.process", "runtime_version" = "${runtime_version}", @@ -56,6 +57,7 @@ suite("test_pythonudtf_pkg_isolation") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_x.process", "runtime_version" = "${runtime_version}", @@ -78,6 +80,7 @@ suite("test_pythonudtf_pkg_isolation") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_y.process", "runtime_version" = "${runtime_version}", @@ -100,6 +103,7 @@ suite("test_pythonudtf_pkg_isolation") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_y.process", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_inline.groovy index e15ec67fec9ea9..53a7d1d6fb4138 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_inline.groovy @@ -33,6 +33,8 @@ suite("test_pythonudtf_sql_integration_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "split_with_position", "runtime_version" = "${runtime_version}" ) @@ -53,6 +55,8 @@ def split_with_position(text, delimiter): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "generate_range", "runtime_version" = "${runtime_version}" ) @@ -72,6 +76,8 @@ def generate_range(start, end): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "explode_with_index", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_module.groovy index 9ac0a4aebd2135..215f4a2f9a3840 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_module.groovy @@ -39,6 +39,8 @@ suite("test_pythonudtf_sql_integration_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.sql_integration_udtf.split_with_position", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -52,6 +54,8 @@ suite("test_pythonudtf_sql_integration_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.sql_integration_udtf.generate_range", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -65,6 +69,8 @@ suite("test_pythonudtf_sql_integration_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.sql_integration_udtf.explode_with_index", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/query_p0/show/test_nereids_show_functions.groovy b/regression-test/suites/query_p0/show/test_nereids_show_functions.groovy index 12f727f7ee4e01..3eea7c2e271fa0 100644 --- a/regression-test/suites/query_p0/show/test_nereids_show_functions.groovy +++ b/regression-test/suites/query_p0/show/test_nereids_show_functions.groovy @@ -110,6 +110,8 @@ suite("test_nereids_show_functions") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -135,6 +137,8 @@ suite("test_nereids_show_functions") { "file"="file://${pyudfPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "pool_size"="1", + "auto_suspend"="0", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -158,6 +162,8 @@ suite("test_nereids_show_functions") { "file"="file://${pyudfPath}", "symbol"="sum_int.SumInt", "type"="PYTHON_UDF", + "pool_size"="1", + "auto_suspend"="0", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -181,6 +187,8 @@ suite("test_nereids_show_functions") { "file" = "file://${pyudtfPath}", "symbol" = "pyudtf_module.basic_udtf.split_string_udtf", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ From eee46a1dc20a35cc3527127931322cb01d5dc4ff Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Thu, 22 Oct 2026 21:03:32 +0800 Subject: [PATCH 5/5] [fix](udf) Bound Python pools and default idle reclamation to four hours ### What problem does this PR solve? Related PR: #69255 Problem Summary: Per-function Python pools could allocate and start as many processes as requested, and their default idle policy retained processes indefinitely. Cap each pool at the executing backend's CPU core count before allocating slots or starting processes, while retaining positive pool_size validation. Set auto_suspend to 14400 seconds consistently across function creation, catalog metadata, Thrift, and backend defaults. Require explicit -1 to disable idle reclamation and reject smaller values. ### Release note Python function pools are capped by each backend's CPU core count and are reclaimed after four hours without active clients by default. The next call recreates the pool. Explicit auto_suspend=-1 disables idle reclamation. ### Check List (For Author) - Test: Full build; 38 FE unit tests; 29 BE unit tests on both host and one-core configurations; four targeted regression suites; all 19 Python UDTF suites; verified pool_size=10000 starts exactly one Python server on a one-core BE; clang-format 16 and clang-tidy checks. - Behavior changed: Yes, cap effective pool_size per backend and default auto_suspend to four hours; reject auto_suspend values below -1. - Does this need documentation: Yes, English and Chinese development-version descriptions are updated in the separate doris-website working directory. --- .../aggregate_function_python_udaf.cpp | 4 +- be/src/exprs/function/function_python_udf.cpp | 3 +- .../table_function/python_udtf_function.cpp | 4 +- be/src/udf/python/python_server.cpp | 8 ++- be/src/udf/python/python_server.h | 4 +- be/src/udf/python/python_udf_meta.h | 4 +- be/test/udf/python/python_server_test.cpp | 66 ++++++++++++++++--- be/test/udf/python/python_udf_meta_test.cpp | 6 ++ .../org/apache/doris/catalog/Function.java | 4 +- .../plans/commands/CreateFunctionCommand.java | 5 +- .../doris/catalog/CreateFunctionTest.java | 30 ++++++--- .../FunctionToThriftConverterTest.java | 7 +- gensrc/thrift/Types.thrift | 2 +- .../test_pythonudaf_auto_suspend.groovy | 31 ++++++++- .../test_pythonudf_auto_suspend.groovy | 21 +++++- .../test_pythonudtf_auto_suspend.groovy | 23 ++++++- 16 files changed, 186 insertions(+), 36 deletions(-) diff --git a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp index ae4bce41fb3fa6..0612e011bd7465 100644 --- a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp +++ b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp @@ -192,7 +192,9 @@ Status AggregatePythonUDAF::open() { // Build function metadata from TFunction _func_meta.id = _fn.id; _func_meta.pool_size = _fn.__isset.pool_size ? _fn.pool_size : 3; - _func_meta.auto_suspend = _fn.__isset.auto_suspend ? _fn.auto_suspend : -1; + _func_meta.auto_suspend = _fn.__isset.auto_suspend + ? _fn.auto_suspend + : PythonUDFMeta::DEFAULT_AUTO_SUSPEND_SECONDS; _func_meta.name = _fn.name.function_name; // For UDAF, symbol is in aggregate_fn diff --git a/be/src/exprs/function/function_python_udf.cpp b/be/src/exprs/function/function_python_udf.cpp index f6d266a7e439d7..a524b44310a607 100644 --- a/be/src/exprs/function/function_python_udf.cpp +++ b/be/src/exprs/function/function_python_udf.cpp @@ -54,7 +54,8 @@ Status PythonFunctionCall::open(FunctionContext* context, PythonUDFMeta func_meta; func_meta.id = _fn.id; func_meta.pool_size = _fn.__isset.pool_size ? _fn.pool_size : 3; - func_meta.auto_suspend = _fn.__isset.auto_suspend ? _fn.auto_suspend : -1; + func_meta.auto_suspend = _fn.__isset.auto_suspend ? _fn.auto_suspend + : PythonUDFMeta::DEFAULT_AUTO_SUSPEND_SECONDS; func_meta.name = _fn.name.function_name; func_meta.symbol = _fn.scalar_fn.symbol; if (!_fn.function_code.empty()) { diff --git a/be/src/exprs/table_function/python_udtf_function.cpp b/be/src/exprs/table_function/python_udtf_function.cpp index e87d40efff706c..4c2b466069510b 100644 --- a/be/src/exprs/table_function/python_udtf_function.cpp +++ b/be/src/exprs/table_function/python_udtf_function.cpp @@ -62,7 +62,9 @@ Status PythonUDTFFunction::open() { PythonUDFMeta python_udf_meta; python_udf_meta.id = _t_fn.id; python_udf_meta.pool_size = _t_fn.__isset.pool_size ? _t_fn.pool_size : 3; - python_udf_meta.auto_suspend = _t_fn.__isset.auto_suspend ? _t_fn.auto_suspend : -1; + python_udf_meta.auto_suspend = _t_fn.__isset.auto_suspend + ? _t_fn.auto_suspend + : PythonUDFMeta::DEFAULT_AUTO_SUSPEND_SECONDS; python_udf_meta.name = _t_fn.name.function_name; python_udf_meta.symbol = _t_fn.scalar_fn.symbol; diff --git a/be/src/udf/python/python_server.cpp b/be/src/udf/python/python_server.cpp index 72209141e68c64..ed6f11654f9238 100644 --- a/be/src/udf/python/python_server.cpp +++ b/be/src/udf/python/python_server.cpp @@ -42,6 +42,7 @@ #include "udf/python/python_udaf_client.h" #include "udf/python/python_udf_client.h" #include "udf/python/python_udtf_client.h" +#include "util/cpu_info.h" #include "util/defer_op.h" namespace doris { @@ -99,13 +100,18 @@ PythonServerManager::_get_or_create_process_pool( return ResultError( Status::InvalidArgument("Python UDF pool_size must be a positive integer")); } + if (meta.auto_suspend < -1) { + return ResultError(Status::InvalidArgument( + "Python UDF auto_suspend must be -1 or a non-negative integer")); + } auto& pool = _process_pools[meta.id]; if (!pool) { pool = std::make_shared(); pool->version = version; pool->process_registry = _process_registry; pool->function_id = meta.id; - pool->pool_size = meta.pool_size; + // The same function may run on BEs with different CPU limits. + pool->pool_size = std::min(meta.pool_size, CpuInfo::num_cores()); pool->auto_suspend = meta.auto_suspend; pool->package_cache_entry = std::move(package_cache_entry); // The first process can serve a client before the remaining slots finish initialization. diff --git a/be/src/udf/python/python_server.h b/be/src/udf/python/python_server.h index 78ae34da9eee12..18b333f28cc934 100644 --- a/be/src/udf/python/python_server.h +++ b/be/src/udf/python/python_server.h @@ -107,7 +107,7 @@ class PythonServerManager { std::shared_ptr process_registry; int64_t function_id = 0; int32_t pool_size = 3; - int64_t auto_suspend = -1; + int64_t auto_suspend = PythonUDFMeta::DEFAULT_AUTO_SUSPEND_SECONDS; size_t active_clients = 0; std::chrono::steady_clock::time_point idle_since = std::chrono::steady_clock::now(); std::shared_ptr package_cache_entry; @@ -151,7 +151,7 @@ class PythonServerManager { void _start_health_check_thread(); // Only reclaim pools with no active clients: STOPPED pools on the next scan; other pools - // after auto_suspend seconds without a client (negative values disable idle reclamation). + // after auto_suspend seconds without a client (-1 disables idle reclamation). void _reclaim_idle_pools(std::chrono::steady_clock::time_point now); /** diff --git a/be/src/udf/python/python_udf_meta.h b/be/src/udf/python/python_udf_meta.h index 8a421d953b6f15..3704278518bf0b 100644 --- a/be/src/udf/python/python_udf_meta.h +++ b/be/src/udf/python/python_udf_meta.h @@ -33,9 +33,11 @@ enum class PythonUDFLoadType : uint8_t { INLINE = 0, MODULE = 1, UNKNOWN = 2 }; enum class PythonClientType : uint8_t { UDF = 0, UDAF = 1, UDTF = 2, UNKNOWN = 3 }; struct PythonUDFMeta { + static constexpr int64_t DEFAULT_AUTO_SUSPEND_SECONDS = 14400; + int64_t id = 0; int32_t pool_size = 3; - int64_t auto_suspend = -1; + int64_t auto_suspend = DEFAULT_AUTO_SUSPEND_SECONDS; std::string name; std::string symbol; std::string location; diff --git a/be/test/udf/python/python_server_test.cpp b/be/test/udf/python/python_server_test.cpp index 43f47ba1c2f15d..3d00540a3d96b0 100644 --- a/be/test/udf/python/python_server_test.cpp +++ b/be/test/udf/python/python_server_test.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -39,6 +40,7 @@ #include "udf/python/python_udf_client.h" #include "udf/python/python_udf_meta.h" #include "udf/python/python_udtf_client.h" +#include "util/cpu_info.h" #include "util/defer_op.h" namespace doris { @@ -813,6 +815,53 @@ TEST_F(PythonServerTest, RejectsInvalidPoolSizeBeforeStartingProcesses) { mgr.shutdown(); } +TEST_F(PythonServerTest, CapsPoolSizeBeforeAllocatingProcessSlots) { + CpuInfo::init(); + const int cores = CpuInfo::num_cores(); + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + for (const int requested : {1, 3, cores, cores + 1, 10000}) { + PythonUDFMeta meta; + meta.id = requested; + meta.pool_size = requested; + auto result = mgr._get_or_create_process_pool(meta, version, nullptr); + ASSERT_TRUE(result.has_value()) << result.error().to_string(); + EXPECT_EQ(std::min(requested, cores), result.value()->pool_size); + EXPECT_TRUE(result.value()->processes.empty()); + } +} + +TEST_F(PythonServerTest, DefaultAutoSuspendReclaimsAfterFourHoursWithoutClients) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + PythonUDFMeta meta; + ASSERT_EQ(14400, meta.auto_suspend); + std::shared_ptr lease; + auto result = mgr._get_or_create_process_pool(meta, version, nullptr, &lease); + ASSERT_TRUE(result.has_value()); + auto pool = result.value(); + ASSERT_EQ(14400, pool->auto_suspend); + EXPECT_EQ(std::min(3, CpuInfo::num_cores()), pool->pool_size); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::hours(5)); + EXPECT_EQ(pool, mgr._get_or_create_process_pool(meta, version, nullptr).value()); + lease.reset(); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::seconds(14399)); + EXPECT_EQ(pool, mgr._get_or_create_process_pool(meta, version, nullptr).value()); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::seconds(14400)); + EXPECT_NE(pool, mgr._get_or_create_process_pool(meta, version, nullptr).value()); +} + +TEST_F(PythonServerTest, RejectsAutoSuspendBelowMinusOneBeforeStartingProcesses) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + PythonUDFMeta meta; + meta.auto_suspend = -2; + auto result = mgr._get_or_create_process_pool(meta, version, nullptr); + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().to_string().find("auto_suspend must be -1 or a non-negative integer"), + std::string::npos); +} + TEST_F(PythonServerTest, GetProcessSelectsLeastLoadedProcess) { PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); @@ -1237,18 +1286,15 @@ TEST_F(PythonServerTest, AutoSuspendWaitsForTheConfiguredIdleDuration) { EXPECT_FALSE(process->is_alive()); } -TEST_F(PythonServerTest, NegativeAutoSuspendKeepsIdleProcessesAlive) { +TEST_F(PythonServerTest, ExplicitMinusOneKeepsIdleProcessesAlive) { PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); - for (int64_t timeout : {-1, -2}) { - auto meta = test_meta(version); - meta.id = 2000 - timeout; - meta.auto_suspend = timeout; - auto process = create_sleep_process(); - mgr.set_process_pool_for_test(meta, version, {process}); - mgr._reclaim_idle_pools(std::chrono::steady_clock::now() + std::chrono::hours(24)); - EXPECT_TRUE(process->is_alive()); - } + auto meta = test_meta(version); + meta.auto_suspend = -1; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now() + std::chrono::hours(24)); + EXPECT_TRUE(process->is_alive()); } TEST_F(PythonServerTest, AutoSuspendWaitsForLastClientAndStartsANewIdlePeriod) { diff --git a/be/test/udf/python/python_udf_meta_test.cpp b/be/test/udf/python/python_udf_meta_test.cpp index 322963319eea0a..f8c1cbfe3093df 100644 --- a/be/test/udf/python/python_udf_meta_test.cpp +++ b/be/test/udf/python/python_udf_meta_test.cpp @@ -59,6 +59,12 @@ class PythonUDFMetaTest : public ::testing::Test { DataTypePtr nullable_double_; }; +TEST_F(PythonUDFMetaTest, DefaultResourceProperties) { + PythonUDFMeta meta; + EXPECT_EQ(3, meta.pool_size); + EXPECT_EQ(14400, meta.auto_suspend); +} + // ============================================================================ // PythonUDFMeta check() tests // ============================================================================ diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java index 1c52f8e13925e3..4b590dd53495f5 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java @@ -41,6 +41,8 @@ */ public class Function implements Writable { + public static final long DEFAULT_AUTO_SUSPEND_SECONDS = 14400; + public enum NullableMode { // Whether output column is nullable is depend on the input column is nullable DEPEND_ON_ARGUMENT, @@ -115,7 +117,7 @@ public enum BinaryType { @SerializedName("ps") protected int poolSize = 3; @SerializedName("as") - protected long autoSuspend = -1; + protected long autoSuspend = DEFAULT_AUTO_SUSPEND_SECONDS; @SerializedName("fc") protected String functionCode; @SerializedName("vol") diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java index 97b51a8cc45de6..80766e4bd2091c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java @@ -191,7 +191,7 @@ public class CreateFunctionCommand extends Command implements ForwardWithSync { private FunctionVolatility volatility = FunctionVolatility.IMMUTABLE; private String runtimeVersion; private int poolSize = 3; - private long autoSuspend = -1; + private long autoSuspend = Function.DEFAULT_AUTO_SUSPEND_SECONDS; private String functionCode; /** @@ -394,6 +394,9 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { } catch (NumberFormatException e) { throw new AnalysisException("Python UDF auto_suspend must be an integer number of seconds"); } + if (autoSuspend < -1) { + throw new AnalysisException("Python UDF auto_suspend must be -1 or a non-negative integer"); + } } if (properties.containsKey(POOL_SIZE)) { try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java index 4b1705b7e92009..4febb6f0e69353 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java @@ -186,14 +186,17 @@ public void testPoolSize() throws Exception { dorisAssert.useDatabase("python_pool_db"); Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("python_pool_db"); for (String kind : new String[]{"", "aggregate ", "tables "}) { - String name = kind.isEmpty() ? "scalar_fn" : kind.trim() + "_fn"; String ret = kind.equals("tables ") ? "array" : "int"; - createFunction("create " + kind + "function python_pool_db." + name - + "(int) returns " + ret + " properties('type'='PYTHON_UDF', " - + "'symbol'='evaluate', 'runtime_version'='3.10.2', 'pool_size'='3');", ctx); - Function fn = findFunction(db, name); - Assertions.assertEquals(3, fn.getPoolSize()); - Assertions.assertTrue(FunctionToSqlConverter.toSql(fn, false).contains("\"POOL_SIZE\"=\"3\"")); + for (int requested : new int[]{3, 10000}) { + String name = (kind.isEmpty() ? "scalar_fn" : kind.trim() + "_fn") + requested; + createFunction("create " + kind + "function python_pool_db." + name + + "(int) returns " + ret + " properties('type'='PYTHON_UDF', " + + "'symbol'='evaluate', 'runtime_version'='3.10.2', 'pool_size'='" + requested + "');", ctx); + Function fn = findFunction(db, name); + Assertions.assertEquals(requested, fn.getPoolSize()); + Assertions.assertTrue(FunctionToSqlConverter.toSql(fn, false) + .contains("\"POOL_SIZE\"=\"" + requested + "\"")); + } } for (String invalid : new String[]{"0", "-1", "abc", "1.5", "2147483648"}) { assertCreateFunctionAnalysisException(ctx, @@ -229,11 +232,11 @@ public void testAutoSuspend() throws Exception { + "'symbol'='evaluate', 'runtime_version'='3.10.2'"; createFunction("create " + kind + "function python_auto_suspend_db." + prefix + "_default" + definition + ");", ctx); - assertAutoSuspendMetadata(findFunction(db, prefix + "_default"), -1); + assertAutoSuspendMetadata(findFunction(db, prefix + "_default"), 14400); if (kind.equals("tables ")) { - assertAutoSuspendMetadata(findFunction(db, prefix + "_default_outer"), -1); + assertAutoSuspendMetadata(findFunction(db, prefix + "_default_outer"), 14400); } - long[] values = {Long.MIN_VALUE, -2, -1, 0, 30, Long.MAX_VALUE}; + long[] values = {-1, 0, 30, 14400, Long.MAX_VALUE}; for (int i = 0; i < values.length; i++) { String name = prefix + "_explicit_" + i; createFunction("create " + kind + "function python_auto_suspend_db." + name @@ -278,6 +281,13 @@ public void testAutoSuspendRejectsInvalidValues() throws Exception { + "'runtime_version'='3.10.2', 'auto_suspend'='" + invalid + "');", "auto_suspend must be an integer number of seconds"); } + for (String invalid : new String[]{"-2", Long.toString(Long.MIN_VALUE)}) { + assertCreateFunctionAnalysisException(ctx, + "create function invalid_auto_suspend_db.invalid_fn(int) returns int " + + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', " + + "'runtime_version'='3.10.2', 'auto_suspend'='" + invalid + "');", + "auto_suspend must be -1 or a non-negative integer"); + } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java index f80730c43ba4a8..3219e47e078ec5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java @@ -57,12 +57,13 @@ void testAutoSuspendSurvivesCopySerializationAndThrift() { Type.INT, null, "init", "update", "merge", null, null, null, null); aggregate.setBinaryType(BinaryType.PYTHON_UDF); for (Function fn : new Function[]{scalar, aggregate}) { - Assertions.assertEquals(-1, fn.getAutoSuspend()); + Assertions.assertEquals(14400, fn.getAutoSuspend()); + Assertions.assertEquals(14400, new TFunction().getAutoSuspend()); JsonObject oldJson = GsonUtils.GSON.toJsonTree(fn).getAsJsonObject(); oldJson.remove("as"); Function oldRestored = GsonUtils.GSON.fromJson(oldJson, fn.getClass()); - Assertions.assertEquals(-1, oldRestored.getAutoSuspend()); - for (long value : new long[]{Long.MIN_VALUE, -2, 0, 30, Long.MAX_VALUE}) { + Assertions.assertEquals(14400, oldRestored.getAutoSuspend()); + for (long value : new long[]{-1, 0, 30, 14400, Long.MAX_VALUE}) { fn.setAutoSuspend(value); Function copy = fn.clone(); Assertions.assertEquals(value, copy.getAutoSuspend()); diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index bd7f2c55719760..066a8c6912b751 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -421,7 +421,7 @@ struct TFunction { 18: optional string runtime_version 19: optional string function_code 20: optional i32 pool_size = 3 - 21: optional i64 auto_suspend = -1 // idle timeout in seconds; negative values disable suspension + 21: optional i64 auto_suspend = 14400 // idle timeout in seconds; -1 disables suspension } enum TJdbcOperation { diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy index a67597f8d1da55..5992a309d4c018 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy @@ -24,7 +24,7 @@ suite("test_pythonudaf_auto_suspend", "nonConcurrent") { sql "INSERT INTO py_auto_suspend_input VALUES (1), (2), (3)" try { // Check results before/after idling; BE unit tests separately verify process termination. - ["0", "1", "-2", null].eachWithIndex { timeout, index -> + ["0", "1", "-1", null].eachWithIndex { timeout, index -> sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" def property = timeout == null ? "" : ", \"auto_suspend\"=\"${timeout}\"" sql """CREATE AGGREGATE FUNCTION py_auto_suspend(INT) RETURNS BIGINT @@ -67,7 +67,36 @@ class SumAgg: qt_result_2("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") Thread.sleep(2500) qt_result_3("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" + sql """CREATE AGGREGATE FUNCTION py_auto_suspend_pid(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "symbol"="PidAgg", + "runtime_version"="${runtime}", "pool_size"="1", "auto_suspend"="0") + AS \$\$ +import os +class PidAgg: + def __init__(self): + self.total = 0 + @property + def aggregate_state(self): + return self.total + def accumulate(self, v): + self.total += 1 + def merge(self, other): + self.total += other + def finish(self): + return os.getpid() +\$\$""" + def queryPid = { sql("SELECT py_auto_suspend_pid(v) FROM py_auto_suspend_input")[0][0] } + def firstPid = queryPid() + def nextPid = firstPid + def deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(120) + while (nextPid == firstPid && System.nanoTime() < deadline) { + Thread.sleep(2000) + nextPid = queryPid() + } + assertNotEquals(firstPid, nextPid) } finally { sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" } } diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy index cdf5ff19d9378d..f69e1d28e5dc0c 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy @@ -24,7 +24,7 @@ suite("test_pythonudf_auto_suspend", "nonConcurrent") { sql "INSERT INTO py_auto_suspend_input VALUES (1), (2), (3)" try { // Check results before/after idling; BE unit tests separately verify process termination. - ["0", "1", "-2", null].eachWithIndex { timeout, index -> + ["0", "1", "-1", null].eachWithIndex { timeout, index -> sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" def property = timeout == null ? "" : ", \"auto_suspend\"=\"${timeout}\"" sql """CREATE FUNCTION py_auto_suspend(INT) RETURNS INT @@ -45,7 +45,26 @@ def evaluate(v): exception "Can not found function" } } + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" + sql """CREATE FUNCTION py_auto_suspend_pid(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + "runtime_version"="${runtime}", "pool_size"="1", "auto_suspend"="0") + AS \$\$ +import os +def evaluate(v): + return os.getpid() +\$\$""" + def queryPid = { sql("SELECT py_auto_suspend_pid(v) FROM py_auto_suspend_input")[0][0] } + def firstPid = queryPid() + def nextPid = firstPid + def deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(120) + while (nextPid == firstPid && System.nanoTime() < deadline) { + Thread.sleep(2000) + nextPid = queryPid() + } + assertNotEquals(firstPid, nextPid) } finally { sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" } } diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy index e48314ebfab820..3544b0b87216e8 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy @@ -24,7 +24,7 @@ suite("test_pythonudtf_auto_suspend", "nonConcurrent") { sql "INSERT INTO py_auto_suspend_input VALUES (1), (2), (3)" try { // Check results before/after idling; BE unit tests separately verify process termination. - ["0", "1", "-2", null].eachWithIndex { timeout, index -> + ["0", "1", "-1", null].eachWithIndex { timeout, index -> sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" def property = timeout == null ? "" : ", \"auto_suspend\"=\"${timeout}\"" sql """CREATE TABLES FUNCTION py_auto_suspend(INT) RETURNS ARRAY @@ -48,7 +48,28 @@ def evaluate(v): exception "Can not found function" } } + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" + sql """CREATE TABLES FUNCTION py_auto_suspend_pid(INT) RETURNS ARRAY + PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + "runtime_version"="${runtime}", "pool_size"="1", "auto_suspend"="0") + AS \$\$ +import os +def evaluate(v): + yield (os.getpid(),) +\$\$""" + def queryPid = { + sql("SELECT x FROM py_auto_suspend_input LATERAL VIEW py_auto_suspend_pid(v) t AS x")[0][0] + } + def firstPid = queryPid() + def nextPid = firstPid + def deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(120) + while (nextPid == firstPid && System.nanoTime() < deadline) { + Thread.sleep(2000) + nextPid = queryPid() + } + assertNotEquals(firstPid, nextPid) } finally { sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" } }