diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index ab51e93e650da4..ef16da1b44c8eb 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -59,6 +59,7 @@ #include "common/metrics/doris_metrics.h" #include "common/status.h" #include "cpp/obj-client/obj_storage_client.h" +#include "cpp/sync_point.h" #include "exec/spill/spill_file_manager.h" #include "io/fs/file_system.h" #include "io/fs/hdfs_file_system.h" @@ -154,6 +155,7 @@ void remove_task_info(const TTaskType::type task_type, int64_t signature) { } void finish_task(const TFinishTaskRequest& finish_task_request) { + TEST_SYNC_POINT_RETURN_WITH_VOID("TaskWorkerPool::finish_task", &finish_task_request); // Return result to FE TMasterResult result; uint32_t try_time = 0; @@ -2579,34 +2581,50 @@ void clean_trash_callback(StorageEngine& engine, const TAgentTaskRequest& req) { } void clean_udf_cache_callback(const TAgentTaskRequest& req) { + Status status; const auto& clean_req = req.clean_udf_cache_req; if (clean_req.__isset.function_id && clean_req.function_id <= 0) { LOG(WARNING) << "skip clean udf cache request with invalid function_id=" << clean_req.function_id << ", function_signature=" << clean_req.function_signature; - return; - } - // Requests from old FEs do not set function_id and must keep signature-based cleanup. - const bool drop_by_function_id = clean_req.__isset.function_id; - - if (doris::config::enable_java_support) { - // The id, not just the signature: it is what the java-udf plugin keys its compiled - // classes by, because the signature carries no database and FE renders a variadic one - // differently here than on the requests that execute the function. - WARN_IF_ERROR( - Jni::PluginRegistry::clean_udf_cache( - drop_by_function_id ? clean_req.function_id : 0, - clean_req.function_signature), - fmt::format("failed to clean Java UDF cache, function_signature={}, function_id={}", + status = Status::InvalidArgument("Invalid function_id for UDF cache cleanup: {}", + clean_req.function_id); + } else { + // Requests from old FEs do not set function_id and must keep signature-based cleanup. + const bool drop_by_function_id = clean_req.__isset.function_id; + + if (doris::config::enable_java_support) { + // The id, not just the signature: it is what the java-udf plugin keys its compiled + // classes by, because the signature carries no database and FE renders a variadic one + // differently here than on the requests that execute the function. + WARN_IF_ERROR( + Jni::PluginRegistry::clean_udf_cache( + drop_by_function_id ? clean_req.function_id : 0, + clean_req.function_signature), + fmt::format( + "failed to clean Java UDF cache, function_signature={}, function_id={}", clean_req.function_signature, clean_req.function_id)); - } - if (drop_by_function_id) { - UserFunctionCache::instance()->drop_function_cache(clean_req.function_id); - PythonServerManager::instance().clear_udaf_state_cache(clean_req.function_id); + } + if (drop_by_function_id) { + UserFunctionCache::instance()->drop_function_cache(clean_req.function_id); + PythonServerManager::instance().drop_function(clean_req.function_id); + } + + LOG(INFO) << "clean udf cache callback finish: function_signature=" + << clean_req.function_signature << ", function_id=" << clean_req.function_id; } - LOG(INFO) << "clean udf cache callback finish: function_signature=" - << clean_req.function_signature << ", function_id=" << clean_req.function_id; + // Report the result only for queued tasks; one-shot tasks use signature -1. + if (req.signature != -1) { + TFinishTaskRequest finish_task_request; + finish_task_request.__set_backend(BackendOptions::get_local_backend()); + finish_task_request.__set_task_type(req.task_type); + finish_task_request.__set_signature(req.signature); + finish_task_request.__set_task_status(status.to_thrift()); + finish_task(finish_task_request); + // Stop reporting this task as running so a failed result submission can trigger a retry. + remove_task_info(req.task_type, req.signature); + } } void report_index_policy_callback(const ClusterInfo* cluster_info) { diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 4c9957dfdfa173..685f3c0a313228 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1198,9 +1198,6 @@ DEFINE_String(python_conda_root_path, ""); DEFINE_String(python_venv_root_path, "${DORIS_HOME}/lib/udf/python"); // python interpreter paths used by venv, e.g. /usr/bin/python3.7:/usr/bin/python3.6 DEFINE_String(python_venv_interpreter_paths, ""); -// max python processes in global shared pool, each version can have up to this many processes -// 0 means use CPU core count as default, otherwise use the specified value -DEFINE_mInt32(max_python_process_num, "0"); // Memory limit in bytes for all Python UDF processes; warning is logged when exceeded // default is 10GB DEFINE_mInt64(python_udf_processes_memory_limit_bytes, "10737418240"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 62becc90f28a1e..805eb41732affc 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1271,8 +1271,6 @@ DECLARE_String(python_conda_root_path); DECLARE_String(python_venv_root_path); // python interpreter paths used by venv, e.g. /usr/bin/python3.7:/usr/bin/python3.6 DECLARE_String(python_venv_interpreter_paths); -// max python processes in global shared pool, each version can have up to this many processes -DECLARE_mInt32(max_python_process_num); // Memory limit in bytes for all Python UDF processes; warning is logged when exceeded DECLARE_mInt64(python_udf_processes_memory_limit_bytes); diff --git a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp index ab6000783ad191..0612e011bd7465 100644 --- a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp +++ b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp @@ -191,6 +191,10 @@ Status AggregatePythonUDAFData::get(IColumn& to, const DataTypePtr& result_type, Status AggregatePythonUDAF::open() { // Build function metadata from TFunction _func_meta.id = _fn.id; + _func_meta.pool_size = _fn.__isset.pool_size ? _fn.pool_size : 3; + _func_meta.auto_suspend = _fn.__isset.auto_suspend + ? _fn.auto_suspend + : PythonUDFMeta::DEFAULT_AUTO_SUSPEND_SECONDS; _func_meta.name = _fn.name.function_name; // For UDAF, symbol is in aggregate_fn @@ -236,7 +240,8 @@ Status AggregatePythonUDAF::open() { if (_func_meta.type == PythonUDFLoadType::MODULE) { RETURN_IF_ERROR(UserFunctionCache::instance()->get_pypath( - _func_meta.id, _func_meta.location, _func_meta.checksum, &_func_meta.location)); + _func_meta.id, _func_meta.location, _func_meta.checksum, &_func_meta.location, + &_package_cache_entry)); } return Status::OK(); @@ -272,7 +277,7 @@ void AggregatePythonUDAF::create(AggregateDataPtr __restrict place) const { DCHECK(reinterpret_cast(place)) << "Place must not be null"; if (Status st = PythonServerManager::instance().get_client( - _func_meta, _python_version, &(this->data(place).client), _schema); + _func_meta, _python_version, &data(place).client, _schema, _package_cache_entry); UNLIKELY(!st.ok())) { this->data(place).~Data(); throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Failed to get Python UDAF client: {}", diff --git a/be/src/exprs/aggregate/aggregate_function_python_udaf.h b/be/src/exprs/aggregate/aggregate_function_python_udaf.h index 43a86aaa5900f7..d6744cb9432b7f 100644 --- a/be/src/exprs/aggregate/aggregate_function_python_udaf.h +++ b/be/src/exprs/aggregate/aggregate_function_python_udaf.h @@ -28,6 +28,8 @@ namespace doris { +struct UserFunctionCacheEntry; + /** * Aggregate state data for Python UDAF * @@ -172,6 +174,7 @@ class AggregatePythonUDAF final DataTypePtr _return_type; PythonUDFMeta _func_meta; PythonVersion _python_version; + std::shared_ptr _package_cache_entry; // Arrow Flight schema: [argument_types..., places: int64, binary_data: binary] // Used for all UDAF RPC operations // - places column is always present (NULL in single-place mode, actual place_id values in GROUP BY mode) diff --git a/be/src/exprs/function/function_python_udf.cpp b/be/src/exprs/function/function_python_udf.cpp index 9f7e40119eda5d..a524b44310a607 100644 --- a/be/src/exprs/function/function_python_udf.cpp +++ b/be/src/exprs/function/function_python_udf.cpp @@ -53,6 +53,9 @@ Status PythonFunctionCall::open(FunctionContext* context, PythonVersion version; PythonUDFMeta func_meta; func_meta.id = _fn.id; + func_meta.pool_size = _fn.__isset.pool_size ? _fn.pool_size : 3; + func_meta.auto_suspend = _fn.__isset.auto_suspend ? _fn.auto_suspend + : PythonUDFMeta::DEFAULT_AUTO_SUSPEND_SECONDS; func_meta.name = _fn.name.function_name; func_meta.symbol = _fn.scalar_fn.symbol; if (!_fn.function_code.empty()) { @@ -85,13 +88,16 @@ Status PythonFunctionCall::open(FunctionContext* context, LOG(INFO) << fmt::format("runtime_version: {}, func_meta: {}", version.to_string(), func_meta.to_string()); + std::shared_ptr package_cache_entry; if (func_meta.type == PythonUDFLoadType::MODULE) { RETURN_IF_ERROR(UserFunctionCache::instance()->get_pypath( - func_meta.id, func_meta.location, func_meta.checksum, &func_meta.location)); + func_meta.id, func_meta.location, func_meta.checksum, &func_meta.location, + &package_cache_entry)); } PythonUDFClientPtr client = nullptr; - RETURN_IF_ERROR(PythonServerManager::instance().get_client(func_meta, version, &client)); + RETURN_IF_ERROR(PythonServerManager::instance().get_client(func_meta, version, &client, nullptr, + std::move(package_cache_entry))); if (!client) { return Status::InternalError("Python UDF client is null"); diff --git a/be/src/exprs/table_function/python_udtf_function.cpp b/be/src/exprs/table_function/python_udtf_function.cpp index a45a0f95f15368..4c2b466069510b 100644 --- a/be/src/exprs/table_function/python_udtf_function.cpp +++ b/be/src/exprs/table_function/python_udtf_function.cpp @@ -61,6 +61,10 @@ PythonUDTFFunction::PythonUDTFFunction(const TFunction& t_fn) : TableFunction(), Status PythonUDTFFunction::open() { PythonUDFMeta python_udf_meta; python_udf_meta.id = _t_fn.id; + python_udf_meta.pool_size = _t_fn.__isset.pool_size ? _t_fn.pool_size : 3; + python_udf_meta.auto_suspend = _t_fn.__isset.auto_suspend + ? _t_fn.auto_suspend + : PythonUDFMeta::DEFAULT_AUTO_SUSPEND_SECONDS; python_udf_meta.name = _t_fn.name.function_name; python_udf_meta.symbol = _t_fn.scalar_fn.symbol; @@ -79,10 +83,11 @@ Status PythonUDTFFunction::open() { python_udf_meta.client_type = PythonClientType::UDTF; + std::shared_ptr package_cache_entry; if (python_udf_meta.type == PythonUDFLoadType::MODULE) { RETURN_IF_ERROR(UserFunctionCache::instance()->get_pypath( python_udf_meta.id, python_udf_meta.location, python_udf_meta.checksum, - &python_udf_meta.location)); + &python_udf_meta.location, &package_cache_entry)); } PythonVersion version; @@ -105,8 +110,8 @@ Status PythonUDTFFunction::open() { python_udf_meta.always_nullable = python_udf_meta.return_type->is_nullable(); RETURN_IF_ERROR(python_udf_meta.check()); - RETURN_IF_ERROR( - PythonServerManager::instance().get_client(python_udf_meta, version, &_udtf_client)); + RETURN_IF_ERROR(PythonServerManager::instance().get_client( + python_udf_meta, version, &_udtf_client, nullptr, std::move(package_cache_entry))); if (!_udtf_client) { return Status::InternalError("Failed to create Python UDTF client"); diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp index 7199cc7e1ff407..28a19d3a0ff026 100644 --- a/be/src/runtime/user_function_cache.cpp +++ b/be/src/runtime/user_function_cache.cpp @@ -498,10 +498,14 @@ Status UserFunctionCache::get_jarpath(int64_t fid, const std::string& url, } Status UserFunctionCache::get_pypath(int64_t fid, const std::string& url, - const std::string& checksum, std::string* libpath) { + const std::string& checksum, std::string* libpath, + std::shared_ptr* cache_entry) { std::shared_ptr entry = nullptr; RETURN_IF_ERROR(_get_cache_entry(fid, url, checksum, entry, LibType::PY_ZIP)); *libpath = entry->lib_file; + if (cache_entry != nullptr) { + *cache_entry = std::move(entry); + } return Status::OK(); } @@ -579,17 +583,11 @@ void UserFunctionCache::drop_function_cache(int64_t fid) { _entry_map.erase(it); } - // For Python UDF, clear module cache in Python server before deleting files - if (entry->type == LibType::PY_ZIP && !entry->lib_file.empty()) { - auto status = PythonServerManager::instance().clear_module_cache(entry->lib_file); - if (!status.ok()) [[unlikely]] { - LOG(WARNING) << "drop_function_cache: failed to clear Python module cache for " - << entry->lib_file << ": " << status.to_string(); - } - } - // Mark for deletion, destructor will delete the files entry->should_delete_library.store(true); + if (entry->type == LibType::PY_ZIP) { + PythonServerManager::instance().drop_function(fid); + } } } // namespace doris diff --git a/be/src/runtime/user_function_cache.h b/be/src/runtime/user_function_cache.h index e606899f969d4d..f71252a346380f 100644 --- a/be/src/runtime/user_function_cache.h +++ b/be/src/runtime/user_function_cache.h @@ -59,8 +59,11 @@ class UserFunctionCache { Status get_jarpath(int64_t fid, const std::string& url, const std::string& checksum, std::string* libpath); + // Optionally return a shared cache entry to pin package files: DROP only removes them + // after the last query/process releases this handle. The handle is for lifetime only. Status get_pypath(int64_t fid, const std::string& url, const std::string& checksum, - std::string* libpath); + std::string* libpath, + std::shared_ptr* cache_entry = nullptr); // Drop the cached function library by function id. void drop_function_cache(int64_t fid); diff --git a/be/src/udf/python/python_client.cpp b/be/src/udf/python/python_client.cpp index 716ae6f15f69ce..555b21a3118150 100644 --- a/be/src/udf/python/python_client.cpp +++ b/be/src/udf/python/python_client.cpp @@ -72,22 +72,20 @@ Status PythonClient::init(const PythonUDFMeta& func_meta, ProcessPtr process) { } Status PythonClient::close() { - if (!_inited || !_writer) { - return Status::OK(); - } - - auto writer_res = _writer->Close(); - if (!writer_res.ok()) { - // Don't propagate error from close, just log it - LOG(WARNING) << "Error closing Python client writer: " << writer_res.message(); + if (_inited && _writer) { + auto writer_res = _writer->Close(); + if (!writer_res.ok()) { + LOG(WARNING) << "Error closing Python client writer: " << writer_res.message(); + } } - + // Error handling may already have reset the writer; still release the pool lease. _inited = false; _begin = false; _arrow_client.reset(); _writer.reset(); _reader.reset(); _process.reset(); + _pool_lease.reset(); return Status::OK(); } diff --git a/be/src/udf/python/python_client.h b/be/src/udf/python/python_client.h index 62d576774e3c84..7a6363a77c99b5 100644 --- a/be/src/udf/python/python_client.h +++ b/be/src/udf/python/python_client.h @@ -28,6 +28,8 @@ namespace doris { +class PythonUDFPoolLease; + /** * Base class for Python UDF/UDAF/UDTF clients * @@ -102,6 +104,9 @@ class PythonClient { */ Status read_batch(std::shared_ptr* output); + // Released after the stream and process, so idle reclamation cannot interrupt cleanup. + std::shared_ptr _pool_lease; + // Common state bool _inited = false; bool _begin = false; // Track if Begin() has been called @@ -112,6 +117,8 @@ class PythonClient { ProcessPtr _process; private: + friend class PythonServerManager; + DISALLOW_COPY_AND_ASSIGN(PythonClient); }; diff --git a/be/src/udf/python/python_server.cpp b/be/src/udf/python/python_server.cpp index 2d06caa311067e..ed6f11654f9238 100644 --- a/be/src/udf/python/python_server.cpp +++ b/be/src/udf/python/python_server.cpp @@ -38,104 +38,173 @@ #include "common/status.h" #include "cpp/sync_point.h" #include "runtime/thread_context.h" +#include "runtime/user_function_cache.h" #include "udf/python/python_udaf_client.h" #include "udf/python/python_udf_client.h" #include "udf/python/python_udtf_client.h" #include "util/cpu_info.h" +#include "util/defer_op.h" namespace doris { -Result> -PythonServerManager::_get_or_create_process_pool(const PythonVersion& version) { - std::lock_guard lock(_pools_mutex); +// A client holds this from pool acquisition through stream/state cleanup. The pool may outlive +// its manager after DROP; releasing a lease must therefore only access the pool itself. +class PythonUDFPoolLease { +public: + explicit PythonUDFPoolLease(std::shared_ptr pool) + : _pool(std::move(pool)) {} + + ~PythonUDFPoolLease() { + std::scoped_lock lock(_pool->mutex); + DCHECK_GT(_pool->active_clients, 0); + if (--_pool->active_clients == 0) { + _pool->idle_since = std::chrono::steady_clock::now(); + } + } + +private: + std::shared_ptr _pool; +}; + +void PythonServerManager::ProcessRegistry::add(const ProcessPtr& process) { + std::scoped_lock lock(mutex); + processes.emplace_back(process); +} + +std::vector PythonServerManager::ProcessRegistry::snapshot() { + std::vector snapshot; + std::scoped_lock lock(mutex); + std::erase_if(processes, [&snapshot](const auto& weak_process) { + if (auto process = weak_process.lock()) { + snapshot.emplace_back(std::move(process)); + return false; + } + return true; + }); + return snapshot; +} + +Result> +PythonServerManager::_get_or_create_process_pool( + const PythonUDFMeta& meta, const PythonVersion& version, + std::shared_ptr package_cache_entry, + std::shared_ptr* lease) { + std::scoped_lock lock(_pools_mutex); // shutdown() owns the manager lifecycle. Once it starts, creating a new pool would let detached // init workers publish Python processes that the manager no longer tracks. if (_shutdown_flag.load(std::memory_order_acquire)) { return ResultError(Status::Error( "Python server manager is shutting down")); } - auto& pool = _process_pools[version]; + if (meta.pool_size <= 0) { + return ResultError( + Status::InvalidArgument("Python UDF pool_size must be a positive integer")); + } + if (meta.auto_suspend < -1) { + return ResultError(Status::InvalidArgument( + "Python UDF auto_suspend must be -1 or a non-negative integer")); + } + auto& pool = _process_pools[meta.id]; if (!pool) { - pool = std::make_shared(); + pool = std::make_shared(); + pool->version = version; + pool->process_registry = _process_registry; + pool->function_id = meta.id; + // The same function may run on BEs with different CPU limits. + pool->pool_size = std::min(meta.pool_size, CpuInfo::num_cores()); + pool->auto_suspend = meta.auto_suspend; + pool->package_cache_entry = std::move(package_cache_entry); + // The first process can serve a client before the remaining slots finish initialization. + _start_health_check_thread(); + } + if (lease) { + auto acquired = std::make_shared(pool); + std::scoped_lock pool_lock(pool->mutex); + ++pool->active_clients; + *lease = std::move(acquired); } return pool; } -std::vector>> +std::vector> PythonServerManager::_snapshot_process_pools() { - std::lock_guard lock(_pools_mutex); - std::vector>> snapshot; + std::scoped_lock lock(_pools_mutex); + std::vector> snapshot; snapshot.reserve(_process_pools.size()); - for (const auto& [version, pool] : _process_pools) { - snapshot.emplace_back(version, pool); + for (const auto& [function_id, pool] : _process_pools) { + snapshot.emplace_back(pool); } return snapshot; } #ifdef BE_TEST -void PythonServerManager::set_process_pool_for_test(const PythonVersion& version, +void PythonServerManager::set_process_pool_for_test(const PythonUDFMeta& meta, + const PythonVersion& version, std::vector processes, bool initialized) { - auto versioned_pool = _get_or_create_process_pool(version).value(); - std::lock_guard lock(versioned_pool->mutex); - versioned_pool->processes = std::move(processes); - versioned_pool->state = initialized ? PoolState::INITIALIZED : PoolState::UNINITIALIZED; - versioned_pool->has_available_process = - std::any_of(versioned_pool->processes.begin(), versioned_pool->processes.end(), + auto function_pool = _get_or_create_process_pool(meta, version, nullptr).value(); + std::scoped_lock lock(function_pool->mutex); + for (const auto& process : processes) { + if (process) { + function_pool->process_registry->add(process); + } + } + function_pool->processes = std::move(processes); + function_pool->state = initialized ? PoolState::INITIALIZED : PoolState::UNINITIALIZED; + function_pool->has_available_process = + std::any_of(function_pool->processes.begin(), function_pool->processes.end(), [](const ProcessPtr& process) { return process && process->is_alive(); }); } -std::vector PythonServerManager::process_pool_snapshot_for_test( - const PythonVersion& version) { - auto versioned_pool = _get_or_create_process_pool(version).value(); - std::lock_guard lock(versioned_pool->mutex); - return versioned_pool->processes; +std::vector PythonServerManager::process_pool_snapshot_for_test(int64_t function_id) { + std::scoped_lock map_lock(_pools_mutex); + auto iter = _process_pools.find(function_id); + if (iter == _process_pools.end()) { + return {}; + } + std::scoped_lock lock(iter->second->mutex); + return iter->second->processes; } -bool PythonServerManager::process_pool_is_initialized_for_test(const PythonVersion& version) { - auto versioned_pool = _get_or_create_process_pool(version).value(); - std::lock_guard lock(versioned_pool->mutex); - return versioned_pool->state == PoolState::INITIALIZED; +bool PythonServerManager::process_pool_is_initialized_for_test(int64_t function_id) { + return wait_for_process_pool_initialized_for_test(function_id, std::chrono::milliseconds(0)); } bool PythonServerManager::wait_for_process_pool_initialized_for_test( - const PythonVersion& version, std::chrono::milliseconds timeout) { - auto versioned_pool_result = _get_or_create_process_pool(version); - if (!versioned_pool_result.has_value()) { - return false; + int64_t function_id, std::chrono::milliseconds timeout) { + std::shared_ptr pool; + { + std::scoped_lock map_lock(_pools_mutex); + pool = _process_pools.at(function_id); } - auto versioned_pool = versioned_pool_result.value(); - std::unique_lock lock(versioned_pool->mutex); - return versioned_pool->cv.wait_for(lock, timeout, [&versioned_pool]() { - return versioned_pool->state == PoolState::INITIALIZED; - }); + std::unique_lock lock(pool->mutex); + return pool->cv.wait_for(lock, timeout, + [&]() { return pool->state == PoolState::INITIALIZED; }); } bool PythonServerManager::wait_for_process_pool_initialization_finished_for_test( - const PythonVersion& version, std::chrono::milliseconds timeout) { - auto versioned_pool_result = _get_or_create_process_pool(version); - if (!versioned_pool_result.has_value()) { - return false; + int64_t function_id, std::chrono::milliseconds timeout) { + std::shared_ptr pool; + { + std::scoped_lock map_lock(_pools_mutex); + pool = _process_pools.at(function_id); } - auto versioned_pool = versioned_pool_result.value(); - std::unique_lock lock(versioned_pool->mutex); - return versioned_pool->cv.wait_for(lock, timeout, [&versioned_pool]() { - return versioned_pool->state != PoolState::INITIALIZING; - }); + std::unique_lock lock(pool->mutex); + return pool->cv.wait_for(lock, timeout, + [&]() { return pool->state != PoolState::INITIALIZING; }); } #endif bool PythonServerManager::_select_alive_process_from_pool(const std::vector& pool, ProcessPtr* process) { - auto alive_iter = std::min_element(pool.begin(), pool.end(), - [](const ProcessPtr& a, const ProcessPtr& b) { - const bool a_alive = a && a->is_alive(); - const bool b_alive = b && b->is_alive(); - if (a_alive != b_alive) { - return a_alive > b_alive; - } - return a.use_count() < b.use_count(); - }); + auto alive_iter = std::ranges::min_element(pool, [](const ProcessPtr& a, const ProcessPtr& b) { + const bool a_alive = a && a->is_alive(); + const bool b_alive = b && b->is_alive(); + if (a_alive != b_alive) { + return a_alive > b_alive; + } + return a.use_count() < b.use_count(); + }); if (alive_iter == pool.end() || !*alive_iter || !(*alive_iter)->is_alive()) { return false; } @@ -144,14 +213,16 @@ bool PythonServerManager::_select_alive_process_from_pool(const std::vector -Status PythonServerManager::get_client(const PythonUDFMeta& func_meta, const PythonVersion& version, - std::shared_ptr* client, - const std::shared_ptr& data_schema) { - std::shared_ptr versioned_pool = - DORIS_TRY(_ensure_pool_initialized(version)); +Status PythonServerManager::get_client( + const PythonUDFMeta& func_meta, const PythonVersion& version, + std::shared_ptr* client, const std::shared_ptr& data_schema, + std::shared_ptr package_cache_entry) { + std::shared_ptr lease; + std::shared_ptr function_pool = DORIS_TRY( + _ensure_pool_initialized(func_meta, version, std::move(package_cache_entry), &lease)); ProcessPtr process; - RETURN_IF_ERROR(_get_process(version, versioned_pool, &process)); + RETURN_IF_ERROR(_get_process(version, function_pool, &process)); if constexpr (std::is_same_v) { RETURN_IF_ERROR(ClientType::create(func_meta, std::move(process), data_schema, client)); @@ -159,117 +230,122 @@ Status PythonServerManager::get_client(const PythonUDFMeta& func_meta, const Pyt RETURN_IF_ERROR(ClientType::create(func_meta, std::move(process), client)); } + (*client)->_pool_lease = std::move(lease); return Status::OK(); } -Result> -PythonServerManager::_ensure_pool_initialized(const PythonVersion& version) { - auto versioned_pool_result = _get_or_create_process_pool(version); - if (!versioned_pool_result.has_value()) { - return ResultError(versioned_pool_result.error()); +void PythonServerManager::_start_pool_initialization( + const std::shared_ptr& function_pool, const PythonVersion& version, + std::chrono::milliseconds process_pool_init_timeout) { + // Called with the pool lock held; workers publish only after the caller starts waiting. + const int max_pool_size = function_pool->pool_size; + function_pool->state = PoolState::INITIALIZING; + function_pool->has_available_process = false; + function_pool->processes.resize(max_pool_size); + auto init_finished_count = std::make_shared>(0); + TEST_SYNC_POINT_CALLBACK("PythonServerManager::_ensure_pool_initialized:generation_started", + &version, init_finished_count.get()); + + LOG(INFO) << "Initializing Python process pool for function " << function_pool->function_id + << " with " << max_pool_size << " processes, runtime=" << version.to_string(); + + std::thread([function_pool, init_finished_count, max_pool_size, process_pool_init_timeout]() { + SCOPED_INIT_THREAD_CONTEXT(); + std::unique_lock lock(function_pool->mutex); + function_pool->cv.wait_for( + lock, process_pool_init_timeout, + [&function_pool, init_finished_count, max_pool_size]() { + return function_pool->state != PoolState::INITIALIZING || + init_finished_count->load(std::memory_order_acquire) >= max_pool_size; + }); + if (function_pool->state == PoolState::INITIALIZING) { + if (function_pool->has_available_process) { + function_pool->state = PoolState::INITIALIZED; + } else { + function_pool->state = PoolState::UNINITIALIZED; + } + } + function_pool->cv.notify_all(); + }).detach(); + + for (int i = 0; i < max_pool_size; ++i) { + std::thread([version, function_pool, i, max_pool_size, init_finished_count]() { + SCOPED_INIT_THREAD_CONTEXT(); + ProcessPtr process; + Status status = PythonServerManager::fork(version, &process); + const bool ok = status.ok() && process; + ProcessPtr process_to_shutdown; + { + std::scoped_lock lock(function_pool->mutex); + // shutdown() and repair can race with detached init workers after timeout. + // Late successful forks only fill slots that are still empty or dead. + if (ok && + (function_pool->state == PoolState::INITIALIZING || + function_pool->state == PoolState::INITIALIZED) && + i < function_pool->processes.size() && + (!function_pool->processes[i] || !function_pool->processes[i]->is_alive())) { + // Register only accepted processes; rejected forks shut down separately. + function_pool->process_registry->add(process); + process->set_package_cache_entry(function_pool->package_cache_entry); + function_pool->processes[i] = std::move(process); + function_pool->has_available_process = true; + } else if (ok) { + process_to_shutdown = std::move(process); + } else [[unlikely]] { + LOG(WARNING) << "Failed to create Python process " << (i + 1) << "/" + << max_pool_size << " for version " << version.to_string() << ": " + << status.to_string(); + } + } + init_finished_count->fetch_add(1, std::memory_order_acq_rel); + function_pool->cv.notify_all(); + if (process_to_shutdown) { + process_to_shutdown->shutdown(); + } + TEST_SYNC_POINT_CALLBACK( + "PythonServerManager::_ensure_pool_initialized:init_worker_finished", &version, + init_finished_count.get()); + }).detach(); } - auto versioned_pool = versioned_pool_result.value(); - const int max_pool_size = config::max_python_process_num > 0 ? config::max_python_process_num - : CpuInfo::num_cores(); +} + +Result> +PythonServerManager::_ensure_pool_initialized( + const PythonUDFMeta& meta, const PythonVersion& version, + std::shared_ptr package_cache_entry, + std::shared_ptr* lease) { + auto function_pool_result = + _get_or_create_process_pool(meta, version, std::move(package_cache_entry), lease); + if (!function_pool_result.has_value()) { + return ResultError(function_pool_result.error()); + } + auto function_pool = function_pool_result.value(); auto process_pool_init_timeout = PROCESS_POOL_INIT_TIMEOUT; TEST_SYNC_POINT_CALLBACK( "PythonServerManager::_ensure_pool_initialized:process_pool_init_timeout", &process_pool_init_timeout); - std::unique_lock lock(versioned_pool->mutex); - if (versioned_pool->state == PoolState::INITIALIZED) { - return versioned_pool; + std::unique_lock lock(function_pool->mutex); + if (function_pool->state == PoolState::INITIALIZED) { + return function_pool; } - if (versioned_pool->state != PoolState::STOPPED) { - if (versioned_pool->state != PoolState::INITIALIZING) { - versioned_pool->state = PoolState::INITIALIZING; - versioned_pool->has_available_process = false; - versioned_pool->processes.resize(max_pool_size); - auto init_finished_count = std::make_shared>(0); - TEST_SYNC_POINT_CALLBACK( - "PythonServerManager::_ensure_pool_initialized:generation_started", &version, - init_finished_count.get()); - - LOG(INFO) << "Initializing Python process pool for version " << version.to_string() - << " with " << max_pool_size << " processes (config::max_python_process_num=" - << config::max_python_process_num << ", CPU cores=" << CpuInfo::num_cores() - << ")"; - - std::thread([this, versioned_pool, init_finished_count, max_pool_size, - process_pool_init_timeout]() { - SCOPED_INIT_THREAD_CONTEXT(); - std::unique_lock lock(versioned_pool->mutex); - versioned_pool->cv.wait_for( - lock, process_pool_init_timeout, - [&versioned_pool, init_finished_count, max_pool_size]() { - return versioned_pool->state != PoolState::INITIALIZING || - init_finished_count->load(std::memory_order_acquire) >= - max_pool_size; - }); - if (versioned_pool->state == PoolState::INITIALIZING) { - if (versioned_pool->has_available_process) { - // Keep this under the pool lock. shutdown() must acquire the same lock - // before it can destroy manager-owned health-check state. - _start_health_check_thread(); - versioned_pool->state = PoolState::INITIALIZED; - } else { - versioned_pool->state = PoolState::UNINITIALIZED; - } - } - versioned_pool->cv.notify_all(); - }).detach(); - - for (int i = 0; i < max_pool_size; ++i) { - std::thread([version, versioned_pool, i, max_pool_size, init_finished_count]() { - SCOPED_INIT_THREAD_CONTEXT(); - ProcessPtr process; - Status status = PythonServerManager::fork(version, &process); - const bool ok = status.ok() && process; - ProcessPtr process_to_shutdown; - { - std::lock_guard lock(versioned_pool->mutex); - // shutdown() and repair can race with detached init workers after timeout. - // Late successful forks only fill slots that are still empty or dead. - if (ok && - (versioned_pool->state == PoolState::INITIALIZING || - versioned_pool->state == PoolState::INITIALIZED) && - i < versioned_pool->processes.size() && - (!versioned_pool->processes[i] || - !versioned_pool->processes[i]->is_alive())) { - versioned_pool->processes[i] = std::move(process); - versioned_pool->has_available_process = true; - } else if (ok) { - process_to_shutdown = std::move(process); - } else [[unlikely]] { - LOG(WARNING) << "Failed to create Python process " << (i + 1) << "/" - << max_pool_size << " for version " << version.to_string() - << ": " << status.to_string(); - } - } - init_finished_count->fetch_add(1, std::memory_order_acq_rel); - versioned_pool->cv.notify_all(); - if (process_to_shutdown) { - process_to_shutdown->shutdown(); - } - TEST_SYNC_POINT_CALLBACK( - "PythonServerManager::_ensure_pool_initialized:init_worker_finished", - &version, init_finished_count.get()); - }).detach(); - } + if (function_pool->state != PoolState::STOPPED) { + if (function_pool->state != PoolState::INITIALIZING) { + _start_pool_initialization(function_pool, version, process_pool_init_timeout); } // Wait only for the first usable process. INITIALIZED is set later by the last init worker // after every slot has attempted initialization. - versioned_pool->cv.wait_for(lock, process_pool_init_timeout, [&versioned_pool]() { - return versioned_pool->has_available_process || - versioned_pool->state == PoolState::STOPPED || - versioned_pool->state != PoolState::INITIALIZING; + function_pool->cv.wait_for(lock, process_pool_init_timeout, [&function_pool]() { + return function_pool->has_available_process || + function_pool->state == PoolState::STOPPED || + function_pool->state != PoolState::INITIALIZING; }); - if (versioned_pool->has_available_process) { - return versioned_pool; + if (function_pool->has_available_process) { + return function_pool; } - versioned_pool->cv.notify_all(); + function_pool->cv.notify_all(); } return ResultError(Status::Error( @@ -278,90 +354,95 @@ PythonServerManager::_ensure_pool_initialized(const PythonVersion& version) { version.to_string(), process_pool_init_timeout.count())); } -Status PythonServerManager::_get_process( - const PythonVersion& version, const std::shared_ptr& versioned_pool, - ProcessPtr* process) { +void PythonServerManager::_start_pool_repair( + const PythonVersion& version, const std::shared_ptr& function_pool) { + // Called with the pool lock held so concurrent requests cannot schedule the same repair. + function_pool->repairing = true; + // Repair is done in the background because fork can be slow. The current request still + // waits briefly below so a transient all-dead pool can recover without failing. + std::thread([version, function_pool]() { + SCOPED_INIT_THREAD_CONTEXT(); + int recreated = PythonServerManager::_repair_process_pool(version, function_pool); + { + std::scoped_lock lock(function_pool->mutex); + function_pool->repairing = false; + } + function_pool->cv.notify_all(); + if (recreated > 0) { + LOG(INFO) << "Repaired Python process pool for version " << version.to_string() + << ": recreated=" << recreated; + } + }).detach(); +} + +Status PythonServerManager::_get_process(const PythonVersion& version, + const std::shared_ptr& function_pool, + ProcessPtr* process) { { - std::unique_lock lock(versioned_pool->mutex); - std::vector& pool = versioned_pool->processes; + std::unique_lock lock(function_pool->mutex); + std::vector& pool = function_pool->processes; - if (versioned_pool->state == PoolState::STOPPED) { - versioned_pool->has_available_process = false; + if (function_pool->state == PoolState::STOPPED) { + function_pool->has_available_process = false; return Status::Error( "Python process pool has stopped for version {}", version.to_string()); } if (_select_alive_process_from_pool(pool, process)) [[likely]] { - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; return Status::OK(); } - versioned_pool->has_available_process = false; + function_pool->has_available_process = false; - if (versioned_pool->state == PoolState::INITIALIZING) { - versioned_pool->cv.wait_for(lock, PROCESS_REPAIR_WAIT_TIMEOUT, [&versioned_pool]() { - return std::any_of(versioned_pool->processes.begin(), - versioned_pool->processes.end(), + if (function_pool->state == PoolState::INITIALIZING) { + function_pool->cv.wait_for(lock, PROCESS_REPAIR_WAIT_TIMEOUT, [&function_pool]() { + return std::any_of(function_pool->processes.begin(), function_pool->processes.end(), [](const ProcessPtr& p) { return p && p->is_alive(); }) || - versioned_pool->state != PoolState::INITIALIZING; + function_pool->state != PoolState::INITIALIZING; }); - if (versioned_pool->state == PoolState::STOPPED) { + if (function_pool->state == PoolState::STOPPED) { return Status::Error( "Python process pool has stopped for version {}", version.to_string()); } if (_select_alive_process_from_pool(pool, process)) { - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; return Status::OK(); } - versioned_pool->has_available_process = false; + function_pool->has_available_process = false; return Status::Error( "Python process pool is initializing but has no available process for version " "{} after waiting {} ms", version.to_string(), PROCESS_REPAIR_WAIT_TIMEOUT.count()); } - if (versioned_pool->state != PoolState::INITIALIZED) { + if (function_pool->state != PoolState::INITIALIZED) { return Status::Error( "Python process pool is not initialized for version {}", version.to_string()); } - if (!versioned_pool->repairing) { - versioned_pool->repairing = true; - // Repair is done in the background because fork can be slow. The current request still - // waits briefly below so a transient all-dead pool can recover without failing. - std::thread([version, versioned_pool]() { - SCOPED_INIT_THREAD_CONTEXT(); - int recreated = PythonServerManager::_repair_process_pool(version, versioned_pool); - { - std::lock_guard lock(versioned_pool->mutex); - versioned_pool->repairing = false; - } - versioned_pool->cv.notify_all(); - if (recreated > 0) { - LOG(INFO) << "Repaired Python process pool for version " << version.to_string() - << ": recreated=" << recreated; - } - }).detach(); + if (!function_pool->repairing) { + _start_pool_repair(version, function_pool); } // Keep the request recoverable in the common case where the Python runtime can fork // normally and only the existing pool entries died. The wait is short so a wedged fork path // still returns SERVICE_UNAVAILABLE promptly. - versioned_pool->cv.wait_for(lock, PROCESS_REPAIR_WAIT_TIMEOUT, [&versioned_pool]() { - return std::any_of(versioned_pool->processes.begin(), versioned_pool->processes.end(), + function_pool->cv.wait_for(lock, PROCESS_REPAIR_WAIT_TIMEOUT, [&function_pool]() { + return std::any_of(function_pool->processes.begin(), function_pool->processes.end(), [](const ProcessPtr& p) { return p && p->is_alive(); }) || - versioned_pool->state == PoolState::STOPPED; + function_pool->state == PoolState::STOPPED; }); - if (versioned_pool->state == PoolState::STOPPED) { - versioned_pool->has_available_process = false; + if (function_pool->state == PoolState::STOPPED) { + function_pool->has_available_process = false; return Status::Error( "Python process pool has stopped for version {}", version.to_string()); } if (_select_alive_process_from_pool(pool, process)) { - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; return Status::OK(); } - versioned_pool->has_available_process = false; + function_pool->has_available_process = false; } return Status::Error( @@ -458,14 +539,54 @@ Status PythonServerManager::fork(const PythonVersion& version, ProcessPtr* proce return Status::OK(); } +void PythonServerManager::_reclaim_idle_pools(std::chrono::steady_clock::time_point now) { + std::vector> retired; + { + std::scoped_lock lock(_pools_mutex); + std::erase_if(_process_pools, [&](const auto& entry) { + const auto& pool = entry.second; + std::scoped_lock pool_lock(pool->mutex); + // 1. Never reclaim a pool while any client still holds it. + // 2. A STOPPED pool is reclaimed on the next scan after its last client closes, + // regardless of auto_suspend. + // 3. Other pools require auto_suspend >= 0 and no clients for that many seconds. + // The last client closing resets idle_since, so each idle period starts afresh. + if (pool->active_clients != 0 || + (pool->state != PoolState::STOPPED && + (pool->auto_suspend < 0 || + std::chrono::duration_cast(now - pool->idle_since).count() < + pool->auto_suspend))) { + return false; + } + // Prevent detached initialization/repair workers from publishing into a retired pool. + pool->state = PoolState::STOPPED; + pool->has_available_process = false; + retired.emplace_back(pool); + return true; + }); + } + for (const auto& pool : retired) { + pool->cv.notify_all(); + std::vector processes; + { + std::scoped_lock lock(pool->mutex); + processes.swap(pool->processes); + } + for (const auto& process : processes) { + if (process) { + process->shutdown(); + } + } + } +} + void PythonServerManager::_start_health_check_thread() { - std::lock_guard lock(_health_check_mutex); + std::scoped_lock lock(_health_check_mutex); if (_health_check_thread || _shutdown_flag.load(std::memory_order_acquire)) return; - LOG(INFO) << "Starting Python process health check thread (interval: 30 seconds)"; + LOG(INFO) << "Starting Python process monitor"; _health_check_thread = std::make_unique([this]() { - // Health check loop while (!_shutdown_flag.load(std::memory_order_acquire)) { // Wait for interval or shutdown signal { @@ -477,6 +598,7 @@ void PythonServerManager::_start_health_check_thread() { if (_shutdown_flag.load(std::memory_order_acquire)) break; + _reclaim_idle_pools(std::chrono::steady_clock::now()); _check_and_recreate_processes(); _refresh_memory_stats(); } @@ -487,22 +609,22 @@ void PythonServerManager::_start_health_check_thread() { void PythonServerManager::_check_and_recreate_processes() { int total_recreated = 0; - for (auto& [version, versioned_pool] : _snapshot_process_pools()) { + for (const auto& function_pool : _snapshot_process_pools()) { { - std::lock_guard lock(versioned_pool->mutex); - if (versioned_pool->state != PoolState::INITIALIZED || versioned_pool->repairing) { + std::scoped_lock lock(function_pool->mutex); + if (function_pool->state != PoolState::INITIALIZED || function_pool->repairing) { continue; } // Share the same repair guard with foreground requests. Otherwise health check and // _get_process() can fork the same empty/dead slots at the same time under failures. - versioned_pool->repairing = true; + function_pool->repairing = true; } - int recreated = _repair_process_pool(version, versioned_pool); + int recreated = _repair_process_pool(function_pool->version, function_pool); { - std::lock_guard lock(versioned_pool->mutex); - versioned_pool->repairing = false; + std::scoped_lock lock(function_pool->mutex); + function_pool->repairing = false; } - versioned_pool->cv.notify_all(); + function_pool->cv.notify_all(); total_recreated += recreated; } @@ -512,17 +634,16 @@ void PythonServerManager::_check_and_recreate_processes() { } int PythonServerManager::_repair_process_pool( - const PythonVersion& version, const std::shared_ptr& versioned_pool) { - const int max_pool_size = config::max_python_process_num > 0 ? config::max_python_process_num - : CpuInfo::num_cores(); + const PythonVersion& version, const std::shared_ptr& function_pool) { + const int max_pool_size = function_pool->pool_size; std::vector died_process_indices; { - std::lock_guard lock(versioned_pool->mutex); - if (versioned_pool->state != PoolState::INITIALIZED) { + std::scoped_lock lock(function_pool->mutex); + if (function_pool->state != PoolState::INITIALIZED) { return 0; } - auto& pool = versioned_pool->processes; + auto& pool = function_pool->processes; died_process_indices.reserve(std::max(pool.size(), max_pool_size)); // Need to fix the following two cases // 1. Existing processes have died @@ -546,8 +667,8 @@ int PythonServerManager::_repair_process_pool( std::vector processes_to_shutdown; for (size_t index : died_process_indices) { { - std::lock_guard lock(versioned_pool->mutex); - if (versioned_pool->state != PoolState::INITIALIZED) { + std::scoped_lock lock(function_pool->mutex); + if (function_pool->state != PoolState::INITIALIZED) { break; } } @@ -557,22 +678,26 @@ int PythonServerManager::_repair_process_pool( if (status.ok() && new_process) { bool published = false; { - std::lock_guard lock(versioned_pool->mutex); - auto& pool = versioned_pool->processes; - if (versioned_pool->state != PoolState::INITIALIZED) { + std::scoped_lock lock(function_pool->mutex); + auto& pool = function_pool->processes; + if (function_pool->state != PoolState::INITIALIZED) { processes_to_shutdown.emplace_back(std::move(new_process)); } else if (index < pool.size()) { if (!pool[index] || !pool[index]->is_alive()) [[likely]] { + function_pool->process_registry->add(new_process); + new_process->set_package_cache_entry(function_pool->package_cache_entry); pool[index] = std::move(new_process); - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; recreated++; published = true; } else { processes_to_shutdown.emplace_back(std::move(new_process)); } } else if (pool.size() < static_cast(max_pool_size)) { + function_pool->process_registry->add(new_process); + new_process->set_package_cache_entry(function_pool->package_cache_entry); pool.emplace_back(std::move(new_process)); - versioned_pool->has_available_process = true; + function_pool->has_available_process = true; recreated++; published = true; } else { @@ -580,7 +705,7 @@ int PythonServerManager::_repair_process_pool( } } if (published) { - versioned_pool->cv.notify_all(); + function_pool->cv.notify_all(); } } else { LOG(ERROR) << "Failed to recreate Python process for version " << version.to_string() @@ -589,21 +714,20 @@ int PythonServerManager::_repair_process_pool( } { - std::lock_guard lock(versioned_pool->mutex); - auto& pool = versioned_pool->processes; + std::scoped_lock lock(function_pool->mutex); + auto& pool = function_pool->processes; - if (versioned_pool->state != PoolState::INITIALIZED) { - versioned_pool->has_available_process = false; + if (function_pool->state != PoolState::INITIALIZED) { + function_pool->has_available_process = false; } else { // Keep empty/dead slots instead of shrinking the vector. Init workers are detached and // publish by original slot index; shrinking here would make a late successful init look // out-of-range and discard a usable process. - versioned_pool->has_available_process = std::any_of( - pool.begin(), pool.end(), - [](const ProcessPtr& process) { return process && process->is_alive(); }); + function_pool->has_available_process = std::ranges::any_of( + pool, [](const ProcessPtr& process) { return process && process->is_alive(); }); } } - versioned_pool->cv.notify_all(); + function_pool->cv.notify_all(); for (auto& process : processes_to_shutdown) { process->shutdown(); } @@ -611,31 +735,31 @@ int PythonServerManager::_repair_process_pool( } void PythonServerManager::shutdown() { - std::vector>> pools_to_shutdown; + std::vector> pools_to_shutdown; { - std::lock_guard lock(_pools_mutex); + std::scoped_lock lock(_pools_mutex); _shutdown_flag.store(true, std::memory_order_release); pools_to_shutdown.reserve(_process_pools.size()); - for (auto& [version, versioned_pool] : _process_pools) { - pools_to_shutdown.emplace_back(version, std::move(versioned_pool)); + for (auto& [function_id, function_pool] : _process_pools) { + pools_to_shutdown.emplace_back(std::move(function_pool)); } _process_pools.clear(); } - for (auto& [version, versioned_pool] : pools_to_shutdown) { - if (!versioned_pool) { + for (const auto& function_pool : pools_to_shutdown) { + if (!function_pool) { continue; } - std::lock_guard lock(versioned_pool->mutex); - versioned_pool->state = PoolState::STOPPED; - versioned_pool->has_available_process = false; - versioned_pool->repairing = false; - versioned_pool->cv.notify_all(); + std::scoped_lock lock(function_pool->mutex); + function_pool->state = PoolState::STOPPED; + function_pool->has_available_process = false; + function_pool->repairing = false; + function_pool->cv.notify_all(); } std::unique_ptr health_check_thread; { - std::lock_guard lock(_health_check_mutex); + std::scoped_lock lock(_health_check_mutex); health_check_thread = std::move(_health_check_thread); } _health_check_cv.notify_one(); @@ -643,23 +767,18 @@ void PythonServerManager::shutdown() { health_check_thread->join(); } - // Shutdown all processes - std::vector processes_to_shutdown; - for (auto& [version, versioned_pool] : pools_to_shutdown) { - if (!versioned_pool) { + // Include processes retained by clients after DROP. Snapshot ownership keeps pool clearing + // from destroying a process under the pool lock; all shutdowns happen outside locks. + auto processes_to_shutdown = _process_registry->snapshot(); + for (const auto& function_pool : pools_to_shutdown) { + if (!function_pool) { continue; } - std::lock_guard lock(versioned_pool->mutex); - auto& pool = versioned_pool->processes; - for (auto& process : pool) { - if (process) { - processes_to_shutdown.emplace_back(std::move(process)); - } - } - pool.clear(); - versioned_pool->cv.notify_all(); + std::scoped_lock lock(function_pool->mutex); + function_pool->processes.clear(); + function_pool->cv.notify_all(); } - for (auto& process : processes_to_shutdown) { + for (const auto& process : processes_to_shutdown) { process->shutdown(); } } @@ -692,21 +811,19 @@ Status PythonServerManager::_read_process_memory(pid_t pid, size_t* rss_bytes) { void PythonServerManager::_refresh_memory_stats() { int64_t total_rss = 0; - for (const auto& [version, versioned_pool] : _snapshot_process_pools()) { - std::lock_guard lock(versioned_pool->mutex); - const auto& pool = versioned_pool->processes; - for (const auto& process : pool) { - if (!process || !process->is_alive()) continue; - - size_t rss_bytes = 0; - Status s = _read_process_memory(process->get_child_pid(), &rss_bytes); + // DROP removes scheduling pools immediately, but active clients can retain their processes. + for (const auto& process : _process_registry->snapshot()) { + if (!process->is_alive()) { + continue; + } - if (s.ok()) { - total_rss += rss_bytes; - } else [[unlikely]] { - LOG(WARNING) << "Failed to read memory info for Python process (pid=" - << process->get_child_pid() << "): " << s.to_string(); - } + size_t rss_bytes = 0; + Status s = _read_process_memory(process->get_child_pid(), &rss_bytes); + if (s.ok()) { + total_rss += rss_bytes; + } else [[unlikely]] { + LOG(WARNING) << "Failed to read memory info for Python process (pid=" + << process->get_child_pid() << "): " << s.to_string(); } } _mem_tracker.set_consumption(total_rss); @@ -719,104 +836,44 @@ void PythonServerManager::_refresh_memory_stats() { } } -Status PythonServerManager::clear_module_cache(const std::string& location) { - if (location.empty()) { - return Status::InvalidArgument("Empty location for clear_module_cache"); - } - - std::string body = fmt::format(R"({{"location": "{}"}})", location); - return _broadcast_action_to_processes("clear_module_cache", body, - fmt::format("location={}", location)); -} - -void PythonServerManager::clear_udaf_state_cache(int64_t function_id) { - std::string body = fmt::format(R"({{"function_id": {}}})", function_id); - WARN_IF_ERROR(_broadcast_action_to_processes("clear_udaf_state_cache", body, - fmt::format("function_id={}", function_id)), - "failed to clear Python UDAF state cache"); -} - -Status PythonServerManager::_broadcast_action_to_processes(const std::string& action_type, - const std::string& body, - const std::string& log_name) { - int success_count = 0; - int fail_count = 0; - bool has_active_process = false; - - for (auto& [version, versioned_pool] : _snapshot_process_pools()) { - std::lock_guard lock(versioned_pool->mutex); - auto& pool = versioned_pool->processes; - for (auto& process : pool) { - if (!process || !process->is_alive()) { - continue; - } - has_active_process = true; - try { - auto loc_result = arrow::flight::Location::Parse(process->get_uri()); - if (!loc_result.ok()) [[unlikely]] { - fail_count++; - continue; - } - - auto client_result = arrow::flight::FlightClient::Connect(*loc_result); - if (!client_result.ok()) [[unlikely]] { - fail_count++; - continue; - } - auto client = std::move(*client_result); - - arrow::flight::Action action; - action.type = action_type; - action.body = arrow::Buffer::FromString(body); - - auto result_stream = client->DoAction(action); - if (!result_stream.ok()) { - fail_count++; - continue; - } - - auto result = (*result_stream)->Next(); - if (result.ok() && *result) { - success_count++; - } else { - fail_count++; - } - - } catch (...) { - fail_count++; - } +void PythonServerManager::drop_function(int64_t function_id) { + std::shared_ptr pool; + { + std::scoped_lock map_lock(_pools_mutex); + auto iter = _process_pools.find(function_id); + if (iter == _process_pools.end()) { + return; + } + pool = iter->second; + // Keep the STOPPED pool in the map until the health check sees no active clients. + // For example, a GROUP BY UDAF may process group A, receive DROP, then encounter group B. + // B's create() must find this STOPPED pool and fail instead of starting a new pool while + // A's client is still alive. Existing clients may finish on their current processes. + { + std::scoped_lock pool_lock(pool->mutex); + pool->state = PoolState::STOPPED; + pool->has_available_process = false; } } - - if (!has_active_process) { - return Status::OK(); - } - - LOG(INFO) << action_type << " completed for " << log_name << ", success=" << success_count - << ", failed=" << fail_count; - - if (fail_count > 0) { - return Status::InternalError("{} failed for {}, success={}, failed={}", action_type, - log_name, success_count, fail_count); - } - - return Status::OK(); + pool->cv.notify_all(); } // Explicit template instantiation for UDF, UDAF and UDTF clients template Status PythonServerManager::get_client( const PythonUDFMeta& func_meta, const PythonVersion& version, - std::shared_ptr* client, - const std::shared_ptr& data_schema); + std::shared_ptr* client, const std::shared_ptr& data_schema, + std::shared_ptr package_cache_entry); template Status PythonServerManager::get_client( const PythonUDFMeta& func_meta, const PythonVersion& version, std::shared_ptr* client, - const std::shared_ptr& data_schema); + const std::shared_ptr& data_schema, + std::shared_ptr package_cache_entry); template Status PythonServerManager::get_client( const PythonUDFMeta& func_meta, const PythonVersion& version, std::shared_ptr* client, - const std::shared_ptr& data_schema); + const std::shared_ptr& data_schema, + std::shared_ptr package_cache_entry); } // namespace doris diff --git a/be/src/udf/python/python_server.h b/be/src/udf/python/python_server.h index c72a2db43d1b94..18b333f28cc934 100644 --- a/be/src/udf/python/python_server.h +++ b/be/src/udf/python/python_server.h @@ -32,6 +32,8 @@ #include "udf/python/python_udf_runtime.h" namespace doris { +struct UserFunctionCacheEntry; +class PythonUDFPoolLease; class PythonServerManager { public: PythonServerManager() = default; @@ -46,15 +48,13 @@ class PythonServerManager { template Status get_client(const PythonUDFMeta& func_meta, const PythonVersion& version, std::shared_ptr* client, - const std::shared_ptr& data_schema = nullptr); + const std::shared_ptr& data_schema = nullptr, + std::shared_ptr package_cache_entry = nullptr); static Status fork(const PythonVersion& version, ProcessPtr* process); - // Clear Python module cache for a specific UDF location across all processes - Status clear_module_cache(const std::string& location); - - // Clear Python UDAF runtime state after DROP FUNCTION - void clear_udaf_state_cache(int64_t function_id); + // Stop accepting new clients. Health check removes the pool after its last client closes. + void drop_function(int64_t function_id); void shutdown(); @@ -62,40 +62,55 @@ class PythonServerManager { // For unit testing only. void check_and_recreate_processes_for_test() { _check_and_recreate_processes(); } - void set_process_pool_for_test(const PythonVersion& version, std::vector processes, - bool initialized = true); + void set_process_pool_for_test(const PythonUDFMeta& meta, const PythonVersion& version, + std::vector processes, bool initialized = true); - std::vector process_pool_snapshot_for_test(const PythonVersion& version); + std::vector process_pool_snapshot_for_test(int64_t function_id); - bool process_pool_is_initialized_for_test(const PythonVersion& version); + bool process_pool_is_initialized_for_test(int64_t function_id); - bool wait_for_process_pool_initialized_for_test(const PythonVersion& version, + bool wait_for_process_pool_initialized_for_test(int64_t function_id, std::chrono::milliseconds timeout); - bool wait_for_process_pool_initialization_finished_for_test(const PythonVersion& version, + bool wait_for_process_pool_initialization_finished_for_test(int64_t function_id, std::chrono::milliseconds timeout); - Status broadcast_action_to_processes_for_test(const std::string& action_type, - const std::string& body, - const std::string& log_name) { - return _broadcast_action_to_processes(action_type, body, log_name); - } #endif private: + friend class PythonUDFPoolLease; + enum class PoolState { UNINITIALIZED, INITIALIZING, // `Initialized` means: // 1. All process slots have attempted initialization. // 2. At least one live process is available for requests. - // 3. The health-check thread has been started. INITIALIZED, - // Prevent the pool from being incorrectly reused after `shutdown()` + // Reject new clients and late fork publication after DROP, shutdown or idle reclamation. STOPPED, }; - struct VersionedProcessPool { + // Includes processes held by clients after DROP, without extending their lifetime. + struct ProcessRegistry { + void add(const ProcessPtr& process); + std::vector snapshot(); + + std::mutex mutex; + std::vector> processes; + }; + + struct FunctionProcessPool { + // Interpreter information needed to replace dead processes in this function pool. + PythonVersion version; + // Detached fork/repair workers can outlive the manager. + std::shared_ptr process_registry; + int64_t function_id = 0; + int32_t pool_size = 3; + int64_t auto_suspend = PythonUDFMeta::DEFAULT_AUTO_SUSPEND_SECONDS; + size_t active_clients = 0; + std::chrono::steady_clock::time_point idle_since = std::chrono::steady_clock::now(); + std::shared_ptr package_cache_entry; std::mutex mutex; // Coordinates initialization and repair workers with foreground requests. std::condition_variable cv; @@ -107,25 +122,38 @@ class PythonServerManager { bool repairing = false; }; - /** - * Lazily initialize and return the process pool for specific Python version. + /** + * Lazily initialize and return the process pool for specific function. */ - Result> _ensure_pool_initialized( - const PythonVersion& version); + Result> _ensure_pool_initialized( + const PythonUDFMeta& meta, const PythonVersion& version, + std::shared_ptr package_cache_entry = nullptr, + std::shared_ptr* lease = nullptr); /** * Pick an available process from specific pool, recreating one on demand if needed. */ + static void _start_pool_initialization( + const std::shared_ptr& function_pool, const PythonVersion& version, + std::chrono::milliseconds process_pool_init_timeout); + + static void _start_pool_repair(const PythonVersion& version, + const std::shared_ptr& function_pool); + Status _get_process(const PythonVersion& version, - const std::shared_ptr& versioned_pool, + const std::shared_ptr& function_pool, ProcessPtr* process); /** - * Start health check background thread (called once by ensure_pool_initialized) - * Thread periodically checks process health and refreshes memory stats + * Start health check background thread when the first function pool is created + * Thread periodically reclaims pools, checks process health and refreshes memory stats. */ void _start_health_check_thread(); + // Only reclaim pools with no active clients: STOPPED pools on the next scan; other pools + // after auto_suspend seconds without a client (-1 disables idle reclamation). + void _reclaim_idle_pools(std::chrono::steady_clock::time_point now); + /** * Check process health and recreate dead processes */ @@ -145,7 +173,7 @@ class PythonServerManager { static constexpr std::chrono::milliseconds PROCESS_REPAIR_WAIT_TIMEOUT {1000}; #endif static int _repair_process_pool(const PythonVersion& version, - const std::shared_ptr& versioned_pool); + const std::shared_ptr& function_pool); /** * Read resident set size (RSS) for a single process from /proc/{pid}/statm @@ -157,18 +185,21 @@ class PythonServerManager { */ void _refresh_memory_stats(); - Result> _get_or_create_process_pool( - const PythonVersion& version); - std::vector>> - _snapshot_process_pools(); - Status _broadcast_action_to_processes(const std::string& action_type, const std::string& body, - const std::string& log_name); + Result> _get_or_create_process_pool( + const PythonUDFMeta& meta, const PythonVersion& version, + std::shared_ptr package_cache_entry, + std::shared_ptr* lease = nullptr); + std::vector> _snapshot_process_pools(); static bool _select_alive_process_from_pool(const std::vector& pool, ProcessPtr* process); - std::unordered_map> _process_pools; - // Protects the version -> pool handle map only. Per-version process operations are guarded - // by VersionedProcessPool::mutex. + // Weak references keep dropped-but-active processes visible without keeping them alive. + std::shared_ptr _process_registry = std::make_shared(); + + // function_id -> process_pool + std::unordered_map> _process_pools; + // Protects the function id -> pool map. DROP also stops the pool under this lock; + // when taking both locks, acquire this before FunctionProcessPool::mutex. std::mutex _pools_mutex; // Health check background thread std::unique_ptr _health_check_thread; diff --git a/be/src/udf/python/python_server.py b/be/src/udf/python/python_server.py index 14c28d6ef3c198..01b6c6801dc57d 100644 --- a/be/src/udf/python/python_server.py +++ b/be/src/udf/python/python_server.py @@ -31,7 +31,6 @@ import pickle import uuid from abc import ABC, abstractmethod -from contextlib import contextmanager from typing import Any, Callable, Optional, Tuple, get_origin, Dict from datetime import datetime from enum import Enum @@ -168,30 +167,6 @@ def monitor_parent_exit(): monitor_thread.start() -@contextmanager -def temporary_sys_path(path: str): - """ - Context manager to temporarily add a path to sys.path. - Ensures the path is removed after use to avoid pollution. - - Args: - path: Directory path to add to sys.path - - Yields: - None - """ - path_added = False - if path not in sys.path: - sys.path.insert(0, path) - path_added = True - - try: - yield - finally: - if path_added and path in sys.path: - sys.path.remove(path) - - def int32_to_uint32(value: int) -> int: """ Convert a signed int32 to unsigned uint32 representation. @@ -962,9 +937,8 @@ class ModuleUDFLoader(UDFLoader): """Loads a UDF from a Python module file (.py).""" # Module names that are forbidden for UDFs because they conflict with - # modules already imported by the server process. Loading a user module - # with one of these names would overwrite the entry in sys.modules and - # could break the server itself. + # modules already imported by the server process. A conflicting name could + # resolve to the server module instead of the requested UDF module. _FORBIDDEN_MODULE_NAMES: frozenset = frozenset({ "argparse", "base64", "gc", "importlib", "inspect", "ipaddress", "json", "sys", "os", "traceback", "logging", "time", "threading", @@ -973,48 +947,6 @@ class ModuleUDFLoader(UDFLoader): "logging.handlers", }) - # Class-level lock dictionary for thread-safe module imports - # Using RLock allows the same thread to acquire the lock multiple times - - # Key for _import_locks: module_name only (not location) - # sys.modules is a global dict keyed by module name. - # we need to ensure that imports with the same module name - # do not interfere with each other across different threads, - # even if they come from different file paths. - _import_locks: Dict[str, threading.Lock] = {} - _import_locks_lock = threading.Lock() - - # Key for _module_cache: location only - # since location already contains a unique function_id - _module_cache: Dict[str, Any] = {} - _module_cache_lock = threading.Lock() - - @classmethod - def _get_import_lock(cls, module_name: str) -> threading.Lock: - """ - Get or create an import lock for the given module namespace. - - Uses double-checked locking pattern for optimal performance: - - Fast path: return existing lock without acquiring global lock - - Slow path: create new lock under global lock protection - """ - # Lock by top-level package to avoid concurrent imports mutating shared - # parent entries in sys.modules. If we lock by full module name instead, - # pkg.mod.func1 and pkg.mod.func2 can import in parallel and race while - # initializing pkg/pkg.mod, causing flaky import failures (for example KeyError). - cache_key = module_name.split(".", 1)[0] - - # Fast path: check without lock (read-only, safe for most cases) - if cache_key in cls._import_locks: - return cls._import_locks[cache_key] - - # Slow path: create lock under protection - with cls._import_locks_lock: - # Double-check: another thread might have created it while we waited - if cache_key not in cls._import_locks: - cls._import_locks[cache_key] = threading.Lock() - return cls._import_locks[cache_key] - def load(self) -> AdaptivePythonUDF: """ Loads a UDF from a Python module file. @@ -1080,29 +1012,8 @@ def parse_symbol(self, symbol: str): return package_name, module_name, func_name - @staticmethod - def _clear_modules_from_sys(full_module_name: str) -> None: - """Remove a module and all its ancestor packages from sys.modules. - - To prevent the same module from being polluted by old caches - when loaded from different paths. - e.g., the pkg under path_a affecting the pkg.mdu_a under path_b, - the ancestor chain is cleared after each import. - - This ensures that subsequent imports always start from a fresh state. - """ - parts = full_module_name.split(".") - for i in range(len(parts)): - ancestor = ".".join(parts[: i + 1]) - sys.modules.pop(ancestor, None) - - def _get_or_import_module(self, location: str, full_module_name: str) -> Any: - """ - Get module from cache or import it (thread-safe). - - The cache is keyed by location alone, which already contains a unique - function_id assigned by the FE catalog. - """ + def _get_or_import_module(self, full_module_name: str) -> Any: + """Use Python's module cache and import locks within the function process.""" # Reject module names that would shadow server-critical modules top_level_name = full_module_name.split(".")[0] if top_level_name in ModuleUDFLoader._FORBIDDEN_MODULE_NAMES: @@ -1113,39 +1024,7 @@ def _get_or_import_module(self, location: str, full_module_name: str) -> Any: f"server-critical modules." ) - cache_key = location - - # Use a per-module lock to prevent race conditions during import - import_lock = ModuleUDFLoader._get_import_lock(full_module_name) - - with import_lock: - # Fast path: check cache first - if cache_key in ModuleUDFLoader._module_cache: - cached_module = ModuleUDFLoader._module_cache[cache_key] - if cached_module is not None and ( - hasattr(cached_module, "__file__") - or hasattr(cached_module, "__path__") - ): - return cached_module - else: - del ModuleUDFLoader._module_cache[cache_key] - - self._clear_modules_from_sys(full_module_name) - - with temporary_sys_path(location): - try: - module = importlib.import_module(full_module_name) - ModuleUDFLoader._module_cache[cache_key] = module - # Evict from sys.modules so future imports from a - # different location are not poisoned by this one. - self._clear_modules_from_sys(full_module_name) - return module - except Exception: - # Clean up any partially-imported modules - self._clear_modules_from_sys(full_module_name) - if cache_key in ModuleUDFLoader._module_cache: - del ModuleUDFLoader._module_cache[cache_key] - raise + return importlib.import_module(full_module_name) def _extract_function( self, module: Any, func_name: str, module_name: str @@ -1196,7 +1075,7 @@ def _load_single_file_udf( raise ImportError(f"Python file not found: {py_file}") try: - udf_module = self._get_or_import_module(location, module_name) + udf_module = self._get_or_import_module(module_name) return self._extract_function(udf_module, func_name, module_name) except (ImportError, AttributeError, TypeError) as e: raise ImportError( @@ -1244,7 +1123,7 @@ def _load_package_udf( try: full_module_name = self._build_full_module_name(package_name, module_name) - udf_module = self._get_or_import_module(location, full_module_name) + udf_module = self._get_or_import_module(full_module_name) return self._extract_function(udf_module, func_name, full_module_name) except (ImportError, AttributeError, TypeError) as e: raise ImportError( @@ -1737,6 +1616,9 @@ def __init__(self, location: str): # cannot reuse a class loaded from old inline code. self.udaf_state_managers: Dict[str, UDAFStateManager] = {} self.udaf_managers_lock = threading.Lock() + self._binding_lock = threading.Lock() + self._bound_runtime_version: Optional[str] = None + self._bound_function_id: Optional[int] = None def _get_udaf_state_manager( self, python_udaf_meta: PythonUDFMeta @@ -1855,6 +1737,29 @@ def parse_python_udf_meta( return python_udf_meta + def _validate_process_binding(self, python_udf_meta: PythonUDFMeta) -> None: + """Bind once and reject requests for a different function id.""" + with self._binding_lock: + if self._bound_function_id is None: + # One function owns this process. Keep its module root available + # for imports during evaluate and for pickle's module lookups. + if python_udf_meta.udf_load_type == 1: + sys.path.insert(0, python_udf_meta.location) + self._bound_runtime_version = python_udf_meta.runtime_version + self._bound_function_id = python_udf_meta.id + return + + if self._bound_runtime_version != python_udf_meta.runtime_version: + raise ValueError( + "Python process runtime mismatch: " + f"bound={self._bound_runtime_version}, requested={python_udf_meta.runtime_version}" + ) + if self._bound_function_id != python_udf_meta.id: + raise ValueError( + "Python process function id mismatch: " + f"bound={self._bound_function_id}, requested={python_udf_meta.id}" + ) + @staticmethod def check_schema( record_batch: pa.RecordBatch, expected_schema: pa.Schema @@ -2694,6 +2599,8 @@ def do_exchange( if not python_udf_meta: raise ValueError("Invalid or missing metadata in descriptor") + self._validate_process_binding(python_udf_meta) + if python_udf_meta.is_udf(): self._handle_exchange_udf(python_udf_meta, reader, writer) elif python_udf_meta.is_udaf(): @@ -2712,16 +2619,12 @@ def do_action( Handle Flight actions for cache management. Supported actions: - - "clear_module_cache": Clear Python module cache for a specific location - Body: JSON with "location" field (the UDF cache directory path) - "clear_udaf_state_cache": Clear UDAF runtime state for a dropped function id Body: JSON with "function_id" field """ action_type = action.type - if action_type == "clear_module_cache": - yield from self._handle_clear_module_cache(action.body.to_pybytes()) - elif action_type == "clear_udaf_state_cache": + if action_type == "clear_udaf_state_cache": yield from self._handle_clear_udaf_state_cache(action.body.to_pybytes()) else: raise flight.FlightUnavailableError(f"Unknown action: {action_type}") @@ -2750,87 +2653,6 @@ def _handle_clear_udaf_state_cache(self, body: bytes): "error": str(e) }).encode("utf-8")) - def _handle_clear_module_cache(self, body: bytes): - """ - Clear Python module cache for a specific UDF location. - - This removes modules from sys.modules that were loaded from the specified - location, allowing fresh imports when a new UDF with the same module name - is created. - """ - try: - params = json.loads(body.decode("utf-8")) - location = params.get("location", "") - - if not location: - yield flight.Result(b'{"success": false, "error": "empty location"}') - return - - cleared_modules = self._clear_modules_from_location(location) - - result = { - "success": True, - "cleared_modules": cleared_modules, - "location": location, - } - yield flight.Result(json.dumps(result).encode("utf-8")) - - except Exception as e: - logging.error("clear_module_cache failed: %s", e) - yield flight.Result(json.dumps({ - "success": False, - "error": str(e) - }).encode("utf-8")) - - def _clear_modules_from_location(self, location: str) -> list: - """ - Clear module cache for the given location. - - Acquires per-module import locks to ensure no concurrent import is - in progress for the modules being cleared, preventing race conditions - where sys.modules entries are removed mid-import. - - Returns list of cleared module names. - """ - cleared = [] - - with ModuleUDFLoader._module_cache_lock: - keys_to_remove = [ - key for key in ModuleUDFLoader._module_cache - if key[0] == location - ] - - # For each module, acquire its import lock before clearing. - # This ensures no concurrent _get_or_import_module is in progress - # for this (location, module_name) pair. - for key in keys_to_remove: - _, module_name = key - import_lock = ModuleUDFLoader._get_import_lock(module_name) - - with import_lock: - with ModuleUDFLoader._module_cache_lock: - if key in ModuleUDFLoader._module_cache: - del ModuleUDFLoader._module_cache[key] - - modules_to_remove = [ - name for name, mod in sys.modules.items() - if name == module_name or name.startswith(module_name + ".") - or ( - hasattr(mod, "__file__") and mod.__file__ is not None - and mod.__file__.startswith(location) - ) - ] - for mod_name in modules_to_remove: - del sys.modules[mod_name] - if mod_name not in cleared: - cleared.append(mod_name) - - if module_name not in cleared: - cleared.append(module_name) - - return cleared - - class UDAFOperationType(Enum): """Enum representing UDAF operation types.""" diff --git a/be/src/udf/python/python_udaf_client.cpp b/be/src/udf/python/python_udaf_client.cpp index 8b39524afa5696..a053da1e0197df 100644 --- a/be/src/udf/python/python_udaf_client.cpp +++ b/be/src/udf/python/python_udaf_client.cpp @@ -412,7 +412,9 @@ Status PythonUDAFClient::destroy(int64_t place_id) { } Status PythonUDAFClient::close() { - if (!_inited || !_writer) return Status::OK(); + if (!_inited || !_writer) { + return PythonClient::close(); + } // Destroy the place if it exists (cleanup on client destruction) if (_created_place_id.has_value()) { diff --git a/be/src/udf/python/python_udf_meta.h b/be/src/udf/python/python_udf_meta.h index 55c49abb30ad07..3704278518bf0b 100644 --- a/be/src/udf/python/python_udf_meta.h +++ b/be/src/udf/python/python_udf_meta.h @@ -33,7 +33,11 @@ enum class PythonUDFLoadType : uint8_t { INLINE = 0, MODULE = 1, UNKNOWN = 2 }; enum class PythonClientType : uint8_t { UDF = 0, UDAF = 1, UDTF = 2, UNKNOWN = 3 }; struct PythonUDFMeta { + static constexpr int64_t DEFAULT_AUTO_SUSPEND_SECONDS = 14400; + int64_t id = 0; + int32_t pool_size = 3; + int64_t auto_suspend = DEFAULT_AUTO_SUSPEND_SECONDS; std::string name; std::string symbol; std::string location; diff --git a/be/src/udf/python/python_udf_runtime.h b/be/src/udf/python/python_udf_runtime.h index 2d635dc11a1a21..9665a110ec131b 100644 --- a/be/src/udf/python/python_udf_runtime.h +++ b/be/src/udf/python/python_udf_runtime.h @@ -19,11 +19,14 @@ #include #include +#include #include "udf/python/python_env.h" namespace doris { +struct UserFunctionCacheEntry; + static const char* UNIX_SOCKET_PREFIX = "grpc+unix://"; // Use /tmp directory for Unix socket to avoid path length limit (max 107 chars) static const char* BASE_UNIX_SOCKET_PATH_TEMPLATE = "/tmp/doris_python_udf"; @@ -89,6 +92,10 @@ class PythonUDFProcess { pid_t get_child_pid() const { return _child_pid; } + void set_package_cache_entry(std::shared_ptr entry) { + _package_cache_entry = std::move(entry); + } + bool operator==(const PythonUDFProcess& other) const { return _child_pid == other._child_pid; } bool operator!=(const PythonUDFProcess& other) const { return !(*this == other); } @@ -104,6 +111,8 @@ class PythonUDFProcess { private: constexpr static size_t MAX_ACCUMULATED_LOG_SIZE = 65536; + // Keep package files available for lazy imports until this process is destroyed. + std::shared_ptr _package_cache_entry; bool _is_shutdown {false}; pid_t _child_pid; std::string _uri; diff --git a/be/test/agent/task_worker_pool_test.cpp b/be/test/agent/task_worker_pool_test.cpp index b8b2de0fc4b097..46893edd4daa15 100644 --- a/be/test/agent/task_worker_pool_test.cpp +++ b/be/test/agent/task_worker_pool_test.cpp @@ -19,16 +19,23 @@ #include #include +#include #include #include #include +#include +#include #include +#include #include "agent/agent_server.h" #include "cloud/cloud_storage_engine.h" +#include "common/config.h" +#include "cpp/sync_point.h" #include "runtime/cluster_info.h" #include "runtime/exec_env.h" +#include "runtime/user_function_cache.h" #include "storage/options.h" #include "storage/storage_engine.h" @@ -36,6 +43,90 @@ namespace doris { using namespace std::chrono_literals; +class CleanUDFCacheWorkerTest : public ::testing::Test { +protected: + UserFunctionCache cache; + UserFunctionCache* original_cache = nullptr; + bool original_java_support = false; + std::vector acknowledgements; + + void SetUp() override { + original_cache = ExecEnv::GetInstance()->_user_function_cache; + ExecEnv::GetInstance()->_user_function_cache = &cache; + original_java_support = config::enable_java_support; + config::enable_java_support = false; + auto* sync_point = SyncPoint::get_instance(); + sync_point->set_call_back("TaskWorkerPool::finish_task", [this](auto&& args) { + acknowledgements.push_back(*try_any_cast(args[0])); + *try_any_cast(args.back()) = true; + }); + sync_point->enable_processing(); + } + + void TearDown() override { + auto* sync_point = SyncPoint::get_instance(); + sync_point->disable_processing(); + sync_point->clear_call_back("TaskWorkerPool::finish_task"); + config::enable_java_support = original_java_support; + ExecEnv::GetInstance()->_user_function_cache = original_cache; + } + + TAgentTaskRequest request(int64_t signature, int64_t function_id = 938812) { + TAgentTaskRequest req; + req.__set_task_type(TTaskType::CLEAN_UDF_CACHE); + req.__set_signature(signature); + TCleanUDFCacheReq clean; + clean.__set_function_id(function_id); + clean.__set_function_signature("sum(INT)"); + req.__set_clean_udf_cache_req(clean); + return req; + } +}; + +TEST_F(CleanUDFCacheWorkerTest, OneShotCleanupDoesNotAcknowledge) { + clean_udf_cache_callback(request(-1)); + EXPECT_TRUE(acknowledgements.empty()); +} + +TEST_F(CleanUDFCacheWorkerTest, TrackedCleanupCanRunAgainAfterLostAcknowledgement) { + std::mutex mutex; + std::condition_variable cv; + int completed = 0; + TaskWorkerPool workers("test_clean_udf", 1, [&](const auto& req) { + clean_udf_cache_callback(req); + { + std::scoped_lock lock(mutex); + ++completed; + } + cv.notify_all(); + }); + auto req = request(938812); + ASSERT_TRUE(workers.submit_task(req).ok()); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, 5s, [&] { return completed == 1; })); + } + // A lost execution result must not prevent the same task from running again. + ASSERT_TRUE(workers.submit_task(req).ok()); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, 5s, [&] { return completed == 2; })); + } + ASSERT_EQ(2, acknowledgements.size()); + for (const auto& ack : acknowledgements) { + EXPECT_EQ(TTaskType::CLEAN_UDF_CACHE, ack.task_type); + EXPECT_EQ(938812, ack.signature); + EXPECT_EQ(TStatusCode::OK, ack.task_status.status_code); + } +} + +TEST_F(CleanUDFCacheWorkerTest, InvalidTrackedCleanupReportsFailure) { + clean_udf_cache_callback(request(938813, 0)); + ASSERT_EQ(1, acknowledgements.size()); + EXPECT_EQ(938813, acknowledgements.front().signature); + EXPECT_EQ(TStatusCode::INVALID_ARGUMENT, acknowledgements.front().task_status.status_code); +} + TEST(TaskWorkerPoolTest, TaskWorkerPool) { std::atomic_int count {0}; TaskWorkerPool workers("test", 2, [&](auto&& task) { diff --git a/be/test/runtime/user_function_cache_test.cpp b/be/test/runtime/user_function_cache_test.cpp index 57cd3149547e9a..de964ffc4afb8c 100644 --- a/be/test/runtime/user_function_cache_test.cpp +++ b/be/test/runtime/user_function_cache_test.cpp @@ -27,6 +27,7 @@ #include #include "common/status.h" +#include "util/md5.h" namespace doris { @@ -99,6 +100,16 @@ class UserFunctionCacheTest : public ::testing::Test { ofs.close(); } + std::string file_md5(const std::string& path) { + std::ifstream input(path, std::ios::binary); + std::string bytes((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + Md5Digest digest; + digest.update(bytes.data(), bytes.size()); + digest.digest(); + return digest.hex(); + } + private: const char* original_doris_home_ = nullptr; }; @@ -541,4 +552,25 @@ TEST_F(UserFunctionCacheTest, LoadEntryFromLibInvalidFileName) { EXPECT_FALSE(status.ok()); } -} // namespace doris \ No newline at end of file +TEST_F(UserFunctionCacheTest, PythonZipDeletionWaitsForRuntimeReference) { + const std::string cache_dir = test_dir_ + "/cache"; + ASSERT_TRUE(ufc.init(cache_dir).ok()); + + const std::string zip_path = test_dir_ + "/shared.zip"; + ASSERT_TRUE(create_zip_file(zip_path, {{"shared.py", "def evaluate(x): return x"}})); + const std::string checksum = file_md5(zip_path); + const std::string source_location = "file://" + zip_path; + + std::string local_path; + std::shared_ptr runtime_reference; + ASSERT_TRUE( + ufc.get_pypath(2001, source_location, checksum, &local_path, &runtime_reference).ok()); + + ufc.drop_function_cache(2001); + EXPECT_TRUE(std::filesystem::exists(local_path)); + runtime_reference.reset(); + EXPECT_FALSE(std::filesystem::exists(local_path)); + EXPECT_FALSE(std::filesystem::exists(local_path + ".zip")); +} + +} // namespace doris diff --git a/be/test/udf/python/python_server_test.cpp b/be/test/udf/python/python_server_test.cpp index 39e6143bc144db..3d00540a3d96b0 100644 --- a/be/test/udf/python/python_server_test.cpp +++ b/be/test/udf/python/python_server_test.cpp @@ -21,11 +21,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -34,8 +36,11 @@ #include "common/status.h" #include "cpp/sync_point.h" #include "udf/python/python_env.h" +#include "udf/python/python_udaf_client.h" #include "udf/python/python_udf_client.h" #include "udf/python/python_udf_meta.h" +#include "udf/python/python_udtf_client.h" +#include "util/cpu_info.h" #include "util/defer_op.h" namespace doris { @@ -47,7 +52,7 @@ class PythonServerTest : public ::testing::Test { protected: std::string test_dir_; std::optional original_doris_home_; - int original_max_python_process_num_ = 0; + int test_pool_size_ = 1; void SetUp() override { test_dir_ = fs::temp_directory_path().string() + "/python_server_test_" + @@ -57,12 +62,10 @@ class PythonServerTest : public ::testing::Test { if (const char* doris_home = std::getenv("DORIS_HOME")) { original_doris_home_ = doris_home; } - original_max_python_process_num_ = config::max_python_process_num; } void TearDown() override { // Restore configuration - config::max_python_process_num = original_max_python_process_num_; if (!test_dir_.empty() && fs::exists(test_dir_)) { fs::remove_all(test_dir_); @@ -75,6 +78,53 @@ class PythonServerTest : public ::testing::Test { } } + void check_dropped_process_memory(bool repair) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + SCOPED_TRACE(repair ? "repaired process" : "initial process"); + PythonServerManager mgr; + auto meta = test_meta(version); + std::shared_ptr lease; + auto pool = mgr._ensure_pool_initialized(meta, version, nullptr, &lease); + ASSERT_TRUE(pool.has_value()) << pool.error().to_string(); + ASSERT_TRUE( + mgr.wait_for_process_pool_initialized_for_test(meta.id, std::chrono::seconds(5))); + // Model the ProcessPtr retained by PythonClient during an active exchange. + ProcessPtr client_process; + ASSERT_TRUE(mgr._get_process(version, pool.value(), &client_process).ok()); + if (repair) { + client_process->shutdown(); + client_process.reset(); + mgr._check_and_recreate_processes(); + ASSERT_TRUE(mgr._get_process(version, pool.value(), &client_process).ok()); + } + mgr._refresh_memory_stats(); + ASSERT_GT(mgr._mem_tracker.consumption(), 0); + + std::weak_ptr process_lifetime = client_process; + mgr.drop_function(meta.id); + EXPECT_EQ(mgr.process_pool_snapshot_for_test(meta.id).size(), 1); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + ASSERT_TRUE(client_process->is_alive()); + mgr._refresh_memory_stats(); + EXPECT_GT(mgr._mem_tracker.consumption(), 0); + + client_process.reset(); + lease.reset(); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_TRUE(process_lifetime.expired()); + mgr._refresh_memory_stats(); + EXPECT_EQ(mgr._mem_tracker.consumption(), 0); + } + + PythonUDFMeta test_meta(const PythonVersion& version) const { + PythonUDFMeta meta; + meta.id = static_cast(std::hash()(version.to_string()) & INT64_MAX); + meta.pool_size = test_pool_size_; + return meta; + } + // Create a fake Python script that creates a socket file and keeps running // Arg: script prints "Python X.Y.Z" for version detection std::string create_fake_python_with_socket_creation(const std::string& version = "3.9.16") { @@ -339,11 +389,11 @@ TEST_F(PythonServerTest, ForkEnqueuesBackgroundReapWhenKilledStartFailureIsNotRe TEST_F(PythonServerTest, EnsurePoolInitializedWithInvalidVersionFails) { PythonServerManager mgr; - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonVersion invalid_version("3.99.99", "/non/existent/path", "/non/existent/python"); - auto result = mgr._ensure_pool_initialized(invalid_version); + auto result = mgr._ensure_pool_initialized(test_meta(invalid_version), invalid_version); // Verify: invalid version should cause initialization to fail EXPECT_FALSE(result.has_value()); @@ -355,7 +405,7 @@ TEST_F(PythonServerTest, EnsurePoolInitializedWithInvalidVersionFails) { TEST_F(PythonServerTest, EnsurePoolInitializedRetriesAfterRuntimeIsRepaired) { setup_doris_home(); - config::max_python_process_num = 1; + test_pool_size_ = 1; // The first executable starts but never publishes its Flight socket. This reproduces a real // worker-start failure and verifies that the pool does not remain stuck in INITIALIZING. @@ -407,7 +457,7 @@ TEST_F(PythonServerTest, EnsurePoolInitializedRetriesAfterRuntimeIsRepaired) { }); sync_point->enable_processing(); - auto failed_result = mgr._ensure_pool_initialized(version); + auto failed_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_FALSE(failed_result.has_value()); // Wait for both parts of the failed generation to finish. Otherwise an old detached worker can @@ -419,12 +469,12 @@ TEST_F(PythonServerTest, EnsurePoolInitializedRetriesAfterRuntimeIsRepaired) { })); } ASSERT_TRUE(mgr.wait_for_process_pool_initialization_finished_for_test( - version, std::chrono::seconds(5))); + test_meta(version).id, std::chrono::seconds(5))); // Repair the runtime in place and retry the same version key. Production can hit this when an // environment or server entry is fixed after a transient initialization failure. ASSERT_EQ(create_fake_python_with_socket_creation("3.9.16"), python_path); - auto recovered_result = mgr._ensure_pool_initialized(version); + auto recovered_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(recovered_result.has_value()) << recovered_result.error().to_string(); ProcessPtr process; @@ -443,7 +493,7 @@ TEST_F(PythonServerTest, EnsurePoolInitializedAfterShutdownReturnsServiceUnavail std::string python_path = create_fake_python_with_socket_creation("3.9.16"); PythonVersion version("3.9.16", test_dir_, python_path); - auto result = mgr._ensure_pool_initialized(version); + auto result = mgr._ensure_pool_initialized(test_meta(version), version); EXPECT_FALSE(result.has_value()); EXPECT_NE(result.error().to_string().find("shutting down"), std::string::npos); @@ -466,11 +516,11 @@ TEST_F(PythonServerTest, ShutdownCalledMultipleTimesDoesNotCrash) { TEST_F(PythonServerTest, ShutdownAfterFailedInitializationDoesNotCrash) { PythonServerManager mgr; - config::max_python_process_num = 1; + test_pool_size_ = 1; // Try initialization first (expected to fail) PythonVersion invalid_version("3.99.99", "/bad/path", "/bad/python"); - auto result = mgr._ensure_pool_initialized(invalid_version); + auto result = mgr._ensure_pool_initialized(test_meta(invalid_version), invalid_version); EXPECT_FALSE(result.has_value()); // Verify: calling shutdown after failed initialization does not crash @@ -481,11 +531,11 @@ TEST_F(PythonServerTest, GetProcessFromStoppedPoolReturnsUnavailable) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto pool_result = mgr._ensure_pool_initialized(version); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); mgr.shutdown(); @@ -498,46 +548,15 @@ TEST_F(PythonServerTest, GetProcessFromStoppedPoolReturnsUnavailable) { EXPECT_NE(status.to_string().find("stopped"), std::string::npos); } -TEST_F(PythonServerTest, ClearUdafStateCacheWithoutProcessesIsNoOp) { - PythonServerManager mgr; - - EXPECT_NO_THROW(mgr.clear_udaf_state_cache(12345)); -} - -TEST_F(PythonServerTest, ClearModuleCacheWithoutProcessesIsNoOp) { +TEST_F(PythonServerTest, DropUnknownFunctionIsNoOp) { PythonServerManager mgr; - auto status = mgr.clear_module_cache("/tmp/python_udf_cache"); - EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_NO_THROW(mgr.drop_function(12345)); } -TEST_F(PythonServerTest, BroadcastActionWithInvalidProcessUriReturnsError) { - PythonServerManager mgr; - PythonVersion version("3.9.16", test_dir_, test_dir_ + "/bin/python3"); - ProcessPtr process = create_sleep_process(); - ASSERT_NE(process, nullptr); - ASSERT_TRUE(process->is_alive()); - process->set_uri_for_test("invalid-python-flight-uri"); - - mgr.set_process_pool_for_test(version, {process}); - auto status = mgr.broadcast_action_to_processes_for_test( - "clear_udaf_state_cache", R"({"function_id": 12345})", "function_id=12345"); - - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("clear_udaf_state_cache failed for function_id=12345"), - std::string::npos); - EXPECT_NE(status.to_string().find("success=0, failed=1"), std::string::npos); - - mgr.shutdown(); -} - -// ============================================================================ -// PythonServerManager::get_client() - client retrieval test -// ============================================================================ - TEST_F(PythonServerTest, GetClientWithInvalidVersionFails) { PythonServerManager mgr; - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonVersion invalid_version("3.9.16", "/invalid/path", "/invalid/python"); PythonUDFMeta meta; @@ -562,14 +581,14 @@ TEST_F(PythonServerTest, GetClientWithInvalidVersionFails) { TEST_F(PythonServerTest, DestructorCleansUpResources) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; ProcessPtr process; std::string socket_path; { PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto pool_result = mgr._ensure_pool_initialized(version); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); ASSERT_TRUE(mgr._get_process(version, pool_result.value(), &process).ok()); ASSERT_NE(process, nullptr); @@ -617,12 +636,12 @@ TEST_F(PythonServerTest, EnsurePoolInitializedSuccess) { std::string python_path = create_fake_python_with_socket_creation("3.9.16"); // Limit process pool to 1 to speed up the test - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto result = mgr._ensure_pool_initialized(version); + auto result = mgr._ensure_pool_initialized(test_meta(version), version); // Verify pool initialization succeeded EXPECT_TRUE(result.has_value()) << result.error().to_string(); @@ -636,12 +655,12 @@ TEST_F(PythonServerTest, EnsurePoolInitializedSucceedsWithOneStuckWorkerAndOneUs std::string python_path = create_fake_python_with_one_stuck_and_others_socket("python3.mixed", "3.9.16"); - config::max_python_process_num = 2; + test_pool_size_ = 2; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto result = mgr._ensure_pool_initialized(version); + auto result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(result.has_value()) << result.error().to_string(); @@ -652,9 +671,9 @@ TEST_F(PythonServerTest, EnsurePoolInitializedSucceedsWithOneStuckWorkerAndOneUs // Wait on the same condition variable used by the production coordinator instead of polling // or asserting on a racy intermediate state. - EXPECT_TRUE(mgr.wait_for_process_pool_initialized_for_test(version, + EXPECT_TRUE(mgr.wait_for_process_pool_initialized_for_test(test_meta(version).id, std::chrono::milliseconds(2000))); - EXPECT_TRUE(mgr.process_pool_is_initialized_for_test(version)); + EXPECT_TRUE(mgr.process_pool_is_initialized_for_test(test_meta(version).id)); mgr.shutdown(); } @@ -663,24 +682,24 @@ TEST_F(PythonServerTest, EnsurePoolInitializedIdempotent) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); // First initialization - auto result1 = mgr._ensure_pool_initialized(version); + auto result1 = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(result1.has_value()) << result1.error().to_string(); - auto first_snapshot = mgr.process_pool_snapshot_for_test(version); + auto first_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(first_snapshot.size(), 1); ASSERT_NE(first_snapshot[0], nullptr); pid_t first_pid = first_snapshot[0]->get_child_pid(); // Re-initialization must reuse both the versioned pool and its live process. - auto result2 = mgr._ensure_pool_initialized(version); + auto result2 = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(result2.has_value()) << result2.error().to_string(); EXPECT_EQ(result2.value(), result1.value()); - auto second_snapshot = mgr.process_pool_snapshot_for_test(version); + auto second_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(second_snapshot.size(), 1); EXPECT_EQ(second_snapshot[0], first_snapshot[0]); EXPECT_EQ(second_snapshot[0]->get_child_pid(), first_pid); @@ -692,13 +711,13 @@ TEST_F(PythonServerTest, GetProcessFromInitializedPool) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); // Initialize the pool first - auto init_result = mgr._ensure_pool_initialized(version); + auto init_result = mgr._ensure_pool_initialized(test_meta(version), version); EXPECT_TRUE(init_result.has_value()) << init_result.error().to_string(); // Get a process @@ -716,12 +735,12 @@ TEST_F(PythonServerTest, GetProcessRecreatesDeadProcessWhenNoAliveProcess) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); - auto pool_result = mgr._ensure_pool_initialized(version); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); ProcessPtr first_process; @@ -748,7 +767,7 @@ TEST_F(PythonServerTest, GetProcessSkipsDeadProcessWhenAliveProcessExists) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 3; + test_pool_size_ = 3; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); @@ -765,8 +784,8 @@ TEST_F(PythonServerTest, GetProcessSkipsDeadProcessWhenAliveProcessExists) { dead_process->shutdown(); ASSERT_FALSE(dead_process->is_alive()); - mgr.set_process_pool_for_test(version, {alive_process, dead_process}); - auto pool_result = mgr._ensure_pool_initialized(version); + mgr.set_process_pool_for_test(test_meta(version), version, {alive_process, dead_process}); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); ProcessPtr selected; @@ -774,7 +793,7 @@ TEST_F(PythonServerTest, GetProcessSkipsDeadProcessWhenAliveProcessExists) { EXPECT_TRUE(status.ok()) << status.to_string(); EXPECT_EQ(selected, alive_process); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(pool_snapshot.size(), 2); EXPECT_FALSE(pool_snapshot[1]->is_alive()); EXPECT_EQ(pool_snapshot[1]->get_child_pid(), dead_pid); @@ -782,15 +801,77 @@ TEST_F(PythonServerTest, GetProcessSkipsDeadProcessWhenAliveProcessExists) { mgr.shutdown(); } -TEST_F(PythonServerTest, GetProcessSelectsLeastSharedProcess) { +TEST_F(PythonServerTest, RejectsInvalidPoolSizeBeforeStartingProcesses) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + for (const int pool_size : {0, -1}) { + meta.pool_size = pool_size; + auto result = mgr._ensure_pool_initialized(meta, version); + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().to_string().find("pool_size must be a positive integer"), + std::string::npos); + } + mgr.shutdown(); +} + +TEST_F(PythonServerTest, CapsPoolSizeBeforeAllocatingProcessSlots) { + CpuInfo::init(); + const int cores = CpuInfo::num_cores(); + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + for (const int requested : {1, 3, cores, cores + 1, 10000}) { + PythonUDFMeta meta; + meta.id = requested; + meta.pool_size = requested; + auto result = mgr._get_or_create_process_pool(meta, version, nullptr); + ASSERT_TRUE(result.has_value()) << result.error().to_string(); + EXPECT_EQ(std::min(requested, cores), result.value()->pool_size); + EXPECT_TRUE(result.value()->processes.empty()); + } +} + +TEST_F(PythonServerTest, DefaultAutoSuspendReclaimsAfterFourHoursWithoutClients) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + PythonUDFMeta meta; + ASSERT_EQ(14400, meta.auto_suspend); + std::shared_ptr lease; + auto result = mgr._get_or_create_process_pool(meta, version, nullptr, &lease); + ASSERT_TRUE(result.has_value()); + auto pool = result.value(); + ASSERT_EQ(14400, pool->auto_suspend); + EXPECT_EQ(std::min(3, CpuInfo::num_cores()), pool->pool_size); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::hours(5)); + EXPECT_EQ(pool, mgr._get_or_create_process_pool(meta, version, nullptr).value()); + lease.reset(); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::seconds(14399)); + EXPECT_EQ(pool, mgr._get_or_create_process_pool(meta, version, nullptr).value()); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::seconds(14400)); + EXPECT_NE(pool, mgr._get_or_create_process_pool(meta, version, nullptr).value()); +} + +TEST_F(PythonServerTest, RejectsAutoSuspendBelowMinusOneBeforeStartingProcesses) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + PythonUDFMeta meta; + meta.auto_suspend = -2; + auto result = mgr._get_or_create_process_pool(meta, version, nullptr); + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().to_string().find("auto_suspend must be -1 or a non-negative integer"), + std::string::npos); +} + +TEST_F(PythonServerTest, GetProcessSelectsLeastLoadedProcess) { PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); - mgr.set_process_pool_for_test(version, {create_sleep_process(), create_sleep_process()}); + mgr.set_process_pool_for_test(test_meta(version), version, + {create_sleep_process(), create_sleep_process()}); - auto init_result = mgr._ensure_pool_initialized(version); + auto init_result = mgr._ensure_pool_initialized(test_meta(version), version); EXPECT_TRUE(init_result.has_value()) << init_result.error().to_string(); - // Holding p1 increases its shared ownership count, so the next client must select the other + // Keeping the first process shared_ptr raises its use_count, so select the other // live process. An implementation that always returns pool[0] fails this assertion. ProcessPtr p1, p2; ASSERT_TRUE(mgr._get_process(version, init_result.value(), &p1).ok()); @@ -806,13 +887,13 @@ TEST_F(PythonServerTest, ShutdownWithRunningProcesses) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 2; + test_pool_size_ = 2; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); // Initialize the pool - auto init_result = mgr._ensure_pool_initialized(version); + auto init_result = mgr._ensure_pool_initialized(test_meta(version), version); EXPECT_TRUE(init_result.has_value()) << init_result.error().to_string(); // Get a process reference @@ -859,15 +940,15 @@ TEST_F(PythonServerTest, MultipleVersionPools) { } fs::permissions(python310_path, fs::perms::owner_all); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version39("3.9.16", test_dir_, python39_path); PythonVersion version310("3.10.0", test_dir_, python310_path); // Initialize pools for two versions - auto pool39_result = mgr._ensure_pool_initialized(version39); - auto pool310_result = mgr._ensure_pool_initialized(version310); + auto pool39_result = mgr._ensure_pool_initialized(test_meta(version39), version39); + auto pool310_result = mgr._ensure_pool_initialized(test_meta(version310), version310); EXPECT_TRUE(pool39_result.has_value()) << pool39_result.error().to_string(); EXPECT_TRUE(pool310_result.has_value()) << pool310_result.error().to_string(); @@ -885,7 +966,7 @@ TEST_F(PythonServerTest, MultipleVersionPools) { TEST_F(PythonServerTest, EnsurePoolInitializedForDifferentVersionsDoesNotShareVersionLock) { setup_doris_home(); - config::max_python_process_num = 1; + test_pool_size_ = 1; std::string python39_path = create_fake_python_with_socket_creation("3.9.16"); std::string python310_path = test_dir_ + "/bin/python3.10"; @@ -940,10 +1021,12 @@ TEST_F(PythonServerTest, EnsurePoolInitializedForDifferentVersionsDoesNotShareVe }); sync_point->enable_processing(); - auto future39 = std::async(std::launch::async, - [&]() { return mgr._ensure_pool_initialized(version39); }); - auto future310 = std::async(std::launch::async, - [&]() { return mgr._ensure_pool_initialized(version310); }); + auto future39 = std::async(std::launch::async, [&]() { + return mgr._ensure_pool_initialized(test_meta(version39), version39); + }); + auto future310 = std::async(std::launch::async, [&]() { + return mgr._ensure_pool_initialized(test_meta(version310), version310); + }); bool both_forks_entered = false; { @@ -965,15 +1048,370 @@ TEST_F(PythonServerTest, EnsurePoolInitializedForDifferentVersionsDoesNotShareVe << "Both version pools must enter fork before either process starts"; } -// ============================================================================ -// PythonServerManager::_check_and_recreate_processes() - health-check recreation test -// ============================================================================ +TEST_F(PythonServerTest, FunctionsWithSameRuntimeHaveIndependentFixedPools) { + PythonServerManager mgr; + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonUDFMeta first; + first.id = 101; + first.pool_size = 1; + PythonUDFMeta second; + second.id = 102; + second.pool_size = 2; + auto first_result = mgr._ensure_pool_initialized(first, version); + ASSERT_TRUE(first_result.has_value()) << first_result.error().to_string(); + auto second_result = mgr._ensure_pool_initialized(second, version); + ASSERT_TRUE(second_result.has_value()) << second_result.error().to_string(); + ASSERT_TRUE(mgr.wait_for_process_pool_initialized_for_test(101, std::chrono::seconds(5))); + ASSERT_TRUE(mgr.wait_for_process_pool_initialized_for_test(102, std::chrono::seconds(5))); + auto first_pool = mgr.process_pool_snapshot_for_test(101); + auto second_pool = mgr.process_pool_snapshot_for_test(102); + ASSERT_EQ(first_pool.size(), 1); + ASSERT_EQ(second_pool.size(), 2); + EXPECT_NE(first_pool[0]->get_child_pid(), second_pool[0]->get_child_pid()); + EXPECT_NE(first_pool[0]->get_child_pid(), second_pool[1]->get_child_pid()); + ProcessPtr first_client, second_client; + ASSERT_TRUE(mgr._get_process(version, first_result.value(), &first_client).ok()); + ASSERT_TRUE(mgr._get_process(version, second_result.value(), &second_client).ok()); + EXPECT_NE(first_client, second_client); +} + +TEST_F(PythonServerTest, DropRejectsNewClientsUntilHealthCheckReclaimsThePool) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + std::shared_ptr lease; + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr, &lease).value(); + + mgr.drop_function(meta.id); + ASSERT_TRUE(mgr._process_pools.contains(meta.id)); + EXPECT_EQ(PythonServerManager::PoolState::STOPPED, pool->state); + EXPECT_TRUE(process->is_alive()); + + // UDAF can encounter a new GROUP BY key after DROP while an earlier group's client is alive. + // All three client factories must reject this pool instead of creating a replacement. + std::shared_ptr udf; + std::shared_ptr udaf; + std::shared_ptr udtf; + EXPECT_FALSE(mgr.get_client(meta, version, &udf).ok()); + EXPECT_FALSE(mgr.get_client(meta, version, &udaf).ok()); + EXPECT_FALSE(mgr.get_client(meta, version, &udtf).ok()); + EXPECT_EQ(pool, mgr._process_pools.at(meta.id)); + EXPECT_EQ(1, pool->active_clients); + + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_TRUE(mgr._process_pools.contains(meta.id)); + EXPECT_TRUE(process->is_alive()); + mgr.drop_function(meta.id); // Duplicate cleanup must keep the same STOPPED pool. + EXPECT_EQ(pool, mgr._process_pools.at(meta.id)); + + lease.reset(); + EXPECT_TRUE(mgr._process_pools.contains(meta.id)); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(mgr._process_pools.contains(meta.id)); + EXPECT_FALSE(process->is_alive()); +} + +TEST_F(PythonServerTest, HealthCheckReclaimsStoppedPoolsRegardlessOfAutoSuspend) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + for (int64_t timeout : {-1, 0, 3600}) { + auto meta = test_meta(version); + meta.id = 3000 + timeout; + meta.auto_suspend = timeout; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + mgr.drop_function(meta.id); + ASSERT_TRUE(mgr._process_pools.contains(meta.id)); + EXPECT_TRUE(process->is_alive()); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(mgr._process_pools.contains(meta.id)); + EXPECT_FALSE(process->is_alive()); + } +} + +TEST_F(PythonServerTest, MonitorReclaimsDroppedPoolAfterLastClientCloses) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + PythonClient client; + ASSERT_TRUE(mgr._get_or_create_process_pool(meta, version, nullptr, &client._pool_lease) + .has_value()); + client._process = process; + mgr.drop_function(meta.id); + std::this_thread::sleep_for(std::chrono::seconds(2)); + EXPECT_TRUE(process->is_alive()); + ASSERT_TRUE(client.close().ok()); + + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(40); + while (process->is_alive() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + EXPECT_FALSE(process->is_alive()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); +} + +TEST_F(PythonServerTest, DropDefersProcessCleanupAndDoesNotAffectOtherFunction) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + PythonUDFMeta first; + first.id = 101; + PythonUDFMeta second; + second.id = 102; + auto active = create_sleep_process(); + mgr.set_process_pool_for_test(first, version, {active}); + mgr.set_process_pool_for_test(second, version, {create_sleep_process()}); + mgr.drop_function(101); + EXPECT_TRUE(active->is_alive()); + EXPECT_EQ(mgr.process_pool_snapshot_for_test(101).size(), 1); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(active->is_alive()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(101).empty()); + EXPECT_EQ(mgr.process_pool_snapshot_for_test(102).size(), 1); + active.reset(); +} + +TEST_F(PythonServerTest, DroppedClientProcessesRemainCountedUntilReleased) { + check_dropped_process_memory(false); +} + +TEST_F(PythonServerTest, RepairedDroppedClientProcessesRemainCountedUntilReleased) { + check_dropped_process_memory(true); +} + +TEST_F(PythonServerTest, DropDuringInitializationKeepsPeriodicMemoryAccounting) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonServerManager mgr; + struct ForkBarrier { + std::mutex mutex; + std::condition_variable cv; + int entries = 0; + bool released = false; + }; + auto barrier = std::make_shared(); + auto* sync_point = SyncPoint::get_instance(); + Defer clear_sync_point {[barrier, sync_point]() { + { + std::scoped_lock lock(barrier->mutex); + barrier->released = true; + } + barrier->cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_call_back("PythonServerManager::fork:before_process_start"); + }}; + sync_point->set_call_back("PythonServerManager::fork:before_process_start", [barrier](auto&&) { + std::unique_lock lock(barrier->mutex); + if (++barrier->entries == 2) { + barrier->cv.wait(lock, [&]() { return barrier->released; }); + } + }); + sync_point->enable_processing(); + auto meta = test_meta(version); + meta.pool_size = 2; + std::shared_ptr lease; + auto pool = mgr._ensure_pool_initialized(meta, version, nullptr, &lease); + ASSERT_TRUE(pool.has_value()) << pool.error().to_string(); + ProcessPtr client_process; + ASSERT_TRUE(mgr._get_process(version, pool.value(), &client_process).ok()); + mgr.drop_function(meta.id); + ASSERT_TRUE(client_process->is_alive()); + + // Exercise the real periodic sampler: no explicit _refresh_memory_stats() call. + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(35); + while (mgr._mem_tracker.consumption() == 0 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + EXPECT_GT(mgr._mem_tracker.consumption(), 0); +} + +TEST_F(PythonServerTest, ShutdownStopsDroppedClientProcesses) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonServerManager mgr; + auto meta = test_meta(version); + auto pool = mgr._ensure_pool_initialized(meta, version); + ASSERT_TRUE(pool.has_value()) << pool.error().to_string(); + ASSERT_TRUE(mgr.wait_for_process_pool_initialized_for_test(meta.id, std::chrono::seconds(5))); + ProcessPtr client_process; + ASSERT_TRUE(mgr._get_process(version, pool.value(), &client_process).ok()); + mgr.drop_function(meta.id); + ASSERT_TRUE(client_process->is_alive()); + + mgr.shutdown(); + EXPECT_FALSE(client_process->is_alive()); +} + +TEST_F(PythonServerTest, AutoSuspendDestroysIdleProcessesAndAllowsRecreation) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonServerManager mgr; + auto meta = test_meta(version); + meta.auto_suspend = 0; + auto result = mgr._ensure_pool_initialized(meta, version); + ASSERT_TRUE(result.has_value()) << result.error().to_string(); + ASSERT_TRUE(mgr.wait_for_process_pool_initialized_for_test(meta.id, std::chrono::seconds(5))); + auto old_pool = result.value(); + auto process = mgr.process_pool_snapshot_for_test(meta.id).front(); + ASSERT_TRUE(process->is_alive()); + + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(process->is_alive()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); + auto reopened = mgr._ensure_pool_initialized(meta, version); + ASSERT_TRUE(reopened.has_value()) << reopened.error().to_string(); + EXPECT_NE(old_pool, reopened.value()); +} + +TEST_F(PythonServerTest, AutoSuspendWaitsForTheConfiguredIdleDuration) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + meta.auto_suspend = 3; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr).value(); + auto idle_since = pool->idle_since; + mgr._reclaim_idle_pools(idle_since + std::chrono::seconds(2)); + EXPECT_TRUE(process->is_alive()); + mgr._reclaim_idle_pools(idle_since + std::chrono::seconds(3)); + EXPECT_FALSE(process->is_alive()); +} + +TEST_F(PythonServerTest, ExplicitMinusOneKeepsIdleProcessesAlive) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + meta.auto_suspend = -1; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now() + std::chrono::hours(24)); + EXPECT_TRUE(process->is_alive()); +} + +TEST_F(PythonServerTest, AutoSuspendWaitsForLastClientAndStartsANewIdlePeriod) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + meta.auto_suspend = 3; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + std::shared_ptr first; + std::shared_ptr second; + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr, &first).value(); + ASSERT_TRUE(mgr._get_or_create_process_pool(meta, version, nullptr, &second).has_value()); + auto future = std::chrono::steady_clock::now() + std::chrono::hours(24); + mgr._reclaim_idle_pools(future); + EXPECT_TRUE(process->is_alive()); + first.reset(); + mgr._reclaim_idle_pools(future); + EXPECT_TRUE(process->is_alive()); + second.reset(); + EXPECT_EQ(0, pool->active_clients); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::seconds(2)); + EXPECT_TRUE(process->is_alive()); + mgr._reclaim_idle_pools(pool->idle_since + std::chrono::seconds(3)); + EXPECT_FALSE(process->is_alive()); +} + +TEST_F(PythonServerTest, AutoSuspendClientCloseReleasesLeaseWithoutAWriter) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + meta.auto_suspend = 0; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + PythonClient client; + auto pool = + mgr._get_or_create_process_pool(meta, version, nullptr, &client._pool_lease).value(); + client._process = process; + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_TRUE(process->is_alive()); + ASSERT_TRUE(client.close().ok()); + EXPECT_EQ(0, pool->active_clients); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_FALSE(process->is_alive()); +} + +TEST_F(PythonServerTest, AutoSuspendInitializationFailureReleasesLease) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/missing_python"); + auto meta = test_meta(version); + meta.auto_suspend = 0; + std::shared_ptr client; + EXPECT_FALSE(mgr.get_client(meta, version, &client).ok()); + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr).value(); + EXPECT_EQ(0, pool->active_clients); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); +} + +TEST_F(PythonServerTest, AutoSuspendProtectsClientInitialization) { + setup_doris_home(); + auto python_path = create_fake_python_with_socket_creation(); + PythonVersion version("3.9.16", test_dir_, python_path); + PythonServerManager mgr; + auto meta = test_meta(version); + meta.auto_suspend = 0; + std::promise started; + std::promise resume; + auto resumed = resume.get_future().share(); + auto* sync_point = SyncPoint::get_instance(); + sync_point->set_call_back("PythonServerManager::fork:before_process_start", [&](auto&&) { + started.set_value(); + resumed.wait(); + }); + sync_point->enable_processing(); + Defer cleanup {[&]() { + sync_point->disable_processing(); + sync_point->clear_call_back("PythonServerManager::fork:before_process_start"); + }}; + auto opening = std::async(std::launch::async, [&]() { + std::shared_ptr client; + return mgr.get_client(meta, version, &client); + }); + EXPECT_EQ(std::future_status::ready, started.get_future().wait_for(std::chrono::seconds(5))); + auto pool = mgr._get_or_create_process_pool(meta, version, nullptr).value(); + mgr._reclaim_idle_pools(std::chrono::steady_clock::now() + std::chrono::hours(24)); + EXPECT_EQ(1, pool->active_clients); + EXPECT_NE(PythonServerManager::PoolState::STOPPED, pool->state); + resume.set_value(); + // Either an initialization error or client destruction must release usage. + (void)opening.get(); + EXPECT_EQ(0, pool->active_clients); +} + +TEST_F(PythonServerTest, AutoSuspendMonitorTerminatesAnIdleProcess) { + PythonServerManager mgr; + PythonVersion version("3.9.16", test_dir_, test_dir_ + "/unused_python"); + auto meta = test_meta(version); + meta.auto_suspend = 0; + auto process = create_sleep_process(); + mgr.set_process_pool_for_test(meta, version, {process}); + // A zero timeout becomes eligible immediately, but reclamation waits for the monitor scan. + std::this_thread::sleep_for(std::chrono::seconds(2)); + EXPECT_TRUE(process->is_alive()); + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(40); + while (process->is_alive() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + EXPECT_FALSE(process->is_alive()); + EXPECT_TRUE(mgr.process_pool_snapshot_for_test(meta.id).empty()); +} TEST_F(PythonServerTest, CheckAndRecreateProcessesRecreatesDeadProcess) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 3; + test_pool_size_ = 3; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); @@ -990,11 +1428,12 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesRecreatesDeadProcess) { dead_process->shutdown(); ASSERT_FALSE(dead_process->is_alive()); - mgr.set_process_pool_for_test(version, {alive_process, dead_process, nullptr}); + mgr.set_process_pool_for_test(test_meta(version), version, + {alive_process, dead_process, nullptr}); mgr.check_and_recreate_processes_for_test(); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(pool_snapshot.size(), 3); EXPECT_EQ(pool_snapshot[0], alive_process); @@ -1012,7 +1451,7 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsRepairingPool) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); @@ -1024,8 +1463,8 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsRepairingPool) { dead_process->shutdown(); ASSERT_FALSE(dead_process->is_alive()); - mgr.set_process_pool_for_test(version, {dead_process}); - auto pool_result = mgr._ensure_pool_initialized(version); + mgr.set_process_pool_for_test(test_meta(version), version, {dead_process}); + auto pool_result = mgr._ensure_pool_initialized(test_meta(version), version); ASSERT_TRUE(pool_result.has_value()) << pool_result.error().to_string(); { std::lock_guard lock(pool_result.value()->mutex); @@ -1034,7 +1473,7 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsRepairingPool) { mgr.check_and_recreate_processes_for_test(); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(pool_snapshot.size(), 1); ASSERT_NE(pool_snapshot[0], nullptr); EXPECT_FALSE(pool_snapshot[0]->is_alive()); @@ -1051,7 +1490,7 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsUninitializedPool) { setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 1; + test_pool_size_ = 1; PythonServerManager mgr; PythonVersion version("3.9.16", test_dir_, python_path); @@ -1063,11 +1502,11 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesSkipsUninitializedPool) { dead_process->shutdown(); ASSERT_FALSE(dead_process->is_alive()); - mgr.set_process_pool_for_test(version, {dead_process}, false); + mgr.set_process_pool_for_test(test_meta(version), version, {dead_process}, false); mgr.check_and_recreate_processes_for_test(); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(version).id); ASSERT_EQ(pool_snapshot.size(), 1); ASSERT_NE(pool_snapshot[0], nullptr); EXPECT_FALSE(pool_snapshot[0]->is_alive()); @@ -1080,7 +1519,7 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesKeepsDeadSlotsWhenRecreateFail setup_doris_home(); std::string python_path = create_fake_python_with_socket_creation("3.9.16"); - config::max_python_process_num = 2; + test_pool_size_ = 2; PythonServerManager mgr; PythonVersion live_version("3.9.16", test_dir_, python_path); @@ -1098,11 +1537,12 @@ TEST_F(PythonServerTest, CheckAndRecreateProcessesKeepsDeadSlotsWhenRecreateFail ASSERT_FALSE(dead_process_2->is_alive()); PythonVersion invalid_version("3.9.16", test_dir_, test_dir_ + "/bin/nonexistent_python"); - mgr.set_process_pool_for_test(invalid_version, {dead_process_1, dead_process_2}); + mgr.set_process_pool_for_test(test_meta(invalid_version), invalid_version, + {dead_process_1, dead_process_2}); mgr.check_and_recreate_processes_for_test(); - auto pool_snapshot = mgr.process_pool_snapshot_for_test(invalid_version); + auto pool_snapshot = mgr.process_pool_snapshot_for_test(test_meta(invalid_version).id); ASSERT_EQ(pool_snapshot.size(), 2); EXPECT_FALSE(pool_snapshot[0]->is_alive()); EXPECT_FALSE(pool_snapshot[1]->is_alive()); diff --git a/be/test/udf/python/python_udf_meta_test.cpp b/be/test/udf/python/python_udf_meta_test.cpp index 322963319eea0a..f8c1cbfe3093df 100644 --- a/be/test/udf/python/python_udf_meta_test.cpp +++ b/be/test/udf/python/python_udf_meta_test.cpp @@ -59,6 +59,12 @@ class PythonUDFMetaTest : public ::testing::Test { DataTypePtr nullable_double_; }; +TEST_F(PythonUDFMetaTest, DefaultResourceProperties) { + PythonUDFMeta meta; + EXPECT_EQ(3, meta.pool_size); + EXPECT_EQ(14400, meta.auto_suspend); +} + // ============================================================================ // PythonUDFMeta check() tests // ============================================================================ diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java index 93783a47725d73..4b590dd53495f5 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java @@ -41,6 +41,8 @@ */ public class Function implements Writable { + public static final long DEFAULT_AUTO_SUSPEND_SECONDS = 14400; + public enum NullableMode { // Whether output column is nullable is depend on the input column is nullable DEPEND_ON_ARGUMENT, @@ -112,6 +114,10 @@ public enum BinaryType { protected long expirationTime = 360; // default 6 hours; @SerializedName("rv") protected String runtimeVersion; + @SerializedName("ps") + protected int poolSize = 3; + @SerializedName("as") + protected long autoSuspend = DEFAULT_AUTO_SUSPEND_SECONDS; @SerializedName("fc") protected String functionCode; @SerializedName("vol") @@ -175,6 +181,8 @@ public Function(Function other) { this.isStaticLoad = other.isStaticLoad; this.expirationTime = other.expirationTime; this.runtimeVersion = other.runtimeVersion; + this.poolSize = other.poolSize; + this.autoSuspend = other.autoSuspend; this.functionCode = other.functionCode; this.volatility = other.getVolatility(); } @@ -288,6 +296,22 @@ public void setGlobal(boolean global) { isGlobal = global; } + public int getPoolSize() { + return poolSize; + } + + public void setPoolSize(int poolSize) { + this.poolSize = poolSize; + } + + public long getAutoSuspend() { + return autoSuspend; + } + + public void setAutoSuspend(long autoSuspend) { + this.autoSuspend = autoSuspend; + } + public String getRuntimeVersion() { return runtimeVersion; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index dcea067295c5d1..c3b4e47dd90d30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -322,13 +322,14 @@ private void eraseDatabase(long currentTimeMs, int keepNum) { // 2. erase each expired database one at a time for (Long dbId : expiredIds) { + Database db; writeLock(); try { RecycleDatabaseInfo dbInfo = idToDatabase.remove(dbId); if (dbInfo == null) { continue; } - Database db = dbInfo.getDb(); + db = dbInfo.getDb(); idToRecycleTime.remove(dbId); dbNameToIds.computeIfPresent(db.getFullName(), (k, v) -> { @@ -342,6 +343,7 @@ private void eraseDatabase(long currentTimeMs, int keepNum) { } finally { writeUnlock(); } + FunctionUtil.cleanUdfCache(db); } // 3. erase exceed number @@ -375,6 +377,7 @@ private void eraseDatabaseWithSameName(String dbName, long currentTimeMs, readUnlock(); } for (Long dbId : dbIdToErase) { + Database db; writeLock(); try { RecycleDatabaseInfo dbInfo = idToDatabase.get(dbId); @@ -384,6 +387,7 @@ private void eraseDatabaseWithSameName(String dbName, long currentTimeMs, if (!eraseAllTables(dbInfo)) { continue; } + db = dbInfo.getDb(); idToDatabase.remove(dbId); idToRecycleTime.remove(dbId); @@ -397,6 +401,7 @@ private void eraseDatabaseWithSameName(String dbName, long currentTimeMs, } finally { writeUnlock(); } + FunctionUtil.cleanUdfCache(db); } } @@ -1222,6 +1227,9 @@ public void eraseDatabaseInstantly(long dbId) throws DdlException { } finally { writeUnlock(); } + if (dbInfo != null) { + FunctionUtil.cleanUdfCache(dbInfo.getDb()); + } // 2. collect tables with same dbId List tableIdToErase = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java index 590a6582594e35..8db2644e331fa6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java @@ -84,6 +84,8 @@ public static String toSql(ScalarFunction fn, boolean ifNotExists) { boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE; sb.append(",\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\""); sb.append(",\n \"RUNTIME_VERSION\"=").append("\"" + Strings.nullToEmpty(fn.getRuntimeVersion()) + "\""); + sb.append(",\n \"POOL_SIZE\"=\"").append(fn.getPoolSize()).append("\""); + sb.append(",\n \"AUTO_SUSPEND\"=\"").append(fn.getAutoSuspend()).append("\""); appendExpirationTimeIfNeeded(sb, fn); sb.append(",\n \"VOLATILITY\"=").append("\"" + fn.getVolatility().toSql() + "\""); } else { @@ -173,6 +175,8 @@ public static String toSql(AggregateFunction fn, boolean ifNotExists) { sb.append("\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\","); sb.append("\n \"RUNTIME_VERSION\"=") .append("\"" + Strings.nullToEmpty(fn.getRuntimeVersion()) + "\","); + sb.append("\n \"POOL_SIZE\"=\"").append(fn.getPoolSize()).append("\","); + sb.append("\n \"AUTO_SUSPEND\"=\"").append(fn.getAutoSuspend()).append("\","); if (fn.getExpirationTime() != DEFAULT_EXPIRATION_TIME) { sb.append("\n \"EXPIRATION_TIME\"=").append("\"" + fn.getExpirationTime() + "\","); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java index 5c4ca6f8be9994..b0af83a6806ecd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java @@ -97,6 +97,8 @@ public static TFunction toThrift(ScalarFunction fn, Type realReturnType, Type[] tfn.setFunctionCode(fn.getFunctionCode()); } tfn.setRuntimeVersion(fn.getRuntimeVersion()); + tfn.setPoolSize(fn.getPoolSize()); + tfn.setAutoSuspend(fn.getAutoSuspend()); } if (fn.getDictFunction() != null) { tfn.setDictFunction(fn.getDictFunction()); @@ -144,6 +146,8 @@ public static TFunction toThrift(AggregateFunction fn, Type realReturnType, Type tfn.setFunctionCode(fn.getFunctionCode()); } tfn.setRuntimeVersion(fn.getRuntimeVersion()); + tfn.setPoolSize(fn.getPoolSize()); + tfn.setAutoSuspend(fn.getAutoSuspend()); } return tfn; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java index 76d454e9ed1b3a..b20e69fd2e90e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java @@ -30,6 +30,10 @@ import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdf; import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdtf; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.task.AgentBatchTask; +import org.apache.doris.task.AgentTaskExecutor; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.CleanUDFCacheTask; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -48,6 +52,40 @@ public class FunctionUtil { private static final Logger LOG = LogManager.getLogger(FunctionUtil.class); + public static void cleanUdfCache(Map functionSignatures) { + if (functionSignatures.isEmpty()) { + // No function generation was removed. Never fall back to signature-based cleanup, + // which could delete a newly created function's cache when an old task arrives late. + return; + } + AgentBatchTask batchTask = new AgentBatchTask(); + for (long backendId : Env.getCurrentSystemInfo().getAllBackendIds(false)) { + for (Map.Entry function : functionSignatures.entrySet()) { + CleanUDFCacheTask task = new CleanUDFCacheTask(backendId, function.getValue(), function.getKey(), true); + // DROP removes the metadata even if a backend is unavailable, so missed cleanup needs retries. + if (AgentTaskQueue.addTask(task)) { + batchTask.addTask(task); + } + LOG.info("clean udf cache in be {}, functionId {}", backendId, function.getKey()); + } + } + if (batchTask.getTaskNum() > 0) { + AgentTaskExecutor.submit(batchTask); + } + } + + public static void cleanUdfCache(Database db) { + Map functionSignatures; + db.readLock(); + try { + functionSignatures = db.getFunctions().stream() + .filter(function -> function.getBinaryType() == Function.BinaryType.PYTHON_UDF) + .collect(Collectors.toMap(Function::getId, Function::signatureString)); + } finally { + db.readUnlock(); + } + cleanUdfCache(functionSignatures); + } /** * @param function diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 1b879531b12acd..178c92e0fe14b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -478,6 +478,7 @@ public void replayCreateDb(Database db, String newDbName) { public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlException { LOG.info("begin drop database[{}], is force : {}", dbName, force); + Map functionSignatures = new HashMap<>(); // 1. check if database exists if (!tryLock(false)) { throw new DdlException("Failed to acquire catalog lock. Try again"); @@ -537,6 +538,11 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc Env.getCurrentRecycleBin().recycleDatabase(db, tableNames, tableIds, false, force, 0); recycleTime = Env.getCurrentRecycleBin().getRecycleTimeById(db.getId()); + if (force) { + functionSignatures = db.getFunctions().stream() + .filter(function -> function.getBinaryType() == Function.BinaryType.PYTHON_UDF) + .collect(Collectors.toMap(Function::getId, Function::signatureString)); + } } finally { db.writeUnlock(); } @@ -553,6 +559,8 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc unlock(); } + // Recoverable databases retain their function generations until the recycle bin erases them. + FunctionUtil.cleanUdfCache(functionSignatures); LOG.info("finish drop database[{}], is force : {}", dbName, force); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java index 14bba10acc53e3..e72d88890787ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java @@ -199,6 +199,9 @@ public TMasterResult finishTaskInternal(TFinishTaskRequest request) { case DROP: finishDropReplica(task); break; + case CLEAN_UDF_CACHE: + AgentTaskQueue.removeTask(backendId, taskType, signature); + break; case SCHEMA_CHANGE: case ROLLUP: throw new RuntimeException("Schema change and rollup job is not used any more," diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java index 39a27b72834d52..056d6ffa3092d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java @@ -66,6 +66,8 @@ public class PythonUdaf extends AggregateFunction implements ExplicitlyCastableS private final String checkSum; private final boolean isStaticLoad; private final long expirationTime; + private final int poolSize; + private final long autoSuspend; private final String runtimeVersion; private final String functionCode; @@ -80,7 +82,7 @@ public PythonUdaf(String name, long functionId, String dbName, Function.BinaryTy String initFn, String updateFn, String mergeFn, String serializeFn, String finalizeFn, String getValueFn, String removeFn, boolean isDistinct, String checkSum, boolean isStaticLoad, long expirationTime, - String runtimeVersion, String functionCode, Expression... args) { + String runtimeVersion, String functionCode, int poolSize, long autoSuspend, Expression... args) { super(name, isDistinct, args); this.dbName = dbName; this.functionId = functionId; @@ -102,6 +104,8 @@ public PythonUdaf(String name, long functionId, String dbName, Function.BinaryTy this.checkSum = checkSum; this.isStaticLoad = isStaticLoad; this.expirationTime = expirationTime; + this.poolSize = poolSize; + this.autoSuspend = autoSuspend; this.runtimeVersion = runtimeVersion; this.functionCode = functionCode; } @@ -135,7 +139,7 @@ public PythonUdaf withDistinctAndChildren(boolean isDistinct, List c return new PythonUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode, volatility, volatileIdentity, objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn, - isDistinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, + isDistinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @@ -150,7 +154,7 @@ public PythonUdaf withIgnoreUniqueId(boolean ignoreUniqueId) { return new PythonUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode, volatility, volatileIdentity.withIgnoreUniqueId(ignoreUniqueId), objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn, - distinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, + distinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @@ -162,7 +166,7 @@ public PythonUdaf withFreshVolatileIdentity() { return new PythonUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode, volatility, VolatileIdentity.newVolatileIdentity(), objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn, - distinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, + distinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @@ -223,7 +227,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca aggregate.isStaticLoad(), aggregate.getExpirationTime(), aggregate.getRuntimeVersion(), - aggregate.getFunctionCode(), + aggregate.getFunctionCode(), aggregate.getPoolSize(), aggregate.getAutoSuspend(), arguments); PythonUdafBuilder builder = new PythonUdafBuilder(udaf); @@ -261,6 +265,8 @@ public Function getCatalogFunction() { expr.setStaticLoad(isStaticLoad); expr.setExpirationTime(expirationTime); expr.setRuntimeVersion(runtimeVersion); + expr.setPoolSize(poolSize); + expr.setAutoSuspend(autoSuspend); expr.setFunctionCode(functionCode); expr.setVolatility(volatility); return expr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java index 8bf05a8a6afaea..3c939df341e121 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java @@ -60,6 +60,8 @@ public class PythonUdf extends ScalarFunction implements ExplicitlyCastableSigna private final String checkSum; private final boolean isStaticLoad; private final long expirationTime; + private final int poolSize; + private final long autoSuspend; private final String runtimeVersion; private final String functionCode; @@ -71,7 +73,7 @@ public PythonUdf(String name, long functionId, String dbName, Function.BinaryTyp NullableMode nullableMode, FunctionVolatility volatility, VolatileIdentity volatileIdentity, String objectFile, String symbol, String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, - String runtimeVersion, String functionCode, Expression... args) { + String runtimeVersion, String functionCode, int poolSize, long autoSuspend, Expression... args) { super(name, args); this.dbName = dbName; this.functionId = functionId; @@ -87,6 +89,8 @@ public PythonUdf(String name, long functionId, String dbName, Function.BinaryTyp this.checkSum = checkSum; this.isStaticLoad = isStaticLoad; this.expirationTime = expirationTime; + this.poolSize = poolSize; + this.autoSuspend = autoSuspend; this.runtimeVersion = runtimeVersion; this.functionCode = functionCode; } @@ -120,7 +124,7 @@ public PythonUdf withChildren(List children) { return new PythonUdf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, volatileIdentity, objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -134,7 +138,7 @@ public PythonUdf withIgnoreUniqueId(boolean ignoreUniqueId) { return new PythonUdf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, volatileIdentity.withIgnoreUniqueId(ignoreUniqueId), objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } /** Return a copy with a new per-call identity when this UDF is VOLATILE. */ @@ -146,7 +150,7 @@ public PythonUdf withFreshVolatileIdentity() { return new PythonUdf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, VolatileIdentity.newVolatileIdentity(), objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -190,7 +194,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca scalar.getCloseFnSymbol(), scalar.getChecksum(), scalar.isStaticLoad(), scalar.getExpirationTime(), scalar.getRuntimeVersion(), - scalar.getFunctionCode(), + scalar.getFunctionCode(), scalar.getPoolSize(), scalar.getAutoSuspend(), arguments); PythonUdfBuilder builder = new PythonUdfBuilder(udf); @@ -222,6 +226,8 @@ public Function getCatalogFunction() { expr.setStaticLoad(isStaticLoad); expr.setExpirationTime(expirationTime); expr.setRuntimeVersion(runtimeVersion); + expr.setPoolSize(poolSize); + expr.setAutoSuspend(autoSuspend); expr.setFunctionCode(functionCode); expr.setVolatility(volatility); return expr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java index 61aa694ff8ea60..1c275df3258250 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java @@ -60,6 +60,8 @@ public class PythonUdtf extends TableGeneratingFunction implements ExplicitlyCas private final String checkSum; private final boolean isStaticLoad; private final long expirationTime; + private final int poolSize; + private final long autoSuspend; private final String runtimeVersion; private final String functionCode; @@ -72,7 +74,7 @@ public PythonUdtf(String name, long functionId, String dbName, Function.BinaryTy String objectFile, String symbol, String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, - String runtimeVersion, String functionCode, Expression... args) { + String runtimeVersion, String functionCode, int poolSize, long autoSuspend, Expression... args) { super(name, args); this.dbName = dbName; this.functionId = functionId; @@ -88,6 +90,8 @@ public PythonUdtf(String name, long functionId, String dbName, Function.BinaryTy this.checkSum = checkSum; this.isStaticLoad = isStaticLoad; this.expirationTime = expirationTime; + this.poolSize = poolSize; + this.autoSuspend = autoSuspend; this.runtimeVersion = runtimeVersion; this.functionCode = functionCode; } @@ -101,7 +105,7 @@ public PythonUdtf withChildren(List children) { return new PythonUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, volatileIdentity, objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -115,7 +119,7 @@ public PythonUdtf withIgnoreUniqueId(boolean ignoreUniqueId) { return new PythonUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, volatileIdentity.withIgnoreUniqueId(ignoreUniqueId), objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -126,7 +130,7 @@ public PythonUdtf withFreshVolatileIdentity() { return new PythonUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode, volatility, VolatileIdentity.newVolatileIdentity(), objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, - runtimeVersion, functionCode, children.toArray(new Expression[0])); + runtimeVersion, functionCode, poolSize, autoSuspend, children.toArray(new Expression[0])); } @Override @@ -179,6 +183,8 @@ public Function getCatalogFunction() { expr.setExpirationTime(expirationTime); expr.setUDTFunction(true); expr.setRuntimeVersion(runtimeVersion); + expr.setPoolSize(poolSize); + expr.setAutoSuspend(autoSuspend); expr.setFunctionCode(functionCode); expr.setVolatility(volatility); return expr; @@ -218,7 +224,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca scalar.isStaticLoad(), scalar.getExpirationTime(), scalar.getRuntimeVersion(), - scalar.getFunctionCode(), + scalar.getFunctionCode(), scalar.getPoolSize(), scalar.getAutoSuspend(), arguments); PythonUdtfBuilder builder = new PythonUdtfBuilder(udtf); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java index d04d0c7010f8c6..80766e4bd2091c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java @@ -154,6 +154,8 @@ public class CreateFunctionCommand extends Command implements ForwardWithSync { public static final String IS_STATIC_LOAD = "static_load"; public static final String EXPIRATION_TIME = "expiration_time"; public static final String RUNTIME_VERSION = "runtime_version"; + public static final String POOL_SIZE = "pool_size"; + public static final String AUTO_SUSPEND = "auto_suspend"; public static final String VOLATILITY = "volatility"; private static final Pattern PYTHON_VERSION_PATTERN = Pattern.compile("^3\\.\\d{1,2}(?:\\.\\d{1,2})?$"); @@ -188,6 +190,8 @@ public class CreateFunctionCommand extends Command implements ForwardWithSync { // Keep IMMUTABLE as the UDAF/UDTF default for compatibility with previous behavior. private FunctionVolatility volatility = FunctionVolatility.IMMUTABLE; private String runtimeVersion; + private int poolSize = 3; + private long autoSuspend = Function.DEFAULT_AUTO_SUSPEND_SECONDS; private String functionCode; /** @@ -343,6 +347,12 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { } } if (binaryType == Function.BinaryType.JAVA_UDF) { + if (properties.containsKey(POOL_SIZE)) { + throw new AnalysisException("pool_size property only supports PYTHON_UDF"); + } + if (properties.containsKey(AUTO_SUSPEND)) { + throw new AnalysisException("auto_suspend property only supports PYTHON_UDF"); + } FunctionUtil.checkEnableJavaUdf(); checkUdfSupportedTypes(); volatility = analyzeVolatility(defaultVolatility()); @@ -378,6 +388,26 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { + "'3.X.X' or '3.XX.XX' (e.g. '3.10.2').", runtimeVersionString)); } runtimeVersion = runtimeVersionString; + if (properties.containsKey(AUTO_SUSPEND)) { + try { + autoSuspend = Long.parseLong(properties.get(AUTO_SUSPEND)); + } catch (NumberFormatException e) { + throw new AnalysisException("Python UDF auto_suspend must be an integer number of seconds"); + } + if (autoSuspend < -1) { + throw new AnalysisException("Python UDF auto_suspend must be -1 or a non-negative integer"); + } + } + if (properties.containsKey(POOL_SIZE)) { + try { + poolSize = Integer.parseInt(properties.get(POOL_SIZE)); + } catch (NumberFormatException e) { + throw new AnalysisException("Python UDF pool_size must be a positive integer"); + } + if (poolSize <= 0) { + throw new AnalysisException("Python UDF pool_size must be a positive integer"); + } + } } else if (properties.containsKey(VOLATILITY)) { throw new AnalysisException("volatility property only supports JAVA_UDF and PYTHON_UDF"); } @@ -551,6 +581,8 @@ private void analyzeUdtf() throws AnalysisException { function.setExpirationTime(expirationTime); function.setUDTFunction(true); function.setRuntimeVersion(runtimeVersion); + function.setPoolSize(poolSize); + function.setAutoSuspend(autoSuspend); function.setFunctionCode(functionCode); function.setVolatility(volatility); // Todo: maybe in create tables function, need register two function, one is @@ -626,6 +658,8 @@ private void analyzeUdaf() throws AnalysisException { function.setStaticLoad(isStaticLoad); function.setExpirationTime(expirationTime); function.setRuntimeVersion(runtimeVersion); + function.setPoolSize(poolSize); + function.setAutoSuspend(autoSuspend); function.setFunctionCode(functionCode); function.setVolatility(volatility); } @@ -664,6 +698,8 @@ private void analyzeUdf() throws AnalysisException { function.setStaticLoad(isStaticLoad); function.setExpirationTime(expirationTime); function.setRuntimeVersion(runtimeVersion); + function.setPoolSize(poolSize); + function.setAutoSuspend(autoSuspend); function.setFunctionCode(functionCode); function.setVolatility(volatility); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFunctionCommand.java index 35296b9c29834f..2767c7b6fb98b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropFunctionCommand.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSearchDesc; +import org.apache.doris.catalog.FunctionUtil; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -31,23 +32,17 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.system.Backend; -import org.apache.doris.task.AgentBatchTask; -import org.apache.doris.task.AgentTaskExecutor; -import org.apache.doris.task.CleanUDFCacheTask; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableMap; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * drop a alias or user defined function */ public class DropFunctionCommand extends Command implements ForwardWithSync { - private static final Logger LOG = LogManager.getLogger(DropFunctionCommand.class); private SetType setType; private final boolean ifExists; private final FunctionName functionName; @@ -89,25 +84,12 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { } functionIds = db.dropFunction(function, ifExists); } - if (functionIds.isEmpty()) { - // No function generation was removed. A signature-based cleanup task could arrive - // after a same-signature function is created and delete the new generation's cache. - return; - } - // BE will cache classload, when drop function, BE need clear cache - ImmutableMap backendsInfo = Env.getCurrentSystemInfo().getAllBackendsByAllCluster(); String functionSignature = getSignatureString(); - AgentBatchTask batchTask = new AgentBatchTask(); - for (Backend backend : backendsInfo.values()) { - for (long functionId : functionIds) { - CleanUDFCacheTask cleanUDFCacheTask = new CleanUDFCacheTask( - backend.getId(), functionSignature, functionId); - batchTask.addTask(cleanUDFCacheTask); - LOG.info("clean udf cache in be {}, beId {}, functionId {}", - backend.getHost(), backend.getId(), functionId); - } + Map functionSignatures = new HashMap<>(); + for (long functionId : functionIds) { + functionSignatures.put(functionId, functionSignature); } - AgentTaskExecutor.submit(batchTask); + FunctionUtil.cleanUdfCache(functionSignatures); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CleanUDFCacheTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CleanUDFCacheTask.java index 85871ca202b012..d0a8c2c928bf90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CleanUDFCacheTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CleanUDFCacheTask.java @@ -30,7 +30,13 @@ public class CleanUDFCacheTask extends AgentTask { private long functionId; public CleanUDFCacheTask(long backendId, String signature, long functionId) { - super(null, backendId, TTaskType.CLEAN_UDF_CACHE, -1, -1, -1, -1, -1, -1, -1); + this(backendId, signature, functionId, false); + } + + public CleanUDFCacheTask(long backendId, String signature, long functionId, boolean retry) { + // Use functionId to track retries; -1 means no execution result is expected. + super(null, backendId, TTaskType.CLEAN_UDF_CACHE, -1, -1, -1, -1, -1, + retry ? functionId : -1, -1); this.functionSignature = signature; this.functionId = functionId; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java index c7eeae3949e569..4febb6f0e69353 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java @@ -164,6 +164,7 @@ public void test() throws Exception { + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2');"; createFunction(defaultVolatileSql, ctx); Assertions.assertEquals(FunctionVolatility.VOLATILE, findFunction(db, "py_default").getVolatility()); + Assertions.assertEquals(3, findFunction(db, "py_default").getPoolSize()); String defaultImmutableUdafSql = "create aggregate function db1.py_agg_default(int) returns int " + "properties('type'='PYTHON_UDF', 'symbol'='Agg', 'runtime_version'='3.10.2');"; @@ -177,6 +178,118 @@ public void test() throws Exception { Assertions.assertEquals(FunctionVolatility.STABLE, findFunction(db, "py_table_stable").getVolatility()); } + @Test + public void testPoolSize() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database python_pool_db;"); + dorisAssert = new DorisAssert(ctx); + dorisAssert.useDatabase("python_pool_db"); + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("python_pool_db"); + for (String kind : new String[]{"", "aggregate ", "tables "}) { + String ret = kind.equals("tables ") ? "array" : "int"; + for (int requested : new int[]{3, 10000}) { + String name = (kind.isEmpty() ? "scalar_fn" : kind.trim() + "_fn") + requested; + createFunction("create " + kind + "function python_pool_db." + name + + "(int) returns " + ret + " properties('type'='PYTHON_UDF', " + + "'symbol'='evaluate', 'runtime_version'='3.10.2', 'pool_size'='" + requested + "');", ctx); + Function fn = findFunction(db, name); + Assertions.assertEquals(requested, fn.getPoolSize()); + Assertions.assertTrue(FunctionToSqlConverter.toSql(fn, false) + .contains("\"POOL_SIZE\"=\"" + requested + "\"")); + } + } + for (String invalid : new String[]{"0", "-1", "abc", "1.5", "2147483648"}) { + assertCreateFunctionAnalysisException(ctx, + "create function python_pool_db.invalid_fn(int) returns int " + + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', " + + "'runtime_version'='3.10.2', 'pool_size'='" + invalid + "');", + "pool_size must be a positive integer"); + } + } + + @Test + public void testJavaUdfRejectsPoolSize() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database java_pool_db;"); + createFunction("create function java_pool_db.java_default(timestamp_ns) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='org.apache.doris.catalog.TimestampNsUdf');", ctx); + assertCreateFunctionAnalysisException(ctx, + "create function java_pool_db.java_pool(timestamp_ns) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='org.apache.doris.catalog.TimestampNsUdf', " + + "'pool_size'='3');", + "pool_size property only supports PYTHON_UDF"); + } + + @Test + public void testAutoSuspend() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database python_auto_suspend_db;"); + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("python_auto_suspend_db"); + for (String kind : new String[]{"", "aggregate ", "tables "}) { + String prefix = kind.isEmpty() ? "scalar" : kind.trim(); + String ret = kind.equals("tables ") ? "array" : "int"; + String definition = "(int) returns " + ret + " properties('type'='PYTHON_UDF', " + + "'symbol'='evaluate', 'runtime_version'='3.10.2'"; + createFunction("create " + kind + "function python_auto_suspend_db." + prefix + "_default" + + definition + ");", ctx); + assertAutoSuspendMetadata(findFunction(db, prefix + "_default"), 14400); + if (kind.equals("tables ")) { + assertAutoSuspendMetadata(findFunction(db, prefix + "_default_outer"), 14400); + } + long[] values = {-1, 0, 30, 14400, Long.MAX_VALUE}; + for (int i = 0; i < values.length; i++) { + String name = prefix + "_explicit_" + i; + createFunction("create " + kind + "function python_auto_suspend_db." + name + + definition + ", 'auto_suspend'='" + values[i] + "');", ctx); + assertAutoSuspendMetadata(findFunction(db, name), values[i]); + if (kind.equals("tables ")) { + assertAutoSuspendMetadata(findFunction(db, name + "_outer"), values[i]); + } + } + } + } + + private void assertAutoSuspendMetadata(Function fn, long expected) { + Assertions.assertEquals(expected, fn.getAutoSuspend()); + Assertions.assertTrue(FunctionToSqlConverter.toSql(fn, false) + .contains("\"AUTO_SUSPEND\"=\"" + expected + "\"")); + Assertions.assertEquals(expected, FunctionToThriftConverter.toThrift( + fn, fn.getReturnType(), fn.getArgs(), new Boolean[]{true}).getAutoSuspend()); + } + + @Test + public void testJavaUdfRejectsAutoSuspend() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database java_auto_suspend_db;"); + createFunction("create function java_auto_suspend_db.java_default(timestamp_ns) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='org.apache.doris.catalog.TimestampNsUdf');", ctx); + assertCreateFunctionAnalysisException(ctx, + "create function java_auto_suspend_db.java_auto_suspend(timestamp_ns) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='org.apache.doris.catalog.TimestampNsUdf', " + + "'auto_suspend'='0');", + "auto_suspend property only supports PYTHON_UDF"); + } + + @Test + public void testAutoSuspendRejectsInvalidValues() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database invalid_auto_suspend_db;"); + for (String invalid : new String[]{"abc", "1.5", "9223372036854775808", "-9223372036854775809"}) { + assertCreateFunctionAnalysisException(ctx, + "create function invalid_auto_suspend_db.invalid_fn(int) returns int " + + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', " + + "'runtime_version'='3.10.2', 'auto_suspend'='" + invalid + "');", + "auto_suspend must be an integer number of seconds"); + } + for (String invalid : new String[]{"-2", Long.toString(Long.MIN_VALUE)}) { + assertCreateFunctionAnalysisException(ctx, + "create function invalid_auto_suspend_db.invalid_fn(int) returns int " + + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', " + + "'runtime_version'='3.10.2', 'auto_suspend'='" + invalid + "');", + "auto_suspend must be -1 or a non-negative integer"); + } + } + @Test public void testCreatePythonFunctionRejectsObjectTypes() throws Exception { ConnectContext ctx = UtFrameUtils.createDefaultCtx(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java index 26df84fbbe3885..2a7f1d5cbd4fae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java @@ -19,14 +19,27 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.RecoverDatabaseCommand; +import org.apache.doris.task.AgentBatchTask; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskExecutor; +import org.apache.doris.task.CleanUDFCacheTask; import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; public class DropDbTest extends TestWithFeService { @@ -41,6 +54,138 @@ protected void runBeforeAll() throws Exception { + "distributed by hash(k2) buckets 1 " + "properties('replication_num' = '1');"); } + @Test + public void testDropDbPreservesFunctionsForRecovery() throws Exception { + Database db = createDatabaseWithPythonFunctions("drop_function_cache_db"); + Set functionIds = functionIds(db); + try (MockedStatic executor = Mockito.mockStatic(AgentTaskExecutor.class)) { + dropDatabaseWithSql("drop database drop_function_cache_db"); + RecoverDatabaseCommand recover = (RecoverDatabaseCommand) new NereidsParser() + .parseSingle("recover database drop_function_cache_db"); + recover.run(connectContext, null); + Database recovered = Env.getCurrentInternalCatalog().getDbOrMetaException("drop_function_cache_db"); + Assertions.assertEquals(functionIds, functionIds(recovered)); + executor.verifyNoInteractions(); + } + } + + @Test + public void testForceDropDbSubmitsFunctionCacheCleanup() throws Exception { + Database db = createDatabaseWithPythonFunctions("force_drop_function_cache_db"); + // Java database cleanup is unchanged; only the new Python pools need this notification. + db.getFunctions().stream() + .filter(function -> function.getFunctionName().getFunction().equals("py_scalar")) + .findFirst().orElseThrow().setBinaryType(Function.BinaryType.JAVA_UDF); + try (MockedStatic executor = Mockito.mockStatic(AgentTaskExecutor.class)) { + dropDatabaseWithSql("drop database force_drop_function_cache_db force"); + assertFunctionCacheCleanupSubmitted(executor, db); + } + Assertions.assertNull(Env.getCurrentInternalCatalog().getDbNullable(db.getFullName())); + } + + @Test + public void testExpiredDbSubmitsFunctionCacheCleanup() throws Exception { + assertErasedDbSubmitsFunctionCacheCleanup("expired"); + } + + @Test + public void testSameNameErasedDbSubmitsFunctionCacheCleanup() throws Exception { + assertErasedDbSubmitsFunctionCacheCleanup("same_name"); + } + + @Test + public void testInstantlyErasedDbSubmitsFunctionCacheCleanup() throws Exception { + assertErasedDbSubmitsFunctionCacheCleanup("instantly"); + } + + @Test + public void testDropDbWithoutFunctionsDoesNotSubmitCacheCleanup() throws Exception { + createDatabaseWithSql("create database drop_empty_function_cache_db"); + try (MockedStatic executor = Mockito.mockStatic(AgentTaskExecutor.class)) { + dropDatabaseWithSql("drop database drop_empty_function_cache_db force"); + executor.verifyNoInteractions(); + } + } + + private void assertErasedDbSubmitsFunctionCacheCleanup(String kind) throws Exception { + Database db = createDatabaseWithPythonFunctions("erase_" + kind + "_function_cache_db"); + dropDatabaseWithSql("drop database " + db.getFullName()); + CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin(); + try (MockedStatic executor = Mockito.mockStatic(AgentTaskExecutor.class)) { + switch (kind) { + case "expired": + recycleBin.setRecycleTimeByIdForReplay(db.getId(), 0L); + Deencapsulation.invoke(recycleBin, "eraseDatabase", System.currentTimeMillis(), -1); + break; + case "same_name": + recycleBin.setRecycleTimeByIdForReplay(db.getId(), 1L); + Deencapsulation.invoke(recycleBin, "eraseDatabaseWithSameName", db.getFullName(), + System.currentTimeMillis(), 0, new ArrayList<>(List.of(db.getId()))); + break; + case "instantly": + recycleBin.eraseDatabaseInstantly(db.getId()); + break; + default: + throw new AssertionError(kind); + } + Assertions.assertFalse(recycleBin.isRecycleDatabase(db.getId())); + assertFunctionCacheCleanupSubmitted(executor, db); + } + } + + private Database createDatabaseWithPythonFunctions(String dbName) throws Exception { + createDatabaseWithSql("create database " + dbName); + createFunction("create function " + dbName + ".py_scalar(int) returns int properties(" + + "'type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2')"); + createFunction("create aggregate function " + dbName + ".py_agg(int) returns int properties(" + + "'type'='PYTHON_UDF', 'symbol'='Agg', 'runtime_version'='3.10.2')"); + createFunction("create tables function " + dbName + ".py_table(int) returns array properties(" + + "'type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2')"); + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(dbName); + Assertions.assertEquals(4, functionIds(db).size()); + Assertions.assertTrue(db.getFunctions().stream() + .anyMatch(function -> function.getFunctionName().getFunction().equals("py_table_outer"))); + return db; + } + + private Set functionIds(Database db) { + return db.getFunctions().stream().map(Function::getId).collect(Collectors.toSet()); + } + + private void assertFunctionCacheCleanupSubmitted(MockedStatic executor, Database db) + throws Exception { + Map functions = db.getFunctions().stream() + .filter(function -> function.getBinaryType() == Function.BinaryType.PYTHON_UDF) + .collect(Collectors.toMap(Function::getId, Function::signatureString)); + Set databaseFunctionIds = functionIds(db); + Set backendIds = Env.getCurrentSystemInfo().getAllBackendsByAllCluster().keySet(); + Assertions.assertFalse(backendIds.isEmpty()); + ArgumentCaptor submitted = ArgumentCaptor.forClass(AgentBatchTask.class); + executor.verify(() -> AgentTaskExecutor.submit(submitted.capture()), Mockito.atLeastOnce()); + Map> cleanedFunctions = new HashMap<>(); + int taskCount = 0; + for (AgentBatchTask batch : submitted.getAllValues()) { + for (AgentTask task : batch.getAllTasks()) { + if (task instanceof CleanUDFCacheTask) { + CleanUDFCacheTask cleanup = (CleanUDFCacheTask) task; + long functionId = cleanup.toThrift().getFunctionId(); + // Recycle-bin scans may also erase other databases in the same test environment. + if (databaseFunctionIds.contains(functionId)) { + Assertions.assertTrue(functions.containsKey(functionId), "Only Python pools need cleanup"); + cleanedFunctions.computeIfAbsent(task.getBackendId(), ignored -> new HashMap<>()) + .put(functionId, cleanup.toThrift().getFunctionSignature()); + taskCount++; + } + } + } + } + Assertions.assertEquals(backendIds, cleanedFunctions.keySet()); + for (Map cleaned : cleanedFunctions.values()) { + Assertions.assertEquals(functions, cleaned); + } + Assertions.assertEquals(backendIds.size() * functions.size(), taskCount); + } + @Test public void testNormalDropDb() throws Exception { Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test1"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java index 2aafc93c8ea661..3219e47e078ec5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToThriftConverterTest.java @@ -18,17 +18,67 @@ package org.apache.doris.catalog; import org.apache.doris.catalog.Function.BinaryType; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TAggregateFunction; import org.apache.doris.thrift.TFunction; import org.apache.doris.thrift.TFunctionBinaryType; +import com.google.gson.JsonObject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class FunctionToThriftConverterTest { + @Test + void testPoolSizeSurvivesCopyAndThrift() { + ScalarFunction fn = ScalarFunction.createUdf(BinaryType.PYTHON_UDF, + new FunctionName("db1", "python_fn"), new Type[]{Type.INT}, Type.INT, + false, null, "evaluate", null, null); + Assertions.assertEquals(3, fn.getPoolSize()); + fn.setPoolSize(2); + ScalarFunction copy = new ScalarFunction(fn); + Assertions.assertEquals(2, copy.getPoolSize()); + ScalarFunction restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(fn), ScalarFunction.class); + Assertions.assertEquals(2, restored.getPoolSize()); + TFunction result = FunctionToThriftConverter.toThrift( + copy, Type.INT, new Type[]{Type.INT}, new Boolean[]{true}); + Assertions.assertEquals(2, result.getPoolSize()); + } + // ======================== ScalarFunction Tests ======================== + @Test + void testAutoSuspendSurvivesCopySerializationAndThrift() { + ScalarFunction scalar = ScalarFunction.createUdf(BinaryType.PYTHON_UDF, + new FunctionName("db1", "python_fn"), new Type[]{Type.INT}, Type.INT, + false, null, "evaluate", null, null); + AggregateFunction aggregate = new AggregateFunction( + new FunctionName("db1", "python_agg"), new Type[]{Type.INT}, Type.INT, false, + Type.INT, null, "init", "update", "merge", null, null, null, null); + aggregate.setBinaryType(BinaryType.PYTHON_UDF); + for (Function fn : new Function[]{scalar, aggregate}) { + Assertions.assertEquals(14400, fn.getAutoSuspend()); + Assertions.assertEquals(14400, new TFunction().getAutoSuspend()); + JsonObject oldJson = GsonUtils.GSON.toJsonTree(fn).getAsJsonObject(); + oldJson.remove("as"); + Function oldRestored = GsonUtils.GSON.fromJson(oldJson, fn.getClass()); + Assertions.assertEquals(14400, oldRestored.getAutoSuspend()); + for (long value : new long[]{-1, 0, 30, 14400, Long.MAX_VALUE}) { + fn.setAutoSuspend(value); + Function copy = fn.clone(); + Assertions.assertEquals(value, copy.getAutoSuspend()); + JsonObject json = GsonUtils.GSON.toJsonTree(fn).getAsJsonObject(); + Assertions.assertEquals(value, json.get("as").getAsLong()); + Function restored = GsonUtils.GSON.fromJson(json, fn.getClass()); + Assertions.assertEquals(value, restored.getAutoSuspend()); + TFunction thrift = FunctionToThriftConverter.toThrift( + copy, Type.INT, new Type[]{Type.INT}, new Boolean[]{true}); + Assertions.assertTrue(thrift.isSetAutoSuspend()); + Assertions.assertEquals(value, thrift.getAutoSuspend()); + } + } + } + @Test void testScalarFunctionJavaUdf_symbolIsPopulated() { FunctionName name = new FunctionName("db1", "java_udf_fn"); @@ -41,6 +91,7 @@ void testScalarFunctionJavaUdf_symbolIsPopulated() { Assertions.assertTrue(result.isSetScalarFn()); Assertions.assertFalse(result.isSetAggregateFn()); + Assertions.assertFalse(result.isSetAutoSuspend()); Assertions.assertEquals("com.example.MyFn", result.getScalarFn().getSymbol()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java index 3c54a1acd18b5d..8a7e21911b28da 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java @@ -111,10 +111,47 @@ void testVolatilePythonUdafUsesUniqueIdentity() { Assertions.assertEquals(ignoredFirst, ignoredSecond); } + @Test + void testPoolSizeSurvivesExpressionCopies() { + PythonUdf udf = pythonUdf(FunctionVolatility.IMMUTABLE, VolatileIdentity.NON_VOLATILE); + PythonUdaf udaf = pythonUdaf(FunctionVolatility.IMMUTABLE); + PythonUdtf udtf = pythonUdtf(FunctionVolatility.IMMUTABLE); + Assertions.assertEquals(3, udf.withChildren(udf.children()).getCatalogFunction().getPoolSize()); + Assertions.assertEquals(3, udaf.withDistinctAndChildren(true, udaf.children()) + .getCatalogFunction().getPoolSize()); + Assertions.assertEquals(3, udtf.withChildren(udtf.children()).getCatalogFunction().getPoolSize()); + } + + @Test + void testAutoSuspendSurvivesExpressionCopies() { + for (long value : new long[]{-2, 0, 30}) { + PythonUdf udf = pythonUdf(FunctionVolatility.VOLATILE, + Udf.createVolatileIdentity(FunctionVolatility.VOLATILE), value); + PythonUdaf udaf = pythonUdaf(FunctionVolatility.VOLATILE, value); + PythonUdtf udtf = pythonUdtf(FunctionVolatility.VOLATILE, value); + for (Function fn : new Function[]{ + udf.withChildren(udf.children()).getCatalogFunction(), + udf.withIgnoreUniqueId(true).getCatalogFunction(), + udf.withFreshVolatileIdentity().getCatalogFunction(), + udaf.withDistinctAndChildren(true, udaf.children()).getCatalogFunction(), + udaf.withIgnoreUniqueId(true).getCatalogFunction(), + udaf.withFreshVolatileIdentity().getCatalogFunction(), + udtf.withChildren(udtf.children()).getCatalogFunction(), + udtf.withIgnoreUniqueId(true).getCatalogFunction(), + udtf.withFreshVolatileIdentity().getCatalogFunction()}) { + Assertions.assertEquals(value, fn.getAutoSuspend()); + } + } + } + private PythonUdf pythonUdf(FunctionVolatility volatility, VolatileIdentity volatileIdentity) { + return pythonUdf(volatility, volatileIdentity, -1); + } + + private PythonUdf pythonUdf(FunctionVolatility volatility, VolatileIdentity volatileIdentity, long autoSuspend) { return new PythonUdf("py_fn", 1, "db1", Function.BinaryType.PYTHON_UDF, signature(), NullableMode.ALWAYS_NULLABLE, volatility, volatileIdentity, - null, "evaluate", null, null, "", false, 360, "3.10.2", "", + null, "evaluate", null, null, "", false, 360, "3.10.2", "", 3, autoSuspend, new IntegerLiteral(1)); } @@ -125,16 +162,24 @@ private JavaUdf javaUdf(FunctionVolatility volatility, VolatileIdentity volatile } private PythonUdaf pythonUdaf(FunctionVolatility volatility) { + return pythonUdaf(volatility, -1); + } + + private PythonUdaf pythonUdaf(FunctionVolatility volatility, long autoSuspend) { return new PythonUdaf("py_agg", 1, "db1", Function.BinaryType.PYTHON_UDF, signature(), IntegerType.INSTANCE, NullableMode.ALWAYS_NULLABLE, volatility, Udf.createVolatileIdentity(volatility), null, "Agg", null, null, null, null, null, null, null, false, "", false, 360, - "3.10.2", "", new IntegerLiteral(1)); + "3.10.2", "", 3, autoSuspend, new IntegerLiteral(1)); } private PythonUdtf pythonUdtf(FunctionVolatility volatility) { + return pythonUdtf(volatility, -1); + } + + private PythonUdtf pythonUdtf(FunctionVolatility volatility, long autoSuspend) { return new PythonUdtf("py_table", 1, "db1", Function.BinaryType.PYTHON_UDF, signature(), NullableMode.ALWAYS_NULLABLE, volatility, Udf.createVolatileIdentity(volatility), - null, "evaluate", null, null, "", false, 360, "3.10.2", "", new IntegerLiteral(1)); + null, "evaluate", null, null, "", false, 360, "3.10.2", "", 3, autoSuspend, new IntegerLiteral(1)); } private FunctionSignature signature() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java index 11aeff8bad338e..eb6d590efbc18b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/CacheExpressionTest.java @@ -79,7 +79,7 @@ void testAllUdfVolatilitiesRemainExcluded() { "file:///udf.jar", "F", "", "", "", false, 360, slot); Expression pythonUdf = new PythonUdf("f", 2, "db", Function.BinaryType.PYTHON_UDF, signature, Function.NullableMode.DEPEND_ON_ARGUMENT, volatility, identity, - "", "f", "", "", "", false, 360, "3.12", "", slot); + "", "f", "", "", "", false, 360, "3.12", "", 1, -1, slot); Assertions.assertEquals(volatility == FunctionVolatility.IMMUTABLE, javaUdf.isDeterministic()); Assertions.assertTrue(uncacheable(javaUdf)); Assertions.assertTrue(uncacheable(pythonUdf)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java new file mode 100644 index 00000000000000..1d276b80a6bb5c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/task/CleanUDFCacheTaskTest.java @@ -0,0 +1,266 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.task; + +import org.apache.doris.analysis.SetType; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.FunctionName; +import org.apache.doris.catalog.FunctionSearchDesc; +import org.apache.doris.catalog.FunctionUtil; +import org.apache.doris.common.ClientPool; +import org.apache.doris.common.GenericPool; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.master.MasterImpl; +import org.apache.doris.master.ReportHandler; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.commands.DropFunctionCommand; +import org.apache.doris.nereids.trees.plans.commands.info.FunctionArgTypesInfo; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.BackendService; +import org.apache.doris.thrift.TAgentTaskRequest; +import org.apache.doris.thrift.TBackend; +import org.apache.doris.thrift.TFinishTaskRequest; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TStatus; +import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TTaskType; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class CleanUDFCacheTaskTest { + private static final long BACKEND_ID = 10000L; + private static final long FUNCTION_ID = 20000L; + private static final String HOST = "127.0.0.1"; + private static final int BE_PORT = 9060; + + private final List submissions = new ArrayList<>(); + private Backend backend; + private MasterImpl master; + private MockedStatic envMock; + private MockedStatic executorMock; + private MockedConstruction reportHandlerMock; + + @BeforeEach + public void setUp() { + MetricRepo.init(); + AgentTaskQueue.clearAllTasks(); + SystemInfoService systemInfo = new SystemInfoService(); + backend = new Backend(BACKEND_ID, HOST, 9050); + backend.setBePort(BE_PORT); + systemInfo.addBackend(backend); + envMock = Mockito.mockStatic(Env.class); + envMock.when(Env::getCurrentSystemInfo).thenReturn(systemInfo); + executorMock = Mockito.mockStatic(AgentTaskExecutor.class); + executorMock.when(() -> AgentTaskExecutor.submit(Mockito.any(AgentBatchTask.class))) + .thenAnswer(invocation -> { + submissions.add(invocation.getArgument(0)); + return null; + }); + reportHandlerMock = Mockito.mockConstruction(ReportHandler.class, + (mock, context) -> Mockito.doNothing().when(mock).start()); + master = new MasterImpl(); + } + + @AfterEach + public void tearDown() { + reportHandlerMock.close(); + executorMock.close(); + envMock.close(); + AgentTaskQueue.clearAllTasks(); + } + + @Test + public void unavailableBackendCleanupIsRetriedByTaskReportAfterRecovery() throws Exception { + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)")); + AgentTask queued = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID); + Assertions.assertNotNull(queued); + Assertions.assertFalse(backend.isAlive()); + Assertions.assertEquals(1, submissions.size()); + submissions.get(0).run(); // AgentBatchTask skips the unavailable backend. + Assertions.assertSame(queued, + AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + Assertions.assertEquals(1, queued.getFailedTimes()); + + backend.setAlive(true); + reportTasks(); + Assertions.assertEquals(2, submissions.size()); + Assertions.assertEquals(List.of(queued), submissions.get(1).getAllTasks()); + TAgentTaskRequest request = submissions.get(1).toAgentTaskRequest(queued); + Assertions.assertEquals(FUNCTION_ID, request.getSignature()); + Assertions.assertEquals(FUNCTION_ID, request.getCleanUdfCacheReq().getFunctionId()); + + Assertions.assertEquals(TStatusCode.OK, finish(FUNCTION_ID, TStatusCode.OK)); + Assertions.assertNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + } + + @Test + @SuppressWarnings("unchecked") + public void rpcSubmissionFailureKeepsCleanupForTheNextReport() throws Exception { + backend.setAlive(true); + GenericPool originalPool = ClientPool.backendPool; + GenericPool failingPool = Mockito.mock(GenericPool.class); + Mockito.when(failingPool.borrowObject(Mockito.any(TNetworkAddress.class))) + .thenThrow(new Exception("simulated submit RPC failure")); + ClientPool.backendPool = failingPool; + try { + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)")); + AgentTask queued = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID); + Assertions.assertNotNull(queued); + submissions.get(0).run(); + Mockito.verify(failingPool).borrowObject(Mockito.any(TNetworkAddress.class)); + Assertions.assertSame(queued, + AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + reportTasks(); + Assertions.assertEquals(List.of(queued), submissions.get(1).getAllTasks()); + } finally { + ClientPool.backendPool = originalPool; + } + } + + @Test + public void duplicateCleanupKeepsPendingGenerationAndDistinctIdsDoNotCollide() { + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)", FUNCTION_ID + 1, "sum(INT)")); + AgentTask first = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID); + AgentTask second = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID + 1); + Assertions.assertNotNull(first); + Assertions.assertNotNull(second); + Assertions.assertNotSame(first, second); + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)")); + Assertions.assertEquals(1, submissions.size()); + Assertions.assertSame(first, + AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + } + + @Test + public void failedAcknowledgementKeepsTaskAndSuccessOnlyRemovesItsGeneration() { + FunctionUtil.cleanUdfCache(Map.of(FUNCTION_ID, "sum(INT)", FUNCTION_ID + 1, "sum(INT)")); + AgentTask first = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID); + Assertions.assertNotNull(first); + finish(FUNCTION_ID, TStatusCode.INTERNAL_ERROR); + Assertions.assertSame(first, + AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + Assertions.assertEquals(1, first.getFailedTimes()); + Assertions.assertEquals(TStatusCode.OK, finish(FUNCTION_ID, TStatusCode.OK)); + Assertions.assertNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + Assertions.assertNotNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID + 1)); + // The task is already removed when the same execution result arrives again. + Assertions.assertEquals(TStatusCode.CANCELLED, finish(FUNCTION_ID, TStatusCode.OK)); + } + + @Test + public void recycledDatabaseSnapshotOnlyQueuesPythonFunctions() { + Database database = Mockito.mock(Database.class); + Function python = Mockito.mock(Function.class); + Mockito.when(python.getBinaryType()).thenReturn(Function.BinaryType.PYTHON_UDF); + Mockito.when(python.getId()).thenReturn(FUNCTION_ID); + Mockito.when(python.signatureString()).thenReturn("sum(INT)"); + Function java = Mockito.mock(Function.class); + Mockito.when(java.getBinaryType()).thenReturn(Function.BinaryType.JAVA_UDF); + Mockito.when(database.getFunctions()).thenReturn(List.of(python, java)); + FunctionUtil.cleanUdfCache(database); + Assertions.assertNotNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, FUNCTION_ID)); + Assertions.assertEquals(1, submissions.get(0).getTaskNum()); + Mockito.verify(database).readLock(); + Mockito.verify(database).readUnlock(); + } + + @Test + public void legacyTaskKeepsTheOneShotTaskSignature() { + CleanUDFCacheTask task = new CleanUDFCacheTask(BACKEND_ID, "sum(INT)", FUNCTION_ID); + Assertions.assertEquals(-1L, task.getSignature()); + Assertions.assertEquals(FUNCTION_ID, task.toThrift().getFunctionId()); + Assertions.assertEquals(-1L, new AgentBatchTask().toAgentTaskRequest(task).getSignature()); + } + + @Test + public void localDropRetriesAfterBackendRecovery() throws Exception { + dropAndRetry(SetType.DEFAULT); + } + + @Test + public void globalDropRetriesAfterBackendRecovery() throws Exception { + dropAndRetry(SetType.GLOBAL); + } + + private void dropAndRetry(SetType scope) throws Exception { + Env env = Mockito.mock(Env.class, Mockito.RETURNS_DEEP_STUBS); + envMock.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getAccessManager().checkGlobalPriv( + Mockito.nullable(ConnectContext.class), Mockito.eq(PrivPredicate.ADMIN))).thenReturn(true); + List functionIds = List.of(FUNCTION_ID, FUNCTION_ID + 1); + if (scope == SetType.GLOBAL) { + Mockito.when(env.getGlobalFunctionMgr().dropFunction( + Mockito.any(FunctionSearchDesc.class), Mockito.eq(false))).thenReturn(functionIds); + } else { + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + envMock.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + Database database = Mockito.mock(Database.class); + Mockito.when(catalog.getDbNullable("cleanup_db")).thenReturn(database); + Mockito.when(database.dropFunction(Mockito.any(FunctionSearchDesc.class), Mockito.eq(false))) + .thenReturn(functionIds); + } + DropFunctionCommand command = new DropFunctionCommand(scope, false, + new FunctionName("cleanup_db", "sum"), + new FunctionArgTypesInfo(List.of(IntegerType.INSTANCE), false)); + command.run(new ConnectContext(), null); + Assertions.assertEquals(1, submissions.size()); + submissions.get(0).run(); + for (long functionId : functionIds) { + AgentTask task = AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, functionId); + Assertions.assertNotNull(task); + Assertions.assertEquals(1, task.getFailedTimes()); + Assertions.assertEquals(functionId, ((CleanUDFCacheTask) task).toThrift().getFunctionId()); + } + backend.setAlive(true); + reportTasks(); + Assertions.assertEquals(2, submissions.get(1).getTaskNum()); + for (long functionId : functionIds) { + Assertions.assertEquals(TStatusCode.OK, finish(functionId, TStatusCode.OK)); + Assertions.assertNull(AgentTaskQueue.getTask(BACKEND_ID, TTaskType.CLEAN_UDF_CACHE, functionId)); + } + } + + private void reportTasks() throws Exception { + Method taskReport = ReportHandler.class.getDeclaredMethod("taskReport", long.class, Map.class, long.class); + taskReport.setAccessible(true); + taskReport.invoke(null, BACKEND_ID, Map.of(), 0L); + } + + private TStatusCode finish(long functionId, TStatusCode status) { + TFinishTaskRequest request = new TFinishTaskRequest(new TBackend(HOST, BE_PORT, 0), + TTaskType.CLEAN_UDF_CACHE, functionId, new TStatus(status)); + return master.finishTask(request).getStatus().getStatusCode(); + } +} diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index e18c66ec89125a..066a8c6912b751 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -420,6 +420,8 @@ struct TFunction { 17: optional TDictFunction dict_function 18: optional string runtime_version 19: optional string function_code + 20: optional i32 pool_size = 3 + 21: optional i64 auto_suspend = 14400 // idle timeout in seconds; -1 disables suspension } enum TJdbcOperation { diff --git a/regression-test/data/pythonudaf_p0/test_pythonudaf_auto_suspend.out b/regression-test/data/pythonudaf_p0/test_pythonudaf_auto_suspend.out new file mode 100644 index 00000000000000..c8f9c9befe9f5e --- /dev/null +++ b/regression-test/data/pythonudaf_p0/test_pythonudaf_auto_suspend.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !result_0_0 -- +6 + +-- !result_1_0 -- +6 + +-- !result_0_1 -- +6 + +-- !result_1_1 -- +6 + +-- !result_0_2 -- +6 + +-- !result_1_2 -- +6 + +-- !result_0_3 -- +6 + +-- !result_1_3 -- +6 + +-- !result_2 -- +6 + +-- !result_3 -- +6 + diff --git a/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out b/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out new file mode 100644 index 00000000000000..115891b1272788 --- /dev/null +++ b/regression-test/data/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_0 -- +6 + +-- !after_0 -- +7 + +-- !before_1 -- +6 + +-- !after_1 -- +7 + +-- !before_2 -- +6 + +-- !after_2 -- +7 + +-- !before_3 -- +6 + +-- !after_3 -- +7 + diff --git a/regression-test/data/pythonudf_p0/test_pythonudf_auto_suspend.out b/regression-test/data/pythonudf_p0/test_pythonudf_auto_suspend.out new file mode 100644 index 00000000000000..c8671ef89d9213 --- /dev/null +++ b/regression-test/data/pythonudf_p0/test_pythonudf_auto_suspend.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !result_0_0 -- +11 +12 +13 + +-- !result_1_0 -- +11 +12 +13 + +-- !result_0_1 -- +11 +12 +13 + +-- !result_1_1 -- +11 +12 +13 + +-- !result_0_2 -- +11 +12 +13 + +-- !result_1_2 -- +11 +12 +13 + +-- !result_0_3 -- +11 +12 +13 + +-- !result_1_3 -- +11 +12 +13 + diff --git a/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out b/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out new file mode 100644 index 00000000000000..778262b74cbbc5 --- /dev/null +++ b/regression-test/data/pythonudf_p0/test_pythonudf_database_cleanup_retry.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_0 -- +11 + +-- !after_0 -- +12 + +-- !before_1 -- +11 + +-- !after_1 -- +12 + +-- !before_2 -- +11 + +-- !after_2 -- +12 + +-- !before_3 -- +11 + +-- !after_3 -- +12 + diff --git a/regression-test/data/pythonudf_p0/test_pythonudf_native_module_cache.out b/regression-test/data/pythonudf_p0/test_pythonudf_native_module_cache.out new file mode 100644 index 00000000000000..4000b9b000e55e --- /dev/null +++ b/regression-test/data/pythonudf_p0/test_pythonudf_native_module_cache.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !lazy_import -- +1107 + +-- !concurrent_import -- +12 1107 1107 + +-- !repeated_import -- +1107 \N + diff --git a/regression-test/data/pythonudtf_p0/test_pythonudtf_auto_suspend.out b/regression-test/data/pythonudtf_p0/test_pythonudtf_auto_suspend.out new file mode 100644 index 00000000000000..009a7d1b26408f --- /dev/null +++ b/regression-test/data/pythonudtf_p0/test_pythonudtf_auto_suspend.out @@ -0,0 +1,65 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !result_0_0 -- +11 +12 +13 + +-- !result_1_0 -- +\N + +-- !result_2_0 -- +11 +12 +13 + +-- !result_3_0 -- +\N + +-- !result_0_1 -- +11 +12 +13 + +-- !result_1_1 -- +\N + +-- !result_2_1 -- +11 +12 +13 + +-- !result_3_1 -- +\N + +-- !result_0_2 -- +11 +12 +13 + +-- !result_1_2 -- +\N + +-- !result_2_2 -- +11 +12 +13 + +-- !result_3_2 -- +\N + +-- !result_0_3 -- +11 +12 +13 + +-- !result_1_3 -- +\N + +-- !result_2_3 -- +11 +12 +13 + +-- !result_3_3 -- +\N + diff --git a/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out b/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out new file mode 100644 index 00000000000000..0870b357b7fc48 --- /dev/null +++ b/regression-test/data/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_0 -- +10 +10 + +-- !after_0 -- +11 +11 + +-- !before_1 -- +10 +10 + +-- !after_1 -- +11 +11 + +-- !before_2 -- +10 +10 + +-- !after_2 -- +11 +11 + diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf b/regression-test/pipeline/cloud_p0/conf/be_custom.conf index 86b396dce3e824..1cdc9dc9c13592 100644 --- a/regression-test/pipeline/cloud_p0/conf/be_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/be_custom.conf @@ -65,7 +65,6 @@ fail_when_segment_rows_not_in_rowset_meta=true enable_python_udf_support=true python_env_mode=conda python_conda_root_path=/opt/miniforge3 -max_python_process_num=16 enable_cloud_make_rs_visible_on_be=true cloud_mow_sync_rowsets_when_load_txn_begin=false diff --git a/regression-test/pipeline/cloud_p1/conf/be_custom.conf b/regression-test/pipeline/cloud_p1/conf/be_custom.conf index 973b34376b6cfd..c70a36932d83cb 100644 --- a/regression-test/pipeline/cloud_p1/conf/be_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/be_custom.conf @@ -45,7 +45,6 @@ enable_segment_rows_check_core=true enable_python_udf_support=true python_env_mode=conda python_conda_root_path=/opt/miniconda3 -max_python_process_num=16 enable_cloud_make_rs_visible_on_be=true cloud_mow_sync_rowsets_when_load_txn_begin=false diff --git a/regression-test/pipeline/external/conf/be.conf b/regression-test/pipeline/external/conf/be.conf index 8300afb35939d5..b73e742feba87e 100644 --- a/regression-test/pipeline/external/conf/be.conf +++ b/regression-test/pipeline/external/conf/be.conf @@ -79,6 +79,5 @@ enable_graceful_exit_check=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/pipeline/nonConcurrent/conf/be.conf b/regression-test/pipeline/nonConcurrent/conf/be.conf index f0fc6ae7728f32..0c9549aaa0066f 100644 --- a/regression-test/pipeline/nonConcurrent/conf/be.conf +++ b/regression-test/pipeline/nonConcurrent/conf/be.conf @@ -99,6 +99,5 @@ enable_segment_rows_check_core=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf index d7145b68494773..b2adc781a0f4b8 100644 --- a/regression-test/pipeline/p0/conf/be.conf +++ b/regression-test/pipeline/p0/conf/be.conf @@ -102,6 +102,5 @@ fail_when_segment_rows_not_in_rowset_meta=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/pipeline/p1/conf/be.conf b/regression-test/pipeline/p1/conf/be.conf index 74bcff45aaf27e..02f7e183b27d54 100644 --- a/regression-test/pipeline/p1/conf/be.conf +++ b/regression-test/pipeline/p1/conf/be.conf @@ -85,6 +85,5 @@ enable_segment_rows_check_core=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/pipeline/performance/conf/be_custom.conf b/regression-test/pipeline/performance/conf/be_custom.conf index 3e2ba2aac4cc3f..e2565519772507 100644 --- a/regression-test/pipeline/performance/conf/be_custom.conf +++ b/regression-test/pipeline/performance/conf/be_custom.conf @@ -30,6 +30,5 @@ enable_graceful_exit_check=true enable_python_udf_support = true python_env_mode = venv python_venv_interpreter_paths = /usr/bin/python -max_python_process_num = 64 enable_cloud_random_segment_id=true \ No newline at end of file diff --git a/regression-test/pipeline/vault_p0/conf/be_custom.conf b/regression-test/pipeline/vault_p0/conf/be_custom.conf index ccfdac33c885cb..a5e682d04a7efe 100644 --- a/regression-test/pipeline/vault_p0/conf/be_custom.conf +++ b/regression-test/pipeline/vault_p0/conf/be_custom.conf @@ -45,6 +45,5 @@ enable_brpc_connection_check=true enable_python_udf_support=true python_env_mode=venv python_venv_interpreter_paths=/usr/bin/python -max_python_process_num=16 enable_cloud_random_segment_id=true diff --git a/regression-test/suites/pythonudaf_p0/test_python_raise_error_propagation.groovy b/regression-test/suites/pythonudaf_p0/test_python_raise_error_propagation.groovy index 4fadf5f4c2508f..589fa0d725782a 100644 --- a/regression-test/suites/pythonudaf_p0/test_python_raise_error_propagation.groovy +++ b/regression-test/suites/pythonudaf_p0/test_python_raise_error_propagation.groovy @@ -57,6 +57,8 @@ suite("test_python_raise_error_propagation") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -77,6 +79,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${udfPath}", "symbol" = "udf_errors.raise_in_module", "runtime_version" = "${runtime_version}", @@ -95,6 +99,8 @@ def evaluate(x): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineFinishErrorUDAF", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -130,6 +136,8 @@ class InlineFinishErrorUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineAccumulateErrorUDAF", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -166,6 +174,8 @@ class InlineAccumulateErrorUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineMergeErrorUDAF", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -202,6 +212,8 @@ class InlineMergeErrorUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${udafPath}", "symbol" = "udaf_errors.ModuleFinishErrorUDAF", "runtime_version" = "${runtime_version}", @@ -220,6 +232,8 @@ class InlineMergeErrorUDAF: RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "inline_raise_udtf", "runtime_version" = "${runtime_version}" ) AS \$\$ @@ -245,6 +259,8 @@ def inline_raise_udtf(x): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${udtfPath}", "symbol" = "pyudtf_module.exceptions_udtf.raise_in_module_udtf", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy new file mode 100644 index 00000000000000..5992a309d4c018 --- /dev/null +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_auto_suspend.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudaf_auto_suspend", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + sql "DROP TABLE IF EXISTS py_auto_suspend_input" + sql """CREATE TABLE py_auto_suspend_input(v INT) + DUPLICATE KEY(v) DISTRIBUTED BY HASH(v) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO py_auto_suspend_input VALUES (1), (2), (3)" + try { + // Check results before/after idling; BE unit tests separately verify process termination. + ["0", "1", "-1", null].eachWithIndex { timeout, index -> + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + def property = timeout == null ? "" : ", \"auto_suspend\"=\"${timeout}\"" + sql """CREATE AGGREGATE FUNCTION py_auto_suspend(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "symbol"="SumAgg", + "runtime_version"="${runtime}", "pool_size"="1" ${property}) + AS \$\$ +class SumAgg: + def __init__(self): + self.total = 0 + @property + def aggregate_state(self): + return self.total + def accumulate(self, v): + if v is not None: + self.total += v + def merge(self, other): + self.total += other + def finish(self): + return self.total +\$\$""" + "qt_result_0_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + Thread.sleep(2500) + "qt_result_1_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + // DROP now leaves a STOPPED pool until the monitor reclaims it. The next + // iteration recreates the same SQL name with a new function id and must still work. + sql "DROP FUNCTION py_auto_suspend(INT)" + test { + sql "SELECT py_auto_suspend(v) FROM py_auto_suspend_input" + exception "Can not found function" + } + } + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + def zip = "${context.file.parent}/udaf_scripts/python_udaf_drop_a/python_udaf_drop_test.zip" + scp_udf_file_to_all_be(zip) + scp_udf_file_to_all_fe(zip) + sql """CREATE AGGREGATE FUNCTION py_auto_suspend(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "symbol"="drop_udaf.SumAgg", + "file"="file://${zip}", "runtime_version"="${runtime}", + "pool_size"="1", "auto_suspend"="0")""" + qt_result_2("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + Thread.sleep(2500) + qt_result_3("SELECT py_auto_suspend(v) FROM py_auto_suspend_input") + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" + sql """CREATE AGGREGATE FUNCTION py_auto_suspend_pid(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "symbol"="PidAgg", + "runtime_version"="${runtime}", "pool_size"="1", "auto_suspend"="0") + AS \$\$ +import os +class PidAgg: + def __init__(self): + self.total = 0 + @property + def aggregate_state(self): + return self.total + def accumulate(self, v): + self.total += 1 + def merge(self, other): + self.total += other + def finish(self): + return os.getpid() +\$\$""" + def queryPid = { sql("SELECT py_auto_suspend_pid(v) FROM py_auto_suspend_input")[0][0] } + def firstPid = queryPid() + def nextPid = firstPid + def deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(120) + while (nextPid == firstPid && System.nanoTime() < deadline) { + Thread.sleep(2000) + nextPid = queryPid() + } + assertNotEquals(firstPid, nextPid) + } finally { + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" + } +} diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_basic.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_basic.groovy index 642fa2fabaa7f4..d07ac9970b0130 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_basic.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_basic.groovy @@ -60,6 +60,8 @@ suite("test_pythonudaf_basic") { "file"="file://${pyPath}", "symbol"="sum_int.SumInt", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -101,6 +103,8 @@ suite("test_pythonudaf_basic") { "file"="file://${pyPath}", "symbol"="sum_int.AvgDouble", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -148,6 +152,8 @@ suite("test_pythonudaf_basic") { "file"="file://${pyPath}", "symbol"="sum_int.SumInt", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_inline.groovy index 46c6f80ce4c91d..d026760619aec7 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_inline.groovy @@ -67,6 +67,8 @@ suite("test_pythonudaf_complex_aggregation_inline") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "VarianceUDAF", "runtime_version" = "${runtime_version}" ) @@ -115,6 +117,8 @@ class VarianceUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "StdDevUDAF", "runtime_version" = "${runtime_version}" ) @@ -165,6 +169,8 @@ class StdDevUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MedianUDAF", "runtime_version" = "${runtime_version}" ) @@ -210,6 +216,8 @@ class MedianUDAF: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CollectListUDAF", "runtime_version" = "${runtime_version}" ) @@ -250,6 +258,8 @@ class CollectListUDAF: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RangeUDAF", "runtime_version" = "${runtime_version}" ) @@ -299,6 +309,8 @@ class RangeUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "GeometricMeanUDAF", "runtime_version" = "${runtime_version}" ) @@ -344,6 +356,8 @@ class GeometricMeanUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WeightedAvgUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_module.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_module.groovy index 4d214c5b6c89b5..6e8d58ed516438 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_module.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_aggregation_module.groovy @@ -73,6 +73,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.VarianceUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -93,6 +95,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.StdDevUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -113,6 +117,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.MedianUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -133,6 +139,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.CollectListUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -153,6 +161,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.RangeUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -173,6 +183,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.GeometricMeanUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -193,6 +205,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.WeightedAvgUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -304,6 +318,8 @@ suite("test_pythonudaf_complex_aggregation_module") { "file" = "file://${pyPath}", "symbol" = "stats_udaf.VarianceUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_inline.groovy index 9f974917df764d..ca643c32e34d19 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_inline.groovy @@ -81,6 +81,8 @@ suite("test_pythonudaf_complex_state_objects_inline") { RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "UserProfileUDAF", "runtime_version" = "${runtime_version}" ) @@ -157,6 +159,8 @@ class UserProfileUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "ProductStatsUDAF", "runtime_version" = "${runtime_version}" ) @@ -247,6 +251,8 @@ class ProductStatsUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "TransactionTimelineUDAF", "runtime_version" = "${runtime_version}" ) @@ -303,6 +309,8 @@ class TransactionTimelineUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "UniqueTrackerUDAF", "runtime_version" = "${runtime_version}" ) @@ -357,6 +365,8 @@ class UniqueTrackerUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CategorySummaryUDAF", "runtime_version" = "${runtime_version}" ) @@ -439,6 +449,8 @@ class CategorySummaryUDAF: RETURNS VARCHAR PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "HierarchicalAggUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_module.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_module.groovy index d0d1edbee052b2..609f08750867ea 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_module.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_complex_state_objects_module.groovy @@ -81,6 +81,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.UserProfileUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -97,6 +99,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.ProductStatsUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -113,6 +117,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.TransactionTimelineUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -129,6 +135,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.UniqueTrackerUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -145,6 +153,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.CategorySummaryUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -161,6 +171,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.HierarchicalAggUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -326,6 +338,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.UserProfileUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -347,6 +361,8 @@ suite("test_pythonudaf_complex_state_objects_module") { "file" = "file://${pyPath}", "symbol" = "complex_state_udaf.UserProfileUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_concurrent.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_concurrent.groovy index 961fa6984126cb..6ed48c47fcfaaa 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_concurrent.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_concurrent.groovy @@ -58,6 +58,8 @@ suite("test_pythonudaf_concurrent") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumUDAF", "runtime_version" = "${runtime_version}" ) @@ -89,6 +91,8 @@ class SumUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountUDAF", "runtime_version" = "${runtime_version}" ) @@ -120,6 +124,8 @@ class CountUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -157,6 +163,8 @@ class AvgUDAF: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MaxUDAF", "runtime_version" = "${runtime_version}" ) @@ -191,6 +199,8 @@ class MaxUDAF: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MinUDAF", "runtime_version" = "${runtime_version}" ) @@ -225,6 +235,8 @@ class MinUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumDoubleUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_data_types.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_data_types.groovy index 614446a6b91169..0c7cedfc69b6f0 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_data_types.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_data_types.groovy @@ -63,6 +63,8 @@ suite("test_pythonudaf_data_types") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumTinyIntUDAF", "runtime_version" = "${runtime_version}" ) @@ -98,6 +100,8 @@ class SumTinyIntUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumSmallIntUDAF", "runtime_version" = "${runtime_version}" ) @@ -133,6 +137,8 @@ class SumSmallIntUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumBigIntUDAF", "runtime_version" = "${runtime_version}" ) @@ -168,6 +174,8 @@ class SumBigIntUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumFloatUDAF", "runtime_version" = "${runtime_version}" ) @@ -203,6 +211,8 @@ class SumFloatUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumDecimalUDAF", "runtime_version" = "${runtime_version}" ) @@ -238,6 +248,8 @@ class SumDecimalUDAF: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "ConcatStrUDAF", "runtime_version" = "${runtime_version}" ) @@ -276,6 +288,8 @@ class ConcatStrUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountTrueUDAF", "runtime_version" = "${runtime_version}" ) @@ -326,6 +340,8 @@ class CountTrueUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "IntToDoubleSumUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy new file mode 100644 index 00000000000000..9cc4b7d60d06b8 --- /dev/null +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_database_cleanup_retry.groovy @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudaf_database_cleanup_retry", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + def originalDb = sql("SELECT DATABASE()")[0][0] + def heartbeatPoint = "HeartbeatMgr.BackendHeartbeatHandler" + def zipA = "${context.file.parent}/udaf_scripts/python_udaf_drop_a/python_udaf_drop_test.zip" + def zipB = "${context.file.parent}/udaf_scripts/python_udaf_drop_b/python_udaf_drop_test.zip" + [zipA, zipB].each { zip -> + scp_udf_file_to_all_be(zip) + scp_udf_file_to_all_fe(zip) + } + def await = { String message, Closure ready -> + def deadline = System.currentTimeMillis() + 120000 + while (!ready()) { + if (System.currentTimeMillis() >= deadline) { + throw new IllegalStateException(message) + } + Thread.sleep(1000) + } + } + def cleanupTasks = { + sql_return_maparray("SHOW PROC '/tasks'").find { it.TaskType == "CLEAN_UDF_CACHE" } + } + def purgeDatabase = { db -> + sql "DROP DATABASE IF EXISTS ${db} FORCE" + // Remove leftover recycle-bin records before the next case. + def ids = sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .collect { it.DbId }.unique() + ids.each { id -> sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${id}" } + } + def createFunction = { zip, scope -> + sql """CREATE ${scope} AGGREGATE FUNCTION py_cleanup_retry(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "symbol"="drop_udaf.SumAgg", + "file"="file://${zip}", "runtime_version"="${runtime}", "pool_size"="1")""" + } + ["force", "recycle", "function", "global"].eachWithIndex { mode, index -> + def db = "py_udaf_cleanup_retry_${mode}" + def scope = mode == "global" ? "GLOBAL" : "" + try { + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } + purgeDatabase(db) + await("earlier database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + sql "CREATE DATABASE ${db}" + sql "USE ${db}" + def dbId = getDbId(db) + createFunction(zipA, scope) + "qt_before_${index}"("SELECT py_cleanup_retry(v) FROM (SELECT 1 v UNION ALL SELECT 2 UNION ALL SELECT 3) s") + + def beIds = sql_return_maparray("SHOW BACKENDS").collect { it.BackendId.toString() } + // Simulate heartbeat failures while Python processes keep running. + GetDebugPoint().enableDebugPointForAllFEs(heartbeatPoint, + [deadBeIds: beIds.join(","), timeout: "180"]) + await("backends were not marked unavailable", { + sql_return_maparray("SHOW BACKENDS").every { !it.Alive.toBoolean() } + }) + if (mode in ["function", "global"]) { + sql "DROP ${scope} FUNCTION py_cleanup_retry(INT)" + } else if (mode == "recycle") { + sql "USE ${originalDb}" + sql "DROP DATABASE ${db}" + // Recoverable databases retain their function pools. + assertEquals(0, cleanupTasks().TotalNum.toInteger()) + sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${dbId}" + assertTrue(sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .every { it.DbId.toString() != dbId.toString() }) + } else { + sql "USE ${originalDb}" + sql "DROP DATABASE ${db} FORCE" + } + await("function cleanup was lost while backend was unavailable", { + def tasks = cleanupTasks() + tasks.TotalNum.toInteger() == beIds.size() && tasks.FailedNum.toInteger() == beIds.size() + }) + beIds.each { id -> + def tasks = sql_return_maparray("SHOW PROC '/tasks/CLEAN_UDF_CACHE/${id}'") + assertEquals(1, tasks.size()) + assertTrue(tasks[0].TaskSignature.toLong() > 0) + } + + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + await("backends did not recover", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + // The task leaves the queue only after cleanup succeeds and its result is reported. + await("backend recovery did not complete function cleanup", { + cleanupTasks().TotalNum.toInteger() == 0 + }) + if (mode in ["force", "recycle"]) { + sql "CREATE DATABASE ${db}" + } + sql "USE ${db}" + createFunction(zipB, scope) + "qt_after_${index}"("SELECT py_cleanup_retry(v) FROM (SELECT 1 v UNION ALL SELECT 2 UNION ALL SELECT 3) s") + } finally { + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + sql "USE ${originalDb}" + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } + purgeDatabase(db) + await("backends did not recover during cleanup", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + await("final database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + } + } +} diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_drop.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_drop.groovy index 1745a14864d46c..9d67b8fb57f0c6 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_drop.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_drop.groovy @@ -55,6 +55,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { sql """ CREATE AGGREGATE FUNCTION py_drop_sum_once(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udaf.SumAgg", "runtime_version" = "${runtime_version}" @@ -74,6 +76,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { sql """ CREATE AGGREGATE FUNCTION py_drop_sum_a(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udaf.SumAgg", "runtime_version" = "${runtime_version}" @@ -82,6 +86,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { sql """ CREATE AGGREGATE FUNCTION py_drop_sum_b(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipB}", "symbol" = "drop_udaf.SumAgg", "runtime_version" = "${runtime_version}" @@ -109,6 +115,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { sql """ CREATE AGGREGATE FUNCTION py_drop_sum_reconnect(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udaf.SumAgg", "runtime_version" = "${runtime_version}" @@ -136,6 +144,8 @@ suite('test_pythonudaf_drop', "nonConcurrent") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineDropRecreateUdaf", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -165,6 +175,8 @@ class InlineDropRecreateUdaf: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "InlineDropRecreateUdaf", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_edge_cases.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_edge_cases.groovy index 9bd7ed3890a79f..9ce752e3084e35 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_edge_cases.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_edge_cases.groovy @@ -64,6 +64,8 @@ suite("test_pythonudaf_edge_cases") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumLargeUDAF", "runtime_version" = "${runtime_version}" ) @@ -102,6 +104,8 @@ class SumLargeUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MinValUDAF", "runtime_version" = "${runtime_version}" ) @@ -140,6 +144,8 @@ class MinValUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MaxValUDAF", "runtime_version" = "${runtime_version}" ) @@ -178,6 +184,8 @@ class MaxValUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountDistinctUDAF", "runtime_version" = "${runtime_version}" ) @@ -217,6 +225,8 @@ class CountDistinctUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "ProductUDAF", "runtime_version" = "${runtime_version}" ) @@ -260,6 +270,8 @@ class ProductUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AbsSumUDAF", "runtime_version" = "${runtime_version}" ) @@ -298,6 +310,8 @@ class AbsSumUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SafeAvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -342,6 +356,8 @@ class SafeAvgUDAF: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SignSummaryUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_forbidden_module.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_forbidden_module.groovy index 4892a249f61056..7d9d08b55a02a6 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_forbidden_module.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_forbidden_module.groovy @@ -50,6 +50,8 @@ suite("test_pythonudaf_forbidden_module") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "${forbiddenCase.symbol}", "runtime_version" = "${runtime_version}" @@ -68,6 +70,8 @@ suite("test_pythonudaf_forbidden_module") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "safepkg_udaf.pathlib.SafePathlibUDAF", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline.groovy index 4c6955ca99a937..99ca41a8785bff 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline.groovy @@ -54,6 +54,8 @@ suite("test_pythonudaf_inline") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumUDAF", "runtime_version" = "${runtime_version}" ) @@ -100,6 +102,8 @@ class SumUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -150,6 +154,8 @@ class AvgUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountUDAF", "runtime_version" = "${runtime_version}" ) @@ -194,6 +200,8 @@ class CountUDAF: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MaxUDAF", "runtime_version" = "${runtime_version}" ) @@ -288,6 +296,8 @@ class MaxUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumUDAF", "runtime_version" = "${runtime_version}" ) @@ -361,6 +371,8 @@ class SumUDAF: RETURNS LARGEINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumLargeIntUDAF", "runtime_version" = "${runtime_version}" ) @@ -404,6 +416,8 @@ class SumLargeIntUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountPrivateIPv4UDAF", "runtime_version" = "${runtime_version}" ) @@ -447,6 +461,8 @@ class CountPrivateIPv4UDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountLoopbackIPv6UDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline_simple.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline_simple.groovy index 90e216f2704deb..0199c4ceb70288 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline_simple.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_inline_simple.groovy @@ -49,6 +49,8 @@ suite("test_pythonudaf_inline_simple") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MySum", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_nested_query.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_nested_query.groovy index bdedf6e810c3fc..98c3b9dc8ae9b9 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_nested_query.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_nested_query.groovy @@ -86,6 +86,8 @@ suite("test_pythonudaf_nested_query") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "TotalRevenueUDAF", "runtime_version" = "${runtime_version}" ) @@ -117,6 +119,8 @@ class TotalRevenueUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "OrderCountUDAF", "runtime_version" = "${runtime_version}" ) @@ -148,6 +152,8 @@ class OrderCountUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgOrderValueUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_null_handling.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_null_handling.groovy index 36107e2b5c7513..8f8d19360f006d 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_null_handling.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_null_handling.groovy @@ -62,6 +62,8 @@ suite("test_pythonudaf_null_handling") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountNonNullUDAF", "runtime_version" = "${runtime_version}" ) @@ -99,6 +101,8 @@ class CountNonNullUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumNullUDAF", "runtime_version" = "${runtime_version}" ) @@ -140,6 +144,8 @@ class SumNullUDAF: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "FirstNonNullUDAF", "runtime_version" = "${runtime_version}" ) @@ -178,6 +184,8 @@ class FirstNonNullUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountNullUDAF", "runtime_version" = "${runtime_version}" ) @@ -215,6 +223,8 @@ class CountNullUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgCoalesceUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_object_types_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_object_types_inline.groovy index d37f5d33074a4a..8e49c74ad84292 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_object_types_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_object_types_inline.groovy @@ -24,6 +24,8 @@ suite("test_pythonudaf_object_types_inline") { RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -51,6 +53,8 @@ class Agg: RETURNS HLL PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -79,6 +83,8 @@ class Agg: INTERMEDIATE BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -107,6 +113,8 @@ class Agg: INTERMEDIATE BITMAP PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -134,6 +142,8 @@ class Agg: RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) @@ -161,6 +171,8 @@ class Agg: RETURNS STRUCT>> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Agg", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy index 9523d2c76784f8..446af2fdf2fce9 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_pkg_isolation.groovy @@ -43,17 +43,21 @@ suite('test_pythonudaf_pkg_isolation') { sql """ CREATE AGGREGATE FUNCTION py_pkg_a_sum_x(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_x.SumAgg", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ sql """ CREATE AGGREGATE FUNCTION py_pkg_b_sum_x(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_x.SumAgg", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -64,9 +68,11 @@ suite('test_pythonudaf_pkg_isolation') { sql """ CREATE AGGREGATE FUNCTION py_pkg_a_sum_y(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_y.SumAgg", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -77,9 +83,11 @@ suite('test_pythonudaf_pkg_isolation') { sql """ CREATE AGGREGATE FUNCTION py_pkg_b_sum_y(INT) RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_y.SumAgg", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_simple.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_simple.groovy index ab97a86dcafb6e..204433ccf64ef9 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_simple.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_simple.groovy @@ -53,6 +53,8 @@ suite("test_pythonudaf_simple") { "file"="file://${pyPath}", "symbol"="sum_int.SumInt", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_inline.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_inline.groovy index 3a329299850682..dc1deef0b0c73c 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_inline.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_inline.groovy @@ -67,6 +67,8 @@ suite("test_pythonudaf_window_advanced_inline") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MovingAvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -103,6 +105,8 @@ class MovingAvgUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WindowStdDevUDAF", "runtime_version" = "${runtime_version}" ) @@ -143,6 +147,8 @@ class WindowStdDevUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "LastValueUDAF", "runtime_version" = "${runtime_version}" ) @@ -177,6 +183,8 @@ class LastValueUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WindowMinUDAF", "runtime_version" = "${runtime_version}" ) @@ -210,6 +218,8 @@ class WindowMinUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WindowMaxUDAF", "runtime_version" = "${runtime_version}" ) @@ -429,6 +439,8 @@ class WindowMaxUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "Percentile50UDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_module.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_module.groovy index 90894a11af9044..8a9deb8daae432 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_module.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_advanced_module.groovy @@ -73,6 +73,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.MovingAvgUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -89,6 +91,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.WindowStdDevUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -105,6 +109,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.LastValueUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -121,6 +127,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.WindowMinUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -134,6 +142,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.WindowMaxUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -333,6 +343,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.Percentile50UDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -385,6 +397,8 @@ suite("test_pythonudaf_window_advanced_module") { "file" = "file://${pyPath}", "symbol" = "window_udaf.MovingAvgUDAF", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_functions.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_functions.groovy index 1159f906b0c8d4..9af547eca94eaa 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_functions.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudaf_window_functions.groovy @@ -71,6 +71,8 @@ suite("test_pythonudaf_window_functions") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningSumUDAF", "runtime_version" = "${runtime_version}" ) @@ -104,6 +106,8 @@ class RunningSumUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningCountUDAF", "runtime_version" = "${runtime_version}" ) @@ -137,6 +141,8 @@ class RunningCountUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningAvgUDAF", "runtime_version" = "${runtime_version}" ) @@ -407,6 +413,8 @@ class RunningAvgUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "WindowFirstUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudf_udaf_mixed.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudf_udaf_mixed.groovy index 10d9370d4cdf3c..8d20bc3fd15d6a 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudf_udaf_mixed.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudf_udaf_mixed.groovy @@ -58,6 +58,8 @@ suite("test_pythonudf_udaf_mixed") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "final_price", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -77,6 +79,8 @@ def final_price(price, discount): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "double_value", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -96,6 +100,8 @@ def double_value(x): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "get_prefix", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -117,6 +123,8 @@ def get_prefix(s): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "SumUDAF", "runtime_version" = "${runtime_version}" ) @@ -148,6 +156,8 @@ class SumUDAF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "CountUDAF", "runtime_version" = "${runtime_version}" ) @@ -179,6 +189,8 @@ class CountUDAF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "AvgDoubleUDAF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudaf_p0/test_pythonudwf_comprehensive.groovy b/regression-test/suites/pythonudaf_p0/test_pythonudwf_comprehensive.groovy index 970b7517574bbb..8ece5850564429 100644 --- a/regression-test/suites/pythonudaf_p0/test_pythonudwf_comprehensive.groovy +++ b/regression-test/suites/pythonudaf_p0/test_pythonudwf_comprehensive.groovy @@ -151,6 +151,8 @@ suite("test_pythonudwf_comprehensive") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningSumUDWF", "runtime_version" = "${runtime_version}" ) @@ -182,6 +184,8 @@ class RunningSumUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningAvgUDWF", "runtime_version" = "${runtime_version}" ) @@ -219,6 +223,8 @@ class RunningAvgUDWF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RunningCountUDWF", "runtime_version" = "${runtime_version}" ) @@ -250,6 +256,8 @@ class RunningCountUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MovingAvgUDWF", "runtime_version" = "${runtime_version}" ) @@ -284,6 +292,8 @@ class MovingAvgUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "StdDevUDWF", "runtime_version" = "${runtime_version}" ) @@ -322,6 +332,8 @@ class StdDevUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MinUDWF", "runtime_version" = "${runtime_version}" ) @@ -356,6 +368,8 @@ class MinUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "MaxUDWF", "runtime_version" = "${runtime_version}" ) @@ -390,6 +404,8 @@ class MaxUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "FirstValueUDWF", "runtime_version" = "${runtime_version}" ) @@ -426,6 +442,8 @@ class FirstValueUDWF: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "LastValueUDWF", "runtime_version" = "${runtime_version}" ) @@ -458,6 +476,8 @@ class LastValueUDWF: RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "RankUDWF", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udaf_complex.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udaf_complex.groovy index c7c3c87c3e512e..539c20a0d4506e 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udaf_complex.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udaf_complex.groovy @@ -70,6 +70,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.WeightedAverageUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -92,6 +94,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.SkewnessUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -114,6 +118,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.KurtosisUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -136,6 +142,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.CorrelationUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -158,6 +166,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.CovarianceUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -180,6 +190,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.RetentionRateUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -228,6 +240,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.ConversionFunnelUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -278,6 +292,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.GiniCoefficientUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -324,6 +340,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.HerfindahlIndexUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -372,6 +390,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.TrendSlopeUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -425,6 +445,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.VolatilityUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -473,6 +495,8 @@ suite("test_python_udaf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udaf.JsonArrayAggUDAF", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udf_business_logic.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udf_business_logic.groovy index e63e2014bfe8fc..171fd214ed8204 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udf_business_logic.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udf_business_logic.groovy @@ -37,6 +37,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_moving_average", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -56,6 +58,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_ewma", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -75,6 +79,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.detect_anomalies_zscore", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -94,6 +100,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_trend", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -134,6 +142,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_percentiles", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -153,6 +163,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_statistics", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -172,6 +184,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_correlation", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -194,6 +208,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_rfm_score", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -241,6 +257,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_ltv", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -262,6 +280,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_churn_probability", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -309,6 +329,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_npv", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -330,6 +352,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_irr", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -349,6 +373,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.calculate_sharpe_ratio", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -368,6 +394,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.normalize_values", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -389,6 +417,8 @@ suite("test_python_udf_business_logic") { "file" = "file://${pyPath}", "symbol" = "business_logic.bin_values", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udf_external_api.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udf_external_api.groovy index 931d52b99e1d8e..de94234d0c33b8 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udf_external_api.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udf_external_api.groovy @@ -30,28 +30,28 @@ suite("test_python_udf_external_api") { sql """ DROP FUNCTION IF EXISTS py_fetch_post_title(INT); """ sql """ CREATE FUNCTION py_fetch_post_title(INT) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_post_title", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_post_title", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ qt_fetch_post_title """ SELECT py_fetch_post_title(1); """ sql """ DROP FUNCTION IF EXISTS py_fetch_user_email(INT); """ sql """ CREATE FUNCTION py_fetch_user_email(INT) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_user_email", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_user_email", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ qt_fetch_user_email """ SELECT py_fetch_user_email(1); """ sql """ DROP FUNCTION IF EXISTS py_fetch_user_name(INT); """ sql """ CREATE FUNCTION py_fetch_user_name(INT) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_user_name", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_user_name", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ qt_fetch_user_name """ SELECT py_fetch_user_name(1); """ sql """ DROP FUNCTION IF EXISTS py_fetch_comments_count(INT); """ sql """ CREATE FUNCTION py_fetch_comments_count(INT) RETURNS INT - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_comments_count", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.fetch_comments_count", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ qt_fetch_comments_count """ SELECT py_fetch_comments_count(1); """ @@ -59,7 +59,7 @@ suite("test_python_udf_external_api") { sql """ DROP FUNCTION IF EXISTS py_get_ip_location(STRING); """ sql """ CREATE FUNCTION py_get_ip_location(STRING) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.get_ip_location", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.get_ip_location", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ def ip_result = sql """ SELECT py_get_ip_location('8.8.8.8') """ assertTrue(ip_result.size() == 1) @@ -68,7 +68,7 @@ suite("test_python_udf_external_api") { sql """ DROP FUNCTION IF EXISTS py_get_weather(DOUBLE, DOUBLE); """ sql """ CREATE FUNCTION py_get_weather(DOUBLE, DOUBLE) RETURNS STRING - PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.get_weather", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}"); + PROPERTIES ("file"="file://${pyPath}", "symbol"="external_api.get_weather", "type"="PYTHON_UDF", "runtime_version"="${runtime_version}", "pool_size"="1", "auto_suspend"="0"); """ def weather_result = sql """ SELECT py_get_weather(39.9042, 116.4074) """ assertTrue(weather_result.size() == 1) diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy index d2f6a804edc7d3..bb60f301df14a7 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udf_nlp_chinese.groovy @@ -37,6 +37,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.chinese_word_segment", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -78,6 +80,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.chinese_word_segment_json", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -100,6 +104,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.chinese_word_count", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -121,6 +127,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.extract_keywords_tfidf", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -159,6 +167,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.pos_tagging", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -181,6 +191,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.extract_nouns", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -203,6 +215,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.extract_verbs", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -225,6 +239,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.extract_all_entities", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -264,6 +280,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.chinese_text_similarity", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -305,6 +323,8 @@ suite("test_python_udf_nlp_chinese") { "file" = "file://${pyPath}", "symbol" = "nlp_chinese.split_sentences", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_complex_p0/test_python_udtf_complex.groovy b/regression-test/suites/pythonudf_complex_p0/test_python_udtf_complex.groovy index deeeb6ca64b28b..bd0e1b7bf7bef7 100644 --- a/regression-test/suites/pythonudf_complex_p0/test_python_udtf_complex.groovy +++ b/regression-test/suites/pythonudf_complex_p0/test_python_udtf_complex.groovy @@ -37,6 +37,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_date_range", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -77,6 +79,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_time_slots", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -99,6 +103,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_json_array", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -139,6 +145,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_json_object_keys", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -178,6 +186,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.flatten_nested_json", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -216,6 +226,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_histogram_bins", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -238,6 +250,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_quartile_distribution", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -260,6 +274,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_ngrams", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -298,6 +314,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_sentences", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -320,6 +338,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_order_items", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -359,6 +379,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_user_events", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ @@ -397,6 +419,8 @@ suite("test_python_udtf_complex") { "file" = "file://${pyPath}", "symbol" = "complex_udtf.expand_bbox_to_tiles", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertequal.groovy b/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertequal.groovy index 44cbaa9a6716fa..fa7fd1eaebe76c 100644 --- a/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertequal.groovy +++ b/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertequal.groovy @@ -43,6 +43,8 @@ suite("test_pythonudf_assertequal") { "file"="file://${pyPath}", "symbol"="assert_equal_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertlessthan.groovy b/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertlessthan.groovy index 19b85f25c0dc9c..1191f535bde45a 100644 --- a/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertlessthan.groovy +++ b/regression-test/suites/pythonudf_p0/sanity/test_pythonudf_assertlessthan.groovy @@ -44,6 +44,8 @@ suite("test_pythonudf_assertlessthan") { "file"="file://${pyPath}", "symbol"="assert_lessthan_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_python_arrow_convertor_timezone.groovy b/regression-test/suites/pythonudf_p0/test_python_arrow_convertor_timezone.groovy index 0112e61ebb5a33..840bb9171801df 100644 --- a/regression-test/suites/pythonudf_p0/test_python_arrow_convertor_timezone.groovy +++ b/regression-test/suites/pythonudf_p0/test_python_arrow_convertor_timezone.groovy @@ -32,7 +32,7 @@ suite("test_python_arrow_convertor_timezone") { sql "DROP FUNCTION IF EXISTS ${name}(DATETIME(6))" } sql """CREATE FUNCTION ${scalar}(DATETIME(6)) RETURNS STRING - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(value): return None if value is None else value.strftime('%Y-%m-%d %H:%M:%S.%f') @@ -40,13 +40,13 @@ def evaluate(value): // Declare the yielded string as the array element; ARRAY> emits a struct. sql """CREATE TABLES FUNCTION ${tableFunction}(DATETIME(6)) RETURNS ARRAY - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(value): yield (None if value is None else value.strftime('%Y-%m-%d %H:%M:%S.%f'),) \$\$""" sql """CREATE AGGREGATE FUNCTION ${aggregate}(DATETIME(6)) RETURNS DATETIME(6) - PROPERTIES("type"="PYTHON_UDF", "symbol"="Maximum", + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="Maximum", "runtime_version"="${runtimeVersion}") AS \$\$ class Maximum: def __init__(self): diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_aggregate.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_aggregate.groovy index 59940e4ed417f9..e214085ac24596 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_aggregate.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_aggregate.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_aggregate") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "volatility" = "immutable" @@ -120,6 +122,8 @@ def evaluate(score): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "volatility" = "immutable" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_always_nullable.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_always_nullable.groovy index 21589f0c53c0fc..0a48ea89b4cc89 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_always_nullable.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_always_nullable.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_always_nullable") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -52,6 +54,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "false", "runtime_version" = "${runtime_version}" @@ -75,6 +79,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "false", "runtime_version" = "${runtime_version}" @@ -152,6 +158,8 @@ def evaluate(x): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_array.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_array.groovy index c5b71db5e7fc50..d3edb4ab88f94b 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_array.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_array.groovy @@ -56,6 +56,8 @@ suite("test_pythonudf_array") { "file"="file://${pyPath}", "symbol"="array_int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -68,6 +70,8 @@ suite("test_pythonudf_array") { "file"="file://${pyPath}", "symbol"="array_return_array_int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -81,6 +85,8 @@ suite("test_pythonudf_array") { "file"="file://${pyPath}", "symbol"="array_return_array_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -93,6 +99,8 @@ suite("test_pythonudf_array") { "file"="file://${pyPath}", "symbol"="array_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy new file mode 100644 index 00000000000000..f69e1d28e5dc0c --- /dev/null +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_auto_suspend.groovy @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudf_auto_suspend", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + sql "DROP TABLE IF EXISTS py_auto_suspend_input" + sql """CREATE TABLE py_auto_suspend_input(v INT) + DUPLICATE KEY(v) DISTRIBUTED BY HASH(v) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO py_auto_suspend_input VALUES (1), (2), (3)" + try { + // Check results before/after idling; BE unit tests separately verify process termination. + ["0", "1", "-1", null].eachWithIndex { timeout, index -> + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + def property = timeout == null ? "" : ", \"auto_suspend\"=\"${timeout}\"" + sql """CREATE FUNCTION py_auto_suspend(INT) RETURNS INT + PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + "runtime_version"="${runtime}", "pool_size"="1" ${property}) + AS \$\$ +def evaluate(v): + return v + 10 if v is not None else None +\$\$""" + "qt_result_0_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input ORDER BY v") + Thread.sleep(2500) + "qt_result_1_${index}"("SELECT py_auto_suspend(v) FROM py_auto_suspend_input ORDER BY v") + // DROP now leaves a STOPPED pool until the monitor reclaims it. The next + // iteration recreates the same SQL name with a new function id and must still work. + sql "DROP FUNCTION py_auto_suspend(INT)" + test { + sql "SELECT py_auto_suspend(v) FROM py_auto_suspend_input" + exception "Can not found function" + } + } + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" + sql """CREATE FUNCTION py_auto_suspend_pid(INT) RETURNS BIGINT + PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + "runtime_version"="${runtime}", "pool_size"="1", "auto_suspend"="0") + AS \$\$ +import os +def evaluate(v): + return os.getpid() +\$\$""" + def queryPid = { sql("SELECT py_auto_suspend_pid(v) FROM py_auto_suspend_input")[0][0] } + def firstPid = queryPid() + def nextPid = firstPid + def deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(120) + while (nextPid == firstPid && System.nanoTime() < deadline) { + Thread.sleep(2000) + nextPid = queryPid() + } + assertNotEquals(firstPid, nextPid) + } finally { + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" + } +} diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_base_data_type.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_base_data_type.groovy index 32114872eabbb9..ff90d560ea6486 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_base_data_type.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_base_data_type.groovy @@ -66,6 +66,8 @@ CREATE FUNCTION row_to_csv_all( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "row_to_csv_all_impl", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -224,6 +226,8 @@ def row_to_csv_all_impl( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_data_type.row_to_csv_all_impl", "always_nullable" = "true", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_boolean.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_boolean.groovy index f3f14532dd53c6..19705a2fd7450a 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_boolean.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_boolean.groovy @@ -48,6 +48,8 @@ suite("test_pythonudf_boolean") { "file"="file://${pyPath}", "symbol"="boolean_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_complex_data_type.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_complex_data_type.groovy index 76206717135e97..33d79cf62a4631 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_complex_data_type.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_complex_data_type.groovy @@ -40,6 +40,8 @@ CREATE FUNCTION array_to_csv( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "array_to_csv_impl", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -117,6 +119,8 @@ def array_to_csv_impl(int_arr, str_arr, nested_arr): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file"="file://${pyPath}", "symbol" = "python_udf_array_type.array_to_csv_impl", "always_nullable" = "true", @@ -172,6 +176,8 @@ CREATE FUNCTION map_to_csv( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "map_to_csv_impl", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -237,6 +243,8 @@ def map_to_csv_impl(map1, map2): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file"="file://${pyPath}", "symbol" = "python_udf_map_type.map_to_csv_impl", "always_nullable" = "true", @@ -289,6 +297,8 @@ CREATE FUNCTION struct_to_csv( RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "struct_to_csv_impl", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -371,6 +381,8 @@ def struct_to_csv_impl(person, point): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file"="file://${pyPath}", "symbol" = "python_udf_struct_type.struct_to_csv_impl", "always_nullable" = "true", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_concurrent.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_concurrent.groovy index 2de5e2a9249662..8b1b97b08558f5 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_concurrent.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_concurrent.groovy @@ -56,6 +56,8 @@ suite("test_pythonudf_concurrent") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "add_int", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -75,6 +77,8 @@ def add_int(a, b): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "multiply_int", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -94,6 +98,8 @@ def multiply_int(a, b): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "divide_double", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -113,6 +119,8 @@ def divide_double(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "concat_str", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -132,6 +140,8 @@ def concat_str(s1, s2): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "str_len", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -186,6 +196,8 @@ def str_len(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "vec_add", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -204,6 +216,8 @@ def vec_add(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "vec_sub", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -222,6 +236,8 @@ def vec_sub(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "vec_max", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -287,6 +303,8 @@ def vec_max(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "int_to_str", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -342,4 +360,3 @@ def int_to_str(n): } } - diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_data_types.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_data_types.groovy index 6b15d9382bd953..0100f72186e1d8 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_data_types.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_data_types.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_data_types") { RETURNS TINYINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -47,6 +49,8 @@ def evaluate(x): RETURNS SMALLINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -67,6 +71,8 @@ def evaluate(x): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -87,6 +93,8 @@ def evaluate(x): RETURNS DECIMAL(10,2) PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -108,6 +116,8 @@ def evaluate(x): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -128,6 +138,8 @@ def evaluate(d): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy new file mode 100644 index 00000000000000..837f04393a6e72 --- /dev/null +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_database_cleanup_retry.groovy @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudf_database_cleanup_retry", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + def originalDb = sql("SELECT DATABASE()")[0][0] + def heartbeatPoint = "HeartbeatMgr.BackendHeartbeatHandler" + def zipA = "${context.file.parent}/udf_scripts/python_udf_drop_a/python_udf_drop_test.zip" + def zipB = "${context.file.parent}/udf_scripts/python_udf_drop_b/python_udf_drop_test.zip" + [zipA, zipB].each { zip -> + scp_udf_file_to_all_be(zip) + scp_udf_file_to_all_fe(zip) + } + def await = { String message, Closure ready -> + def deadline = System.currentTimeMillis() + 120000 + while (!ready()) { + if (System.currentTimeMillis() >= deadline) { + throw new IllegalStateException(message) + } + Thread.sleep(1000) + } + } + def cleanupTasks = { + sql_return_maparray("SHOW PROC '/tasks'").find { it.TaskType == "CLEAN_UDF_CACHE" } + } + def purgeDatabase = { db -> + sql "DROP DATABASE IF EXISTS ${db} FORCE" + // Remove leftover recycle-bin records before the next case. + def ids = sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .collect { it.DbId }.unique() + ids.each { id -> sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${id}" } + } + def createFunction = { zip, scope -> + sql """CREATE ${scope} FUNCTION py_cleanup_retry(INT) RETURNS INT + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "symbol"="drop_udf.evaluate", + "file"="file://${zip}", "runtime_version"="${runtime}", "pool_size"="1")""" + } + ["force", "recycle", "function", "global"].eachWithIndex { mode, index -> + def db = "py_udf_cleanup_retry_${mode}" + def scope = mode == "global" ? "GLOBAL" : "" + try { + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } + purgeDatabase(db) + await("earlier database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + sql "CREATE DATABASE ${db}" + sql "USE ${db}" + def dbId = getDbId(db) + createFunction(zipA, scope) + "qt_before_${index}"("SELECT py_cleanup_retry(10)") + + def beIds = sql_return_maparray("SHOW BACKENDS").collect { it.BackendId.toString() } + // Simulate heartbeat failures while Python processes keep running. + GetDebugPoint().enableDebugPointForAllFEs(heartbeatPoint, + [deadBeIds: beIds.join(","), timeout: "180"]) + await("backends were not marked unavailable", { + sql_return_maparray("SHOW BACKENDS").every { !it.Alive.toBoolean() } + }) + if (mode in ["function", "global"]) { + sql "DROP ${scope} FUNCTION py_cleanup_retry(INT)" + } else if (mode == "recycle") { + sql "USE ${originalDb}" + sql "DROP DATABASE ${db}" + // Recoverable databases retain their function pools. + assertEquals(0, cleanupTasks().TotalNum.toInteger()) + sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${dbId}" + assertTrue(sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .every { it.DbId.toString() != dbId.toString() }) + } else { + sql "USE ${originalDb}" + sql "DROP DATABASE ${db} FORCE" + } + await("function cleanup was lost while backend was unavailable", { + def tasks = cleanupTasks() + tasks.TotalNum.toInteger() == beIds.size() && tasks.FailedNum.toInteger() == beIds.size() + }) + beIds.each { id -> + def tasks = sql_return_maparray("SHOW PROC '/tasks/CLEAN_UDF_CACHE/${id}'") + assertEquals(1, tasks.size()) + assertTrue(tasks[0].TaskSignature.toLong() > 0) + } + + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + await("backends did not recover", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + // The task leaves the queue only after cleanup succeeds and its result is reported. + await("backend recovery did not complete function cleanup", { + cleanupTasks().TotalNum.toInteger() == 0 + }) + if (mode in ["force", "recycle"]) { + sql "CREATE DATABASE ${db}" + } + sql "USE ${db}" + createFunction(zipB, scope) + "qt_after_${index}"("SELECT py_cleanup_retry(10)") + } finally { + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + sql "USE ${originalDb}" + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } + purgeDatabase(db) + await("backends did not recover during cleanup", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + await("final database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + } + } +} diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_drop.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_drop.groovy index 8596855ebdb50b..c196a2d2bee8e4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_drop.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_drop.groovy @@ -55,6 +55,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { sql """ CREATE FUNCTION py_drop_once(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udf.evaluate", "runtime_version" = "${runtime_version}" @@ -74,6 +76,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { sql """ CREATE FUNCTION py_drop_a(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udf.evaluate", "runtime_version" = "${runtime_version}" @@ -82,6 +86,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { sql """ CREATE FUNCTION py_drop_b(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipB}", "symbol" = "drop_udf.evaluate", "runtime_version" = "${runtime_version}" @@ -109,6 +115,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { sql """ CREATE FUNCTION py_drop_reconnect(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udf.evaluate", "runtime_version" = "${runtime_version}" @@ -134,6 +142,8 @@ suite("test_pythonudf_drop", "nonConcurrent") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -154,6 +164,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -184,6 +196,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -213,6 +227,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_error_handling.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_error_handling.groovy index 9b4def35e9c478..5a1049ecc327b7 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_error_handling.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_error_handling.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_error_handling") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -52,6 +54,8 @@ def evaluate(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -77,6 +81,8 @@ def evaluate(s, index): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -103,6 +109,8 @@ def evaluate(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -163,6 +171,8 @@ def evaluate(arr, index): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "always_nullable" = "true", "runtime_version" = "${runtime_version}" @@ -186,6 +196,8 @@ def evaluate(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "module_only", "runtime_version" = "${runtime_version}" ) @@ -204,6 +216,8 @@ def evaluate(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = ".evaluate", "runtime_version" = "${runtime_version}" ) @@ -224,6 +238,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "nonexistent_func", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_file_protocol.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_file_protocol.groovy index 004737662a8ba2..b767e404e7fa55 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_file_protocol.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_file_protocol.groovy @@ -32,6 +32,8 @@ suite("test_pythonudf_file_protocol") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "int_test.evaluate", "runtime_version" = "${runtime_version}" @@ -47,6 +49,8 @@ suite("test_pythonudf_file_protocol") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "string_test.evaluate", "runtime_version" = "${runtime_version}" @@ -62,6 +66,8 @@ suite("test_pythonudf_file_protocol") { RETURNS FLOAT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "float_test.evaluate", "runtime_version" = "${runtime_version}", @@ -78,6 +84,8 @@ suite("test_pythonudf_file_protocol") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "boolean_test.evaluate", "runtime_version" = "${runtime_version}" @@ -127,6 +135,8 @@ suite("test_pythonudf_file_protocol") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${context.file.parent}/udf_scripts/missing_inline_precedence.zip", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_float.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_float.groovy index 2c61719cb57e51..6d91279b375cc4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_float.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_float.groovy @@ -53,6 +53,8 @@ suite("test_pythonudf_float") { "file"="file://${pyPath}", "symbol"="float_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}", "always_nullable" = "true", "volatility" = "immutable" @@ -73,6 +75,8 @@ suite("test_pythonudf_float") { "file"="file://${pyPath}", "symbol"="double_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}", "always_nullable" = "true" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_forbidden_module.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_forbidden_module.groovy index 2cc93b24ca0f88..065c049fd19512 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_forbidden_module.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_forbidden_module.groovy @@ -39,6 +39,8 @@ suite("test_pythonudf_forbidden_module") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "${forbiddenCase.symbol}", "runtime_version" = "${runtime_version}", @@ -58,6 +60,8 @@ suite("test_pythonudf_forbidden_module") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "safepkg_udf.logging.evaluate", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_global_function.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_global_function.groovy index ae57a0afcb6f3d..e924f4ef019bf4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_global_function.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_global_function.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_global_function") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -47,6 +49,8 @@ def evaluate(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -67,6 +71,8 @@ def evaluate(s): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -122,6 +128,8 @@ def evaluate(a, b): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_complex.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_complex.groovy index 8a84223221d80b..e68d0cf7932112 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_complex.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_complex.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_inline_complex") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -47,6 +49,8 @@ def evaluate(arr): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -67,6 +71,8 @@ def evaluate(s): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -90,6 +96,8 @@ def evaluate(val1, weight1, val2, weight2): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -110,6 +118,8 @@ def evaluate(first_name, last_name): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy index 339ceb47cb7ce4..41a5d74cde2209 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_scalar.groovy @@ -19,6 +19,16 @@ suite("test_pythonudf_inline_basic") { // Test basic Python UDF using Inline mode def runtime_version = getPythonUdfRuntimeVersion() + for (def invalidPoolSize : ["0", "-1", "abc", "1.5", "2147483648"]) { + test { + sql """CREATE FUNCTION py_invalid_pool_size(INT) RETURNS INT PROPERTIES ( + "type" = "PYTHON_UDF", "auto_suspend"="0", "symbol" = "evaluate", + "runtime_version" = "${runtime_version}", "pool_size" = "${invalidPoolSize}" + ) AS \$\$def evaluate(x): return x\$\$""" + exception "pool_size must be a positive integer" + } + } + try { // Test 1: Simple integer addition sql """ DROP FUNCTION IF EXISTS py_add(INT, INT); """ @@ -27,8 +37,10 @@ suite("test_pythonudf_inline_basic") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "2" ) AS \$\$ def evaluate(a, b): @@ -45,8 +57,10 @@ def evaluate(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) AS \$\$ def evaluate(s1, s2): @@ -66,8 +80,10 @@ def evaluate(s1, s2): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) AS \$\$ def evaluate(x): @@ -87,8 +103,10 @@ def evaluate(x): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) AS \$\$ def evaluate(num): @@ -109,8 +127,10 @@ def evaluate(num): RETURNS LARGEINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "symbol" = "evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) AS \$\$ def evaluate(val): diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_vector.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_vector.groovy index c5398227262aa5..1818e03fdba694 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_inline_vector.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_inline_vector.groovy @@ -54,6 +54,8 @@ suite("test_pythonudf_inline_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "add", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -83,6 +85,8 @@ def add(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "multiply", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -112,6 +116,8 @@ def multiply(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "concat", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -141,6 +147,8 @@ def concat(s1: pd.Series, s2: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "get_max", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -171,6 +179,8 @@ def get_max(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "sqrt", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -200,6 +210,8 @@ def sqrt(x: pd.Series) -> pd.Series: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "to_upper", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -228,6 +240,8 @@ def to_upper(s: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "weighted_sum", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -257,6 +271,8 @@ def weighted_sum(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "negate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -285,6 +301,8 @@ def negate(b: pd.Series) -> pd.Series: RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "greater", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -314,6 +332,8 @@ def greater(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "str_len", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -342,6 +362,8 @@ def str_len(s: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "fill_null", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -370,6 +392,8 @@ def fill_null(x: pd.Series) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "cumsum", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_int.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_int.groovy index 126f4e2bfa97fb..c9d20bd8dc6465 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_int.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_int.groovy @@ -58,6 +58,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -70,6 +72,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -82,6 +86,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -94,6 +100,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -105,6 +113,8 @@ suite("test_pythonudf_int") { "file"="file://${pyPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_ip.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_ip.groovy index 7c1221c6f1f9e3..b821c4b427a83e 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_ip.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_ip.groovy @@ -50,6 +50,8 @@ suite("test_pythonudf_ip") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -75,6 +77,8 @@ def evaluate(ip): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -101,6 +105,8 @@ def evaluate(ip): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -133,6 +139,8 @@ def evaluate(ip, cidr_str): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -164,6 +172,8 @@ def evaluate(cidr_str): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -195,6 +205,8 @@ def evaluate(cidr_str): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -222,6 +234,8 @@ def evaluate(ip): RETURNS IPv4 PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -250,6 +264,8 @@ def evaluate(ip_int): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -275,6 +291,8 @@ def evaluate(ip): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -301,6 +319,8 @@ def evaluate(ip): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_map.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_map.groovy index effa9ccecd326f..fd1d983ed3c0ab 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_map.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_map.groovy @@ -47,6 +47,8 @@ suite("test_pythonudf_map") { "file"="file://${pyPath}", "symbol"="map_int_int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ @@ -72,6 +74,8 @@ suite("test_pythonudf_map") { "file"="file://${pyPath}", "symbol"="map_string_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_mixed_params.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_mixed_params.groovy index ae849c90b2788c..ab2e58d84ff1d4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_mixed_params.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_mixed_params.groovy @@ -68,6 +68,8 @@ suite("test_pythonudf_mixed_params") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_multiply_constant", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -100,6 +102,8 @@ def py_vec_multiply_constant(values: pd.Series, multiplier: float) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_calc_total", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -136,6 +140,8 @@ def py_vec_calc_total(price: pd.Series, quantity: pd.Series, tax_rate: float) -> RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_apply_discount", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -170,6 +176,8 @@ def py_vec_apply_discount(price: pd.Series, discount_rate: pd.Series) -> pd.Seri RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_complex_calc", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -211,6 +219,8 @@ def py_vec_complex_calc(price: pd.Series, quantity: pd.Series, discount_rate: pd RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_add_prefix", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -244,6 +254,8 @@ def py_vec_add_prefix(categories: pd.Series, prefix: str) -> pd.Series: RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_add_int_inline", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -277,6 +289,8 @@ def py_vec_add_int_inline(quantities: pd.Series, bonus: int) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_conditional_discount", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -317,6 +331,8 @@ def py_vec_conditional_discount(price: pd.Series, discount_rate: pd.Series, thre RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_scale_and_add", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -352,6 +368,8 @@ def py_vec_scale_and_add(scale_factor: float, prices: pd.Series, quantities: pd. RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_alternating", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -394,6 +412,8 @@ def py_vec_alternating(prices: pd.Series, markup: float, quantities: pd.Series, RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_multi_scalar", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_module.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_module.groovy index 14835ebe17421c..c2088e9a6f4de0 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_module.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_module.groovy @@ -28,6 +28,8 @@ suite("test_pythonudf_module") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file"="file://${pyPath}", "symbol" = "python_udf_module_test.main.safe_ltv", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_module_advanced.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_module_advanced.groovy index 19d386e801a10a..4f0e6ce05d0794 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_module_advanced.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_module_advanced.groovy @@ -32,6 +32,8 @@ suite("test_pythonudf_module_advanced") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipPath}", "symbol" = "python_udf_module_test.main.safe_ltv", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_module_scalar.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_module_scalar.groovy index 51fc8dfe299a3a..566048b951e587 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_module_scalar.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_module_scalar.groovy @@ -66,6 +66,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.add_three_numbers", "runtime_version" = "${runtime_version}", @@ -89,6 +91,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.safe_divide_with_precision", "runtime_version" = "${runtime_version}", @@ -112,6 +116,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.calculate_discount_price", "runtime_version" = "${runtime_version}", @@ -136,6 +142,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.compound_interest", "runtime_version" = "${runtime_version}", @@ -159,6 +167,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.calculate_bmi", "runtime_version" = "${runtime_version}", @@ -182,6 +192,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.fibonacci", "runtime_version" = "${runtime_version}", @@ -206,6 +218,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.is_prime", "runtime_version" = "${runtime_version}", @@ -231,6 +245,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.gcd", "runtime_version" = "${runtime_version}", @@ -254,6 +270,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.lcm", "runtime_version" = "${runtime_version}", @@ -279,6 +297,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.reverse_string", "runtime_version" = "${runtime_version}", @@ -302,6 +322,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.count_vowels", "runtime_version" = "${runtime_version}", @@ -325,6 +347,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.count_words", "runtime_version" = "${runtime_version}", @@ -348,6 +372,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.capitalize_words", "runtime_version" = "${runtime_version}", @@ -371,6 +397,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.is_palindrome", "runtime_version" = "${runtime_version}", @@ -394,6 +422,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.string_similarity", "runtime_version" = "${runtime_version}", @@ -417,6 +447,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.mask_email", "runtime_version" = "${runtime_version}", @@ -440,6 +472,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.extract_domain", "runtime_version" = "${runtime_version}", @@ -463,6 +497,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.levenshtein_distance", "runtime_version" = "${runtime_version}", @@ -488,6 +524,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.days_between_dates", "runtime_version" = "${runtime_version}", @@ -511,6 +549,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.is_weekend", "runtime_version" = "${runtime_version}", @@ -534,6 +574,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.get_quarter", "runtime_version" = "${runtime_version}", @@ -557,6 +599,8 @@ suite("test_pythonudf_module_scalar") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.age_in_years", "runtime_version" = "${runtime_version}", @@ -581,6 +625,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.is_in_range", "runtime_version" = "${runtime_version}", @@ -604,6 +650,8 @@ suite("test_pythonudf_module_scalar") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.xor_operation", "runtime_version" = "${runtime_version}", @@ -629,6 +677,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.calculate_grade", "runtime_version" = "${runtime_version}", @@ -652,6 +702,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.categorize_age", "runtime_version" = "${runtime_version}", @@ -675,6 +727,8 @@ suite("test_pythonudf_module_scalar") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.calculate_tax", "runtime_version" = "${runtime_version}", @@ -698,6 +752,8 @@ suite("test_pythonudf_module_scalar") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_scalar_ops.truncate_string", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_module_vector.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_module_vector.groovy index 231c18243e0b50..438da1fd43b63c 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_module_vector.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_module_vector.groovy @@ -61,6 +61,8 @@ suite("test_pythonudf_module_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_add_with_constant", "runtime_version" = "${runtime_version}", @@ -85,6 +87,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_multiply_and_round", "runtime_version" = "${runtime_version}", @@ -109,6 +113,8 @@ suite("test_pythonudf_module_vector") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_string_concat_with_separator", "runtime_version" = "${runtime_version}", @@ -133,6 +139,8 @@ suite("test_pythonudf_module_vector") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_string_title_case", "runtime_version" = "${runtime_version}", @@ -156,6 +164,8 @@ suite("test_pythonudf_module_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_conditional_value", "runtime_version" = "${runtime_version}", @@ -180,6 +190,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_percentage_calculation", "runtime_version" = "${runtime_version}", @@ -204,6 +216,8 @@ suite("test_pythonudf_module_vector") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_is_in_range", "runtime_version" = "${runtime_version}", @@ -227,6 +241,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_safe_divide", "runtime_version" = "${runtime_version}", @@ -251,6 +267,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_exponential_decay", "runtime_version" = "${runtime_version}", @@ -275,6 +293,8 @@ suite("test_pythonudf_module_vector") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_string_extract_first_word", "runtime_version" = "${runtime_version}", @@ -298,6 +318,8 @@ suite("test_pythonudf_module_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_abs_difference", "runtime_version" = "${runtime_version}", @@ -322,6 +344,8 @@ suite("test_pythonudf_module_vector") { RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_power", "runtime_version" = "${runtime_version}", @@ -345,6 +369,8 @@ suite("test_pythonudf_module_vector") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_boolean_and", "runtime_version" = "${runtime_version}", @@ -369,6 +395,8 @@ suite("test_pythonudf_module_vector") { RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_boolean_or", "runtime_version" = "${runtime_version}", @@ -393,6 +421,8 @@ suite("test_pythonudf_module_vector") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "python_udf_vector_ops.vec_clip_values", "runtime_version" = "${runtime_version}", diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_multiline_inline.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_multiline_inline.groovy index cc59150367f891..dcc1977602b63d 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_multiline_inline.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_multiline_inline.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_multiline_inline") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -51,6 +53,8 @@ def evaluate(a, b): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -99,6 +103,8 @@ def evaluate(customer_type, amount, quantity): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -132,6 +138,8 @@ def evaluate(text): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy new file mode 100644 index 00000000000000..5deb0594f773f2 --- /dev/null +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_native_module_cache.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.CountDownLatch + +suite("test_pythonudf_native_module_cache") { + def runtime_version = getPythonUdfRuntimeVersion() + def pyPath = """${context.file.parent}/udf_scripts/pyudf.zip""" + scp_udf_file_to_all_be(pyPath) + scp_udf_file_to_all_fe(pyPath) + + sql """DROP FUNCTION IF EXISTS py_native_module_cache(INT)""" + sql """ + CREATE FUNCTION py_native_module_cache(INT) RETURNS INT PROPERTIES ( + "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "file" = "file://${pyPath}", + "symbol" = "native_module_cache_test.evaluate", + "runtime_version" = "${runtime_version}", + "pool_size" = "1" + ) + """ + sql """DROP TABLE IF EXISTS test_pythonudf_native_module_cache_results""" + sql """ + CREATE TABLE test_pythonudf_native_module_cache_results (value INT) + DISTRIBUTED BY HASH(value) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + + try { + // The first call fails if the module directory is removed before evaluate. + qt_lazy_import """SELECT py_native_module_cache(7)""" + def ready = new CountDownLatch(4) + def start = new CountDownLatch(1) + def workers = (0..<4).collect { worker -> + thread("native-module-cache-${worker}") { + ready.countDown() + start.await() + 3.times { + sql """INSERT INTO test_pythonudf_native_module_cache_results + SELECT py_native_module_cache(7)""" + } + } + } + ready.await() + start.countDown() + workers.each { it.get() } + // Every client must observe exactly one execution of module-level code. + qt_concurrent_import """SELECT COUNT(*), MIN(value), MAX(value) + FROM test_pythonudf_native_module_cache_results""" + qt_repeated_import """SELECT py_native_module_cache(7), py_native_module_cache(NULL)""" + } finally { + try_sql("DROP FUNCTION IF EXISTS py_native_module_cache(INT)") + } +} diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_nested_complex_type.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_nested_complex_type.groovy index ca62629237518b..3bc36906a7e322 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_nested_complex_type.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_nested_complex_type.groovy @@ -82,6 +82,8 @@ suite("test_pythonudf_nested_complex_type") { RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -198,6 +200,8 @@ def evaluate(array_map, map_array_map, struct_nested, map_struct_nested): RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -296,6 +300,8 @@ def evaluate(array_maps: list, map_array_maps: list, struct_nesteds: list, map_s RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -394,6 +400,8 @@ def evaluate(array_maps: pd.Series, map_array_maps: pd.Series, struct_nesteds: p RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_no_input.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_no_input.groovy index a0d78f4629dff0..0de051774c29c4 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_no_input.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_no_input.groovy @@ -27,6 +27,8 @@ suite("test_pythonudf_no_input") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_object_types_inline.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_object_types_inline.groovy index b141e6d503ea26..e02e0d11925993 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_object_types_inline.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_object_types_inline.groovy @@ -24,6 +24,8 @@ suite("test_pythonudf_object_types_inline") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -41,6 +43,8 @@ def evaluate(v): RETURNS HLL PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -58,6 +62,8 @@ def evaluate(v): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -75,6 +81,8 @@ def evaluate(v): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -92,6 +100,8 @@ def evaluate(v): RETURNS STRUCT>> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy index 0107b4b582a89d..93903d38170906 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_pkg_isolation.groovy @@ -32,17 +32,21 @@ suite("test_pythonudf_pkg_isolation") { sql """ CREATE FUNCTION py_pkg_a_mod_x(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_x.evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ sql """ CREATE FUNCTION py_pkg_b_mod_x(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_x.evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -53,9 +57,11 @@ suite("test_pythonudf_pkg_isolation") { sql """ CREATE FUNCTION py_pkg_a_mod_y(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_y.evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -66,9 +72,11 @@ suite("test_pythonudf_pkg_isolation") { sql """ CREATE FUNCTION py_pkg_b_mod_y(INT) RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_y.evaluate", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_ret_map.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_ret_map.groovy index 7ed7317210744e..b7db5ac57ddbca 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_ret_map.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_ret_map.groovy @@ -67,6 +67,8 @@ suite("test_pythonudf_ret_map") { "file"="file://${pyPath}", "symbol"="map_ret_int_int_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -79,6 +81,8 @@ suite("test_pythonudf_ret_map") { "file"="file://${pyPath}", "symbol"="map_ret_string_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -92,6 +96,8 @@ suite("test_pythonudf_ret_map") { "file"="file://${pyPath}", "symbol"="map_ret_int_double_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -104,6 +110,8 @@ suite("test_pythonudf_ret_map") { "file"="file://${pyPath}", "symbol"="map_int_double_ret_string_string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_schema_check.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_schema_check.groovy index 15e8620bd06a91..4da7f5795e9330 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_schema_check.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_schema_check.groovy @@ -62,6 +62,8 @@ suite("test_pythonudf_schema_check") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_add_int_sc", "runtime_version" = "${runtime_version}" ) @@ -106,6 +108,8 @@ def py_add_int_sc(a, b): RETURNS BIGINT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_add_bigint", "runtime_version" = "${runtime_version}" ) @@ -134,6 +138,8 @@ def py_add_bigint(a, b): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_add_double", "runtime_version" = "${runtime_version}" ) @@ -162,6 +168,8 @@ def py_add_double(a, b): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_sum_three", "runtime_version" = "${runtime_version}" ) @@ -191,6 +199,8 @@ def py_sum_three(a, b, c): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_multiply", "runtime_version" = "${runtime_version}", "vectorized" = "true" @@ -223,6 +233,8 @@ def py_vec_multiply(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_divide", "runtime_version" = "${runtime_version}", "vectorized" = "true" @@ -255,6 +267,8 @@ def py_vec_divide(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_calc", "runtime_version" = "${runtime_version}", "vectorized" = "true" @@ -287,6 +301,8 @@ def py_vec_calc(a: pd.Series, b: pd.Series) -> pd.Series: RETURNS STRING PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_string_upper", "runtime_version" = "${runtime_version}" ) @@ -314,6 +330,8 @@ def py_string_upper(s): RETURNS BOOLEAN PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_bool_not", "runtime_version" = "${runtime_version}" ) @@ -354,6 +372,8 @@ def py_bool_not(b): RETURNS DOUBLE PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_vec_scale", "runtime_version" = "${runtime_version}", "vectorized" = "true" @@ -411,6 +431,8 @@ def py_vec_scale(values: pd.Series, factor: float) -> pd.Series: RETURNS FLOAT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "py_add_float", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_string.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_string.groovy index 4a0498ecd85309..39f5b725d0625c 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_string.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_string.groovy @@ -61,6 +61,8 @@ suite("test_pythonudf_string") { "file"="file://${pyPath}", "symbol"="string_test.evaluate", "type"="PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_timestamp_ns.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_timestamp_ns.groovy index 64d4f8fcd06b03..8a6ce75d4b841c 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_timestamp_ns.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_timestamp_ns.groovy @@ -67,6 +67,8 @@ suite("test_pythonudf_timestamp_ns") { RETURNS TIMESTAMP_NS PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "always_nullable" = "true" @@ -82,6 +84,8 @@ def evaluate(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "always_nullable" = "true" @@ -97,6 +101,8 @@ def evaluate(value): RETURNS MAP PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "always_nullable" = "true" @@ -112,6 +118,8 @@ def evaluate(value): RETURNS STRUCT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "always_nullable" = "true" diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_uuid_binary.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_uuid_binary.groovy index f6b6d48e27d575..19d9e5c36c8d16 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_uuid_binary.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_uuid_binary.groovy @@ -44,7 +44,7 @@ suite("test_pythonudf_uuid_binary") { sql "DROP FUNCTION IF EXISTS ${definition[0]}(${definition[1]})" sql """CREATE FUNCTION ${definition[0]}(${definition[1]}) RETURNS ${definition[1]} - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def validate(value): @@ -72,7 +72,7 @@ def evaluate(value): sql "DROP FUNCTION IF EXISTS py_uuid_binary_list(UUID)" sql """CREATE FUNCTION py_uuid_binary_list(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def evaluate(values: list): @@ -83,7 +83,7 @@ def evaluate(values: list): \$\$""" sql "DROP FUNCTION IF EXISTS py_uuid_binary_series(UUID)" sql """CREATE FUNCTION py_uuid_binary_series(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import pandas as pd import uuid @@ -95,7 +95,7 @@ def evaluate(values: pd.Series): \$\$""" sql "DROP FUNCTION IF EXISTS py_uuid_binary_vector_array(ARRAY)" sql """CREATE FUNCTION py_uuid_binary_vector_array(ARRAY) RETURNS ARRAY - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def evaluate(rows: list): @@ -110,7 +110,7 @@ def evaluate(rows: list): py_uuid_binary_vector_array(a) FROM python_uuid_binary ORDER BY id""" sql "DROP FUNCTION IF EXISTS py_uuid_binary_broadcast(UUID)" sql """CREATE FUNCTION py_uuid_binary_broadcast(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def evaluate(values: list): @@ -119,7 +119,7 @@ def evaluate(values: list): qt_broadcast "SELECT id, py_uuid_binary_broadcast(u) FROM python_uuid_binary ORDER BY id" sql "DROP FUNCTION IF EXISTS py_uuid_binary_string(STRING)" sql """CREATE FUNCTION py_uuid_binary_string(STRING) RETURNS STRING - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(value): if value is not None and not isinstance(value, str): @@ -141,7 +141,7 @@ def evaluate(value): ["value", "values: list"].eachWithIndex { parameter, index -> sql "DROP FUNCTION IF EXISTS py_uuid_binary_not_nullable_${index}(UUID)" sql """CREATE FUNCTION py_uuid_binary_not_nullable_${index}(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "always_nullable"="false", + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "always_nullable"="false", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(${parameter}): return None @@ -155,7 +155,7 @@ def evaluate(${parameter}): def functionName = "py_uuid_binary_invalid_${index}" sql "DROP FUNCTION IF EXISTS ${functionName}(UUID)" sql """CREATE FUNCTION ${functionName}(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ def evaluate(value): return ${expression} @@ -167,7 +167,7 @@ def evaluate(value): } sql "DROP FUNCTION IF EXISTS py_uuid_binary_min(UUID)" sql """CREATE AGGREGATE FUNCTION py_uuid_binary_min(UUID) RETURNS UUID - PROPERTIES("type"="PYTHON_UDF", "symbol"="UuidMin", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="UuidMin", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid class UuidMin: @@ -192,7 +192,7 @@ class UuidMin: qt_udaf_null "SELECT py_uuid_binary_min(u) FROM python_uuid_binary WHERE id = 5" sql "DROP FUNCTION IF EXISTS py_uuid_binary_rows(UUID)" sql """CREATE TABLES FUNCTION py_uuid_binary_rows(UUID) RETURNS ARRAY - PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "pool_size"="1", "symbol"="evaluate", "runtime_version"="${runtimeVersion}") AS \$\$ import uuid def evaluate(value): diff --git a/regression-test/suites/pythonudf_p0/test_pythonudf_volatility.groovy b/regression-test/suites/pythonudf_p0/test_pythonudf_volatility.groovy index 691e21efedb6c8..df7c118271aa3a 100644 --- a/regression-test/suites/pythonudf_p0/test_pythonudf_volatility.groovy +++ b/regression-test/suites/pythonudf_p0/test_pythonudf_volatility.groovy @@ -54,6 +54,8 @@ suite("test_pythonudf_volatility") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "volatility" = "immutable", @@ -72,6 +74,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "volatility" = "stable" @@ -89,6 +93,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}", "volatility" = "volatile" @@ -106,6 +112,8 @@ def evaluate(x): RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtimeVersion}" ) diff --git a/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_helper.py b/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_helper.py new file mode 100644 index 00000000000000..290343e4f801ab --- /dev/null +++ b/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_helper.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +def add_offset(value): + return value + 100 diff --git a/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_test.py b/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_test.py new file mode 100644 index 00000000000000..07b3b02f1ef93f --- /dev/null +++ b/regression-test/suites/pythonudf_p0/udf_scripts/native_module_cache_test.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import builtins + + +# Keep the counter outside this module so an unintended reimport is observable. +builtins._doris_native_module_loads = getattr(builtins, "_doris_native_module_loads", 0) + 1 + + +def evaluate(value): + if value is None: + return None + # This dependency must remain importable after the loader returns. + from native_module_cache_helper import add_offset + return builtins._doris_native_module_loads * 1000 + add_offset(value) diff --git a/regression-test/suites/pythonudf_p0/udf_scripts/pyudf.zip b/regression-test/suites/pythonudf_p0/udf_scripts/pyudf.zip index 7a33bc3e20d22a..f5a576d2549016 100644 Binary files a/regression-test/suites/pythonudf_p0/udf_scripts/pyudf.zip and b/regression-test/suites/pythonudf_p0/udf_scripts/pyudf.zip differ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy new file mode 100644 index 00000000000000..3544b0b87216e8 --- /dev/null +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_auto_suspend.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudtf_auto_suspend", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + sql "DROP TABLE IF EXISTS py_auto_suspend_input" + sql """CREATE TABLE py_auto_suspend_input(v INT) + DUPLICATE KEY(v) DISTRIBUTED BY HASH(v) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql "INSERT INTO py_auto_suspend_input VALUES (1), (2), (3)" + try { + // Check results before/after idling; BE unit tests separately verify process termination. + ["0", "1", "-1", null].eachWithIndex { timeout, index -> + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + def property = timeout == null ? "" : ", \"auto_suspend\"=\"${timeout}\"" + sql """CREATE TABLES FUNCTION py_auto_suspend(INT) RETURNS ARRAY + PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + "runtime_version"="${runtime}", "pool_size"="1" ${property}) + AS \$\$ +def evaluate(v): + if v is not None: + yield (v + 10,) +\$\$""" + "qt_result_0_${index}"("SELECT x FROM py_auto_suspend_input LATERAL VIEW py_auto_suspend(v) t AS x ORDER BY x") + "qt_result_1_${index}"("SELECT x FROM (SELECT CAST(NULL AS INT) v) s LATERAL VIEW py_auto_suspend_outer(v) t AS x") + Thread.sleep(2500) + "qt_result_2_${index}"("SELECT x FROM py_auto_suspend_input LATERAL VIEW py_auto_suspend(v) t AS x ORDER BY x") + "qt_result_3_${index}"("SELECT x FROM (SELECT CAST(NULL AS INT) v) s LATERAL VIEW py_auto_suspend_outer(v) t AS x") + // DROP now leaves a STOPPED pool until the monitor reclaims it. The next + // iteration recreates the same SQL name with a new function id and must still work. + sql "DROP FUNCTION py_auto_suspend(INT)" + test { + sql "SELECT x FROM py_auto_suspend_input LATERAL VIEW py_auto_suspend(v) t AS x" + exception "Can not found function" + } + } + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" + sql """CREATE TABLES FUNCTION py_auto_suspend_pid(INT) RETURNS ARRAY + PROPERTIES("type"="PYTHON_UDF", "symbol"="evaluate", + "runtime_version"="${runtime}", "pool_size"="1", "auto_suspend"="0") + AS \$\$ +import os +def evaluate(v): + yield (os.getpid(),) +\$\$""" + def queryPid = { + sql("SELECT x FROM py_auto_suspend_input LATERAL VIEW py_auto_suspend_pid(v) t AS x")[0][0] + } + def firstPid = queryPid() + def nextPid = firstPid + def deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(120) + while (nextPid == firstPid && System.nanoTime() < deadline) { + Thread.sleep(2000) + nextPid = queryPid() + } + assertNotEquals(firstPid, nextPid) + } finally { + sql "DROP FUNCTION IF EXISTS py_auto_suspend(INT)" + sql "DROP FUNCTION IF EXISTS py_auto_suspend_pid(INT)" + } +} diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_inline.groovy index b99c52dfacd459..3e9815e1e31d67 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_inline.groovy @@ -33,6 +33,8 @@ suite("test_pythonudtf_basic_inline") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "split_string_udtf", "runtime_version" = "${runtime_version}" ) @@ -78,6 +80,8 @@ def split_string_udtf(input_str): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "generate_series_udtf", "runtime_version" = "${runtime_version}" ) @@ -130,6 +134,8 @@ def generate_series_udtf(start, end): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "running_sum_udtf", "runtime_version" = "${runtime_version}" ) @@ -179,6 +185,8 @@ def running_sum_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "explode_json_udtf", "runtime_version" = "${runtime_version}" ) @@ -229,6 +237,8 @@ def explode_json_udtf(json_str): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "top_n_udtf", "runtime_version" = "${runtime_version}" ) @@ -281,6 +291,8 @@ def top_n_udtf(value, n): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "duplicate_udtf", "runtime_version" = "${runtime_version}" ) @@ -325,6 +337,8 @@ def duplicate_udtf(text, n): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "filter_positive_udtf", "runtime_version" = "${runtime_version}" ) @@ -369,6 +383,8 @@ def filter_positive_udtf(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "cartesian_udtf", "runtime_version" = "${runtime_version}" ) @@ -418,6 +434,8 @@ def cartesian_udtf(list1, list2): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "filter_negative_udtf", "runtime_version" = "${runtime_version}" ) @@ -512,6 +530,8 @@ def filter_negative_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "nullable_processor_udtf", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -561,6 +581,8 @@ def nullable_processor_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "non_nullable_processor_udtf", "runtime_version" = "${runtime_version}", "always_nullable" = "false" @@ -643,6 +665,8 @@ def non_nullable_processor_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "default_nullable_udtf", "runtime_version" = "${runtime_version}" ) @@ -689,6 +713,8 @@ def default_nullable_udtf(text): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "nullable_explode_udtf", "runtime_version" = "${runtime_version}", "always_nullable" = "true" @@ -746,6 +772,8 @@ def nullable_explode_udtf(csv_string): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "scalar_int_udtf", "runtime_version" = "${runtime_version}" ) @@ -772,6 +800,8 @@ def scalar_int_udtf(start, end): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "scalar_string_udtf", "runtime_version" = "${runtime_version}" ) @@ -798,6 +828,8 @@ def scalar_string_udtf(text): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "mixed_style_udtf", "runtime_version" = "${runtime_version}" ) @@ -828,6 +860,8 @@ def mixed_style_udtf(n): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "return_scalar_udtf", "runtime_version" = "${runtime_version}" ) @@ -852,6 +886,8 @@ def return_scalar_udtf(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "multi_field_udtf", "runtime_version" = "${runtime_version}" ) @@ -883,6 +919,8 @@ def multi_field_udtf(n): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_value_udtf", "runtime_version" = "${runtime_version}" ) @@ -938,6 +976,8 @@ def process_value_udtf(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "split_words_udtf", "runtime_version" = "${runtime_version}" ) @@ -995,6 +1035,8 @@ def split_words_udtf(text): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "expand_range_udtf", "runtime_version" = "${runtime_version}" ) @@ -1051,6 +1093,8 @@ def expand_range_udtf(n): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "parse_csv_udtf", "runtime_version" = "${runtime_version}" ) @@ -1211,6 +1255,8 @@ def parse_csv_udtf(csv_line): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "largeint_expand_udtf", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_module.groovy index 44c121cfdd724d..0904e31257babd 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_basic_module.groovy @@ -38,6 +38,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.split_string_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -76,6 +78,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.generate_series_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -122,6 +126,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.running_sum_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -164,6 +170,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.explode_json_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -201,6 +209,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.top_n_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -247,6 +257,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.duplicate_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -285,6 +297,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.filter_positive_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -323,6 +337,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.cartesian_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -362,6 +378,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.filter_negative_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -450,6 +468,8 @@ suite("test_pythonudtf_basic_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.basic_udtf.nullable_processor_udtf", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}", "always_nullable" = "true" ); diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_inline.groovy index d3e5b6a3bbc162..cc8739bf72c932 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_inline.groovy @@ -31,6 +31,8 @@ suite("test_pythonudtf_data_types_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_tinyint", "runtime_version" = "${runtime_version}" ) @@ -73,6 +75,8 @@ def process_tinyint(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_smallint", "runtime_version" = "${runtime_version}" ) @@ -115,6 +119,8 @@ def process_smallint(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_bigint", "runtime_version" = "${runtime_version}" ) @@ -157,6 +163,8 @@ def process_bigint(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_float", "runtime_version" = "${runtime_version}" ) @@ -199,6 +207,8 @@ def process_float(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_double", "runtime_version" = "${runtime_version}" ) @@ -243,6 +253,8 @@ def process_double(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_boolean", "runtime_version" = "${runtime_version}" ) @@ -285,6 +297,8 @@ def process_boolean(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_string", "runtime_version" = "${runtime_version}" ) @@ -327,6 +341,8 @@ def process_string(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_date", "runtime_version" = "${runtime_version}" ) @@ -370,6 +386,8 @@ def process_date(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_datetime", "runtime_version" = "${runtime_version}" ) @@ -416,6 +434,8 @@ def process_datetime(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_array_int", "runtime_version" = "${runtime_version}" ) @@ -463,6 +483,8 @@ def process_array_int(arr): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_array_string", "runtime_version" = "${runtime_version}" ) @@ -509,6 +531,8 @@ def process_array_string(arr): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_struct", "runtime_version" = "${runtime_version}" ) @@ -559,6 +583,8 @@ def process_struct(person): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_multi_types", "runtime_version" = "${runtime_version}" ) @@ -602,6 +628,8 @@ def process_multi_types(num, text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_decimal", "runtime_version" = "${runtime_version}" ) @@ -650,6 +678,8 @@ def process_decimal(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_map_string", "runtime_version" = "${runtime_version}" ) @@ -699,6 +729,8 @@ def process_map_string(map_str): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_nested_array", "runtime_version" = "${runtime_version}" ) @@ -751,6 +783,8 @@ def process_nested_array(nested_str): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_array_structs", "runtime_version" = "${runtime_version}" ) @@ -800,6 +834,8 @@ def process_array_structs(data): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_struct_array", "runtime_version" = "${runtime_version}" ) @@ -844,6 +880,8 @@ def process_struct_array(data): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "extract_json_fields", "runtime_version" = "${runtime_version}" ) @@ -894,6 +932,8 @@ def extract_json_fields(json_str): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_complex_struct", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_module.groovy index e055dbb47e8eba..b1d079c661e425 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_data_types_module.groovy @@ -37,6 +37,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_tinyint", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -74,6 +76,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_smallint", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -111,6 +115,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_bigint", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -148,6 +154,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_float", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -185,6 +193,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_double", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -222,6 +232,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_boolean", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -259,6 +271,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -296,6 +310,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_date", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -333,6 +349,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_datetime", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -373,6 +391,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_array_int", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -413,6 +433,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_array_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -452,6 +474,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_struct", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -492,6 +516,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_multi_types", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -530,6 +556,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_decimal", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -570,6 +598,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_map_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -607,6 +637,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_nested_array", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -644,6 +676,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_array_structs", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -681,6 +715,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_struct_array", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -718,6 +754,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.extract_json_fields", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -755,6 +793,8 @@ suite("test_pythonudtf_data_types_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.data_types_udtf.process_complex_struct", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy new file mode 100644 index 00000000000000..134e52f229157d --- /dev/null +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_database_cleanup_retry.groovy @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_pythonudtf_database_cleanup_retry", "nonConcurrent") { + def runtime = getPythonUdfRuntimeVersion() + def originalDb = sql("SELECT DATABASE()")[0][0] + def heartbeatPoint = "HeartbeatMgr.BackendHeartbeatHandler" + def zipA = "${context.file.parent}/udtf_scripts/python_udtf_drop_a/python_udtf_drop_test.zip" + def zipB = "${context.file.parent}/udtf_scripts/python_udtf_drop_b/python_udtf_drop_test.zip" + [zipA, zipB].each { zip -> + scp_udf_file_to_all_be(zip) + scp_udf_file_to_all_fe(zip) + } + def await = { String message, Closure ready -> + def deadline = System.currentTimeMillis() + 120000 + while (!ready()) { + if (System.currentTimeMillis() >= deadline) { + throw new IllegalStateException(message) + } + Thread.sleep(1000) + } + } + def cleanupTasks = { + sql_return_maparray("SHOW PROC '/tasks'").find { it.TaskType == "CLEAN_UDF_CACHE" } + } + def purgeDatabase = { db -> + sql "DROP DATABASE IF EXISTS ${db} FORCE" + // Remove leftover recycle-bin records before the next case. + def ids = sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .collect { it.DbId }.unique() + ids.each { id -> sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${id}" } + } + def createFunction = { zip, scope -> + sql """CREATE ${scope} TABLES FUNCTION py_cleanup_retry(INT) RETURNS ARRAY + PROPERTIES("type"="PYTHON_UDF", "auto_suspend"="0", "symbol"="drop_udtf.process", + "file"="file://${zip}", "runtime_version"="${runtime}", "pool_size"="1")""" + } + ["force", "recycle", "function"].eachWithIndex { mode, index -> + def db = "py_udtf_cleanup_retry_${mode}" + def scope = mode == "global" ? "GLOBAL" : "" + try { + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } + purgeDatabase(db) + await("earlier database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + sql "CREATE DATABASE ${db}" + sql "USE ${db}" + def dbId = getDbId(db) + createFunction(zipA, scope) + // Normal and outer functions each need a cleanup task on every backend. + def functionCount = sql("SHOW ${scope} FUNCTIONS LIKE 'py_cleanup_retry%'").size() + "qt_before_${index}"("SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry(v) t AS x UNION ALL SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry_outer(v) t AS x ORDER BY x") + + def beIds = sql_return_maparray("SHOW BACKENDS").collect { it.BackendId.toString() } + // Simulate heartbeat failures while Python processes keep running. + GetDebugPoint().enableDebugPointForAllFEs(heartbeatPoint, + [deadBeIds: beIds.join(","), timeout: "180"]) + await("backends were not marked unavailable", { + sql_return_maparray("SHOW BACKENDS").every { !it.Alive.toBoolean() } + }) + if (mode in ["function", "global"]) { + sql "DROP ${scope} FUNCTION py_cleanup_retry(INT)" + } else if (mode == "recycle") { + sql "USE ${originalDb}" + sql "DROP DATABASE ${db}" + // Recoverable databases retain their function pools. + assertEquals(0, cleanupTasks().TotalNum.toInteger()) + sql "DROP CATALOG RECYCLE BIN WHERE 'DbId' = ${dbId}" + assertTrue(sql_return_maparray("SHOW CATALOG RECYCLE BIN WHERE NAME = '${db}'") + .every { it.DbId.toString() != dbId.toString() }) + } else { + sql "USE ${originalDb}" + sql "DROP DATABASE ${db} FORCE" + } + await("function cleanup was lost while backend was unavailable", { + def tasks = cleanupTasks() + tasks.TotalNum.toInteger() == beIds.size() * functionCount + && tasks.FailedNum.toInteger() == beIds.size() * functionCount + }) + beIds.each { id -> + def tasks = sql_return_maparray("SHOW PROC '/tasks/CLEAN_UDF_CACHE/${id}'") + assertEquals(functionCount, tasks.size()) + assertTrue(tasks.every { it.TaskSignature.toLong() > 0 }) + } + + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + await("backends did not recover", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + // The task leaves the queue only after cleanup succeeds and its result is reported. + await("backend recovery did not complete function cleanup", { + cleanupTasks().TotalNum.toInteger() == 0 + }) + if (mode in ["force", "recycle"]) { + sql "CREATE DATABASE ${db}" + } + sql "USE ${db}" + createFunction(zipB, scope) + "qt_after_${index}"("SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry(v) t AS x UNION ALL SELECT x FROM (SELECT 10 v) s LATERAL VIEW py_cleanup_retry_outer(v) t AS x ORDER BY x") + } finally { + GetDebugPoint().disableDebugPointForAllFEs(heartbeatPoint) + sql "USE ${originalDb}" + if (scope) { + sql "DROP GLOBAL FUNCTION IF EXISTS py_cleanup_retry(INT)" + } + purgeDatabase(db) + await("backends did not recover during cleanup", { + sql_return_maparray("SHOW BACKENDS").every { it.Alive.toBoolean() } + }) + await("final database cleanups did not finish", { cleanupTasks().TotalNum.toInteger() == 0 }) + } + } +} diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_drop.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_drop.groovy index 7eeb9de281a43f..25290aa34be90f 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_drop.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_drop.groovy @@ -57,6 +57,8 @@ suite("test_pythonudtf_drop", "nonConcurrent") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udtf.process", "runtime_version" = "${runtime_version}" @@ -87,6 +89,8 @@ suite("test_pythonudtf_drop", "nonConcurrent") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udtf.process", "runtime_version" = "${runtime_version}" @@ -97,6 +101,8 @@ suite("test_pythonudtf_drop", "nonConcurrent") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipB}", "symbol" = "drop_udtf.process", "runtime_version" = "${runtime_version}" @@ -145,6 +151,8 @@ suite("test_pythonudtf_drop", "nonConcurrent") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${zipA}", "symbol" = "drop_udtf.process", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_inline.groovy index cbdd84b86f143b..bbdbc7b3538f6a 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_inline.groovy @@ -33,6 +33,8 @@ suite("test_pythonudtf_edge_cases_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "handle_null_int", "runtime_version" = "${runtime_version}" ) @@ -75,6 +77,8 @@ def handle_null_int(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "handle_null_string", "runtime_version" = "${runtime_version}" ) @@ -119,6 +123,8 @@ def handle_null_string(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "handle_empty_array", "runtime_version" = "${runtime_version}" ) @@ -166,6 +172,8 @@ def handle_empty_array(arr): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "handle_null_struct", "runtime_version" = "${runtime_version}" ) @@ -230,6 +238,8 @@ def handle_null_struct(person): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_empty_table", "runtime_version" = "${runtime_version}" ) @@ -267,6 +277,8 @@ def process_empty_table(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_single_row", "runtime_version" = "${runtime_version}" ) @@ -308,6 +320,8 @@ def process_single_row(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_long_string", "runtime_version" = "${runtime_version}" ) @@ -353,6 +367,8 @@ def process_long_string(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_large_array", "runtime_version" = "${runtime_version}" ) @@ -399,6 +415,8 @@ def process_large_array(arr): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "output_explosion", "runtime_version" = "${runtime_version}" ) @@ -445,6 +463,8 @@ def output_explosion(n): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_special_numbers", "runtime_version" = "${runtime_version}" ) @@ -503,6 +523,8 @@ def process_special_numbers(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_special_doubles", "runtime_version" = "${runtime_version}" ) @@ -565,6 +587,8 @@ def process_special_doubles(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_special_strings", "runtime_version" = "${runtime_version}" ) @@ -626,6 +650,8 @@ def process_special_strings(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_boundary_dates", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_module.groovy index 8940a6983ffd16..dfb1becba5937c 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_edge_cases_module.groovy @@ -39,6 +39,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.handle_null_int", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -74,6 +76,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.handle_null_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -109,6 +113,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.handle_empty_array", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -147,6 +153,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.handle_null_struct", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -190,6 +198,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_empty_table", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -222,6 +232,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_single_row", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -257,6 +269,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_long_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -294,6 +308,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_large_array", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -331,6 +347,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.output_explosion", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -371,6 +389,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_special_numbers", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -412,6 +432,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_special_doubles", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -452,6 +474,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_special_strings", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -492,6 +516,8 @@ suite("test_pythonudtf_edge_cases_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.edge_cases_udtf.process_boundary_dates", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_inline.groovy index ee8d8bafc47baf..765b485c4b1db4 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_inline.groovy @@ -33,6 +33,8 @@ suite("test_pythonudtf_exceptions_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "safe_divide", "runtime_version" = "${runtime_version}" ) @@ -84,6 +86,8 @@ def safe_divide(a, b): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "check_overflow", "runtime_version" = "${runtime_version}" ) @@ -144,6 +148,8 @@ def check_overflow(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "parse_number", "runtime_version" = "${runtime_version}" ) @@ -196,6 +202,8 @@ def parse_number(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "check_type", "runtime_version" = "${runtime_version}" ) @@ -251,6 +259,8 @@ def check_type(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "safe_array_access", "runtime_version" = "${runtime_version}" ) @@ -303,6 +313,8 @@ def safe_array_access(arr, position): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "compute_stats", "runtime_version" = "${runtime_version}" ) @@ -358,6 +370,8 @@ def compute_stats(arr): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "access_struct_fields", "runtime_version" = "${runtime_version}" ) @@ -415,6 +429,8 @@ def access_struct_fields(person): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "slice_string", "runtime_version" = "${runtime_version}" ) @@ -476,6 +492,8 @@ def slice_string(text, start, end): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "check_text_encoding", "runtime_version" = "${runtime_version}" ) @@ -531,6 +549,8 @@ def check_text_encoding(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "process_conditional", "runtime_version" = "${runtime_version}" ) @@ -588,6 +608,8 @@ def process_conditional(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "conditional_yield", "runtime_version" = "${runtime_version}" ) @@ -639,6 +661,8 @@ def conditional_yield(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "classify_number_range", "runtime_version" = "${runtime_version}" ) @@ -701,6 +725,8 @@ def classify_number_range(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "validate_date", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_module.groovy index 29443b8214b0b1..25b44c60a10a9b 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_exceptions_module.groovy @@ -39,6 +39,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.safe_divide", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -79,6 +81,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.check_overflow", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -122,6 +126,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.parse_number", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -163,6 +169,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.check_type", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -206,6 +214,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.safe_array_access", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -247,6 +257,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.compute_stats", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -290,6 +302,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.access_struct_fields", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -333,6 +347,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.slice_string", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -375,6 +391,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.check_text_encoding", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -419,6 +437,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.process_conditional", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -459,6 +479,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.conditional_yield", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -504,6 +526,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.classify_number_range", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -545,6 +569,8 @@ suite("test_pythonudtf_exceptions_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.exceptions_udtf.validate_date", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_forbidden_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_forbidden_module.groovy index a25852fc69b507..4a5ccd0f203f74 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_forbidden_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_forbidden_module.groovy @@ -53,6 +53,8 @@ suite("test_pythonudtf_forbidden_module") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "${forbiddenCase.symbol}", "runtime_version" = "${runtime_version}" @@ -76,6 +78,8 @@ suite("test_pythonudtf_forbidden_module") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "file" = "file://${pyPath}", "symbol" = "safepkg_udtf.inspect.safe_udtf", "runtime_version" = "${runtime_version}" diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_inline.groovy index b5a1bd5479173e..bc7d07bab69f94 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_inline.groovy @@ -32,6 +32,8 @@ suite("test_pythonudtf_io_patterns_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "one_to_one", "runtime_version" = "${runtime_version}" ) @@ -76,6 +78,8 @@ def one_to_one(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "one_to_many", "runtime_version" = "${runtime_version}" ) @@ -121,6 +125,8 @@ def one_to_many(n): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "one_to_zero", "runtime_version" = "${runtime_version}" ) @@ -166,6 +172,8 @@ def one_to_zero(value): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "one_to_variable", "runtime_version" = "${runtime_version}" ) @@ -222,6 +230,8 @@ def one_to_variable(text): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "aggregate_pattern", "runtime_version" = "${runtime_version}" ) @@ -274,6 +284,8 @@ def aggregate_pattern(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "explosive", "runtime_version" = "${runtime_version}" ) @@ -321,6 +333,8 @@ def explosive(all_rows, all_cols): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "conditional", "runtime_version" = "${runtime_version}" ) @@ -375,6 +389,8 @@ def conditional(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "all_or_nothing", "runtime_version" = "${runtime_version}" ) @@ -428,6 +444,8 @@ def all_or_nothing(text, min_length): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "empty_input", "runtime_version" = "${runtime_version}" ) @@ -468,6 +486,8 @@ def empty_input(value): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "batch_process", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_module.groovy index d84725f13ea0b3..367a2b5067bf67 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_io_patterns_module.groovy @@ -38,6 +38,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.one_to_one", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -76,6 +78,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.one_to_many", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -114,6 +118,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.one_to_zero", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -152,6 +158,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.one_to_variable", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -195,6 +203,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.aggregate_pattern", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -235,6 +245,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.explosive", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -274,6 +286,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.conditional", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -312,6 +326,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.all_or_nothing", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -354,6 +370,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.empty_input", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -389,6 +407,8 @@ suite("test_pythonudtf_io_patterns_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.io_patterns_udtf.batch_process", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_no_input.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_no_input.groovy index e5b90a6ca9ffc7..76aa26e3c611cd 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_no_input.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_no_input.groovy @@ -27,6 +27,8 @@ suite("test_pythonudtf_no_input") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "emit_values", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_object_types_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_object_types_inline.groovy index 0d4259a6e6e02d..70b73ab4f3b858 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_object_types_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_object_types_inline.groovy @@ -24,6 +24,8 @@ suite("test_pythonudtf_object_types_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -41,6 +43,8 @@ def evaluate(v): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -58,6 +62,8 @@ def evaluate(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -75,6 +81,8 @@ def evaluate(v): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -92,6 +100,8 @@ def evaluate(v): RETURNS ARRAY>>> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy index e27030fd083e28..359e63c7934978 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_pkg_isolation.groovy @@ -45,9 +45,11 @@ suite("test_pythonudtf_pkg_isolation") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_x.process", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ sql """ @@ -55,9 +57,11 @@ suite("test_pythonudtf_pkg_isolation") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_x.process", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -76,9 +80,11 @@ suite("test_pythonudtf_pkg_isolation") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipA}", "symbol" = "mypkg.mod_y.process", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ @@ -97,9 +103,11 @@ suite("test_pythonudtf_pkg_isolation") { RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", "file" = "file://${zipB}", "symbol" = "mypkg.mod_y.process", - "runtime_version" = "${runtime_version}" + "runtime_version" = "${runtime_version}", + "pool_size" = "1" ) """ diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_inline.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_inline.groovy index e15ec67fec9ea9..53a7d1d6fb4138 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_inline.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_inline.groovy @@ -33,6 +33,8 @@ suite("test_pythonudtf_sql_integration_inline") { RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "split_with_position", "runtime_version" = "${runtime_version}" ) @@ -53,6 +55,8 @@ def split_with_position(text, delimiter): RETURNS ARRAY PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "generate_range", "runtime_version" = "${runtime_version}" ) @@ -72,6 +76,8 @@ def generate_range(start, end): RETURNS ARRAY> PROPERTIES ( "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "symbol" = "explode_with_index", "runtime_version" = "${runtime_version}" ) diff --git a/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_module.groovy b/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_module.groovy index 9ac0a4aebd2135..215f4a2f9a3840 100644 --- a/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_module.groovy +++ b/regression-test/suites/pythonudtf_p0/test_pythonudtf_sql_integration_module.groovy @@ -39,6 +39,8 @@ suite("test_pythonudtf_sql_integration_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.sql_integration_udtf.split_with_position", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -52,6 +54,8 @@ suite("test_pythonudtf_sql_integration_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.sql_integration_udtf.generate_range", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ @@ -65,6 +69,8 @@ suite("test_pythonudtf_sql_integration_module") { "file" = "file://${pyPath}", "symbol" = "pyudtf_module.sql_integration_udtf.explode_with_index", "type" = "PYTHON_UDF", + "auto_suspend" = "0", + "pool_size" = "1", "runtime_version" = "${runtime_version}" ); """ diff --git a/regression-test/suites/query_p0/show/test_nereids_show_functions.groovy b/regression-test/suites/query_p0/show/test_nereids_show_functions.groovy index 12f727f7ee4e01..3eea7c2e271fa0 100644 --- a/regression-test/suites/query_p0/show/test_nereids_show_functions.groovy +++ b/regression-test/suites/query_p0/show/test_nereids_show_functions.groovy @@ -110,6 +110,8 @@ suite("test_nereids_show_functions") { RETURNS INT PROPERTIES ( "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "symbol" = "evaluate", "runtime_version" = "${runtime_version}" ) @@ -135,6 +137,8 @@ suite("test_nereids_show_functions") { "file"="file://${pyudfPath}", "symbol"="int_test.evaluate", "type"="PYTHON_UDF", + "pool_size"="1", + "auto_suspend"="0", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -158,6 +162,8 @@ suite("test_nereids_show_functions") { "file"="file://${pyudfPath}", "symbol"="sum_int.SumInt", "type"="PYTHON_UDF", + "pool_size"="1", + "auto_suspend"="0", "always_nullable" = "true", "runtime_version" = "${runtime_version}" ); @@ -181,6 +187,8 @@ suite("test_nereids_show_functions") { "file" = "file://${pyudtfPath}", "symbol" = "pyudtf_module.basic_udtf.split_string_udtf", "type" = "PYTHON_UDF", + "pool_size" = "1", + "auto_suspend" = "0", "runtime_version" = "${runtime_version}" ); """