diff --git a/include/exec/static_thread_pool.hpp b/include/exec/static_thread_pool.hpp index b4d570a49..574a28a0a 100644 --- a/include/exec/static_thread_pool.hpp +++ b/include/exec/static_thread_pool.hpp @@ -241,10 +241,10 @@ namespace experimental::execution class CvSender> auto operator()(Tag, Data&& data, CvSender&& sndr) const { - auto [pol, shape, fun] = static_cast(data); - using policy_t = std::remove_cvref_t; - constexpr bool parallelize = std::same_as - || std::same_as; + auto [pol, shape, fun] = static_cast(data); + using policy_t = std::remove_cvref_t; + constexpr bool parallelize = + __one_of; if constexpr (__same_as) { @@ -291,11 +291,14 @@ namespace experimental::execution constexpr auto transform_sender(STDEXEC::set_value_t, Sender&& sndr, Env const & env) const noexcept { - if constexpr (__completes_on) + auto& [tag, data, child] = sndr; + auto& [pol, shape, fun] = data; + + if constexpr (__completes_on) { - auto sched = STDEXEC::get_completion_scheduler(get_env(sndr), + auto sched = STDEXEC::get_completion_scheduler(get_env(child), env); - static_assert(std::is_same_v); + static_assert(std::is_same_v); return __apply(_transform_bulk{*sched.pool_}, static_cast(sndr)); } else @@ -316,7 +319,7 @@ namespace experimental::execution constexpr auto transform_sender(STDEXEC::set_value_t, Sender&& sndr, Env const & env) const noexcept { - if constexpr (__completes_on) + if constexpr (__completes_on) { auto sched = STDEXEC::get_completion_scheduler(get_env(sndr), env); @@ -357,10 +360,9 @@ namespace experimental::execution struct attrs { template - auto query(get_completion_scheduler_t, __ignore = {}) const noexcept - -> _static_thread_pool::scheduler + auto query(get_completion_scheduler_t, __ignore = {}) const noexcept -> scheduler { - return _static_thread_pool::scheduler{pool_, *queue_}; + return scheduler{pool_, *queue_}; } template @@ -402,7 +404,7 @@ namespace experimental::execution } private: - friend struct _static_thread_pool::scheduler; + friend struct scheduler; explicit _sender(_static_thread_pool& pool, remote_queue* queue, @@ -1215,7 +1217,7 @@ namespace experimental::execution } private: - friend class ::exec::_pool_::_static_thread_pool::scheduler::_sender; + friend class scheduler::_sender; explicit _opstate(_static_thread_pool& pool, remote_queue* queue, @@ -1491,26 +1493,25 @@ namespace experimental::execution STDEXEC_TRY { shared_state_.data_.template emplace<__decayed_tuple>(static_cast(as)...); + + if (Shape{} < shared_state_.shape_) + { + enqueue(); + } + else + { + __visit(STDEXEC::__flip(std::bind_front(__apply, STDEXEC::set_value)), + std::move(shared_state_.data_), + static_cast(shared_state_.rcvr_)); + } } STDEXEC_CATCH_ALL { - if constexpr (!__nothrow_decay_copyable) - { + if constexpr (__nothrow_decay_copyable) + STDEXEC::__std::unreachable(); + else STDEXEC::set_error(static_cast(shared_state_.rcvr_), std::current_exception()); - return; - } - } - - if (Shape{} < shared_state_.shape_) - { - enqueue(); - } - else - { - __visit(STDEXEC::__flip(std::bind_front(__apply, STDEXEC::set_value)), - std::move(shared_state_.data_), - static_cast(shared_state_.rcvr_)); } } diff --git a/include/exec/thread_pool_base.hpp b/include/exec/thread_pool_base.hpp index 60dc62cda..2018a81e1 100644 --- a/include/exec/thread_pool_base.hpp +++ b/include/exec/thread_pool_base.hpp @@ -19,9 +19,6 @@ #include "../stdexec/__detail/__config.hpp" -#include "sender_for.hpp" -#include "static_thread_pool.hpp" - #if STDEXEC_USE_MODULES() import std; import stdexec; @@ -37,6 +34,7 @@ import stdexec; # include "../stdexec/__detail/__schedulers.hpp" # include "../stdexec/__detail/__transform_completion_signatures.hpp" # include "../stdexec/__detail/__type_traits.hpp" +# include "../stdexec/__detail/__utility.hpp" # include # include @@ -49,6 +47,10 @@ import stdexec; # include #endif +#include "completion_signatures.hpp" +#include "sender_for.hpp" +#include "static_thread_pool.hpp" + namespace experimental::execution { struct CANNOT_DISPATCH_BULK_ALGORITHM_TO_THE_POOL_SCHEDULER; @@ -68,9 +70,6 @@ namespace experimental::execution template struct opstate; - template - using __decay_ref_t = STDEXEC::__decay_t<_Ty>&; - using task_base = exec::static_thread_pool::task_base; } // namespace _pool_ @@ -92,21 +91,24 @@ namespace experimental::execution { auto& [tag, data, child] = sndr; auto& [pol, shape, fun] = data; + using tag_t = decltype(tag); + using child_t = decltype(child); + using shape_t = decltype(shape); + using fun_t = decltype(fun); - if constexpr (STDEXEC::__completes_on) + if constexpr (STDEXEC::__completes_on) { auto sch = STDEXEC::get_completion_scheduler(STDEXEC::get_env(child), env); - using policy_type = STDEXEC::__decay_t; - constexpr bool parallelize = - STDEXEC::__same_as - || STDEXEC::__same_as; + using policy_type = STDEXEC::__decay_t; + constexpr bool parallelize = STDEXEC::__one_of; if constexpr (sender_for) { - using sender_t = - scheduler::template bulk_sender_t; + using sender_t = scheduler::template bulk_sender_t; return sender_t{*sch.pool_, STDEXEC::__forward_like(child), shape, @@ -115,13 +117,12 @@ namespace experimental::execution } else { - using fun_t = STDEXEC::__bulk::__as_bulk_chunked_fn>; - using sender_t = - scheduler::template bulk_sender_t; + using chunked_fun_t = STDEXEC::__bulk::__as_bulk_chunked_fn; + using sender_t = scheduler::template bulk_sender_t; return sender_t{*sch.pool_, STDEXEC::__forward_like(child), shape, - fun_t{STDEXEC::__forward_like(fun)}, + chunked_fun_t{STDEXEC::__forward_like(fun)}, parallelize}; } } @@ -130,7 +131,7 @@ namespace experimental::execution return STDEXEC::__not_a_sender< STDEXEC::_WHAT_(CANNOT_DISPATCH_BULK_ALGORITHM_TO_THE_POOL_SCHEDULER), STDEXEC::_WHY_(BECAUSE_THERE_IS_NO_POOL_SCHEDULER_IN_THE_ENVIRONMENT), - STDEXEC::_WHERE_(STDEXEC::_IN_ALGORITHM_, STDEXEC::tag_of_t), + STDEXEC::_WHERE_(STDEXEC::_IN_ALGORITHM_, tag_t), STDEXEC::_TO_FIX_THIS_ERROR_( ADD_A_CONTINUES_ON_TRANSITION_TO_THE_POOL_SCHEDULER_BEFORE_THE_BULK_ALGORITHM), STDEXEC::_WITH_SENDER_, @@ -144,13 +145,14 @@ namespace experimental::execution using scheduler_concept = STDEXEC::scheduler_tag; private: - template + template friend struct _pool_::opstate; class sender { public: using sender_concept = STDEXEC::sender_tag; + template static consteval auto get_completion_signatures() noexcept { @@ -166,8 +168,8 @@ namespace experimental::execution } template - auto - query(STDEXEC::get_completion_scheduler_t) const noexcept -> DerivedPoolType::scheduler + auto query(STDEXEC::get_completion_scheduler_t) const noexcept // + -> DerivedPoolType::scheduler { return pool_.get_scheduler(); } @@ -186,8 +188,7 @@ namespace experimental::execution template auto connect(Receiver rcvr) const -> _pool_::opstate { - return _pool_::opstate{this->pool_, - static_cast(rcvr)}; + return _pool_::opstate{pool_, std::move(rcvr)}; } private: @@ -201,194 +202,142 @@ namespace experimental::execution DerivedPoolType& pool_; }; - template - using bulk_non_throwing = STDEXEC::__mbool< - // If function invocation doesn't throw - STDEXEC::__nothrow_callable...> && - // and emplacing a tuple doesn't throw - noexcept(STDEXEC::__decayed_std_tuple(STDEXEC::__declval()...)) - // there's no need to advertise completion with `exception_ptr` - >; - - template + template struct bulk_shared_state : _pool_::task_base { - using variant_t = STDEXEC::__value_types_of_t, - STDEXEC::__q, - STDEXEC::__q>; + using variant_t = STDEXEC::value_types_of_t, + STDEXEC::__decayed_tuple, + STDEXEC::__variant>; - variant_t data_; - DerivedPoolType& pool_; - Receiver rcvr_; - Shape shape_; - Fun fun_; - bool parallelize_; - - std::atomic finished_threads_{0}; - std::atomic thread_with_exception_{0}; - std::exception_ptr exception_; + explicit bulk_shared_state(DerivedPoolType& pool, + Receiver rcvr, + Shape shape, + Fun fun, + bool parallelize) + noexcept(STDEXEC::__nothrow_decay_copyable) + : _pool_::task_base{&do_execute_} + , pool_(pool) + , rcvr_{std::move(rcvr)} + , shape_{shape} + , fun_{std::move(fun)} + , parallelize_{parallelize} + , thread_with_exception_{num_agents_required()} + {} [[nodiscard]] - auto num_agents_required() const -> std::uint32_t + auto num_agents_required() const noexcept -> std::uint32_t { using common_shape_t = std::common_type_t; - auto const parallelism = parallelize_ ? pool_.available_parallelism() - : static_cast(1); + auto const parallelism = parallelize_ ? pool_.available_parallelism() : std::uint32_t(1); // With work stealing, is std::min necessary, or can we feel free to // ask for more agents (tasks) than we can actually deal with at one // time? - return Shape{} < shape_ - ? static_cast((std::min) (shape_, parallelism)) - : 0; + return Shape{} < shape_ ? std::uint32_t(std::min(shape_, parallelism)) + : std::uint32_t(0); } - template - void apply(F f) + static void do_execute_(_pool_::task_base* task, std::uint32_t const tid) noexcept { - std::visit([&](auto& tupl) -> void - { std::apply([&](auto&... args) -> void { f(args...); }, tupl); }, - data_); - } - - explicit bulk_shared_state(DerivedPoolType& pool, - Receiver rcvr, - Shape shape, - Fun fun, - bool parallelize) - : pool_(pool) - , rcvr_{static_cast(rcvr)} - , shape_{shape} - , fun_{fun} - , parallelize_{parallelize} - , thread_with_exception_{num_agents_required()} - { - this->execute_ = [](_pool_::task_base* t, std::uint32_t tid) noexcept + using namespace STDEXEC; + auto& self = *static_cast(task); + auto total_threads = self.num_agents_required(); + + // Each computation does one or more call to the the bulk function. In + // the case that the shape is much larger than the total number of + // threads, then each call to computation will call the function many + // times. + auto const [begin, end] = _pool_::even_share(self.shape_, tid, total_threads); + auto const applicator = std::bind_front(std::ref(self.fun_), begin, end); + auto const computation = std::bind_front(__apply, applicator); + auto const completion = STDEXEC::__flip(std::bind_front(__apply, STDEXEC::set_value)); + + if constexpr (noexcept(__visit(computation, self.data_))) { - auto& self = *static_cast(t); - auto total_threads = self.num_agents_required(); - - auto computation = [&](auto&... args) - { - auto [begin, end] = exec::_pool_::even_share(self.shape_, tid, total_threads); - self.fun_(begin, end, args...); - }; - - auto completion = [&](auto&... args) + __visit(computation, self.data_); + if (self.finished_threads_.fetch_add(1) + 1 == total_threads) // last thread? + __visit(completion, std::move(self.data_), std::move(self.rcvr_)); + } + else + { + STDEXEC_TRY { - STDEXEC::set_value(static_cast(self.rcvr_), std::move(args)...); - }; - - if constexpr (MayThrow) + __visit(computation, self.data_); + } + STDEXEC_CATCH_ALL { - STDEXEC_TRY + std::uint32_t expected = total_threads; + if (self.thread_with_exception_.compare_exchange_strong(expected, + tid, + std::memory_order_relaxed, + std::memory_order_relaxed)) { - self.apply(computation); - } - STDEXEC_CATCH_ALL - { - std::uint32_t expected = total_threads; - - if (self.thread_with_exception_.compare_exchange_strong(expected, - tid, - std::memory_order_relaxed, - std::memory_order_relaxed)) - { - self.exception_ = std::current_exception(); - } - } - - bool const is_last_thread = self.finished_threads_.fetch_add(1) - == (total_threads - 1); - - if (is_last_thread) - { - if (self.exception_) - { - STDEXEC::set_error(static_cast(self.rcvr_), - std::move(self.exception_)); - } - else - { - self.apply(completion); - } + self.exception_ = std::current_exception(); } } - else - { - self.apply(computation); - bool const is_last_thread = self.finished_threads_.fetch_add(1) - == (total_threads - 1); - - if (is_last_thread) - { - self.apply(completion); - } + if (self.finished_threads_.fetch_add(1) + 1 == total_threads) // last thread? + { + if (self.exception_) + STDEXEC::set_error(std::move(self.rcvr_), std::move(self.exception_)); + else + __visit(completion, std::move(self.data_), std::move(self.rcvr_)); } - }; + } } + + variant_t data_{STDEXEC::__no_init}; + DerivedPoolType& pool_; + Receiver rcvr_; + Shape shape_; + Fun fun_; + bool parallelize_; + + std::atomic finished_threads_{0}; + std::atomic thread_with_exception_{0}; + std::exception_ptr exception_; }; - template + template struct bulk_receiver { using receiver_concept = STDEXEC::receiver_tag; - - using shared_state = bulk_shared_state; - - void enqueue() noexcept - { - shared_state_.pool_.bulk_enqueue(&shared_state_, shared_state_.num_agents_required()); - } + using shared_state_t = bulk_shared_state; template void set_value(As&&... as) noexcept { - using tuple_t = STDEXEC::__decayed_std_tuple; - - shared_state& state = shared_state_; - - if constexpr (MayThrow) - { - STDEXEC_TRY - { - state.data_.template emplace(static_cast(as)...); - } - STDEXEC_CATCH_ALL - { - STDEXEC::set_error(std::move(state.rcvr_), std::current_exception()); - return; - } - } - else + using namespace STDEXEC; + STDEXEC_TRY { - state.data_.template emplace(static_cast(as)...); - } + shared_state_.data_.template emplace<__decayed_tuple>(static_cast(as)...); - if (Shape{} < state.shape_) - { - enqueue(); + if (Shape{} < shared_state_.shape_) + shared_state_.pool_.bulk_enqueue(&shared_state_, shared_state_.num_agents_required()); + else + __visit(STDEXEC::__flip(std::bind_front(__apply, STDEXEC::set_value)), + std::move(shared_state_.data_), + std::move(shared_state_.rcvr_)); } - else + STDEXEC_CATCH_ALL { - state.apply([&](auto&... args) - { STDEXEC::set_value(std::move(state.rcvr_), std::move(args)...); }); + if constexpr (__nothrow_decay_copyable) + __std::unreachable(); + else + STDEXEC::set_error(std::move(shared_state_.rcvr_), std::current_exception()); } } template void set_error(Error&& err) noexcept { - shared_state& state = shared_state_; - STDEXEC::set_error(static_cast(state.rcvr_), static_cast(err)); + STDEXEC::set_error(std::move(shared_state_.rcvr_), static_cast(err)); } void set_stopped() noexcept { - shared_state& state = shared_state_; - STDEXEC::set_stopped(static_cast(state.rcvr_)); + STDEXEC::set_stopped(std::move(shared_state_.rcvr_)); } auto get_env() const noexcept -> STDEXEC::env_of_t @@ -396,39 +345,34 @@ namespace experimental::execution return STDEXEC::get_env(shared_state_.rcvr_); } - shared_state& shared_state_; + shared_state_t& shared_state_; }; template struct bulk_opstate { - static constexpr bool may_throw = - !STDEXEC::__value_types_of_t, - STDEXEC::__mbind_front_q, - STDEXEC::__q>::value; - - using bulk_rcvr = bulk_receiver; - using shared_state = bulk_shared_state; - using inner_opstate = STDEXEC::connect_result_t; + using receiver_t = bulk_receiver; + + explicit bulk_opstate(DerivedPoolType& pool, + Shape shape, + Fun fun, + bool parallelize, + CvSender&& sndr, + Receiver rcvr) + noexcept(STDEXEC::__nothrow_decay_copyable + && STDEXEC::__nothrow_connectable) + : shared_state_(pool, std::move(rcvr), shape, std::move(fun), parallelize) + , inner_op_{STDEXEC::connect(static_cast(sndr), receiver_t{shared_state_})} + {} void start() & noexcept { STDEXEC::start(inner_op_); } - bulk_opstate(DerivedPoolType& pool, - Shape shape, - Fun fun, - bool parallelize, - CvSender&& sndr, - Receiver rcvr) - : shared_state_(pool, static_cast(rcvr), shape, fun, parallelize) - , inner_op_{STDEXEC::connect(static_cast(sndr), bulk_rcvr{shared_state_})} - {} - - shared_state shared_state_; - inner_opstate inner_op_; + private: + bulk_shared_state shared_state_; + STDEXEC::connect_result_t inner_op_; }; template @@ -436,23 +380,6 @@ namespace experimental::execution { using sender_concept = STDEXEC::sender_tag; - template - using _with_error_invoke_t = STDEXEC::__eptr_completion_unless_t, Env...>, - STDEXEC::__mtransform, - STDEXEC::__mbind_front_q>, - STDEXEC::__q>>; - - template - using _set_value_t = - STDEXEC::completion_signatures...)>; - - template - using _completion_signatures_t = STDEXEC::__transform_completion_signatures_t< - STDEXEC::__completion_signatures_of_t, Env...>, - _with_error_invoke_t, - _set_value_t>; - template using bulk_opstate_t = bulk_opstate, Receiver, Shape, Fun>; @@ -470,31 +397,49 @@ namespace experimental::execution {} template Self, STDEXEC::receiver Receiver> - requires STDEXEC::receiver_of>> STDEXEC_EXPLICIT_THIS_BEGIN(auto connect)(this Self&& self, Receiver rcvr) - noexcept(STDEXEC::__nothrow_constructible_from, - DerivedPoolType&, - Shape, - Fun, - Sender, - Receiver>) - -> bulk_opstate_t - { - return bulk_opstate_t{self.pool_, - self.shape_, - static_cast(self).fun_, - self.parallelize_, - static_cast(self).sndr_, - static_cast(rcvr)}; - } + STDEXEC_AUTO_RETURN(bulk_opstate_t(self.pool_, + Shape(self.shape_), + static_cast(self).fun_, + self.parallelize_, + static_cast(self).sndr_, + std::move(rcvr))); STDEXEC_EXPLICIT_THIS_END(connect) template Self, class... Env> - static consteval auto get_completion_signatures() // - -> _completion_signatures_t + static consteval auto get_completion_signatures() { - return {}; + using namespace STDEXEC; + return exec::transform_completion_signatures( + STDEXEC::get_completion_signatures, Env...>(), + []() + { + if constexpr (!__nothrow_decay_copyable) + { + return exec::throw_compile_time_error< + _WHAT_(_PREDECESSOR_RESULTS_ARE_NOT_DECAY_COPYABLE_), + _WHERE_(_IN_ALGORITHM_, bulk_t), + _WITH_SENDER_<__copy_cvref_t>, + __fn_t<_WITH_ENVIRONMENT_, Env>...>(); + } + else if constexpr (!__callable&...>) + { + return exec::throw_compile_time_error< + _WHAT_(_FUNCTION_IS_NOT_CALLABLE_WITH_THE_GIVEN_ARGUMENTS_), + _WHERE_(_IN_ALGORITHM_, bulk_t), + _WITH_FUNCTION_(Fun&), + _WITH_ARGUMENTS_(Shape, Shape, __decay_t & ...)>(); + } + else if constexpr (__nothrow_callable&...>) + { + return completion_signatures...)>(); + } + else + { + return completion_signatures...), + set_error_t(std::exception_ptr)>(); + } + }); } struct attrs @@ -590,14 +535,16 @@ namespace experimental::execution private: void enqueue(_pool_::task_base* task, std::uint32_t tid = 0) noexcept { - static_cast(*this).enqueue(task, tid); + auto& self = static_cast(*this); + static_assert(noexcept(self.enqueue(task, tid))); + self.enqueue(task, tid); } void bulk_enqueue(_pool_::task_base* task, std::uint32_t n_threads) noexcept { for (std::uint32_t tid = 0; tid < n_threads; ++tid) { - this->enqueue(task, tid); + enqueue(task, tid); } } }; @@ -607,44 +554,39 @@ namespace experimental::execution template struct opstate : _pool_::task_base { - friend class thread_pool_base; - - PoolType& pool_; - Receiver rcvr_; - - explicit opstate(PoolType& pool, Receiver rcvr) - : pool_(pool) + explicit opstate(PoolType& pool, Receiver rcvr) noexcept + : _pool_::task_base{&do_execute_} + , pool_(pool) , rcvr_(std::move(rcvr)) - { - this->execute_ = [](_pool_::task_base* t, std::uint32_t) noexcept - { - auto& op = *static_cast(t); - auto stoken = STDEXEC::get_stop_token(STDEXEC::get_env(op.rcvr_)); - - if constexpr (STDEXEC::unstoppable_token) - { // NOLINT(bugprone-branch-clone) - STDEXEC::set_value(static_cast(op.rcvr_)); - } - else if (stoken.stop_requested()) - { - STDEXEC::set_stopped(static_cast(op.rcvr_)); - } - else - { - STDEXEC::set_value(static_cast(op.rcvr_)); - } - }; - } + {} - void enqueue() noexcept + void start() & noexcept { pool_.enqueue(this); } - void start() & noexcept + private: + static void do_execute_(_pool_::task_base* task, std::uint32_t) noexcept { - enqueue(); + auto& self = *static_cast(task); + auto stoken = STDEXEC::get_stop_token(STDEXEC::get_env(self.rcvr_)); + + if constexpr (STDEXEC::unstoppable_token) + { // NOLINT(bugprone-branch-clone) + STDEXEC::set_value(std::move(self.rcvr_)); + } + else if (stoken.stop_requested()) + { + STDEXEC::set_stopped(std::move(self.rcvr_)); + } + else + { + STDEXEC::set_value(std::move(self.rcvr_)); + } } + + PoolType& pool_; + Receiver rcvr_; }; } // namespace _pool_ } // namespace experimental::execution