From 0914dc002a88b613628c22ffd64b778e0799eaf3 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 17:08:08 -0700 Subject: [PATCH 01/18] Skip the ARM-only core probes on other CPUs get_num_performant_cores() reads two files that only ARM systems have: the SoC image version under /sys/devices/soc0, to spot a phone model with a known core count, and each CPU's MIDR register, to find small ARM cores. On x86 and other CPUs neither file exists. Each call logs the failed reads, including "Failed to open midr file", and returns 0, meaning the count is unknown. Skip both probes on those CPUs and return 0 directly. The result is the same and those log lines are gone. ARM runs both probes as before. --- extension/threadpool/cpuinfo_utils.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extension/threadpool/cpuinfo_utils.cpp b/extension/threadpool/cpuinfo_utils.cpp index 599527a885e..26e89c13a76 100644 --- a/extension/threadpool/cpuinfo_utils.cpp +++ b/extension/threadpool/cpuinfo_utils.cpp @@ -60,6 +60,7 @@ bool is_non_performant_core(const struct cpuinfo_uarch_info* uarch_info) { return false; } +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 std::vector* get_static_cpu_midr_vector() { static std::vector cpu_midrs; return &cpu_midrs; @@ -140,17 +141,20 @@ uint32_t _get_num_performant_cores() { } return num_possible_cores; } +#endif } // namespace uint32_t get_num_performant_cores() { ET_CHECK_MSG(cpuinfo_initialize(), "cpuinfo cannot be initialized."); +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 // First try and see if we have number of cores profiled for this specific // device uint32_t model_specific_num_cores = _get_model_specific_num_cores(); if (model_specific_num_cores > 0) { return model_specific_num_cores; } +#endif // Else looks at either the # of litte cores if found // Or parse the midr in "Something seems wrong" section. @@ -193,7 +197,12 @@ uint32_t get_num_performant_cores() { } } #endif +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 return _get_num_performant_cores(); +#else + // MIDR registers exist only on ARM. Elsewhere the count is unknown. + return 0; +#endif } } From 058be5cc09aeb68236b94a3ccb710468e1825118 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 17:08:08 -0700 Subject: [PATCH 02/18] Size the default thread pool by the allowed CPUs on Linux when the core count is unknown On x86 Linux the default thread pool starts one thread per CPU in the machine, even when the process may only run on a few of them, for example under taskset or in a container with a CPU set. The extra threads compete for the same CPUs and slow CPU inference down. The pool size comes from get_num_performant_cores(), which returns 0 when it cannot tell fast cores from slow ones, as on x86. The 0 was passed on to pthreadpool, which then counted every online CPU. That count ignores the CPU affinity mask, and it also skips the 63-thread cap applied here. On Linux, when the count is unknown, use the number of CPUs in the affinity mask of the thread that creates the pool, which the pool's threads inherit, and then apply the cap. A machine with more than 63 CPUs and no limit now gets 63 threads instead of one per CPU. The threadpool.h notes describe the fallback. Nothing else changes. A known performance core count, which ARM usually has, is used as before. Android keeps pthreadpool's count, because it moves apps between CPU sets as they go to the background and back, while the pool is sized only once. Other systems keep pthreadpool's count as well. It is the right one there: on Emscripten, for example, pthreadpool can only make a pool of one thread. The new test sizes the pool in a fresh process limited to one CPU and expects one thread. It fails without this change. --- extension/threadpool/test/threadpool_test.cpp | 38 +++++++++++++++++++ extension/threadpool/threadpool.cpp | 25 ++++++++++++ extension/threadpool/threadpool.h | 3 +- 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/extension/threadpool/test/threadpool_test.cpp b/extension/threadpool/test/threadpool_test.cpp index 052e6c22f5e..cd7eaec9c0a 100644 --- a/extension/threadpool/test/threadpool_test.cpp +++ b/extension/threadpool/test/threadpool_test.cpp @@ -17,6 +17,15 @@ #include +#if defined(__linux__) && !defined(__ANDROID__) && \ + defined(EXECUTORCH_THREADPOOL_USE_PERFORMANCE_CORES) +#include + +#include + +#include +#endif + using namespace ::testing; namespace { @@ -201,3 +210,32 @@ TEST(TestNoThreadPoolGuard, TestRunWithGuard) { } ASSERT_EQ(inner, 6); } + +#if defined(__linux__) && !defined(__ANDROID__) && \ + defined(EXECUTORCH_THREADPOOL_USE_PERFORMANCE_CORES) +// The pool is created once per process, so it is checked in a fresh one whose +// exit code is the pool's thread count. Failing to limit the CPUs aborts, so +// it cannot pass for a pool size. +TEST(ThreadPoolDeathTest, DefaultSizeHonorsCpuAffinity) { + executorch::runtime::runtime_init(); + + if (::executorch::extension::cpuinfo::get_num_performant_cores() != 0) { + GTEST_SKIP() << "The performance core count sizes the pool here."; + } + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + EXPECT_EXIT( + { + cpu_set_t cpus; + CPU_ZERO(&cpus); + CPU_SET(sched_getcpu(), &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus) != 0) { + std::abort(); + } + std::exit(static_cast( + ::executorch::extension::threadpool::get_threadpool() + ->get_thread_count())); + }, + ExitedWithCode(1), + ""); +} +#endif diff --git a/extension/threadpool/threadpool.cpp b/extension/threadpool/threadpool.cpp index 2bf13b380be..d6067d83280 100644 --- a/extension/threadpool/threadpool.cpp +++ b/extension/threadpool/threadpool.cpp @@ -18,6 +18,10 @@ #include +#if defined(__linux__) && !defined(__ANDROID__) +#include +#endif + // At most one mode should be set. #if ( \ defined(EXECUTORCH_THREADPOOL_USE_ALL_LOGICAL_CORES) && \ @@ -122,6 +126,22 @@ void ThreadPool::run( 0u); } +namespace { +// The CPUs this process may run on, which taskset and cgroup cpusets can make +// fewer than the online CPUs, or 0 where that is not known. Android is left +// out: it moves apps between cpusets as they go to the background and back, +// while the pool is sized only once. +uint32_t get_num_allowed_cpus() { +#if defined(__linux__) && !defined(__ANDROID__) + cpu_set_t cpus; + if (sched_getaffinity(0, sizeof(cpus), &cpus) == 0) { + return static_cast(CPU_COUNT(&cpus)); + } +#endif + return 0; +} +} // namespace + // get_threadpool is not thread safe due to leak_corrupted_threadpool // Make this part threadsafe: TODO(kimishpatel) ThreadPool* get_threadpool() { @@ -140,6 +160,11 @@ ThreadPool* get_threadpool() { // Set threads equal to the number of performance cores. auto result = ::executorch::extension::cpuinfo::get_num_performant_cores(); #endif + // 0 means the count is unknown, which pthreadpool fills in with every + // online CPU, past the cap below. Prefer the CPUs this process may use. + if (result == 0) { + result = get_num_allowed_cpus(); + } /* * For llvm-tsan, holding limit for the number of locks for a single thread diff --git a/extension/threadpool/threadpool.h b/extension/threadpool/threadpool.h index b1f01687473..eb5060431db 100644 --- a/extension/threadpool/threadpool.h +++ b/extension/threadpool/threadpool.h @@ -26,7 +26,8 @@ * * - EXECUTORCH_THREADPOOL_USE_PERFORMANCE_CORES (flag) - Sizes the threadpool * equal to the number of performance cores on the system. This is the default - * behavior. + * behavior. When that number is unknown, as on x86, Linux other than Android + * uses the number of CPUs this process may run on. * - EXECUTORCH_THREADPOOL_USE_ALL_LOGICAL_CORES (flag) - Sizes the threadpool * equal to the number of logical cores on system. This is the historical * behavior. From b63f976e80cf127476e05efdb2642a89d5eaafcb Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 20:21:47 -0700 Subject: [PATCH 03/18] Read the CPU affinity mask on hosts with more than 1024 CPUs A cpu_set_t holds 1024 CPUs. On a host whose kernel allows more, sched_getaffinity() rejects it with EINVAL, so the allowed-CPU count was 0 and the pool fell back to pthreadpool's count, ignoring both the mask and the 63-thread cap. Allocate the mask with CPU_ALLOC and double its size until the kernel accepts it. --- extension/threadpool/threadpool.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/extension/threadpool/threadpool.cpp b/extension/threadpool/threadpool.cpp index d6067d83280..6f1c15fc5d8 100644 --- a/extension/threadpool/threadpool.cpp +++ b/extension/threadpool/threadpool.cpp @@ -20,6 +20,8 @@ #if defined(__linux__) && !defined(__ANDROID__) #include + +#include #endif // At most one mode should be set. @@ -133,9 +135,21 @@ namespace { // while the pool is sized only once. uint32_t get_num_allowed_cpus() { #if defined(__linux__) && !defined(__ANDROID__) - cpu_set_t cpus; - if (sched_getaffinity(0, sizeof(cpus), &cpus) == 0) { - return static_cast(CPU_COUNT(&cpus)); + // The kernel rejects a mask smaller than its possible CPU count, which can + // exceed the CPU_SETSIZE CPUs a cpu_set_t holds, so grow it until it fits. + for (int num_cpus = CPU_SETSIZE; num_cpus <= (1 << 20); num_cpus *= 2) { + std::unique_ptr cpus( + CPU_ALLOC(num_cpus), [](cpu_set_t* set) { CPU_FREE(set); }); + if (!cpus) { + break; + } + const size_t size = CPU_ALLOC_SIZE(num_cpus); + if (sched_getaffinity(0, size, cpus.get()) == 0) { + return static_cast(CPU_COUNT_S(size, cpus.get())); + } + if (errno != EINVAL) { + break; + } } #endif return 0; From be9d7ac951079dc4079ffbb65caf4b5d75f71a6f Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 20:23:17 -0700 Subject: [PATCH 04/18] Limit the default thread pool to the allowed CPUs on Linux Until now only an unknown core count was replaced by the number of CPUs in the affinity mask. A known count was used as is, so homogeneous ARM servers, and builds that size the pool by all logical cores (the Buck default), still started one thread per CPU in the machine under taskset or a CPU set. On Linux other than Android, take the smaller of the core count and the number of CPUs in the mask, and use the mask alone when the count is unknown. When the process may run on every online CPU, a known count stays as it was, so big.LITTLE systems keep their performance core count. The new test now runs in every Linux build, not only in x86 CMake ones. ParallelAdd and ParallelReduce expect more than one thread only when the process may use more than one CPU, so they pass under a one-CPU limit. The threadpool.h notes and the FAQ now say what the pool reads. --- docs/source/using-executorch-faqs.md | 2 +- extension/threadpool/test/threadpool_test.cpp | 29 ++++++++++++------- extension/threadpool/threadpool.cpp | 18 +++++++----- extension/threadpool/threadpool.h | 9 ++++-- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/docs/source/using-executorch-faqs.md b/docs/source/using-executorch-faqs.md index 05cf2897269..5a1b27d878f 100644 --- a/docs/source/using-executorch-faqs.md +++ b/docs/source/using-executorch-faqs.md @@ -41,7 +41,7 @@ If building the runtime from source, ensure that the build is done in release mo Ensure the model is delegated. If not targeting a specific accelerator, use the XNNPACK delegate for CPU performance. Undelegated operators will typically fall back to the ExecuTorch portable library, which is designed as a fallback, and is not intended for performance sensitive operators. To target XNNPACK, pass an `XnnpackPartitioner` to `to_edge_transform_and_lower`. See [Building and Running ExecuTorch with XNNPACK Backend](tutorial-xnnpack-delegate-lowering.md) for more information. -Thread count can have a significant impact on CPU performance. The optimal thread count may depend on the model and application. By default, ExecuTorch will currently use as many threads as there are cores. Consider setting the thread count to cores / 2, or just set to 4 on mobile CPUs. +Thread count can have a significant impact on CPU performance. The optimal thread count may depend on the model and application. By default, ExecuTorch uses one thread per performance core, and on Linux (other than Android) no more threads than the CPUs the process may run on, for example under `taskset` or in a container with a CPU set. Consider setting the thread count to cores / 2, or just set to 4 on mobile CPUs. Thread count can be set with the following function. Ensure this is done prior to loading or running a model. ``` diff --git a/extension/threadpool/test/threadpool_test.cpp b/extension/threadpool/test/threadpool_test.cpp index cd7eaec9c0a..31dd1d96598 100644 --- a/extension/threadpool/test/threadpool_test.cpp +++ b/extension/threadpool/test/threadpool_test.cpp @@ -17,13 +17,10 @@ #include -#if defined(__linux__) && !defined(__ANDROID__) && \ - defined(EXECUTORCH_THREADPOOL_USE_PERFORMANCE_CORES) +#if defined(__linux__) && !defined(__ANDROID__) #include #include - -#include #endif using namespace ::testing; @@ -78,6 +75,16 @@ void run_lambda_with_size( auto threadpool = ::executorch::extension::threadpool::get_threadpool(); threadpool->run(f, range); } + +// The pool has no more threads than the CPUs this process may run on. +bool may_use_more_than_one_cpu() { +#if defined(__linux__) && !defined(__ANDROID__) + cpu_set_t cpus; + return sched_getaffinity(0, sizeof(cpus), &cpus) != 0 || CPU_COUNT(&cpus) > 1; +#else + return true; +#endif +} } // namespace TEST(ThreadPoolTest, ParallelAdd) { @@ -97,7 +104,9 @@ TEST(ThreadPoolTest, ParallelAdd) { }; auto threadpool = ::executorch::extension::threadpool::get_threadpool(); - EXPECT_GT(threadpool->get_thread_count(), 1); + if (may_use_more_than_one_cpu()) { + EXPECT_GT(threadpool->get_thread_count(), 1); + } generate_add_test_inputs(a, b, c_ref, c, vector_size); run_lambda_with_size(add_lambda, vector_size, grain_size); @@ -142,7 +151,9 @@ TEST(ThreadPoolTest, ParallelReduce) { }; auto threadpool = ::executorch::extension::threadpool::get_threadpool(); - EXPECT_GT(threadpool->get_thread_count(), 1); + if (may_use_more_than_one_cpu()) { + EXPECT_GT(threadpool->get_thread_count(), 1); + } generate_reduce_test_inputs(a, c_ref, vector_size); run_lambda_with_size(reduce_lambda, vector_size, grain_size); @@ -211,17 +222,13 @@ TEST(TestNoThreadPoolGuard, TestRunWithGuard) { ASSERT_EQ(inner, 6); } -#if defined(__linux__) && !defined(__ANDROID__) && \ - defined(EXECUTORCH_THREADPOOL_USE_PERFORMANCE_CORES) +#if defined(__linux__) && !defined(__ANDROID__) // The pool is created once per process, so it is checked in a fresh one whose // exit code is the pool's thread count. Failing to limit the CPUs aborts, so // it cannot pass for a pool size. TEST(ThreadPoolDeathTest, DefaultSizeHonorsCpuAffinity) { executorch::runtime::runtime_init(); - if (::executorch::extension::cpuinfo::get_num_performant_cores() != 0) { - GTEST_SKIP() << "The performance core count sizes the pool here."; - } ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; EXPECT_EXIT( { diff --git a/extension/threadpool/threadpool.cpp b/extension/threadpool/threadpool.cpp index 6f1c15fc5d8..f131db14492 100644 --- a/extension/threadpool/threadpool.cpp +++ b/extension/threadpool/threadpool.cpp @@ -129,10 +129,10 @@ void ThreadPool::run( } namespace { -// The CPUs this process may run on, which taskset and cgroup cpusets can make -// fewer than the online CPUs, or 0 where that is not known. Android is left -// out: it moves apps between cpusets as they go to the background and back, -// while the pool is sized only once. +// The number of CPUs in the calling thread's affinity mask, or 0 where that is +// not known. taskset and cgroup cpusets shrink the mask, CPU quotas do not. +// Android is left out: it moves apps between cpusets as they go to the +// background and back, while the pool is sized only once. uint32_t get_num_allowed_cpus() { #if defined(__linux__) && !defined(__ANDROID__) // The kernel rejects a mask smaller than its possible CPU count, which can @@ -174,10 +174,12 @@ ThreadPool* get_threadpool() { // Set threads equal to the number of performance cores. auto result = ::executorch::extension::cpuinfo::get_num_performant_cores(); #endif - // 0 means the count is unknown, which pthreadpool fills in with every - // online CPU, past the cap below. Prefer the CPUs this process may use. - if (result == 0) { - result = get_num_allowed_cpus(); + // Never start more threads than the CPUs this process may run on, and use + // that number when the count is unknown (0), which pthreadpool would fill + // in with every online CPU, past the cap below. + const auto allowed = get_num_allowed_cpus(); + if (allowed > 0) { + result = result == 0 ? allowed : std::min(result, allowed); } /* diff --git a/extension/threadpool/threadpool.h b/extension/threadpool/threadpool.h index eb5060431db..828f1ffa7e3 100644 --- a/extension/threadpool/threadpool.h +++ b/extension/threadpool/threadpool.h @@ -26,11 +26,16 @@ * * - EXECUTORCH_THREADPOOL_USE_PERFORMANCE_CORES (flag) - Sizes the threadpool * equal to the number of performance cores on the system. This is the default - * behavior. When that number is unknown, as on x86, Linux other than Android - * uses the number of CPUs this process may run on. + * behavior. * - EXECUTORCH_THREADPOOL_USE_ALL_LOGICAL_CORES (flag) - Sizes the threadpool * equal to the number of logical cores on system. This is the historical * behavior. + * + * On Linux other than Android, the threadpool starts with no more threads than + * the CPUs in the affinity mask of the thread that first calls + * get_threadpool(). taskset and cgroup cpusets shrink that mask, CPU quotas do + * not. The mask also sizes the threadpool when the performance core count is + * unknown, as on x86. */ namespace executorch::extension::threadpool { From 07fc43949f3f2b3d43ef8a08b13922d64f23ec5f Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 20:23:37 -0700 Subject: [PATCH 05/18] Skip the sysfs core probes on ARM systems other than Linux The soc0 image version and the MIDR registers are read from /sys, which only Linux has. On macOS and iOS on ARM, and on Windows on ARM, the probes still ran, failed, logged the failure, and could only return 0. Run them only on ARM Linux, which still includes Android. The core counts on the other systems do not change. --- extension/threadpool/cpuinfo_utils.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extension/threadpool/cpuinfo_utils.cpp b/extension/threadpool/cpuinfo_utils.cpp index 26e89c13a76..eb831f3bb6a 100644 --- a/extension/threadpool/cpuinfo_utils.cpp +++ b/extension/threadpool/cpuinfo_utils.cpp @@ -60,7 +60,7 @@ bool is_non_performant_core(const struct cpuinfo_uarch_info* uarch_info) { return false; } -#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 +#if defined(__linux__) && (CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64) std::vector* get_static_cpu_midr_vector() { static std::vector cpu_midrs; return &cpu_midrs; @@ -147,7 +147,7 @@ uint32_t _get_num_performant_cores() { uint32_t get_num_performant_cores() { ET_CHECK_MSG(cpuinfo_initialize(), "cpuinfo cannot be initialized."); -#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 +#if defined(__linux__) && (CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64) // First try and see if we have number of cores profiled for this specific // device uint32_t model_specific_num_cores = _get_model_specific_num_cores(); @@ -197,10 +197,10 @@ uint32_t get_num_performant_cores() { } } #endif -#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 +#if defined(__linux__) && (CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64) return _get_num_performant_cores(); #else - // MIDR registers exist only on ARM. Elsewhere the count is unknown. + // The MIDR files exist only on ARM Linux. Elsewhere the count is unknown. return 0; #endif } From 13117889f9df4e74aa717dc51f9ea48194e23af9 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 20:24:30 -0700 Subject: [PATCH 06/18] Use the default pool size in the Gemma 4 runner when the core count is unknown When get_num_performant_cores() returns 0, the runner reset the pool to std::min(std::thread::hardware_concurrency(), 8). hardware_concurrency() counts every online CPU, so under taskset the runner started more threads than the process may use, undoing the default pool's limit. Cap the default pool's own size at 8 instead. It keeps the runner's limit and follows the CPU affinity mask. --- examples/models/gemma4/e2e_runner.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/models/gemma4/e2e_runner.cpp b/examples/models/gemma4/e2e_runner.cpp index ba423b7bc17..4018dc2bb6f 100644 --- a/examples/models/gemma4/e2e_runner.cpp +++ b/examples/models/gemma4/e2e_runner.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include @@ -57,7 +56,11 @@ int32_t main(int32_t argc, char** argv) { ? ::executorch::extension::cpuinfo::get_num_performant_cores() : static_cast(FLAGS_cpu_threads); if (num_threads == 0) { - num_threads = std::min(std::thread::hardware_concurrency(), 8u); + // The default pool already follows the CPUs this process may run on. + num_threads = static_cast(std::min( + ::executorch::extension::threadpool::get_threadpool() + ->get_thread_count(), + 8)); } ET_LOG(Info, "Setting threadpool to %d threads", (int)num_threads); ::executorch::extension::threadpool::get_threadpool() From e6faa575583c26207f93340623c67430e45a91b5 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 21:14:29 -0700 Subject: [PATCH 07/18] Limit get_num_performant_cores() to the allowed CPUs on Linux The default pool now stays within the CPU affinity mask, but the example runners and other callers size the pool themselves with get_num_performant_cores(), which ignored the mask. On ARM Linux they still started one thread per core under taskset or a CPU set. Move the allowed-CPU count into cpuinfo_utils as get_num_allowed_cpus(), next to get_num_performant_cores(), and have the latter return the smaller of its count and the allowed CPUs on Linux other than Android. An unknown count (0) stays 0, so callers keep falling back to the default pool. When the process may run on every online CPU, the count is unchanged. The affinity test now also checks get_num_performant_cores() and limits itself to its current CPU with a mask sized for any CPU index. ParallelAdd and ParallelReduce read the allowed CPUs from get_num_allowed_cpus(), which handles hosts with more than 1024 CPUs. --- extension/threadpool/cpuinfo_utils.cpp | 42 +++++++++++++-- extension/threadpool/cpuinfo_utils.h | 15 ++++++ extension/threadpool/test/targets.bzl | 1 + extension/threadpool/test/threadpool_test.cpp | 52 +++++++++++-------- extension/threadpool/threadpool.cpp | 37 +------------ 5 files changed, 87 insertions(+), 60 deletions(-) diff --git a/extension/threadpool/cpuinfo_utils.cpp b/extension/threadpool/cpuinfo_utils.cpp index eb831f3bb6a..0a2b0a23840 100644 --- a/extension/threadpool/cpuinfo_utils.cpp +++ b/extension/threadpool/cpuinfo_utils.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -20,6 +21,13 @@ #include #endif +#if defined(__linux__) && !defined(__ANDROID__) +#include + +#include +#include +#endif + namespace executorch::extension::cpuinfo { // Ignore revisions (last digit (4 LSBs)) @@ -143,9 +151,7 @@ uint32_t _get_num_performant_cores() { } #endif -} // namespace - -uint32_t get_num_performant_cores() { +uint32_t count_performant_cores() { ET_CHECK_MSG(cpuinfo_initialize(), "cpuinfo cannot be initialized."); #if defined(__linux__) && (CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64) // First try and see if we have number of cores profiled for this specific @@ -206,4 +212,34 @@ uint32_t get_num_performant_cores() { } } +} // namespace + +uint32_t get_num_allowed_cpus() { +#if defined(__linux__) && !defined(__ANDROID__) + // The kernel rejects a mask smaller than its possible CPU count, which can + // exceed the CPU_SETSIZE CPUs a cpu_set_t holds, so grow it until it fits. + for (int num_cpus = CPU_SETSIZE; num_cpus <= (1 << 20); num_cpus *= 2) { + std::unique_ptr cpus( + CPU_ALLOC(num_cpus), [](cpu_set_t* set) { CPU_FREE(set); }); + if (!cpus) { + break; + } + const size_t size = CPU_ALLOC_SIZE(num_cpus); + if (sched_getaffinity(0, size, cpus.get()) == 0) { + return static_cast(CPU_COUNT_S(size, cpus.get())); + } + if (errno != EINVAL) { + break; + } + } +#endif + return 0; +} + +uint32_t get_num_performant_cores() { + const uint32_t count = count_performant_cores(); + const uint32_t allowed = get_num_allowed_cpus(); + return count > 0 && allowed > 0 ? std::min(count, allowed) : count; +} + } // namespace executorch::extension::cpuinfo diff --git a/extension/threadpool/cpuinfo_utils.h b/extension/threadpool/cpuinfo_utils.h index d559738b728..b9dbf69c592 100644 --- a/extension/threadpool/cpuinfo_utils.h +++ b/extension/threadpool/cpuinfo_utils.h @@ -12,8 +12,23 @@ namespace executorch::extension::cpuinfo { +/** + * Returns the number of performance cores, or 0 when they cannot be told + * apart, as on x86. On Linux other than Android, the count is limited to + * get_num_allowed_cpus(). + */ uint32_t get_num_performant_cores(); +/** + * Returns the number of CPUs in the calling thread's affinity mask on Linux + * other than Android, or 0 elsewhere. taskset and cgroup cpusets shrink the + * mask, CPU quotas do not. Android is left out because it moves apps between + * cpusets as they go to the background and back, and a thread pool sized in + * the background would stay small. Other Linux systems rarely change the + * cpuset of a running process. + */ +uint32_t get_num_allowed_cpus(); + } // namespace executorch::extension::cpuinfo namespace torch::executorch::cpuinfo { // DEPRECATED diff --git a/extension/threadpool/test/targets.bzl b/extension/threadpool/test/targets.bzl index 8bdf776c825..5614d203911 100644 --- a/extension/threadpool/test/targets.bzl +++ b/extension/threadpool/test/targets.bzl @@ -15,6 +15,7 @@ def define_common_targets(): name = "threadpool_test", srcs = _THREADPOOL_TESTS, deps = [ + "//executorch/extension/threadpool:cpuinfo_utils", "//executorch/extension/threadpool:threadpool", ], ) diff --git a/extension/threadpool/test/threadpool_test.cpp b/extension/threadpool/test/threadpool_test.cpp index 31dd1d96598..801972da082 100644 --- a/extension/threadpool/test/threadpool_test.cpp +++ b/extension/threadpool/test/threadpool_test.cpp @@ -6,6 +6,7 @@ * LICENSE file in the root directory of this source tree. */ +#include #include #include @@ -78,13 +79,35 @@ void run_lambda_with_size( // The pool has no more threads than the CPUs this process may run on. bool may_use_more_than_one_cpu() { + return ::executorch::extension::cpuinfo::get_num_allowed_cpus() != 1; +} + #if defined(__linux__) && !defined(__ANDROID__) - cpu_set_t cpus; - return sched_getaffinity(0, sizeof(cpus), &cpus) != 0 || CPU_COUNT(&cpus) > 1; -#else - return true; -#endif +// Limits this thread to the CPU it runs on, then exits with the larger of the +// default pool size and the performance core count, at most 255. Exiting with +// 1 means both follow the limit. Failing to set the limit aborts instead. +[[noreturn]] void exit_with_counts_on_one_cpu() { + const int cpu = sched_getcpu(); + if (cpu < 0) { + std::abort(); + } + std::unique_ptr cpus( + CPU_ALLOC(cpu + 1), [](cpu_set_t* set) { CPU_FREE(set); }); + if (!cpus) { + std::abort(); + } + const size_t size = CPU_ALLOC_SIZE(cpu + 1); + CPU_ZERO_S(size, cpus.get()); + CPU_SET_S(cpu, size, cpus.get()); + if (sched_setaffinity(0, size, cpus.get()) != 0) { + std::abort(); + } + const size_t count = std::max( + ::executorch::extension::threadpool::get_threadpool()->get_thread_count(), + ::executorch::extension::cpuinfo::get_num_performant_cores()); + std::exit(static_cast(std::min(count, 255))); } +#endif } // namespace TEST(ThreadPoolTest, ParallelAdd) { @@ -223,26 +246,11 @@ TEST(TestNoThreadPoolGuard, TestRunWithGuard) { } #if defined(__linux__) && !defined(__ANDROID__) -// The pool is created once per process, so it is checked in a fresh one whose -// exit code is the pool's thread count. Failing to limit the CPUs aborts, so -// it cannot pass for a pool size. +// The pool is created once per process, so it is checked in a fresh one. TEST(ThreadPoolDeathTest, DefaultSizeHonorsCpuAffinity) { executorch::runtime::runtime_init(); ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; - EXPECT_EXIT( - { - cpu_set_t cpus; - CPU_ZERO(&cpus); - CPU_SET(sched_getcpu(), &cpus); - if (sched_setaffinity(0, sizeof(cpus), &cpus) != 0) { - std::abort(); - } - std::exit(static_cast( - ::executorch::extension::threadpool::get_threadpool() - ->get_thread_count())); - }, - ExitedWithCode(1), - ""); + EXPECT_EXIT(exit_with_counts_on_one_cpu(), ExitedWithCode(1), ""); } #endif diff --git a/extension/threadpool/threadpool.cpp b/extension/threadpool/threadpool.cpp index f131db14492..5bcac84965d 100644 --- a/extension/threadpool/threadpool.cpp +++ b/extension/threadpool/threadpool.cpp @@ -18,12 +18,6 @@ #include -#if defined(__linux__) && !defined(__ANDROID__) -#include - -#include -#endif - // At most one mode should be set. #if ( \ defined(EXECUTORCH_THREADPOOL_USE_ALL_LOGICAL_CORES) && \ @@ -128,34 +122,6 @@ void ThreadPool::run( 0u); } -namespace { -// The number of CPUs in the calling thread's affinity mask, or 0 where that is -// not known. taskset and cgroup cpusets shrink the mask, CPU quotas do not. -// Android is left out: it moves apps between cpusets as they go to the -// background and back, while the pool is sized only once. -uint32_t get_num_allowed_cpus() { -#if defined(__linux__) && !defined(__ANDROID__) - // The kernel rejects a mask smaller than its possible CPU count, which can - // exceed the CPU_SETSIZE CPUs a cpu_set_t holds, so grow it until it fits. - for (int num_cpus = CPU_SETSIZE; num_cpus <= (1 << 20); num_cpus *= 2) { - std::unique_ptr cpus( - CPU_ALLOC(num_cpus), [](cpu_set_t* set) { CPU_FREE(set); }); - if (!cpus) { - break; - } - const size_t size = CPU_ALLOC_SIZE(num_cpus); - if (sched_getaffinity(0, size, cpus.get()) == 0) { - return static_cast(CPU_COUNT_S(size, cpus.get())); - } - if (errno != EINVAL) { - break; - } - } -#endif - return 0; -} -} // namespace - // get_threadpool is not thread safe due to leak_corrupted_threadpool // Make this part threadsafe: TODO(kimishpatel) ThreadPool* get_threadpool() { @@ -177,7 +143,8 @@ ThreadPool* get_threadpool() { // Never start more threads than the CPUs this process may run on, and use // that number when the count is unknown (0), which pthreadpool would fill // in with every online CPU, past the cap below. - const auto allowed = get_num_allowed_cpus(); + const auto allowed = + ::executorch::extension::cpuinfo::get_num_allowed_cpus(); if (allowed > 0) { result = result == 0 ? allowed : std::min(result, allowed); } From c607370f53cc46bffbb0e1855b8aaf52aa048382 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 21:14:29 -0700 Subject: [PATCH 08/18] Describe when the default thread pool reads the affinity mask The threadpool.h notes now say that the mask is read once, so the pool does not follow later changes to it, and that a child created by fork() keeps its parent's size. The FAQ now says that the default is one thread per performance core where core types are known and one per logical core otherwise, limited on Linux by the allowed CPUs and 63, and that CPU quotas are not detected. --- docs/source/using-executorch-faqs.md | 2 +- extension/threadpool/threadpool.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/using-executorch-faqs.md b/docs/source/using-executorch-faqs.md index 5a1b27d878f..84bab0a8f7c 100644 --- a/docs/source/using-executorch-faqs.md +++ b/docs/source/using-executorch-faqs.md @@ -41,7 +41,7 @@ If building the runtime from source, ensure that the build is done in release mo Ensure the model is delegated. If not targeting a specific accelerator, use the XNNPACK delegate for CPU performance. Undelegated operators will typically fall back to the ExecuTorch portable library, which is designed as a fallback, and is not intended for performance sensitive operators. To target XNNPACK, pass an `XnnpackPartitioner` to `to_edge_transform_and_lower`. See [Building and Running ExecuTorch with XNNPACK Backend](tutorial-xnnpack-delegate-lowering.md) for more information. -Thread count can have a significant impact on CPU performance. The optimal thread count may depend on the model and application. By default, ExecuTorch uses one thread per performance core, and on Linux (other than Android) no more threads than the CPUs the process may run on, for example under `taskset` or in a container with a CPU set. Consider setting the thread count to cores / 2, or just set to 4 on mobile CPUs. +Thread count can have a significant impact on CPU performance. The optimal thread count may depend on the model and application. By default, ExecuTorch uses one thread per performance core where it can tell core types apart, and one per logical core otherwise. On Linux other than Android, it also uses no more threads than the CPUs the process may run on, for example under `taskset` or in a container with a CPU set, and at most 63. CPU quotas, such as `docker --cpus`, are not detected. Consider setting the thread count to cores / 2, or just set to 4 on mobile CPUs. Thread count can be set with the following function. Ensure this is done prior to loading or running a model. ``` diff --git a/extension/threadpool/threadpool.h b/extension/threadpool/threadpool.h index 828f1ffa7e3..9abe458a4a1 100644 --- a/extension/threadpool/threadpool.h +++ b/extension/threadpool/threadpool.h @@ -35,7 +35,8 @@ * the CPUs in the affinity mask of the thread that first calls * get_threadpool(). taskset and cgroup cpusets shrink that mask, CPU quotas do * not. The mask also sizes the threadpool when the performance core count is - * unknown, as on x86. + * unknown, as on x86. It is read once: the threadpool does not follow later + * changes to the mask, and a child created by fork() keeps its parent's size. */ namespace executorch::extension::threadpool { From 2ec72422f4f94fa73e5d1a2250a545ea35d59535 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 23:44:39 -0700 Subject: [PATCH 09/18] Keep get_num_allowed_cpus() a plain query and apply the Android rule where it sizes threads get_num_allowed_cpus() returned 0 on Android to keep thread counts there unchanged, which mixed a sizing rule into a query whose name promises the CPU count. It now returns the count in the affinity mask on every Linux system, Android included, and 0 only where the mask is not available. Its comment says why the mask is a poor size for a pool on Android. get_num_performant_cores() and the default pool apply the limit everywhere except Android. Thread counts are the same as before. The get_num_performant_cores() comment now also lists the other cases that return 0, and the Gemma 4 runner comment says the default pool follows the mask on Linux other than Android only. --- examples/models/gemma4/e2e_runner.cpp | 3 ++- extension/threadpool/cpuinfo_utils.cpp | 13 ++++++++++--- extension/threadpool/cpuinfo_utils.h | 16 +++++++--------- extension/threadpool/threadpool.cpp | 6 +++++- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/examples/models/gemma4/e2e_runner.cpp b/examples/models/gemma4/e2e_runner.cpp index 4018dc2bb6f..b4bb295c29a 100644 --- a/examples/models/gemma4/e2e_runner.cpp +++ b/examples/models/gemma4/e2e_runner.cpp @@ -56,7 +56,8 @@ int32_t main(int32_t argc, char** argv) { ? ::executorch::extension::cpuinfo::get_num_performant_cores() : static_cast(FLAGS_cpu_threads); if (num_threads == 0) { - // The default pool already follows the CPUs this process may run on. + // On Linux other than Android, the default pool size follows the CPUs + // this process may run on. num_threads = static_cast(std::min( ::executorch::extension::threadpool::get_threadpool() ->get_thread_count(), diff --git a/extension/threadpool/cpuinfo_utils.cpp b/extension/threadpool/cpuinfo_utils.cpp index 0a2b0a23840..d7ec8a54c79 100644 --- a/extension/threadpool/cpuinfo_utils.cpp +++ b/extension/threadpool/cpuinfo_utils.cpp @@ -21,7 +21,7 @@ #include #endif -#if defined(__linux__) && !defined(__ANDROID__) +#if defined(__linux__) #include #include @@ -215,7 +215,7 @@ uint32_t count_performant_cores() { } // namespace uint32_t get_num_allowed_cpus() { -#if defined(__linux__) && !defined(__ANDROID__) +#if defined(__linux__) // The kernel rejects a mask smaller than its possible CPU count, which can // exceed the CPU_SETSIZE CPUs a cpu_set_t holds, so grow it until it fits. for (int num_cpus = CPU_SETSIZE; num_cpus <= (1 << 20); num_cpus *= 2) { @@ -238,8 +238,15 @@ uint32_t get_num_allowed_cpus() { uint32_t get_num_performant_cores() { const uint32_t count = count_performant_cores(); +#if !defined(__ANDROID__) + // Android moves every app between cpusets as it goes to the background and + // back, so a count taken in the background would be too small later. const uint32_t allowed = get_num_allowed_cpus(); - return count > 0 && allowed > 0 ? std::min(count, allowed) : count; + if (count > 0 && allowed > 0) { + return std::min(count, allowed); + } +#endif + return count; } } // namespace executorch::extension::cpuinfo diff --git a/extension/threadpool/cpuinfo_utils.h b/extension/threadpool/cpuinfo_utils.h index b9dbf69c592..afb439cd140 100644 --- a/extension/threadpool/cpuinfo_utils.h +++ b/extension/threadpool/cpuinfo_utils.h @@ -13,19 +13,17 @@ namespace executorch::extension::cpuinfo { /** - * Returns the number of performance cores, or 0 when they cannot be told - * apart, as on x86. On Linux other than Android, the count is limited to - * get_num_allowed_cpus(). + * Returns the number of performance cores, or 0 if it is unknown, for example + * on x86 or when every core is an efficiency core. On Linux other than + * Android, the count is limited to get_num_allowed_cpus(). */ uint32_t get_num_performant_cores(); /** - * Returns the number of CPUs in the calling thread's affinity mask on Linux - * other than Android, or 0 elsewhere. taskset and cgroup cpusets shrink the - * mask, CPU quotas do not. Android is left out because it moves apps between - * cpusets as they go to the background and back, and a thread pool sized in - * the background would stay small. Other Linux systems rarely change the - * cpuset of a running process. + * Returns the number of CPUs in the calling thread's affinity mask on Linux, + * or 0 if it is not available. taskset and cgroup cpusets shrink the mask, + * CPU quotas do not. On Android the mask changes as an app moves between the + * foreground and the background, so it is a poor size for a long-lived pool. */ uint32_t get_num_allowed_cpus(); diff --git a/extension/threadpool/threadpool.cpp b/extension/threadpool/threadpool.cpp index 5bcac84965d..78bfd60db93 100644 --- a/extension/threadpool/threadpool.cpp +++ b/extension/threadpool/threadpool.cpp @@ -142,12 +142,16 @@ ThreadPool* get_threadpool() { #endif // Never start more threads than the CPUs this process may run on, and use // that number when the count is unknown (0), which pthreadpool would fill - // in with every online CPU, past the cap below. + // in with every online CPU, past the cap below. Android is left out: it + // moves every app between cpusets as it goes to the background and back, + // while the pool keeps the size it was created with. +#if !defined(__ANDROID__) const auto allowed = ::executorch::extension::cpuinfo::get_num_allowed_cpus(); if (allowed > 0) { result = result == 0 ? allowed : std::min(result, allowed); } +#endif /* * For llvm-tsan, holding limit for the number of locks for a single thread From 09e61914c82f07e52d2d6228c4a9c7c2f13d9f2f Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 31 Oct 2026 23:44:40 -0700 Subject: [PATCH 10/18] Report both counts from the affinity test and count CPUs independently The death test now prints the pool size and the performance core count, and matches them exactly, so a failure shows which one was wrong, and a pool that failed to start cannot pass. It also uses gtest's default fork style: the threadsafe style runs the child on a small clone() stack that ASAN overflows now and then. The fork style works here because gtest runs death tests before the other tests, so the forked child creates the pool itself. ParallelAdd and ParallelReduce count the allowed CPUs with sched_getaffinity() themselves instead of asking the code under test. --- extension/threadpool/test/threadpool_test.cpp | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/extension/threadpool/test/threadpool_test.cpp b/extension/threadpool/test/threadpool_test.cpp index 801972da082..baa2ea0a41c 100644 --- a/extension/threadpool/test/threadpool_test.cpp +++ b/extension/threadpool/test/threadpool_test.cpp @@ -21,6 +21,7 @@ #if defined(__linux__) && !defined(__ANDROID__) #include +#include #include #endif @@ -79,14 +80,19 @@ void run_lambda_with_size( // The pool has no more threads than the CPUs this process may run on. bool may_use_more_than_one_cpu() { - return ::executorch::extension::cpuinfo::get_num_allowed_cpus() != 1; +#if defined(__linux__) && !defined(__ANDROID__) + cpu_set_t cpus; + return sched_getaffinity(0, sizeof(cpus), &cpus) == 0 && CPU_COUNT(&cpus) > 1; +#else + return true; +#endif } #if defined(__linux__) && !defined(__ANDROID__) -// Limits this thread to the CPU it runs on, then exits with the larger of the -// default pool size and the performance core count, at most 255. Exiting with -// 1 means both follow the limit. Failing to set the limit aborts instead. -[[noreturn]] void exit_with_counts_on_one_cpu() { +// Limits this thread to the CPU it runs on, then prints the default pool size +// and the performance core count for the test to match. Failing to set the +// limit aborts instead. +[[noreturn]] void print_counts_on_one_cpu() { const int cpu = sched_getcpu(); if (cpu < 0) { std::abort(); @@ -102,10 +108,13 @@ bool may_use_more_than_one_cpu() { if (sched_setaffinity(0, size, cpus.get()) != 0) { std::abort(); } - const size_t count = std::max( + std::fprintf( + stderr, + "pool %zu perf %u end\n", ::executorch::extension::threadpool::get_threadpool()->get_thread_count(), - ::executorch::extension::cpuinfo::get_num_performant_cores()); - std::exit(static_cast(std::min(count, 255))); + static_cast( + ::executorch::extension::cpuinfo::get_num_performant_cores())); + std::exit(EXIT_SUCCESS); } #endif } // namespace @@ -246,11 +255,14 @@ TEST(TestNoThreadPoolGuard, TestRunWithGuard) { } #if defined(__linux__) && !defined(__ANDROID__) -// The pool is created once per process, so it is checked in a fresh one. +// gtest runs death tests before the other tests, so the forked child creates +// the pool itself, and the CPU limit it sets does not reach the other tests. TEST(ThreadPoolDeathTest, DefaultSizeHonorsCpuAffinity) { executorch::runtime::runtime_init(); - ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; - EXPECT_EXIT(exit_with_counts_on_one_cpu(), ExitedWithCode(1), ""); + EXPECT_EXIT( + print_counts_on_one_cpu(), + ExitedWithCode(EXIT_SUCCESS), + "pool 1 perf [01] end"); } #endif From 8f8faa10d386effa5628783f0a242fec54b28a36 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 1 Nov 2026 00:20:45 -0700 Subject: [PATCH 11/18] Keep the affinity test correct when tests repeat in one process Under --gtest_repeat, later runs of the death test fork after other tests created the pool, and the child keeps that pool's size, so the check fails although the code is right. Only the first run checks the pool now. The threadsafe death test style would start a fresh process instead, but it runs gtest's child on a small clone() stack that ASAN overflows now and then. ParallelAdd and ParallelReduce also keep checking for more than one thread when the affinity mask cannot be read, as on other systems. --- extension/threadpool/test/threadpool_test.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/extension/threadpool/test/threadpool_test.cpp b/extension/threadpool/test/threadpool_test.cpp index baa2ea0a41c..2249718cf4a 100644 --- a/extension/threadpool/test/threadpool_test.cpp +++ b/extension/threadpool/test/threadpool_test.cpp @@ -82,7 +82,7 @@ void run_lambda_with_size( bool may_use_more_than_one_cpu() { #if defined(__linux__) && !defined(__ANDROID__) cpu_set_t cpus; - return sched_getaffinity(0, sizeof(cpus), &cpus) == 0 && CPU_COUNT(&cpus) > 1; + return sched_getaffinity(0, sizeof(cpus), &cpus) != 0 || CPU_COUNT(&cpus) > 1; #else return true; #endif @@ -258,6 +258,12 @@ TEST(TestNoThreadPoolGuard, TestRunWithGuard) { // gtest runs death tests before the other tests, so the forked child creates // the pool itself, and the CPU limit it sets does not reach the other tests. TEST(ThreadPoolDeathTest, DefaultSizeHonorsCpuAffinity) { + // A later run, as with --gtest_repeat, forks after the other tests created + // the pool, and the child keeps that pool's size. + static int runs = 0; + if (runs++ > 0) { + GTEST_SKIP() << "The pool may already exist in this process"; + } executorch::runtime::runtime_init(); EXPECT_EXIT( From 1681c195689981495063ab766b00492911b173d3 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 1 Nov 2026 00:20:45 -0700 Subject: [PATCH 12/18] Say which mask sizes the pool, that the core count is only capped, and that the mask is read once The comment in get_threadpool() now says the pool follows the affinity mask of the thread that creates it, which the pool's threads inherit. The get_num_performant_cores() comment says the count is capped at the allowed CPUs without checking which cores the mask allows, and the FAQ says the limit is read once, from the thread that first uses the thread pool. --- docs/source/using-executorch-faqs.md | 2 +- extension/threadpool/cpuinfo_utils.h | 3 ++- extension/threadpool/threadpool.cpp | 11 ++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/source/using-executorch-faqs.md b/docs/source/using-executorch-faqs.md index 84bab0a8f7c..782fea07147 100644 --- a/docs/source/using-executorch-faqs.md +++ b/docs/source/using-executorch-faqs.md @@ -41,7 +41,7 @@ If building the runtime from source, ensure that the build is done in release mo Ensure the model is delegated. If not targeting a specific accelerator, use the XNNPACK delegate for CPU performance. Undelegated operators will typically fall back to the ExecuTorch portable library, which is designed as a fallback, and is not intended for performance sensitive operators. To target XNNPACK, pass an `XnnpackPartitioner` to `to_edge_transform_and_lower`. See [Building and Running ExecuTorch with XNNPACK Backend](tutorial-xnnpack-delegate-lowering.md) for more information. -Thread count can have a significant impact on CPU performance. The optimal thread count may depend on the model and application. By default, ExecuTorch uses one thread per performance core where it can tell core types apart, and one per logical core otherwise. On Linux other than Android, it also uses no more threads than the CPUs the process may run on, for example under `taskset` or in a container with a CPU set, and at most 63. CPU quotas, such as `docker --cpus`, are not detected. Consider setting the thread count to cores / 2, or just set to 4 on mobile CPUs. +Thread count can have a significant impact on CPU performance. The optimal thread count may depend on the model and application. By default, ExecuTorch uses one thread per performance core where it can tell core types apart, and one per logical core otherwise. On Linux other than Android, it also uses no more threads than the CPUs the process may run on, for example under `taskset` or in a container with a CPU set, and at most 63. It reads that limit once, from the thread that first uses the thread pool. CPU quotas, such as `docker --cpus`, are not detected. Consider setting the thread count to cores / 2, or just set to 4 on mobile CPUs. Thread count can be set with the following function. Ensure this is done prior to loading or running a model. ``` diff --git a/extension/threadpool/cpuinfo_utils.h b/extension/threadpool/cpuinfo_utils.h index afb439cd140..80053260f66 100644 --- a/extension/threadpool/cpuinfo_utils.h +++ b/extension/threadpool/cpuinfo_utils.h @@ -15,7 +15,8 @@ namespace executorch::extension::cpuinfo { /** * Returns the number of performance cores, or 0 if it is unknown, for example * on x86 or when every core is an efficiency core. On Linux other than - * Android, the count is limited to get_num_allowed_cpus(). + * Android, the count is capped at get_num_allowed_cpus(), without checking + * which cores the affinity mask allows. */ uint32_t get_num_performant_cores(); diff --git a/extension/threadpool/threadpool.cpp b/extension/threadpool/threadpool.cpp index 78bfd60db93..9df73add3aa 100644 --- a/extension/threadpool/threadpool.cpp +++ b/extension/threadpool/threadpool.cpp @@ -140,11 +140,12 @@ ThreadPool* get_threadpool() { // Set threads equal to the number of performance cores. auto result = ::executorch::extension::cpuinfo::get_num_performant_cores(); #endif - // Never start more threads than the CPUs this process may run on, and use - // that number when the count is unknown (0), which pthreadpool would fill - // in with every online CPU, past the cap below. Android is left out: it - // moves every app between cpusets as it goes to the background and back, - // while the pool keeps the size it was created with. + // On Linux, never start more threads than the CPUs in this thread's + // affinity mask, which the pool's threads inherit, and use that number + // when the count is unknown (0), which pthreadpool would fill in with + // every online CPU, past the cap below. Android is left out: it moves + // every app between cpusets as it goes to the background and back, while + // the pool keeps the size it was created with. #if !defined(__ANDROID__) const auto allowed = ::executorch::extension::cpuinfo::get_num_allowed_cpus(); From edc588814f027afd4febf43ebcc0ab8bed006b78 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 1 Nov 2026 01:13:43 -0700 Subject: [PATCH 13/18] Expect an exact core count in the affinity test and mark the 63-thread cap as current in the FAQ The death test accepted a performance core count of 0 or 1 in the child, so on ARM a limit that wrongly dropped the count to 0 still passed. The parent now works out which one to expect: 1 when the count is known, 0 when it is not. The FAQ now says the default is "currently" at most 63 threads, because that cap comes from a sanitizer limit the code may drop later. --- docs/source/using-executorch-faqs.md | 2 +- extension/threadpool/test/threadpool_test.cpp | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/source/using-executorch-faqs.md b/docs/source/using-executorch-faqs.md index 782fea07147..4430147d27a 100644 --- a/docs/source/using-executorch-faqs.md +++ b/docs/source/using-executorch-faqs.md @@ -41,7 +41,7 @@ If building the runtime from source, ensure that the build is done in release mo Ensure the model is delegated. If not targeting a specific accelerator, use the XNNPACK delegate for CPU performance. Undelegated operators will typically fall back to the ExecuTorch portable library, which is designed as a fallback, and is not intended for performance sensitive operators. To target XNNPACK, pass an `XnnpackPartitioner` to `to_edge_transform_and_lower`. See [Building and Running ExecuTorch with XNNPACK Backend](tutorial-xnnpack-delegate-lowering.md) for more information. -Thread count can have a significant impact on CPU performance. The optimal thread count may depend on the model and application. By default, ExecuTorch uses one thread per performance core where it can tell core types apart, and one per logical core otherwise. On Linux other than Android, it also uses no more threads than the CPUs the process may run on, for example under `taskset` or in a container with a CPU set, and at most 63. It reads that limit once, from the thread that first uses the thread pool. CPU quotas, such as `docker --cpus`, are not detected. Consider setting the thread count to cores / 2, or just set to 4 on mobile CPUs. +Thread count can have a significant impact on CPU performance. The optimal thread count may depend on the model and application. By default, ExecuTorch uses one thread per performance core where it can tell core types apart, and one per logical core otherwise. On Linux other than Android, it also uses no more threads than the CPUs the process may run on, for example under `taskset` or in a container with a CPU set, and currently at most 63. It reads that limit once, from the thread that first uses the thread pool. CPU quotas, such as `docker --cpus`, are not detected. Consider setting the thread count to cores / 2, or just set to 4 on mobile CPUs. Thread count can be set with the following function. Ensure this is done prior to loading or running a model. ``` diff --git a/extension/threadpool/test/threadpool_test.cpp b/extension/threadpool/test/threadpool_test.cpp index 2249718cf4a..207c4e56f2c 100644 --- a/extension/threadpool/test/threadpool_test.cpp +++ b/extension/threadpool/test/threadpool_test.cpp @@ -266,9 +266,12 @@ TEST(ThreadPoolDeathTest, DefaultSizeHonorsCpuAffinity) { } executorch::runtime::runtime_init(); + // A known performance core count drops to 1, and an unknown one stays 0. + const char* expected = + ::executorch::extension::cpuinfo::get_num_performant_cores() > 0 + ? "pool 1 perf 1 end" + : "pool 1 perf 0 end"; EXPECT_EXIT( - print_counts_on_one_cpu(), - ExitedWithCode(EXIT_SUCCESS), - "pool 1 perf [01] end"); + print_counts_on_one_cpu(), ExitedWithCode(EXIT_SUCCESS), expected); } #endif From 9483ce20dc4d1da283c66adfa60d9b92c275b57c Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 1 Nov 2026 07:06:49 -0800 Subject: [PATCH 14/18] Guard the affinity limit by Linux and explain the Android exception once Both places that apply the affinity limit now use the same guard as the test, Linux other than Android, instead of only excluding Android and relying on get_num_allowed_cpus() returning 0 elsewhere. The reason for leaving Android out is now given only in the get_num_allowed_cpus() comment, and moved to the Linux-only includes it serves. --- extension/threadpool/cpuinfo_utils.cpp | 6 ++---- extension/threadpool/threadpool.cpp | 12 +++++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/extension/threadpool/cpuinfo_utils.cpp b/extension/threadpool/cpuinfo_utils.cpp index d7ec8a54c79..04988948319 100644 --- a/extension/threadpool/cpuinfo_utils.cpp +++ b/extension/threadpool/cpuinfo_utils.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -24,6 +23,7 @@ #if defined(__linux__) #include +#include #include #include #endif @@ -238,9 +238,7 @@ uint32_t get_num_allowed_cpus() { uint32_t get_num_performant_cores() { const uint32_t count = count_performant_cores(); -#if !defined(__ANDROID__) - // Android moves every app between cpusets as it goes to the background and - // back, so a count taken in the background would be too small later. +#if defined(__linux__) && !defined(__ANDROID__) const uint32_t allowed = get_num_allowed_cpus(); if (count > 0 && allowed > 0) { return std::min(count, allowed); diff --git a/extension/threadpool/threadpool.cpp b/extension/threadpool/threadpool.cpp index 9df73add3aa..036a6c10e6f 100644 --- a/extension/threadpool/threadpool.cpp +++ b/extension/threadpool/threadpool.cpp @@ -140,13 +140,11 @@ ThreadPool* get_threadpool() { // Set threads equal to the number of performance cores. auto result = ::executorch::extension::cpuinfo::get_num_performant_cores(); #endif - // On Linux, never start more threads than the CPUs in this thread's - // affinity mask, which the pool's threads inherit, and use that number - // when the count is unknown (0), which pthreadpool would fill in with - // every online CPU, past the cap below. Android is left out: it moves - // every app between cpusets as it goes to the background and back, while - // the pool keeps the size it was created with. -#if !defined(__ANDROID__) +#if defined(__linux__) && !defined(__ANDROID__) + // Never start more threads than the CPUs in this thread's affinity mask, + // which the pool's threads inherit, and use that number when the count is + // unknown (0), which pthreadpool would fill in with every online CPU, past + // the cap below. const auto allowed = ::executorch::extension::cpuinfo::get_num_allowed_cpus(); if (allowed > 0) { From 9409f8f11ff3a8e45b0bd3546ce541a91c447bb2 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 1 Nov 2026 07:51:13 -0800 Subject: [PATCH 15/18] Read ARM MIDR values above INT_MAX without throwing populate_available_cpu_mids() parsed each midr_el1 file with std::stoi. A MIDR whose implementer byte is 0x80 or higher, such as AmpereOne (0xC0), is above INT_MAX, so std::stoi threw std::out_of_range and the process ended the first time the thread pool or a runner asked for the performance core count. std::stoul reads the full value. --- extension/threadpool/cpuinfo_utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/threadpool/cpuinfo_utils.cpp b/extension/threadpool/cpuinfo_utils.cpp index 04988948319..7dd823a6826 100644 --- a/extension/threadpool/cpuinfo_utils.cpp +++ b/extension/threadpool/cpuinfo_utils.cpp @@ -112,7 +112,7 @@ bool populate_available_cpu_mids() { if (midr_file.is_open()) { std::string x; std::getline(midr_file, x); - tmp = std::stoi(x, nullptr, 16); + tmp = static_cast(std::stoul(x, nullptr, 16)); (*cpu_midrs)[i] = tmp; } else { ET_LOG(Info, "Failed to open midr file %s", midr_file_path.c_str()); From 64490ded4e36809b6747fbc7b9bcba3194cf7e47 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 1 Nov 2026 09:43:57 -0800 Subject: [PATCH 16/18] Read the affinity mask in the test helper on hosts with more than 1024 CPUs may_use_more_than_one_cpu() read the mask into a fixed cpu_set_t, which the kernel rejects when it has more possible CPUs than the set holds. It then assumed more than one CPU, so ParallelAdd and ParallelReduce failed on such a host under a one-CPU mask. It now grows the mask the same way get_num_allowed_cpus() does. threadpool.h also says what to do when the mask grows after the pool is created: call _unsafe_reset_threadpool(). --- extension/threadpool/test/threadpool_test.cpp | 22 +++++++++++++++---- extension/threadpool/threadpool.h | 1 + 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/extension/threadpool/test/threadpool_test.cpp b/extension/threadpool/test/threadpool_test.cpp index 207c4e56f2c..00a380ba0dd 100644 --- a/extension/threadpool/test/threadpool_test.cpp +++ b/extension/threadpool/test/threadpool_test.cpp @@ -21,6 +21,7 @@ #if defined(__linux__) && !defined(__ANDROID__) #include +#include #include #include #endif @@ -81,11 +82,24 @@ void run_lambda_with_size( // The pool has no more threads than the CPUs this process may run on. bool may_use_more_than_one_cpu() { #if defined(__linux__) && !defined(__ANDROID__) - cpu_set_t cpus; - return sched_getaffinity(0, sizeof(cpus), &cpus) != 0 || CPU_COUNT(&cpus) > 1; -#else - return true; + // Grows the mask as get_num_allowed_cpus() does, for kernels with more + // possible CPUs than a cpu_set_t holds. + for (int num_cpus = CPU_SETSIZE; num_cpus <= (1 << 20); num_cpus *= 2) { + std::unique_ptr cpus( + CPU_ALLOC(num_cpus), [](cpu_set_t* set) { CPU_FREE(set); }); + if (!cpus) { + break; + } + const size_t size = CPU_ALLOC_SIZE(num_cpus); + if (sched_getaffinity(0, size, cpus.get()) == 0) { + return CPU_COUNT_S(size, cpus.get()) > 1; + } + if (errno != EINVAL) { + break; + } + } #endif + return true; } #if defined(__linux__) && !defined(__ANDROID__) diff --git a/extension/threadpool/threadpool.h b/extension/threadpool/threadpool.h index 9abe458a4a1..b7d511dd95f 100644 --- a/extension/threadpool/threadpool.h +++ b/extension/threadpool/threadpool.h @@ -37,6 +37,7 @@ * not. The mask also sizes the threadpool when the performance core count is * unknown, as on x86. It is read once: the threadpool does not follow later * changes to the mask, and a child created by fork() keeps its parent's size. + * If the mask grows later, call _unsafe_reset_threadpool() to use the new CPUs. */ namespace executorch::extension::threadpool { From 6e169c1904980d2591a9a46552f47c4c9d60435b Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 1 Nov 2026 10:24:43 -0800 Subject: [PATCH 17/18] Say to set the thread count before loading a model when the mask can grow The previous note told users to call _unsafe_reset_threadpool() after the mask grows. Loaded XNNPACK delegates keep the pthreadpool they were loaded with, so resetting the pool at that point leaves them with a freed pool. The note now says to set the needed count before loading a model, which matches the FAQ. --- extension/threadpool/threadpool.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extension/threadpool/threadpool.h b/extension/threadpool/threadpool.h index b7d511dd95f..6a22cd286fb 100644 --- a/extension/threadpool/threadpool.h +++ b/extension/threadpool/threadpool.h @@ -37,7 +37,8 @@ * not. The mask also sizes the threadpool when the performance core count is * unknown, as on x86. It is read once: the threadpool does not follow later * changes to the mask, and a child created by fork() keeps its parent's size. - * If the mask grows later, call _unsafe_reset_threadpool() to use the new CPUs. + * If the mask can grow later, set the thread count you need with + * _unsafe_reset_threadpool() before loading any model. */ namespace executorch::extension::threadpool { From 180db83b89d3a6c7c5eaf9e66afd98e8c4aadb77 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 1 Nov 2026 10:24:44 -0800 Subject: [PATCH 18/18] Parse the ARM MIDR without exceptions std::stoul still threw std::invalid_argument when a midr_el1 file opened but read back empty or not as hex. std::strtoul with an end check treats that the same as a file that does not open: the probe gives up and the performance core count stays unknown. ExecuTorch does not use exceptions. --- extension/threadpool/cpuinfo_utils.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/extension/threadpool/cpuinfo_utils.cpp b/extension/threadpool/cpuinfo_utils.cpp index 7dd823a6826..754374096e0 100644 --- a/extension/threadpool/cpuinfo_utils.cpp +++ b/extension/threadpool/cpuinfo_utils.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #endif @@ -112,7 +113,14 @@ bool populate_available_cpu_mids() { if (midr_file.is_open()) { std::string x; std::getline(midr_file, x); - tmp = static_cast(std::stoul(x, nullptr, 16)); + char* end = nullptr; + const unsigned long midr = std::strtoul(x.c_str(), &end, 16); + if (end == x.c_str()) { + ET_LOG(Info, "Failed to parse midr file %s", midr_file_path.c_str()); + cpu_midrs->clear(); + return false; + } + tmp = static_cast(midr); (*cpu_midrs)[i] = tmp; } else { ET_LOG(Info, "Failed to open midr file %s", midr_file_path.c_str());