From 90110d0d9d4568d8456dd6f2c4f22220c3bae972 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:37:20 -0500 Subject: [PATCH 1/4] Implement --disable-persistent-l2-cache If user opts in, persistent L2-cache state is reset before timer starts in measure_cold, and before batch executes in measure_cold. Warm-up runs also reset persistent L2 cache state. Reset is performed before L2 flushing to demote persisting cache lines ahead of flushing to allow them to be flushed as well. --- docs/cli_help.md | 8 ++ nvbench/benchmark_base.cuh | 13 +++ nvbench/benchmark_base.cxx | 7 +- nvbench/detail/measure_cold.cu | 8 ++ nvbench/detail/measure_cold.cuh | 4 + .../detail/measure_cold_launch_timer_core.cuh | 1 + nvbench/detail/measure_hot.cu | 8 ++ nvbench/detail/measure_hot.cuh | 4 + nvbench/detail/persisting_l2_cache_reset.cuh | 87 +++++++++++++++++++ nvbench/option_parser.cu | 17 ++++ nvbench/option_parser.cuh | 1 + nvbench/state.cuh | 9 ++ nvbench/state.cxx | 2 + testing/cleanup_guards.cu | 8 +- testing/option_parser.cu | 53 +++++++++++ 15 files changed, 225 insertions(+), 5 deletions(-) create mode 100644 nvbench/detail/persisting_l2_cache_reset.cuh diff --git a/docs/cli_help.md b/docs/cli_help.md index 59de7b47..ec20e69b 100644 --- a/docs/cli_help.md +++ b/docs/cli_help.md @@ -136,6 +136,14 @@ * Applied to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. +* `--disable-persisting-l2-cache` + * Disable persisting-L2 cache reservation and state before GPU warmups and + measurement work. + * Intended to isolate benchmark iterations from persisting-L2 state left by + earlier work. + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. + * `--batch-target-time ` * Target accumulated GPU time for batched measurements. * Default is 0.5 seconds. diff --git a/nvbench/benchmark_base.cuh b/nvbench/benchmark_base.cuh index d3900f48..a32859b6 100644 --- a/nvbench/benchmark_base.cuh +++ b/nvbench/benchmark_base.cuh @@ -239,6 +239,18 @@ struct benchmark_base } /// @} + /// If true, persisting L2 cache state is disabled before GPU measurement work. @{ + [[nodiscard]] bool get_disable_persisting_l2_cache() const + { + return m_disable_persisting_l2_cache; + } + benchmark_base &set_disable_persisting_l2_cache(bool v) + { + m_disable_persisting_l2_cache = v; + return *this; + } + /// @} + /// If a warmup run finishes in less than `skip_time`, the measurement will /// be skipped. /// Extremely fast kernels (< 5000 ns) often timeout before they can @@ -348,6 +360,7 @@ protected: bool m_is_cpu_only{false}; bool m_run_once{false}; bool m_disable_blocking_kernel{false}; + bool m_disable_persisting_l2_cache{false}; bool m_skip_batched{false}; nvbench::int64_t m_min_samples{10}; diff --git a/nvbench/benchmark_base.cxx b/nvbench/benchmark_base.cxx index d9f25697..fb80d7fe 100644 --- a/nvbench/benchmark_base.cxx +++ b/nvbench/benchmark_base.cxx @@ -45,9 +45,10 @@ std::unique_ptr benchmark_base::clone() const result->m_printer_ptr = m_printer_ptr; - result->m_is_cpu_only = m_is_cpu_only; - result->m_run_once = m_run_once; - result->m_disable_blocking_kernel = m_disable_blocking_kernel; + result->m_is_cpu_only = m_is_cpu_only; + result->m_run_once = m_run_once; + result->m_disable_blocking_kernel = m_disable_blocking_kernel; + result->m_disable_persisting_l2_cache = m_disable_persisting_l2_cache; result->m_min_samples = m_min_samples; result->m_cold_warmup_runs = m_cold_warmup_runs; diff --git a/nvbench/detail/measure_cold.cu b/nvbench/detail/measure_cold.cu index eb63d40d..49c5ca0c 100644 --- a/nvbench/detail/measure_cold.cu +++ b/nvbench/detail/measure_cold.cu @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,7 @@ measure_cold_base::measure_cold_base(state &exec_state) , m_stopping_criterion{nvbench::criterion_manager::get().get_criterion( exec_state.get_stopping_criterion())} , m_disable_blocking_kernel{exec_state.get_disable_blocking_kernel()} + , m_disable_persisting_l2_cache{exec_state.get_disable_persisting_l2_cache()} , m_run_once{exec_state.get_run_once()} , m_check_throttling(!exec_state.get_run_once()) , m_min_samples{exec_state.get_min_samples()} @@ -66,6 +68,12 @@ measure_cold_base::measure_cold_base(state &exec_state) } } +void measure_cold_base::reset_persisting_l2_cache() +{ + nvbench::detail::reset_persisting_l2_cache_if_requested(m_disable_persisting_l2_cache, + m_state.get_device()); +} + void measure_cold_base::check() { const auto device = m_state.get_device(); diff --git a/nvbench/detail/measure_cold.cuh b/nvbench/detail/measure_cold.cuh index df245193..2c939f25 100644 --- a/nvbench/detail/measure_cold.cuh +++ b/nvbench/detail/measure_cold.cuh @@ -85,6 +85,7 @@ protected: void check_skip_time(nvbench::float64_t warmup_time); __forceinline__ void flush_device_l2() { m_l2flush.flush(m_launch.get_stream()); } + void reset_persisting_l2_cache(); __forceinline__ cudaError_t sync_stream_noexcept() const noexcept { @@ -117,6 +118,7 @@ protected: nvbench::detail::gpu_frequency m_gpu_frequency{}; bool m_disable_blocking_kernel{false}; + bool m_disable_persisting_l2_cache{false}; bool m_run_once{false}; bool m_check_throttling{true}; @@ -191,6 +193,8 @@ struct measure_cold_base::kernel_launch_timer __forceinline__ void flush_device_l2() { m_measure.flush_device_l2(); } + __forceinline__ void reset_persisting_l2_cache() { m_measure.reset_persisting_l2_cache(); } + __forceinline__ void sync_stream() { m_measure.sync_stream(); } __forceinline__ cudaError_t sync_stream_noexcept() const noexcept diff --git a/nvbench/detail/measure_cold_launch_timer_core.cuh b/nvbench/detail/measure_cold_launch_timer_core.cuh index d0cd659e..8688ef10 100644 --- a/nvbench/detail/measure_cold_launch_timer_core.cuh +++ b/nvbench/detail/measure_cold_launch_timer_core.cuh @@ -101,6 +101,7 @@ public: { cleanup_guard cleanup{*this}; + m_measure.reset_persisting_l2_cache(); m_measure.flush_device_l2(); m_measure.sync_stream(); diff --git a/nvbench/detail/measure_hot.cu b/nvbench/detail/measure_hot.cu index e5199c25..f8f2080f 100644 --- a/nvbench/detail/measure_hot.cu +++ b/nvbench/detail/measure_hot.cu @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -78,6 +79,7 @@ measure_hot_base::measure_hot_base(state &exec_state) , m_batch_target_time{exec_state.get_batch_target_time()} , m_skip_time{exec_state.get_skip_time()} , m_timeout{exec_state.get_timeout()} + , m_disable_persisting_l2_cache{exec_state.get_disable_persisting_l2_cache()} { try { @@ -96,6 +98,12 @@ measure_hot_base::measure_hot_base(state &exec_state) } } +void measure_hot_base::reset_persisting_l2_cache() +{ + nvbench::detail::reset_persisting_l2_cache_if_requested(m_disable_persisting_l2_cache, + m_state.get_device()); +} + // CUDA-time predictions choose how many launches are needed to reach the // accumulated GPU-time target. Valid small predictions are raised to the // supplied minimum; invalid or overflowing predictions fall back to the diff --git a/nvbench/detail/measure_hot.cuh b/nvbench/detail/measure_hot.cuh index e53088b2..033256e6 100644 --- a/nvbench/detail/measure_hot.cuh +++ b/nvbench/detail/measure_hot.cuh @@ -74,6 +74,7 @@ protected: void check_skip_time(nvbench::float64_t warmup_time); void block_stream(); + void reset_persisting_l2_cache(); static nvbench::int64_t predict_cuda_batch_size(nvbench::float64_t target_time, nvbench::float64_t time_estimate, @@ -114,6 +115,7 @@ protected: nvbench::float64_t m_total_cuda_time{}; bool m_disable_blocking_kernel{false}; + bool m_disable_persisting_l2_cache{false}; bool m_max_time_exceeded{false}; }; @@ -141,6 +143,7 @@ private: { nvbench::detail::stream_cleanup_guard cleanup{*this}; + this->reset_persisting_l2_cache(); m_walltime_timer.start(); { m_cuda_timer.start(m_launch.get_stream()); @@ -182,6 +185,7 @@ private: nvbench::detail::stream_cleanup_guard cleanup{*this}; + this->reset_persisting_l2_cache(); if (!m_disable_blocking_kernel) { // Block stream until some work is queued. diff --git a/nvbench/detail/persisting_l2_cache_reset.cuh b/nvbench/detail/persisting_l2_cache_reset.cuh new file mode 100644 index 00000000..f9348f6f --- /dev/null +++ b/nvbench/detail/persisting_l2_cache_reset.cuh @@ -0,0 +1,87 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#if defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_GCC) +#pragma GCC system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_CLANG) +#pragma clang system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_MSVC) +#pragma system_header +#endif + +#include +#include +#include +#include + +#include + +#include +#include + +namespace nvbench::detail +{ + +struct persisting_l2_cache_reset +{ + explicit persisting_l2_cache_reset(int device_id) + : m_device_id{device_id} + {} + + void operator()() const + { + nvbench::detail::device_scope scope{m_device_id}; + + int max_persisting_l2_cache_size{}; + NVBENCH_CUDA_CALL(cudaDeviceGetAttribute(&max_persisting_l2_cache_size, + cudaDevAttrMaxPersistingL2CacheSize, + m_device_id)); + if (max_persisting_l2_cache_size == 0) + { + return; + } + + NVBENCH_CUDA_CALL(cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, 0)); + NVBENCH_CUDA_CALL(cudaCtxResetPersistingL2Cache()); + } + +private: + int m_device_id{}; +}; + +inline void reset_persisting_l2_cache_if_requested(bool requested, + const std::optional &device) +{ + if (!requested) + { + return; + } + + if (!device) + { + NVBENCH_THROW(std::runtime_error, "{}", "Device required to disable persisting L2 cache."); + } + + nvbench::detail::persisting_l2_cache_reset{device->get_id()}(); +} + +} // namespace nvbench::detail diff --git a/nvbench/option_parser.cu b/nvbench/option_parser.cu index 15164f68..b534daba 100644 --- a/nvbench/option_parser.cu +++ b/nvbench/option_parser.cu @@ -572,6 +572,11 @@ void option_parser::parse_range(option_parser::arg_iterator_t first, this->disable_batched(); first += 1; } + else if (arg == "--disable-persisting-l2-cache") + { + this->disable_persisting_l2_cache(); + first += 1; + } else if (arg == "--quiet" || arg == "-q") { // Setting this flag prevents the default stdout printer from being @@ -889,6 +894,18 @@ void option_parser::disable_batched() bench.set_skip_batched(true); } +void option_parser::disable_persisting_l2_cache() +{ + // If no active benchmark, save args as global + if (m_benchmarks.empty()) + { + m_global_benchmark_args.push_back("--disable-persisting-l2-cache"); + return; + } + benchmark_base &bench = *m_benchmarks.back(); + bench.set_disable_persisting_l2_cache(true); +} + void option_parser::add_benchmark(const std::string &name) try { diff --git a/nvbench/option_parser.cuh b/nvbench/option_parser.cuh index 29b2572b..108f8380 100644 --- a/nvbench/option_parser.cuh +++ b/nvbench/option_parser.cuh @@ -102,6 +102,7 @@ private: void enable_profile(); void disable_batched(); + void disable_persisting_l2_cache(); void add_benchmark(const std::string &name); void replay_global_args(); diff --git a/nvbench/state.cuh b/nvbench/state.cuh index 89316b8d..d81987f5 100644 --- a/nvbench/state.cuh +++ b/nvbench/state.cuh @@ -213,6 +213,14 @@ struct state void set_disable_blocking_kernel(bool v) { m_disable_blocking_kernel = v; } /// @} + /// If true, persisting L2 cache state is disabled before GPU measurement work. @{ + [[nodiscard]] bool get_disable_persisting_l2_cache() const + { + return m_disable_persisting_l2_cache; + } + void set_disable_persisting_l2_cache(bool v) { m_disable_persisting_l2_cache = v; } + /// @} + /// If a warmup run finishes in less than `skip_time`, the measurement will /// be skipped. /// Extremely fast kernels (< 5000 ns) often timeout before they can @@ -357,6 +365,7 @@ private: bool m_is_cpu_only{false}; bool m_run_once{false}; bool m_disable_blocking_kernel{false}; + bool m_disable_persisting_l2_cache{false}; bool m_skip_batched{false}; nvbench::criterion_params m_criterion_params; diff --git a/nvbench/state.cxx b/nvbench/state.cxx index 23166e7f..a0ef6818 100644 --- a/nvbench/state.cxx +++ b/nvbench/state.cxx @@ -44,6 +44,7 @@ state::state(const benchmark_base &bench) , m_is_cpu_only(bench.get_is_cpu_only()) , m_run_once{bench.get_run_once()} , m_disable_blocking_kernel{bench.get_disable_blocking_kernel()} + , m_disable_persisting_l2_cache{bench.get_disable_persisting_l2_cache()} , m_criterion_params{bench.get_criterion_params()} , m_stopping_criterion(bench.get_stopping_criterion()) , m_min_samples{bench.get_min_samples()} @@ -68,6 +69,7 @@ state::state(const benchmark_base &bench, , m_is_cpu_only(bench.get_is_cpu_only()) , m_run_once{bench.get_run_once()} , m_disable_blocking_kernel{bench.get_disable_blocking_kernel()} + , m_disable_persisting_l2_cache{bench.get_disable_persisting_l2_cache()} , m_skip_batched{bench.get_skip_batched()} , m_criterion_params{bench.get_criterion_params()} , m_stopping_criterion(bench.get_stopping_criterion()) diff --git a/testing/cleanup_guards.cu b/testing/cleanup_guards.cu index 7b431ec1..589ef936 100644 --- a/testing/cleanup_guards.cu +++ b/testing/cleanup_guards.cu @@ -140,6 +140,7 @@ static_assert((verify_noexcept_contracts(), true), "Noexcept cleanup contracts m enum class action { flush_device_l2, + reset_persisting_l2_cache, sync_stream, sync_stream_noexcept, cpu_timer_start, @@ -194,6 +195,7 @@ struct fake_measure } void flush_device_l2() { this->record_or_throw(action::flush_device_l2); } + void reset_persisting_l2_cache() { this->record_or_throw(action::reset_persisting_l2_cache); } void sync_stream() { this->record_or_throw(action::sync_stream); } int sync_stream_noexcept() noexcept { @@ -301,7 +303,8 @@ void test_kernel_launch_timer_block_stream_throw() }); assert_actions(measure, - {action::flush_device_l2, + {action::reset_persisting_l2_cache, + action::flush_device_l2, action::sync_stream, action::cpu_timer_start, action::block_stream, @@ -323,7 +326,8 @@ void test_kernel_launch_timer_gpu_frequency_start_throw() }); assert_actions(measure, - {action::flush_device_l2, + {action::reset_persisting_l2_cache, + action::flush_device_l2, action::sync_stream, action::cpu_timer_start, action::block_stream, diff --git a/testing/option_parser.cu b/testing/option_parser.cu index aa386e65..e3967d03 100644 --- a/testing/option_parser.cu +++ b/testing/option_parser.cu @@ -1293,6 +1293,58 @@ void test_timeout() ASSERT(std::abs(states[0].get_timeout() - 12345e2) < 1.); } +void test_disable_persisting_l2_cache() +{ + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "DummyBench"}); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(!states[0].get_disable_persisting_l2_cache()); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "DummyBench", "--disable-persisting-l2-cache"}); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_disable_persisting_l2_cache()); + } + + { + nvbench::option_parser parser; + parser.parse({"--disable-persisting-l2-cache", "--benchmark", "DummyBench"}); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_disable_persisting_l2_cache()); + } + + { + nvbench::option_parser parser; + parser.parse( + {"--disable-persisting-l2-cache", "--benchmark", "DummyBench", "--benchmark", "TestBench"}); + + const auto &benches = parser.get_benchmarks(); + ASSERT(benches.size() == 2); + ASSERT(benches[0] != nullptr); + ASSERT(benches[1] != nullptr); + + const auto dummy_states = nvbench::detail::state_generator::create(*benches[0]); + ASSERT(dummy_states.size() == 1); + ASSERT(dummy_states[0].get_disable_persisting_l2_cache()); + + const auto test_states = nvbench::detail::state_generator::create(*benches[1]); + ASSERT(!test_states.empty()); + for (const auto &state : test_states) + { + ASSERT(state.get_disable_persisting_l2_cache()); + } + } +} + void test_batch_target_time() { { @@ -1814,6 +1866,7 @@ try test_skip_time(); test_cold_max_warmup_walltime(); test_timeout(); + test_disable_persisting_l2_cache(); test_batch_target_time(); test_json_stream_destinations(); test_output_parent_directories_created(); From 30847a027f557c3e20fca49afc1ca300e07d6c62 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:28:15 -0500 Subject: [PATCH 2/4] Restore persisting L2 cache limit after measurements Save the current cudaLimitPersistingL2CacheSize when --disable-persisting-l2-cache is active, reset persisting L2 state before cold/hot measurements, and explicitly restore the saved limit after the measurement scope exits or throws. Keep the reset helper out of public measurement headers to avoid leaking the private fmt dependency through nvbench/nvbench.cuh, and add cleanup guard coverage for the no-device helper path. --- nvbench/detail/measure_cold.cu | 24 +++++++++- nvbench/detail/measure_cold.cuh | 27 +++++++++-- nvbench/detail/measure_hot.cu | 24 +++++++++- nvbench/detail/measure_hot.cuh | 23 ++++++++- nvbench/detail/persisting_l2_cache_reset.cuh | 49 ++++++++++++++++---- testing/cleanup_guards.cu | 22 +++++++++ 6 files changed, 149 insertions(+), 20 deletions(-) diff --git a/nvbench/detail/measure_cold.cu b/nvbench/detail/measure_cold.cu index 49c5ca0c..090e85cd 100644 --- a/nvbench/detail/measure_cold.cu +++ b/nvbench/detail/measure_cold.cu @@ -68,10 +68,30 @@ measure_cold_base::measure_cold_base(state &exec_state) } } +measure_cold_base::~measure_cold_base() = default; + +void measure_cold_base::initialize_persisting_l2_cache_disable() +{ + m_persisting_l2_cache_disable = + nvbench::detail::make_persisting_l2_cache_disable_if_requested(m_disable_persisting_l2_cache, + m_state.get_device()); +} + void measure_cold_base::reset_persisting_l2_cache() { - nvbench::detail::reset_persisting_l2_cache_if_requested(m_disable_persisting_l2_cache, - m_state.get_device()); + if (m_persisting_l2_cache_disable) + { + m_persisting_l2_cache_disable->reset_before_measurement(); + } +} + +void measure_cold_base::restore_persisting_l2_cache() +{ + if (m_persisting_l2_cache_disable) + { + m_persisting_l2_cache_disable->restore(); + m_persisting_l2_cache_disable.reset(); + } } void measure_cold_base::check() diff --git a/nvbench/detail/measure_cold.cuh b/nvbench/detail/measure_cold.cuh index 2c939f25..e4663c73 100644 --- a/nvbench/detail/measure_cold.cuh +++ b/nvbench/detail/measure_cold.cuh @@ -46,6 +46,7 @@ #include #include +#include #include #include #include @@ -59,10 +60,13 @@ struct state; namespace detail { +struct persisting_l2_cache_disable; + // non-templated code goes here: struct measure_cold_base { explicit measure_cold_base(nvbench::state &exec_state); + ~measure_cold_base(); measure_cold_base(const measure_cold_base &) = delete; measure_cold_base(measure_cold_base &&) = delete; measure_cold_base &operator=(const measure_cold_base &) = delete; @@ -85,7 +89,9 @@ protected: void check_skip_time(nvbench::float64_t warmup_time); __forceinline__ void flush_device_l2() { m_l2flush.flush(m_launch.get_stream()); } + void initialize_persisting_l2_cache_disable(); void reset_persisting_l2_cache(); + void restore_persisting_l2_cache(); __forceinline__ cudaError_t sync_stream_noexcept() const noexcept { @@ -112,6 +118,7 @@ protected: nvbench::cpu_timer m_walltime_timer{}; nvbench::detail::l2flush m_l2flush{}; nvbench::blocking_kernel m_blocker{}; + std::unique_ptr m_persisting_l2_cache_disable{}; nvbench::criterion_params m_criterion_params{}; nvbench::stopping_criterion_base &m_stopping_criterion; @@ -254,11 +261,23 @@ struct measure_cold : public measure_cold_base { this->check(); this->initialize(); - this->run_warmup(); + this->initialize_persisting_l2_cache_disable(); + + try + { + this->run_warmup(); + + this->run_trials_prologue(); + this->run_trials(); + this->run_trials_epilogue(); - this->run_trials_prologue(); - this->run_trials(); - this->run_trials_epilogue(); + this->restore_persisting_l2_cache(); + } + catch (...) + { + this->restore_persisting_l2_cache(); + throw; + } this->generate_summaries(); } diff --git a/nvbench/detail/measure_hot.cu b/nvbench/detail/measure_hot.cu index f8f2080f..48fd5f78 100644 --- a/nvbench/detail/measure_hot.cu +++ b/nvbench/detail/measure_hot.cu @@ -98,10 +98,30 @@ measure_hot_base::measure_hot_base(state &exec_state) } } +measure_hot_base::~measure_hot_base() = default; + +void measure_hot_base::initialize_persisting_l2_cache_disable() +{ + m_persisting_l2_cache_disable = + nvbench::detail::make_persisting_l2_cache_disable_if_requested(m_disable_persisting_l2_cache, + m_state.get_device()); +} + void measure_hot_base::reset_persisting_l2_cache() { - nvbench::detail::reset_persisting_l2_cache_if_requested(m_disable_persisting_l2_cache, - m_state.get_device()); + if (m_persisting_l2_cache_disable) + { + m_persisting_l2_cache_disable->reset_before_measurement(); + } +} + +void measure_hot_base::restore_persisting_l2_cache() +{ + if (m_persisting_l2_cache_disable) + { + m_persisting_l2_cache_disable->restore(); + m_persisting_l2_cache_disable.reset(); + } } // CUDA-time predictions choose how many launches are needed to reach the diff --git a/nvbench/detail/measure_hot.cuh b/nvbench/detail/measure_hot.cuh index 033256e6..0227b803 100644 --- a/nvbench/detail/measure_hot.cuh +++ b/nvbench/detail/measure_hot.cuh @@ -39,6 +39,7 @@ #include #include +#include namespace nvbench { @@ -48,10 +49,13 @@ struct state; namespace detail { +struct persisting_l2_cache_disable; + // non-templated code goes here to keep instantiation cost down: struct measure_hot_base { explicit measure_hot_base(nvbench::state &exec_state); + ~measure_hot_base(); measure_hot_base(const measure_hot_base &) = delete; measure_hot_base(measure_hot_base &&) = delete; measure_hot_base &operator=(const measure_hot_base &) = delete; @@ -74,7 +78,9 @@ protected: void check_skip_time(nvbench::float64_t warmup_time); void block_stream(); + void initialize_persisting_l2_cache_disable(); void reset_persisting_l2_cache(); + void restore_persisting_l2_cache(); static nvbench::int64_t predict_cuda_batch_size(nvbench::float64_t target_time, nvbench::float64_t time_estimate, @@ -104,6 +110,7 @@ protected: nvbench::cuda_timer m_cuda_timer; nvbench::cpu_timer m_walltime_timer; nvbench::blocking_kernel m_blocker; + std::unique_ptr m_persisting_l2_cache_disable{}; nvbench::int64_t m_min_samples{}; nvbench::float64_t m_batch_target_time{}; @@ -131,8 +138,20 @@ struct measure_hot : public measure_hot_base { this->check(); this->initialize(); - this->run_warmup(); - this->run_trials(); + this->initialize_persisting_l2_cache_disable(); + + try + { + this->run_warmup(); + this->run_trials(); + this->restore_persisting_l2_cache(); + } + catch (...) + { + this->restore_persisting_l2_cache(); + throw; + } + this->generate_summaries(); } diff --git a/nvbench/detail/persisting_l2_cache_reset.cuh b/nvbench/detail/persisting_l2_cache_reset.cuh index f9348f6f..38d03e27 100644 --- a/nvbench/detail/persisting_l2_cache_reset.cuh +++ b/nvbench/detail/persisting_l2_cache_reset.cuh @@ -35,19 +35,18 @@ #include +#include +#include #include #include namespace nvbench::detail { -struct persisting_l2_cache_reset +struct persisting_l2_cache_disable { - explicit persisting_l2_cache_reset(int device_id) + explicit persisting_l2_cache_disable(int device_id) : m_device_id{device_id} - {} - - void operator()() const { nvbench::detail::device_scope scope{m_device_id}; @@ -60,20 +59,50 @@ struct persisting_l2_cache_reset return; } - NVBENCH_CUDA_CALL(cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, 0)); + NVBENCH_CUDA_CALL(cudaDeviceGetLimit(&m_original_limit, cudaLimitPersistingL2CacheSize)); + m_supported = true; + } + + void reset_before_measurement() + { + if (!m_supported) + { + return; + } + + nvbench::detail::device_scope scope{m_device_id}; + NVBENCH_CUDA_CALL(cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, std::size_t{0})); + m_limit_restored = false; + NVBENCH_CUDA_CALL(cudaCtxResetPersistingL2Cache()); + } + + void restore() + { + if (!m_supported || m_limit_restored) + { + return; + } + + nvbench::detail::device_scope scope{m_device_id}; + NVBENCH_CUDA_CALL(cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, m_original_limit)); NVBENCH_CUDA_CALL(cudaCtxResetPersistingL2Cache()); + m_limit_restored = true; } private: int m_device_id{}; + std::size_t m_original_limit{}; + bool m_supported{false}; + bool m_limit_restored{true}; }; -inline void reset_persisting_l2_cache_if_requested(bool requested, - const std::optional &device) +inline std::unique_ptr +make_persisting_l2_cache_disable_if_requested(bool requested, + const std::optional &device) { if (!requested) { - return; + return nullptr; } if (!device) @@ -81,7 +110,7 @@ inline void reset_persisting_l2_cache_if_requested(bool requested, NVBENCH_THROW(std::runtime_error, "{}", "Device required to disable persisting L2 cache."); } - nvbench::detail::persisting_l2_cache_reset{device->get_id()}(); + return std::make_unique(device->get_id()); } } // namespace nvbench::detail diff --git a/testing/cleanup_guards.cu b/testing/cleanup_guards.cu index 589ef936..c67f62f7 100644 --- a/testing/cleanup_guards.cu +++ b/testing/cleanup_guards.cu @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -358,6 +360,24 @@ void test_kernel_launch_timer_gpu_frequency_stop_throw() action::cpu_timer_stop_noexcept}); } +void test_persisting_l2_cache_disable_not_requested_without_device() +{ + const std::optional no_device; + const auto disable = nvbench::detail::make_persisting_l2_cache_disable_if_requested(false, + no_device); + + ASSERT(!disable); +} + +void test_persisting_l2_cache_disable_requires_device() +{ + const std::optional no_device; + + assert_throws([&no_device] { + (void)nvbench::detail::make_persisting_l2_cache_disable_if_requested(true, no_device); + }); +} + } // namespace int main() @@ -368,6 +388,8 @@ try test_kernel_launch_timer_block_stream_throw(); test_kernel_launch_timer_gpu_frequency_start_throw(); test_kernel_launch_timer_gpu_frequency_stop_throw(); + test_persisting_l2_cache_disable_not_requested_without_device(); + test_persisting_l2_cache_disable_requires_device(); return 0; } From bc6ec03a80edf1a0668940a94618e1d7c9216d9f Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:32:49 -0500 Subject: [PATCH 3/4] Fix persisting L2 reset pointer incomplete-type build Use a custom deleter for the persisting-L2 reset helper pointer stored in cold/hot measurement headers. Older CTK/host compiler combinations instantiate std::default_delete while the helper type is still incomplete, causing CTK 12 matrix builds to fail from unrelated translation units. Keep the reset helper definition out of the measurement headers while defining the deleter where the helper type is complete. --- nvbench/detail/measure_cold.cuh | 6 +-- nvbench/detail/measure_hot.cuh | 6 +-- nvbench/detail/persisting_l2_cache_reset.cuh | 13 ++++-- .../detail/persisting_l2_cache_reset_fwd.cuh | 46 +++++++++++++++++++ 4 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 nvbench/detail/persisting_l2_cache_reset_fwd.cuh diff --git a/nvbench/detail/measure_cold.cuh b/nvbench/detail/measure_cold.cuh index e4663c73..b63cb4a7 100644 --- a/nvbench/detail/measure_cold.cuh +++ b/nvbench/detail/measure_cold.cuh @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -46,7 +47,6 @@ #include #include -#include #include #include #include @@ -60,8 +60,6 @@ struct state; namespace detail { -struct persisting_l2_cache_disable; - // non-templated code goes here: struct measure_cold_base { @@ -118,7 +116,7 @@ protected: nvbench::cpu_timer m_walltime_timer{}; nvbench::detail::l2flush m_l2flush{}; nvbench::blocking_kernel m_blocker{}; - std::unique_ptr m_persisting_l2_cache_disable{}; + nvbench::detail::persisting_l2_cache_disable_ptr m_persisting_l2_cache_disable{}; nvbench::criterion_params m_criterion_params{}; nvbench::stopping_criterion_base &m_stopping_criterion; diff --git a/nvbench/detail/measure_hot.cuh b/nvbench/detail/measure_hot.cuh index 0227b803..562a3d7c 100644 --- a/nvbench/detail/measure_hot.cuh +++ b/nvbench/detail/measure_hot.cuh @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,6 @@ #include #include -#include namespace nvbench { @@ -49,8 +49,6 @@ struct state; namespace detail { -struct persisting_l2_cache_disable; - // non-templated code goes here to keep instantiation cost down: struct measure_hot_base { @@ -110,7 +108,7 @@ protected: nvbench::cuda_timer m_cuda_timer; nvbench::cpu_timer m_walltime_timer; nvbench::blocking_kernel m_blocker; - std::unique_ptr m_persisting_l2_cache_disable{}; + nvbench::detail::persisting_l2_cache_disable_ptr m_persisting_l2_cache_disable{}; nvbench::int64_t m_min_samples{}; nvbench::float64_t m_batch_target_time{}; diff --git a/nvbench/detail/persisting_l2_cache_reset.cuh b/nvbench/detail/persisting_l2_cache_reset.cuh index 38d03e27..2c4e44b6 100644 --- a/nvbench/detail/persisting_l2_cache_reset.cuh +++ b/nvbench/detail/persisting_l2_cache_reset.cuh @@ -30,13 +30,13 @@ #include #include +#include #include #include #include #include -#include #include #include @@ -96,7 +96,13 @@ private: bool m_limit_restored{true}; }; -inline std::unique_ptr +inline void +persisting_l2_cache_disable_deleter::operator()(persisting_l2_cache_disable *ptr) const noexcept +{ + delete ptr; +} + +inline nvbench::detail::persisting_l2_cache_disable_ptr make_persisting_l2_cache_disable_if_requested(bool requested, const std::optional &device) { @@ -110,7 +116,8 @@ make_persisting_l2_cache_disable_if_requested(bool requested, NVBENCH_THROW(std::runtime_error, "{}", "Device required to disable persisting L2 cache."); } - return std::make_unique(device->get_id()); + return nvbench::detail::persisting_l2_cache_disable_ptr{ + new nvbench::detail::persisting_l2_cache_disable{device->get_id()}}; } } // namespace nvbench::detail diff --git a/nvbench/detail/persisting_l2_cache_reset_fwd.cuh b/nvbench/detail/persisting_l2_cache_reset_fwd.cuh new file mode 100644 index 00000000..7acf2b43 --- /dev/null +++ b/nvbench/detail/persisting_l2_cache_reset_fwd.cuh @@ -0,0 +1,46 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#if defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_GCC) +#pragma GCC system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_CLANG) +#pragma clang system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_MSVC) +#pragma system_header +#endif + +#include + +namespace nvbench::detail +{ + +struct persisting_l2_cache_disable; + +struct persisting_l2_cache_disable_deleter +{ + void operator()(persisting_l2_cache_disable *ptr) const noexcept; +}; + +using persisting_l2_cache_disable_ptr = + std::unique_ptr; + +} // namespace nvbench::detail From 2eb19c3271005f980d7c50c57fa1f7c6c3d574af Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:20:49 -0500 Subject: [PATCH 4/4] Add Python bindings to setter/getter of disable-persisting-l2-cache Benchmark has setter, State has setter/getter. Decorators added to cuda.bench.option --- python/cuda/bench/__init__.pyi | 5 +++ python/cuda/bench/_decorators.py | 10 ++++++ python/src/py_nvbench.cpp | 32 +++++++++++++++++++ python/test/test_cuda_bench.py | 55 ++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+) diff --git a/python/cuda/bench/__init__.pyi b/python/cuda/bench/__init__.pyi index c0bc037f..6c9c098b 100644 --- a/python/cuda/bench/__init__.pyi +++ b/python/cuda/bench/__init__.pyi @@ -66,6 +66,7 @@ class Benchmark: def set_throttle_threshold(self, threshold: SupportsFloat) -> Self: ... def set_timeout(self, duration_seconds: SupportsFloat) -> Self: ... def set_batch_target_time(self, duration_seconds: SupportsFloat) -> Self: ... + def set_disable_persisting_l2_cache(self, flag: bool) -> Self: ... def set_stopping_criterion(self, criterion: str) -> Self: ... def set_criterion_param_float64(self, name: str, value: SupportsFloat) -> Self: ... def set_criterion_param_int64(self, name: str, value: SupportsInt) -> Self: ... @@ -122,6 +123,8 @@ class State: def set_cold_max_warmup_walltime(self, duration_seconds: SupportsFloat) -> None: ... def get_disable_blocking_kernel(self) -> bool: ... def set_disable_blocking_kernel(self, flag: bool) -> None: ... + def get_disable_persisting_l2_cache(self) -> bool: ... + def set_disable_persisting_l2_cache(self, flag: bool) -> None: ... def get_run_once(self) -> bool: ... def set_run_once(self, run_once_flag: bool) -> None: ... def get_timeout(self) -> float: ... @@ -210,6 +213,8 @@ class _OptionDecorators: def set_batch_target_time( self, duration_seconds: SupportsFloat ) -> Callable[[_F], _F]: ... + def disable_persisting_l2_cache(self, value: bool = True) -> Callable[[_F], _F]: ... + def set_disable_persisting_l2_cache(self, value: bool) -> Callable[[_F], _F]: ... def stopping_criterion(self, criterion: str) -> Callable[[_F], _F]: ... def set_stopping_criterion(self, criterion: str) -> Callable[[_F], _F]: ... def criterion_param_float64( diff --git a/python/cuda/bench/_decorators.py b/python/cuda/bench/_decorators.py index bee2fbc1..a2b40c8b 100644 --- a/python/cuda/bench/_decorators.py +++ b/python/cuda/bench/_decorators.py @@ -250,6 +250,16 @@ def set_batch_target_time(self, duration_seconds: float) -> Callable[[_F], _F]: lambda benchmark: benchmark.set_batch_target_time(duration_seconds) ) + def disable_persisting_l2_cache(self, value: bool = True) -> Callable[[_F], _F]: + """Set whether persisting-L2 cache state is disabled before GPU measurement work.""" + return self.set_disable_persisting_l2_cache(value) + + def set_disable_persisting_l2_cache(self, value: bool) -> Callable[[_F], _F]: + """Set whether persisting-L2 cache state is disabled before GPU measurement work.""" + return _append_benchmark_action( + lambda benchmark: benchmark.set_disable_persisting_l2_cache(value) + ) + def stopping_criterion(self, criterion: str) -> Callable[[_F], _F]: """Set the benchmark stopping criterion.""" return self.set_stopping_criterion(criterion) diff --git a/python/src/py_nvbench.cpp b/python/src/py_nvbench.cpp index 5d138b15..b2b42fd7 100644 --- a/python/src/py_nvbench.cpp +++ b/python/src/py_nvbench.cpp @@ -720,6 +720,21 @@ Set target accumulated GPU time for batched measurements, in seconds py::return_value_policy::reference, py::arg("duration_seconds")); + // method Benchmark.set_disable_persisting_l2_cache + auto method_set_disable_persisting_l2_cache_impl = [](nvbench::benchmark_base &self, + bool disable_persisting_l2_cache) { + self.set_disable_persisting_l2_cache(disable_persisting_l2_cache); + return std::ref(self); + }; + static constexpr const char *benchmark_method_set_disable_persisting_l2_cache_doc = R"XXXX( +Set whether persisting-L2 cache state is disabled before GPU measurement work +)XXXX"; + py_benchmark_cls.def("set_disable_persisting_l2_cache", + method_set_disable_persisting_l2_cache_impl, + benchmark_method_set_disable_persisting_l2_cache_doc, + py::return_value_policy::reference, + py::arg("disable_persisting_l2_cache")); + // method Benchmark.set_throttle_threshold auto method_set_throttle_threshold_impl = [](nvbench::benchmark_base &self, nvbench::float32_t threshold) { @@ -1233,6 +1248,23 @@ Use argument True to disable use of blocking kernel by NVBench" method_set_disable_blocking_kernel_doc, py::arg("disable_blocking_kernel")); + // method State.get_disable_persisting_l2_cache + static constexpr const char *method_get_disable_persisting_l2_cache_doc = R"XXXX( +Return True if persisting-L2 cache state is disabled before GPU measurement work +)XXXX"; + pystate_cls.def("get_disable_persisting_l2_cache", + &nvbench::state::get_disable_persisting_l2_cache, + method_get_disable_persisting_l2_cache_doc); + + // method State.set_disable_persisting_l2_cache + static constexpr const char *state_method_set_disable_persisting_l2_cache_doc = R"XXXX( +Use argument True to disable persisting-L2 cache state before GPU measurement work +)XXXX"; + pystate_cls.def("set_disable_persisting_l2_cache", + &nvbench::state::set_disable_persisting_l2_cache, + state_method_set_disable_persisting_l2_cache_doc, + py::arg("disable_persisting_l2_cache")); + // method State.get_run_once static constexpr const char *method_get_run_once_doc = R"XXXX(Boolean flag indicating whether configuration should only run once)XXXX"; diff --git a/python/test/test_cuda_bench.py b/python/test/test_cuda_bench.py index b354c6dc..c331c6e6 100644 --- a/python/test/test_cuda_bench.py +++ b/python/test/test_cuda_bench.py @@ -119,6 +119,22 @@ def batch_target_state_probe(state: bench.State): with pytest.raises(ValueError, match="finite and positive"): batch_target_benchmark.set_batch_target_time(duration_seconds) + def persisting_l2_state_probe(state: bench.State): + observed["benchmark_disable_persisting_l2_cache"] = ( + state.get_disable_persisting_l2_cache() + ) + + state.set_disable_persisting_l2_cache(False) + observed["state_disable_persisting_l2_cache"] = ( + state.get_disable_persisting_l2_cache() + ) + + state.exec(lambda launch: None) + + persisting_l2_benchmark = bench.register(persisting_l2_state_probe) + persisting_l2_benchmark.set_is_cpu_only(True) + persisting_l2_benchmark.set_disable_persisting_l2_cache(True) + @bench.register() @bench.option.set_is_cpu_only(True) def external_stream_state_probe(state: bench.State): @@ -197,6 +213,8 @@ def __cuda_stream__(self): "state_walltime": 0.125, "benchmark_batch_target_time": 0.75, "state_batch_target_time": 0.125, + "benchmark_disable_persisting_l2_cache": True, + "state_disable_persisting_l2_cache": False, "external_stream_handle": external_stream_handle, } @@ -245,6 +263,8 @@ def test_decorator_docstrings(): obj_has_docstring_check(bench.option.set_timeout) obj_has_docstring_check(bench.option.batch_target_time) obj_has_docstring_check(bench.option.set_batch_target_time) + obj_has_docstring_check(bench.option.disable_persisting_l2_cache) + obj_has_docstring_check(bench.option.set_disable_persisting_l2_cache) obj_has_docstring_check(bench.option.stopping_criterion) obj_has_docstring_check(bench.option.set_stopping_criterion) obj_has_docstring_check(bench.option.criterion_param_float64) @@ -288,6 +308,10 @@ def set_batch_target_time(self, duration_seconds): self.calls.append(("batch_target_time", duration_seconds)) return self + def set_disable_persisting_l2_cache(self, value): + self.calls.append(("disable_persisting_l2_cache", value)) + return self + fake_benchmark = FakeBenchmark() registered_functions = [] @@ -303,6 +327,7 @@ def fake_register(fn): @bench.option.cold_warmup_runs(7) @bench.option.cold_max_warmup_walltime(0.25) @bench.option.batch_target_time(0.75) + @bench.option.disable_persisting_l2_cache() def decorated(state: bench.State): pass @@ -313,6 +338,7 @@ def decorated(state: bench.State): ("cold_warmup_runs", 7), ("cold_max_warmup_walltime", 0.25), ("batch_target_time", 0.75), + ("disable_persisting_l2_cache", True), ] assert callable(decorated) @@ -350,6 +376,35 @@ def decorated(state: bench.State): assert callable(decorated) +def test_disable_persisting_l2_cache_option_decorators_apply_options(monkeypatch): + class FakeBenchmark: + def __init__(self): + self.calls = [] + + def set_disable_persisting_l2_cache(self, value): + self.calls.append(("disable_persisting_l2_cache", value)) + return self + + fake_benchmark = FakeBenchmark() + + def fake_register(fn): + return fake_benchmark + + monkeypatch.setattr(bench, "_register", fake_register) + + @bench.register() + @bench.option.set_disable_persisting_l2_cache(False) + @bench.option.disable_persisting_l2_cache() + def decorated(state: bench.State): + pass + + assert fake_benchmark.calls == [ + ("disable_persisting_l2_cache", False), + ("disable_persisting_l2_cache", True), + ] + assert callable(decorated) + + def test_register_function_form_applies_decorated_options(monkeypatch): class FakeBenchmark: def __init__(self):