Reset persisting l2 cache

docs/cli_help.md

@@ -136,6 +136,14 @@
   * Applied to the most recent `--benchmark`, or all benchmarks if specified
     before any `--benchmark` arguments.
 
+* `--disable-persisting-l2-cache`
+  * Disable persisting-L2 cache reservation and state before GPU warmups and
+    measurement work.
+  * Intended to isolate benchmark iterations from persisting-L2 state left by
+    earlier work.
+  * Applies to the most recent `--benchmark`, or all benchmarks if specified
+    before any `--benchmark` arguments.
+
 * `--batch-target-time <seconds>`
   * Target accumulated GPU time for batched measurements.
   * Default is 0.5 seconds.

nvbench/benchmark_base.cuh

@@ -239,6 +239,18 @@ struct benchmark_base
   }
   /// @}
 
+  /// If true, persisting L2 cache state is disabled before GPU measurement work. @{
+  [[nodiscard]] bool get_disable_persisting_l2_cache() const
+  {
+    return m_disable_persisting_l2_cache;
+  }
+  benchmark_base &set_disable_persisting_l2_cache(bool v)
+  {
+    m_disable_persisting_l2_cache = v;
+    return *this;
+  }
+  /// @}
+
   /// If a warmup run finishes in less than `skip_time`, the measurement will
   /// be skipped.
   /// Extremely fast kernels (< 5000 ns) often timeout before they can
@@ -348,6 +360,7 @@ protected:
   bool m_is_cpu_only{false};
   bool m_run_once{false};
   bool m_disable_blocking_kernel{false};
+  bool m_disable_persisting_l2_cache{false};
   bool m_skip_batched{false};
 
   nvbench::int64_t m_min_samples{10};

nvbench/benchmark_base.cxx

@@ -45,9 +45,10 @@ std::unique_ptr<benchmark_base> benchmark_base::clone() const
 
   result->m_printer_ptr = m_printer_ptr;
 
-  result->m_is_cpu_only             = m_is_cpu_only;
-  result->m_run_once                = m_run_once;
-  result->m_disable_blocking_kernel = m_disable_blocking_kernel;
+  result->m_is_cpu_only                 = m_is_cpu_only;
+  result->m_run_once                    = m_run_once;
+  result->m_disable_blocking_kernel     = m_disable_blocking_kernel;
+  result->m_disable_persisting_l2_cache = m_disable_persisting_l2_cache;
 
   result->m_min_samples              = m_min_samples;
   result->m_cold_warmup_runs         = m_cold_warmup_runs;

nvbench/detail/measure_cold.cu

@@ -20,6 +20,7 @@
 #include <nvbench/criterion_manager.cuh>
 #include <nvbench/detail/measure_cold.cuh>
 #include <nvbench/detail/measure_timeout_warnings.cuh>
+#include <nvbench/detail/persisting_l2_cache_reset.cuh>
 #include <nvbench/detail/throw.cuh>
 #include <nvbench/device_info.cuh>
 #include <nvbench/printer_base.cuh>
@@ -47,6 +48,7 @@ measure_cold_base::measure_cold_base(state &exec_state)
     , m_stopping_criterion{nvbench::criterion_manager::get().get_criterion(
         exec_state.get_stopping_criterion())}
     , m_disable_blocking_kernel{exec_state.get_disable_blocking_kernel()}
+    , m_disable_persisting_l2_cache{exec_state.get_disable_persisting_l2_cache()}
     , m_run_once{exec_state.get_run_once()}
     , m_check_throttling(!exec_state.get_run_once())
     , m_min_samples{exec_state.get_min_samples()}
@@ -66,6 +68,32 @@ measure_cold_base::measure_cold_base(state &exec_state)
   }
 }
 
+measure_cold_base::~measure_cold_base() = default;
+
+void measure_cold_base::initialize_persisting_l2_cache_disable()
+{
+  m_persisting_l2_cache_disable =
+    nvbench::detail::make_persisting_l2_cache_disable_if_requested(m_disable_persisting_l2_cache,
+                                                                   m_state.get_device());
+}
+
+void measure_cold_base::reset_persisting_l2_cache()
+{
+  if (m_persisting_l2_cache_disable)
+  {
+    m_persisting_l2_cache_disable->reset_before_measurement();
+  }
+}
+
+void measure_cold_base::restore_persisting_l2_cache()
+{
+  if (m_persisting_l2_cache_disable)
+  {
+    m_persisting_l2_cache_disable->restore();
+    m_persisting_l2_cache_disable.reset();
+  }
+}
+
 void measure_cold_base::check()
 {
   const auto device = m_state.get_device();

nvbench/detail/measure_cold.cuh

@@ -36,6 +36,7 @@
 #include <nvbench/detail/kernel_launcher_timer_wrapper.cuh>
 #include <nvbench/detail/l2flush.cuh>
 #include <nvbench/detail/measure_cold_launch_timer_core.cuh>
+#include <nvbench/detail/persisting_l2_cache_reset_fwd.cuh>
 #include <nvbench/detail/statistics.cuh>
 #include <nvbench/device_info.cuh>
 #include <nvbench/exec_tag.cuh>
@@ -63,6 +64,7 @@ namespace detail
 struct measure_cold_base
 {
   explicit measure_cold_base(nvbench::state &exec_state);
+  ~measure_cold_base();
   measure_cold_base(const measure_cold_base &)            = delete;
   measure_cold_base(measure_cold_base &&)                 = delete;
   measure_cold_base &operator=(const measure_cold_base &) = delete;
@@ -85,6 +87,9 @@ protected:
   void check_skip_time(nvbench::float64_t warmup_time);
 
   __forceinline__ void flush_device_l2() { m_l2flush.flush(m_launch.get_stream()); }
+  void initialize_persisting_l2_cache_disable();
+  void reset_persisting_l2_cache();
+  void restore_persisting_l2_cache();
 
   __forceinline__ cudaError_t sync_stream_noexcept() const noexcept
   {
@@ -111,12 +116,14 @@ protected:
   nvbench::cpu_timer m_walltime_timer{};
   nvbench::detail::l2flush m_l2flush{};
   nvbench::blocking_kernel m_blocker{};
+  nvbench::detail::persisting_l2_cache_disable_ptr m_persisting_l2_cache_disable{};
 
   nvbench::criterion_params m_criterion_params{};
   nvbench::stopping_criterion_base &m_stopping_criterion;
   nvbench::detail::gpu_frequency m_gpu_frequency{};
 
   bool m_disable_blocking_kernel{false};
+  bool m_disable_persisting_l2_cache{false};
   bool m_run_once{false};
   bool m_check_throttling{true};
 
@@ -191,6 +198,8 @@ struct measure_cold_base::kernel_launch_timer
 
   __forceinline__ void flush_device_l2() { m_measure.flush_device_l2(); }
 
+  __forceinline__ void reset_persisting_l2_cache() { m_measure.reset_persisting_l2_cache(); }
+
   __forceinline__ void sync_stream() { m_measure.sync_stream(); }
 
   __forceinline__ cudaError_t sync_stream_noexcept() const noexcept
@@ -250,11 +259,23 @@ struct measure_cold : public measure_cold_base
   {
     this->check();
     this->initialize();
-    this->run_warmup();
+    this->initialize_persisting_l2_cache_disable();
+
+    try
+    {
+      this->run_warmup();
+
+      this->run_trials_prologue();
+      this->run_trials();
+      this->run_trials_epilogue();
 
-    this->run_trials_prologue();
-    this->run_trials();
-    this->run_trials_epilogue();
+      this->restore_persisting_l2_cache();
+    }
+    catch (...)
+    {
+      this->restore_persisting_l2_cache();
+      throw;
+    }
 
     this->generate_summaries();
   }

nvbench/detail/measure_cold_launch_timer_core.cuh

@@ -101,6 +101,7 @@ public:
   {
     cleanup_guard cleanup{*this};
 
+    m_measure.reset_persisting_l2_cache();
     m_measure.flush_device_l2();
     m_measure.sync_stream();
 

nvbench/detail/measure_hot.cu

@@ -18,6 +18,7 @@
 
 #include <nvbench/benchmark_base.cuh>
 #include <nvbench/detail/measure_hot.cuh>
+#include <nvbench/detail/persisting_l2_cache_reset.cuh>
 #include <nvbench/detail/throw.cuh>
 #include <nvbench/device_info.cuh>
 #include <nvbench/printer_base.cuh>
@@ -78,6 +79,7 @@ measure_hot_base::measure_hot_base(state &exec_state)
     , m_batch_target_time{exec_state.get_batch_target_time()}
     , m_skip_time{exec_state.get_skip_time()}
     , m_timeout{exec_state.get_timeout()}
+    , m_disable_persisting_l2_cache{exec_state.get_disable_persisting_l2_cache()}
 {
   try
   {
@@ -96,6 +98,32 @@ measure_hot_base::measure_hot_base(state &exec_state)
   }
 }
 
+measure_hot_base::~measure_hot_base() = default;
+
+void measure_hot_base::initialize_persisting_l2_cache_disable()
+{
+  m_persisting_l2_cache_disable =
+    nvbench::detail::make_persisting_l2_cache_disable_if_requested(m_disable_persisting_l2_cache,
+                                                                   m_state.get_device());
+}
+
+void measure_hot_base::reset_persisting_l2_cache()
+{
+  if (m_persisting_l2_cache_disable)
+  {
+    m_persisting_l2_cache_disable->reset_before_measurement();
+  }
+}
+
+void measure_hot_base::restore_persisting_l2_cache()
+{
+  if (m_persisting_l2_cache_disable)
+  {
+    m_persisting_l2_cache_disable->restore();
+    m_persisting_l2_cache_disable.reset();
+  }
+}
+
 // CUDA-time predictions choose how many launches are needed to reach the
 // accumulated GPU-time target. Valid small predictions are raised to the
 // supplied minimum; invalid or overflowing predictions fall back to the

nvbench/detail/measure_hot.cuh

@@ -32,6 +32,7 @@
 #include <nvbench/cpu_timer.cuh>
 #include <nvbench/cuda_call.cuh>
 #include <nvbench/cuda_timer.cuh>
+#include <nvbench/detail/persisting_l2_cache_reset_fwd.cuh>
 #include <nvbench/detail/stream_cleanup_guard.cuh>
 #include <nvbench/exec_tag.cuh>
 #include <nvbench/launch.cuh>
@@ -52,6 +53,7 @@ namespace detail
 struct measure_hot_base
 {
   explicit measure_hot_base(nvbench::state &exec_state);
+  ~measure_hot_base();
   measure_hot_base(const measure_hot_base &)            = delete;
   measure_hot_base(measure_hot_base &&)                 = delete;
   measure_hot_base &operator=(const measure_hot_base &) = delete;
@@ -74,6 +76,9 @@ protected:
   void check_skip_time(nvbench::float64_t warmup_time);
 
   void block_stream();
+  void initialize_persisting_l2_cache_disable();
+  void reset_persisting_l2_cache();
+  void restore_persisting_l2_cache();
 
   static nvbench::int64_t predict_cuda_batch_size(nvbench::float64_t target_time,
                                                   nvbench::float64_t time_estimate,
@@ -103,6 +108,7 @@ protected:
   nvbench::cuda_timer m_cuda_timer;
   nvbench::cpu_timer m_walltime_timer;
   nvbench::blocking_kernel m_blocker;
+  nvbench::detail::persisting_l2_cache_disable_ptr m_persisting_l2_cache_disable{};
 
   nvbench::int64_t m_min_samples{};
   nvbench::float64_t m_batch_target_time{};
@@ -114,6 +120,7 @@ protected:
   nvbench::float64_t m_total_cuda_time{};
 
   bool m_disable_blocking_kernel{false};
+  bool m_disable_persisting_l2_cache{false};
   bool m_max_time_exceeded{false};
 };
 
@@ -129,8 +136,20 @@ struct measure_hot : public measure_hot_base
   {
     this->check();
     this->initialize();
-    this->run_warmup();
-    this->run_trials();
+    this->initialize_persisting_l2_cache_disable();
+
+    try
+    {
+      this->run_warmup();
+      this->run_trials();
+      this->restore_persisting_l2_cache();
+    }
+    catch (...)
+    {
+      this->restore_persisting_l2_cache();
+      throw;
+    }
+
     this->generate_summaries();
   }
 
@@ -141,6 +160,7 @@ private:
   {
     nvbench::detail::stream_cleanup_guard<measure_hot_base> cleanup{*this};
 
+    this->reset_persisting_l2_cache();
     m_walltime_timer.start();
     {
       m_cuda_timer.start(m_launch.get_stream());
@@ -182,6 +202,7 @@ private:
 
       nvbench::detail::stream_cleanup_guard<measure_hot_base> cleanup{*this};
 
+      this->reset_persisting_l2_cache();
       if (!m_disable_blocking_kernel)
       {
         // Block stream until some work is queued.