Add CLI option to set target-time parameter for batch measurement loop

docs/cli_help.md

@@ -136,15 +136,25 @@
   * Applied to the most recent `--benchmark`, or all benchmarks if specified
     before any `--benchmark` arguments.
 
+* `--batch-target-time <seconds>`
+  * Target accumulated GPU time for batched measurements.
+  * Default is 0.5 seconds.
+  * `<seconds>` must be finite and positive.
+  * Batched measurements continue until both `--min-samples` and the
+    accumulated GPU-time target are satisfied, unless `--timeout` is reached
+    first.
+  * Applies to the most recent `--benchmark`, or all benchmarks if specified
+    before any `--benchmark` arguments.
+
 ## Measurement Collection
 
 * `--timeout <seconds>`
   * Measurements will timeout after `<seconds>` have elapsed.
   * Default is 15 seconds.
   * `<seconds>` is walltime, not accumulated sample time.
-  * If a measurement times out, the default markdown log will print a warning to
-    report any outstanding termination criteria (min samples, min time, max
-    noise).
+  * If a measurement times out, the default markdown log will report which
+    termination conditions were still unmet. The exact warnings depend on the
+    measurement type and active stopping criterion.
   * Applies to the most recent `--benchmark`, or all benchmarks if specified
     before any `--benchmark` arguments.
 

nvbench/benchmark_base.cuh

@@ -224,6 +224,11 @@ struct benchmark_base
   }
   /// @}
 
+  /// Target accumulated GPU time for batched measurements. @{
+  [[nodiscard]] nvbench::float64_t get_batch_target_time() const { return m_batch_target_time; }
+  benchmark_base &set_batch_target_time(nvbench::float64_t batch_target_time);
+  /// @}
+
   /// If true, the benchmark does not use the blocking_kernel. This is intended
   /// for use with external profiling tools. @{
   [[nodiscard]] bool get_disable_blocking_kernel() const { return m_disable_blocking_kernel; }
@@ -237,8 +242,8 @@ struct benchmark_base
   /// If a warmup run finishes in less than `skip_time`, the measurement will
   /// be skipped.
   /// Extremely fast kernels (< 5000 ns) often timeout before they can
-  /// accumulate `min_time` measurements, and are often uninteresting. Setting
-  /// this value can help improve performance by skipping time consuming
+  /// accumulate enough measurement time, and are often uninteresting. Setting
+  /// this value can help improve performance by skipping time-consuming
   /// measurement that don't provide much information.
   /// Default value is -1., which disables the feature.
   /// @{
@@ -349,6 +354,7 @@ protected:
   nvbench::int64_t m_cold_warmup_runs{1};
 
   nvbench::float64_t m_cold_max_warmup_walltime{-1.};
+  nvbench::float64_t m_batch_target_time{0.5};
   nvbench::float64_t m_skip_time{-1.};
   nvbench::float64_t m_timeout{15.};
 

nvbench/benchmark_base.cxx

@@ -19,6 +19,7 @@
 #include <nvbench/benchmark_base.cuh>
 #include <nvbench/criterion_manager.cuh>
 #include <nvbench/detail/transform_reduce.cuh>
+#include <nvbench/detail/validate_batch_target_time.cuh>
 
 #include <algorithm>
 #include <cstddef>
@@ -52,8 +53,9 @@ std::unique_ptr<benchmark_base> benchmark_base::clone() const
   result->m_cold_warmup_runs         = m_cold_warmup_runs;
   result->m_cold_max_warmup_walltime = m_cold_max_warmup_walltime;
 
-  result->m_skip_time = m_skip_time;
-  result->m_timeout   = m_timeout;
+  result->m_batch_target_time = m_batch_target_time;
+  result->m_skip_time         = m_skip_time;
+  result->m_timeout           = m_timeout;
 
   result->m_criterion_params        = m_criterion_params;
   result->m_throttle_threshold      = m_throttle_threshold;
@@ -64,6 +66,13 @@ std::unique_ptr<benchmark_base> benchmark_base::clone() const
   return result;
 }
 
+benchmark_base &benchmark_base::set_batch_target_time(nvbench::float64_t batch_target_time)
+{
+  nvbench::detail::validate_batch_target_time(batch_target_time);
+  m_batch_target_time = batch_target_time;
+  return *this;
+}
+
 benchmark_base &benchmark_base::set_devices(std::vector<int> device_ids)
 {
   std::vector<device_info> devices;

nvbench/detail/measure_hot.cu

@@ -27,30 +27,64 @@
 #include <fmt/format.h>
 
 #include <algorithm>
+#include <cmath>
 #include <cstdio>
+#include <limits>
 #include <stdexcept>
 #include <utility>
 #include <variant>
 
 namespace nvbench::detail
 {
 
+namespace
+{
+
+nvbench::int64_t predict_batch_size(nvbench::float64_t target_duration,
+                                    nvbench::float64_t duration_per_launch,
+                                    nvbench::int64_t minimum_size,
+                                    nvbench::int64_t fallback_size_on_invalid_prediction)
+{
+  const auto clamped_min_size = std::max(minimum_size, nvbench::int64_t{1});
+  const auto clamped_fallback = std::max(fallback_size_on_invalid_prediction, nvbench::int64_t{1});
+  if (!std::isfinite(target_duration) || target_duration <= nvbench::float64_t{0} ||
+      !std::isfinite(duration_per_launch) || duration_per_launch <= nvbench::float64_t{0})
+  {
+    return clamped_fallback;
+  }
+
+  const auto predicted_launches = target_duration / duration_per_launch;
+  if (!std::isfinite(predicted_launches) ||
+      predicted_launches >=
+        static_cast<nvbench::float64_t>(std::numeric_limits<nvbench::int64_t>::max()))
+  {
+    return clamped_fallback;
+  }
+
+  if (predicted_launches <= static_cast<nvbench::float64_t>(clamped_min_size))
+  {
+    return clamped_min_size;
+  }
+
+  return static_cast<nvbench::int64_t>(predicted_launches);
+}
+
+} // namespace
+
 measure_hot_base::measure_hot_base(state &exec_state)
     : m_state{exec_state}
     , m_launch{exec_state.get_cuda_stream()}
     , m_min_samples{exec_state.get_min_samples()}
-    , m_min_time{exec_state.get_criterion_params().has_value("min-time")
-                   ? exec_state.get_criterion_params().get_float64("min-time")
-                   : 0.5}
+    , m_batch_target_time{exec_state.get_batch_target_time()}
     , m_skip_time{exec_state.get_skip_time()}
     , m_timeout{exec_state.get_timeout()}
 {
-  // Since cold measures converge to a stable result, increase the min_samples
-  // to match the cold result if available.
   try
   {
     nvbench::int64_t cold_samples = m_state.get_summary("nv/cold/sample_size").get_int64("value");
-    m_min_samples                 = std::max(m_min_samples, cold_samples);
+    // Since cold measures converge to a stable result, increase the min_samples
+    // to match the cold result if available.
+    m_min_samples = std::max(m_min_samples, cold_samples);
 
     // If the cold measurement ran successfully, disable skip_time. It'd just
     // be annoying to skip now.
@@ -59,15 +93,55 @@ measure_hot_base::measure_hot_base(state &exec_state)
   catch (...)
   {
     // If the above threw an exception, we don't have a cold measurement to use.
-    // Estimate a target_time between m_min_time and m_timeout.
-    // Use the average of the min_time and timeout, but don't go over 5x
-    // min_time in case timeout is huge.
-    // We could expose a `target_time` property on benchmark_base/state if
-    // needed.
-    m_min_time = std::min((m_min_time + m_timeout) / 2., m_min_time * 5);
   }
 }
 
+// CUDA-time predictions choose how many launches are needed to reach the
+// accumulated GPU-time target. Valid small predictions are raised to the
+// supplied minimum; invalid or overflowing predictions fall back to the
+// caller-provided conservative batch size, usually m_min_samples.
+nvbench::int64_t
+measure_hot_base::predict_cuda_batch_size(nvbench::float64_t target_time,
+                                          nvbench::float64_t time_estimate,
+                                          nvbench::int64_t minimum_batch_size,
+                                          nvbench::int64_t fallback_on_invalid_prediction)
+{
+  return predict_batch_size(target_time,
+                            time_estimate,
+                            minimum_batch_size,
+                            fallback_on_invalid_prediction);
+}
+
+// Timeout predictions are caps on the CUDA-time batch estimate. They only
+// shrink the CUDA estimate when the wall-time model produces a meaningful
+// finite cap; exhausted budgets return one launch, while non-finite or
+// overflowing predictions return the CUDA estimate.
+nvbench::int64_t measure_hot_base::predict_timeout_batch_cap(nvbench::float64_t target_time,
+                                                             nvbench::float64_t time_estimate,
+                                                             nvbench::int64_t cuda_batch_size)
+{
+  if (target_time <= nvbench::float64_t{0})
+  {
+    return nvbench::int64_t{1};
+  }
+
+  return predict_batch_size(target_time, time_estimate, nvbench::int64_t{1}, cuda_batch_size);
+}
+
+nvbench::int64_t measure_hot_base::grow_batch_size(nvbench::int64_t batch_size,
+                                                   nvbench::int64_t minimum_batch_size)
+{
+  const auto fallback           = std::max(minimum_batch_size, nvbench::int64_t{1});
+  const auto batch              = std::max(batch_size, fallback);
+  constexpr auto max_batch_size = std::numeric_limits<nvbench::int64_t>::max();
+  if (batch > max_batch_size / nvbench::int64_t{2})
+  {
+    return max_batch_size;
+  }
+
+  return std::max(batch * nvbench::int64_t{2}, fallback);
+}
+
 void measure_hot_base::check()
 {
   const auto device = m_state.get_device();
@@ -131,15 +205,15 @@ void measure_hot_base::generate_summaries()
                                 m_total_samples,
                                 m_min_samples));
       }
-      if (m_total_cuda_time < m_min_time)
+      if (m_total_cuda_time < m_batch_target_time)
       {
         printer.log(nvbench::log_level::warn,
                     fmt::format("Current measurement timed out ({:0.2f}s) "
-                                "before accumulating min_time ({:0.2f}s < "
+                                "before accumulating batch target time ({:0.2f}s < "
                                 "{:0.2f}s)",
                                 timeout,
                                 m_total_cuda_time,
-                                m_min_time));
+                                m_batch_target_time));
       }
     }
 

nvbench/detail/measure_hot.cuh

@@ -75,6 +75,18 @@ protected:
 
   void block_stream();
 
+  static nvbench::int64_t predict_cuda_batch_size(nvbench::float64_t target_time,
+                                                  nvbench::float64_t time_estimate,
+                                                  nvbench::int64_t minimum_batch_size,
+                                                  nvbench::int64_t fallback_on_invalid_prediction);
+  static nvbench::int64_t predict_timeout_batch_cap(nvbench::float64_t target_time,
+                                                    nvbench::float64_t time_estimate,
+                                                    nvbench::int64_t cuda_batch_size);
+  static nvbench::int64_t grow_batch_size(nvbench::int64_t batch_size,
+                                          nvbench::int64_t minimum_batch_size);
+
+  static constexpr nvbench::int64_t minimum_hot_batch_size = 4;
+
   __forceinline__ void unblock_stream() { m_blocker.unblock(); }
   __forceinline__ void unblock_stream_noexcept() noexcept { m_blocker.unblock_noexcept(); }
 
@@ -93,7 +105,7 @@ protected:
   nvbench::blocking_kernel m_blocker;
 
   nvbench::int64_t m_min_samples{};
-  nvbench::float64_t m_min_time{};
+  nvbench::float64_t m_batch_target_time{};
 
   nvbench::float64_t m_skip_time{};
   nvbench::float64_t m_timeout{};
@@ -129,29 +141,44 @@ private:
   {
     nvbench::detail::stream_cleanup_guard<measure_hot_base> cleanup{*this};
 
-    m_cuda_timer.start(m_launch.get_stream());
-    this->launch_kernel();
-    m_cuda_timer.stop(m_launch.get_stream());
+    m_walltime_timer.start();
+    {
+      m_cuda_timer.start(m_launch.get_stream());
+      this->launch_kernel();
+      m_cuda_timer.stop(m_launch.get_stream());
 
-    this->sync_stream();
+      this->sync_stream();
+    }
+    // get wall-clock estimate of launch execution
+    m_walltime_timer.stop();
     cleanup.release();
 
     this->check_skip_time(m_cuda_timer.get_duration());
   }
 
   void run_trials()
   {
+    const auto wallclock_time_initial_estimate = m_walltime_timer.get_duration();
+    const auto cuda_time_initial_estimate      = m_cuda_timer.get_duration();
+
     m_walltime_timer.start();
 
     // Use warmup results to estimate the number of iterations to run.
     // The .95 factor here pads the batch_size a bit to avoid needing a second
     // batch due to noise.
-    const auto time_estimate = m_cuda_timer.get_duration() * 0.95;
-    auto batch_size          = static_cast<nvbench::int64_t>(m_min_time / time_estimate);
+    const auto hot_batch_size_floor = std::min(std::max(m_min_samples, nvbench::int64_t{1}),
+                                               minimum_hot_batch_size);
+    const auto time_estimate        = cuda_time_initial_estimate * 0.95;
+    auto batch_size                 = this->predict_cuda_batch_size(m_batch_target_time,
+                                                                    time_estimate,
+                                                                    hot_batch_size_floor,
+                                                                    m_min_samples);
+    auto timeout_batch_size =
+      this->predict_timeout_batch_cap(m_timeout, wallclock_time_initial_estimate, batch_size);
 
     do
     {
-      batch_size = std::max(batch_size, nvbench::int64_t{1});
+      batch_size = std::min(batch_size, timeout_batch_size);
 
       nvbench::detail::stream_cleanup_guard<measure_hot_base> cleanup{*this};
 
@@ -197,23 +224,43 @@ private:
       m_total_cuda_time += m_cuda_timer.get_duration();
       m_total_samples += batch_size;
 
-      // Predict number of remaining iterations:
-      batch_size = static_cast<nvbench::int64_t>(
-        (m_min_time - m_total_cuda_time) /
-        (m_total_cuda_time / static_cast<nvbench::float64_t>(m_total_samples)));
-
-      if (m_total_cuda_time > m_min_time && // min time okay
-          m_total_samples >= m_min_samples) // min samples okay
+      if (m_total_cuda_time >= m_batch_target_time && // batch target time okay
+          m_total_samples >= m_min_samples)           // min samples okay
       {
         break; // Stop iterating
       }
 
+      const auto sample_count = static_cast<nvbench::float64_t>(m_total_samples);
+
+      // Predict number of remaining iterations based on cuda-time budget
+      const auto remaining_time               = m_batch_target_time - m_total_cuda_time;
+      const auto time_per_sample              = m_total_cuda_time / sample_count;
+      const auto remaining_samples_to_minimum = std::max(m_min_samples - m_total_samples,
+                                                         nvbench::int64_t{1});
+      const auto batch_target_time_satisfied  = remaining_time <= nvbench::float64_t{0};
+      const auto fallback_size_on_invalid_cuda_prediction = batch_target_time_satisfied
+                                                              ? remaining_samples_to_minimum
+                                                              : m_min_samples;
+      batch_size =
+        this->predict_cuda_batch_size(remaining_time,
+                                      time_per_sample,
+                                      this->grow_batch_size(batch_size, hot_batch_size_floor),
+                                      fallback_size_on_invalid_cuda_prediction);
+
       m_walltime_timer.stop();
-      if (m_walltime_timer.get_duration() > m_timeout)
+      const auto total_walltime = m_walltime_timer.get_duration();
+      if (total_walltime > m_timeout)
       {
         m_max_time_exceeded = true;
         break;
       }
+
+      // Predict number of remaining iterations based on timeout budget.
+      const auto remaining_walltime  = m_timeout - total_walltime;
+      const auto walltime_per_sample = total_walltime / sample_count;
+      timeout_batch_size =
+        this->predict_timeout_batch_cap(remaining_walltime, walltime_per_sample, batch_size);
+
     } while (true);
 
     m_walltime_timer.stop();