diff --git a/docs/cli_help.md b/docs/cli_help.md index 94eb7cb4..59de7b47 100644 --- a/docs/cli_help.md +++ b/docs/cli_help.md @@ -136,15 +136,25 @@ * Applied to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. +* `--batch-target-time ` + * Target accumulated GPU time for batched measurements. + * Default is 0.5 seconds. + * `` must be finite and positive. + * Batched measurements continue until both `--min-samples` and the + accumulated GPU-time target are satisfied, unless `--timeout` is reached + first. + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. + ## Measurement Collection * `--timeout ` * Measurements will timeout after `` have elapsed. * Default is 15 seconds. * `` is walltime, not accumulated sample time. - * If a measurement times out, the default markdown log will print a warning to - report any outstanding termination criteria (min samples, min time, max - noise). + * If a measurement times out, the default markdown log will report which + termination conditions were still unmet. The exact warnings depend on the + measurement type and active stopping criterion. * Applies to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. diff --git a/nvbench/benchmark_base.cuh b/nvbench/benchmark_base.cuh index 9ceea815..d3900f48 100644 --- a/nvbench/benchmark_base.cuh +++ b/nvbench/benchmark_base.cuh @@ -224,6 +224,11 @@ struct benchmark_base } /// @} + /// Target accumulated GPU time for batched measurements. @{ + [[nodiscard]] nvbench::float64_t get_batch_target_time() const { return m_batch_target_time; } + benchmark_base &set_batch_target_time(nvbench::float64_t batch_target_time); + /// @} + /// If true, the benchmark does not use the blocking_kernel. This is intended /// for use with external profiling tools. @{ [[nodiscard]] bool get_disable_blocking_kernel() const { return m_disable_blocking_kernel; } @@ -237,8 +242,8 @@ struct benchmark_base /// If a warmup run finishes in less than `skip_time`, the measurement will /// be skipped. /// Extremely fast kernels (< 5000 ns) often timeout before they can - /// accumulate `min_time` measurements, and are often uninteresting. Setting - /// this value can help improve performance by skipping time consuming + /// accumulate enough measurement time, and are often uninteresting. Setting + /// this value can help improve performance by skipping time-consuming /// measurement that don't provide much information. /// Default value is -1., which disables the feature. /// @{ @@ -349,6 +354,7 @@ protected: nvbench::int64_t m_cold_warmup_runs{1}; nvbench::float64_t m_cold_max_warmup_walltime{-1.}; + nvbench::float64_t m_batch_target_time{0.5}; nvbench::float64_t m_skip_time{-1.}; nvbench::float64_t m_timeout{15.}; diff --git a/nvbench/benchmark_base.cxx b/nvbench/benchmark_base.cxx index 06db10bb..d9f25697 100644 --- a/nvbench/benchmark_base.cxx +++ b/nvbench/benchmark_base.cxx @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -52,8 +53,9 @@ std::unique_ptr benchmark_base::clone() const result->m_cold_warmup_runs = m_cold_warmup_runs; result->m_cold_max_warmup_walltime = m_cold_max_warmup_walltime; - result->m_skip_time = m_skip_time; - result->m_timeout = m_timeout; + result->m_batch_target_time = m_batch_target_time; + result->m_skip_time = m_skip_time; + result->m_timeout = m_timeout; result->m_criterion_params = m_criterion_params; result->m_throttle_threshold = m_throttle_threshold; @@ -64,6 +66,13 @@ std::unique_ptr benchmark_base::clone() const return result; } +benchmark_base &benchmark_base::set_batch_target_time(nvbench::float64_t batch_target_time) +{ + nvbench::detail::validate_batch_target_time(batch_target_time); + m_batch_target_time = batch_target_time; + return *this; +} + benchmark_base &benchmark_base::set_devices(std::vector device_ids) { std::vector devices; diff --git a/nvbench/detail/measure_hot.cu b/nvbench/detail/measure_hot.cu index fe36a57e..e5199c25 100644 --- a/nvbench/detail/measure_hot.cu +++ b/nvbench/detail/measure_hot.cu @@ -27,7 +27,9 @@ #include #include +#include #include +#include #include #include #include @@ -35,22 +37,54 @@ namespace nvbench::detail { +namespace +{ + +nvbench::int64_t predict_batch_size(nvbench::float64_t target_duration, + nvbench::float64_t duration_per_launch, + nvbench::int64_t minimum_size, + nvbench::int64_t fallback_size_on_invalid_prediction) +{ + const auto clamped_min_size = std::max(minimum_size, nvbench::int64_t{1}); + const auto clamped_fallback = std::max(fallback_size_on_invalid_prediction, nvbench::int64_t{1}); + if (!std::isfinite(target_duration) || target_duration <= nvbench::float64_t{0} || + !std::isfinite(duration_per_launch) || duration_per_launch <= nvbench::float64_t{0}) + { + return clamped_fallback; + } + + const auto predicted_launches = target_duration / duration_per_launch; + if (!std::isfinite(predicted_launches) || + predicted_launches >= + static_cast(std::numeric_limits::max())) + { + return clamped_fallback; + } + + if (predicted_launches <= static_cast(clamped_min_size)) + { + return clamped_min_size; + } + + return static_cast(predicted_launches); +} + +} // namespace + measure_hot_base::measure_hot_base(state &exec_state) : m_state{exec_state} , m_launch{exec_state.get_cuda_stream()} , m_min_samples{exec_state.get_min_samples()} - , m_min_time{exec_state.get_criterion_params().has_value("min-time") - ? exec_state.get_criterion_params().get_float64("min-time") - : 0.5} + , m_batch_target_time{exec_state.get_batch_target_time()} , m_skip_time{exec_state.get_skip_time()} , m_timeout{exec_state.get_timeout()} { - // Since cold measures converge to a stable result, increase the min_samples - // to match the cold result if available. try { nvbench::int64_t cold_samples = m_state.get_summary("nv/cold/sample_size").get_int64("value"); - m_min_samples = std::max(m_min_samples, cold_samples); + // Since cold measures converge to a stable result, increase the min_samples + // to match the cold result if available. + m_min_samples = std::max(m_min_samples, cold_samples); // If the cold measurement ran successfully, disable skip_time. It'd just // be annoying to skip now. @@ -59,15 +93,55 @@ measure_hot_base::measure_hot_base(state &exec_state) catch (...) { // If the above threw an exception, we don't have a cold measurement to use. - // Estimate a target_time between m_min_time and m_timeout. - // Use the average of the min_time and timeout, but don't go over 5x - // min_time in case timeout is huge. - // We could expose a `target_time` property on benchmark_base/state if - // needed. - m_min_time = std::min((m_min_time + m_timeout) / 2., m_min_time * 5); } } +// CUDA-time predictions choose how many launches are needed to reach the +// accumulated GPU-time target. Valid small predictions are raised to the +// supplied minimum; invalid or overflowing predictions fall back to the +// caller-provided conservative batch size, usually m_min_samples. +nvbench::int64_t +measure_hot_base::predict_cuda_batch_size(nvbench::float64_t target_time, + nvbench::float64_t time_estimate, + nvbench::int64_t minimum_batch_size, + nvbench::int64_t fallback_on_invalid_prediction) +{ + return predict_batch_size(target_time, + time_estimate, + minimum_batch_size, + fallback_on_invalid_prediction); +} + +// Timeout predictions are caps on the CUDA-time batch estimate. They only +// shrink the CUDA estimate when the wall-time model produces a meaningful +// finite cap; exhausted budgets return one launch, while non-finite or +// overflowing predictions return the CUDA estimate. +nvbench::int64_t measure_hot_base::predict_timeout_batch_cap(nvbench::float64_t target_time, + nvbench::float64_t time_estimate, + nvbench::int64_t cuda_batch_size) +{ + if (target_time <= nvbench::float64_t{0}) + { + return nvbench::int64_t{1}; + } + + return predict_batch_size(target_time, time_estimate, nvbench::int64_t{1}, cuda_batch_size); +} + +nvbench::int64_t measure_hot_base::grow_batch_size(nvbench::int64_t batch_size, + nvbench::int64_t minimum_batch_size) +{ + const auto fallback = std::max(minimum_batch_size, nvbench::int64_t{1}); + const auto batch = std::max(batch_size, fallback); + constexpr auto max_batch_size = std::numeric_limits::max(); + if (batch > max_batch_size / nvbench::int64_t{2}) + { + return max_batch_size; + } + + return std::max(batch * nvbench::int64_t{2}, fallback); +} + void measure_hot_base::check() { const auto device = m_state.get_device(); @@ -131,15 +205,15 @@ void measure_hot_base::generate_summaries() m_total_samples, m_min_samples)); } - if (m_total_cuda_time < m_min_time) + if (m_total_cuda_time < m_batch_target_time) { printer.log(nvbench::log_level::warn, fmt::format("Current measurement timed out ({:0.2f}s) " - "before accumulating min_time ({:0.2f}s < " + "before accumulating batch target time ({:0.2f}s < " "{:0.2f}s)", timeout, m_total_cuda_time, - m_min_time)); + m_batch_target_time)); } } diff --git a/nvbench/detail/measure_hot.cuh b/nvbench/detail/measure_hot.cuh index c8fb808a..e53088b2 100644 --- a/nvbench/detail/measure_hot.cuh +++ b/nvbench/detail/measure_hot.cuh @@ -75,6 +75,18 @@ protected: void block_stream(); + static nvbench::int64_t predict_cuda_batch_size(nvbench::float64_t target_time, + nvbench::float64_t time_estimate, + nvbench::int64_t minimum_batch_size, + nvbench::int64_t fallback_on_invalid_prediction); + static nvbench::int64_t predict_timeout_batch_cap(nvbench::float64_t target_time, + nvbench::float64_t time_estimate, + nvbench::int64_t cuda_batch_size); + static nvbench::int64_t grow_batch_size(nvbench::int64_t batch_size, + nvbench::int64_t minimum_batch_size); + + static constexpr nvbench::int64_t minimum_hot_batch_size = 4; + __forceinline__ void unblock_stream() { m_blocker.unblock(); } __forceinline__ void unblock_stream_noexcept() noexcept { m_blocker.unblock_noexcept(); } @@ -93,7 +105,7 @@ protected: nvbench::blocking_kernel m_blocker; nvbench::int64_t m_min_samples{}; - nvbench::float64_t m_min_time{}; + nvbench::float64_t m_batch_target_time{}; nvbench::float64_t m_skip_time{}; nvbench::float64_t m_timeout{}; @@ -129,11 +141,16 @@ private: { nvbench::detail::stream_cleanup_guard cleanup{*this}; - m_cuda_timer.start(m_launch.get_stream()); - this->launch_kernel(); - m_cuda_timer.stop(m_launch.get_stream()); + m_walltime_timer.start(); + { + m_cuda_timer.start(m_launch.get_stream()); + this->launch_kernel(); + m_cuda_timer.stop(m_launch.get_stream()); - this->sync_stream(); + this->sync_stream(); + } + // get wall-clock estimate of launch execution + m_walltime_timer.stop(); cleanup.release(); this->check_skip_time(m_cuda_timer.get_duration()); @@ -141,17 +158,27 @@ private: void run_trials() { + const auto wallclock_time_initial_estimate = m_walltime_timer.get_duration(); + const auto cuda_time_initial_estimate = m_cuda_timer.get_duration(); + m_walltime_timer.start(); // Use warmup results to estimate the number of iterations to run. // The .95 factor here pads the batch_size a bit to avoid needing a second // batch due to noise. - const auto time_estimate = m_cuda_timer.get_duration() * 0.95; - auto batch_size = static_cast(m_min_time / time_estimate); + const auto hot_batch_size_floor = std::min(std::max(m_min_samples, nvbench::int64_t{1}), + minimum_hot_batch_size); + const auto time_estimate = cuda_time_initial_estimate * 0.95; + auto batch_size = this->predict_cuda_batch_size(m_batch_target_time, + time_estimate, + hot_batch_size_floor, + m_min_samples); + auto timeout_batch_size = + this->predict_timeout_batch_cap(m_timeout, wallclock_time_initial_estimate, batch_size); do { - batch_size = std::max(batch_size, nvbench::int64_t{1}); + batch_size = std::min(batch_size, timeout_batch_size); nvbench::detail::stream_cleanup_guard cleanup{*this}; @@ -197,23 +224,43 @@ private: m_total_cuda_time += m_cuda_timer.get_duration(); m_total_samples += batch_size; - // Predict number of remaining iterations: - batch_size = static_cast( - (m_min_time - m_total_cuda_time) / - (m_total_cuda_time / static_cast(m_total_samples))); - - if (m_total_cuda_time > m_min_time && // min time okay - m_total_samples >= m_min_samples) // min samples okay + if (m_total_cuda_time >= m_batch_target_time && // batch target time okay + m_total_samples >= m_min_samples) // min samples okay { break; // Stop iterating } + const auto sample_count = static_cast(m_total_samples); + + // Predict number of remaining iterations based on cuda-time budget + const auto remaining_time = m_batch_target_time - m_total_cuda_time; + const auto time_per_sample = m_total_cuda_time / sample_count; + const auto remaining_samples_to_minimum = std::max(m_min_samples - m_total_samples, + nvbench::int64_t{1}); + const auto batch_target_time_satisfied = remaining_time <= nvbench::float64_t{0}; + const auto fallback_size_on_invalid_cuda_prediction = batch_target_time_satisfied + ? remaining_samples_to_minimum + : m_min_samples; + batch_size = + this->predict_cuda_batch_size(remaining_time, + time_per_sample, + this->grow_batch_size(batch_size, hot_batch_size_floor), + fallback_size_on_invalid_cuda_prediction); + m_walltime_timer.stop(); - if (m_walltime_timer.get_duration() > m_timeout) + const auto total_walltime = m_walltime_timer.get_duration(); + if (total_walltime > m_timeout) { m_max_time_exceeded = true; break; } + + // Predict number of remaining iterations based on timeout budget. + const auto remaining_walltime = m_timeout - total_walltime; + const auto walltime_per_sample = total_walltime / sample_count; + timeout_batch_size = + this->predict_timeout_batch_cap(remaining_walltime, walltime_per_sample, batch_size); + } while (true); m_walltime_timer.stop(); diff --git a/nvbench/detail/validate_batch_target_time.cuh b/nvbench/detail/validate_batch_target_time.cuh new file mode 100644 index 00000000..e255eb62 --- /dev/null +++ b/nvbench/detail/validate_batch_target_time.cuh @@ -0,0 +1,50 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#if defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_GCC) +#pragma GCC system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_CLANG) +#pragma clang system_header +#elif defined(NVBENCH_IMPLICIT_SYSTEM_HEADER_MSVC) +#pragma system_header +#endif + +#include + +#include +#include + +namespace nvbench +{ +namespace detail +{ + +inline void validate_batch_target_time(nvbench::float64_t batch_target_time) +{ + if (!std::isfinite(batch_target_time) || batch_target_time <= nvbench::float64_t{0}) + { + throw std::invalid_argument{"batch_target_time must be finite and positive."}; + } +} + +} // namespace detail +} // namespace nvbench diff --git a/nvbench/json_printer.cu b/nvbench/json_printer.cu index 9871eaed..e06f25c9 100644 --- a/nvbench/json_printer.cu +++ b/nvbench/json_printer.cu @@ -476,6 +476,7 @@ void json_printer::do_print_benchmark_results(const benchmark_vector &benches) bench["min_samples"] = bench_ptr->get_min_samples(); bench["cold_warmup_runs"] = bench_ptr->get_cold_warmup_runs(); bench["cold_max_warmup_walltime"] = bench_ptr->get_cold_max_warmup_walltime(); + bench["batch_target_time"] = bench_ptr->get_batch_target_time(); bench["skip_time"] = bench_ptr->get_skip_time(); bench["timeout"] = bench_ptr->get_timeout(); @@ -535,6 +536,7 @@ void json_printer::do_print_benchmark_results(const benchmark_vector &benches) st["min_samples"] = exec_state.get_min_samples(); st["cold_warmup_runs"] = exec_state.get_cold_warmup_runs(); st["cold_max_warmup_walltime"] = exec_state.get_cold_max_warmup_walltime(); + st["batch_target_time"] = exec_state.get_batch_target_time(); st["skip_time"] = exec_state.get_skip_time(); st["timeout"] = exec_state.get_timeout(); diff --git a/nvbench/option_parser.cu b/nvbench/option_parser.cu index 00580838..15164f68 100644 --- a/nvbench/option_parser.cu +++ b/nvbench/option_parser.cu @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -110,6 +111,60 @@ std::string_view submatch_to_sv(const sv_submatch &in) } //============================================================================== +std::string format_criterion_param_options(const nvbench::criterion_params ¶ms) +{ + auto names = params.get_names(); + if (names.empty()) + { + return ""; + } + + fmt::memory_buffer buffer; + bool first = true; + for (const auto &name : names) + { + if (!first) + { + fmt::format_to(fmt::appender(buffer), ", "); + } + fmt::format_to(fmt::appender(buffer), "--{}", name); + first = false; + } + return fmt::to_string(buffer); +} + +[[noreturn]] void +throw_unrecognized_criterion_param(const std::string &prop_arg, + const std::string &criterion_name, + const nvbench::criterion_params &criterion_params) +{ + NVBENCH_THROW(std::runtime_error, + "{} is not valid for the active stopping criterion '{}'.\n" + " Current criterion '{}' accepts: {}.", + prop_arg, + criterion_name, + criterion_name, + format_criterion_param_options(criterion_params)); +} + +std::string current_global_stopping_criterion(const std::vector &global_args) +{ + std::string criterion_name = nvbench::detail::default_stopping_criterion(); + for (auto arg_it = global_args.cbegin(); arg_it != global_args.cend(); ++arg_it) + { + if (*arg_it == "--stopping-criterion") + { + const auto value_it = std::next(arg_it); + if (value_it != global_args.cend()) + { + criterion_name = *value_it; + arg_it = value_it; + } + } + } + return criterion_name; +} + // These numeric overloads /could/ be written in a single function using // std::from_chars, but charconv is a mess on GCC. Even GCC 10 only partially // implements it (missing support for floats). @@ -578,7 +633,8 @@ void option_parser::parse_range(option_parser::arg_iterator_t first, first += 2; } else if (arg == "--skip-time" || arg == "--timeout" || arg == "--cold-max-warmup-walltime" || - arg == "--throttle-threshold" || arg == "--throttle-recovery-delay") + arg == "--batch-target-time" || arg == "--throttle-threshold" || + arg == "--throttle-recovery-delay") { check_params(1); this->update_float64_prop(first[0], first[1]); @@ -1072,20 +1128,11 @@ try // If no active benchmark, save args as global. if (m_benchmarks.empty()) { - // Any global params must either belong to the default criterion or follow a - // `--stopping-criterion` arg: - nvbench::criterion_params params = - criterion_manager::get() - .get_criterion(nvbench::detail::default_stopping_criterion()) - .get_params(); - if (!params.has_value(name) && - std::find(m_global_benchmark_args.cbegin(), - m_global_benchmark_args.cend(), - "--stopping-criterion") == m_global_benchmark_args.cend()) + const auto criterion_name = current_global_stopping_criterion(m_global_benchmark_args); + const auto params = criterion_manager::get().get_criterion(criterion_name).get_params(); + if (!params.has_value(name)) { - NVBENCH_THROW(std::runtime_error, - "Unrecognized stopping criterion parameter: `{}` for default criterion.", - name); + throw_unrecognized_criterion_param(prop_arg, criterion_name, params); } m_global_benchmark_args.push_back(prop_arg); @@ -1097,10 +1144,9 @@ try if (!bench.has_criterion_param(name)) { - NVBENCH_THROW(std::runtime_error, - "Unrecognized stopping criterion parameter: `{}` for `{}`.", - name, - bench.get_stopping_criterion()); + throw_unrecognized_criterion_param(prop_arg, + bench.get_stopping_criterion(), + bench.get_criterion_params()); } if (type == nvbench::named_values::type::float64) @@ -1165,6 +1211,10 @@ try { bench.set_cold_max_warmup_walltime(value); } + else if (prop_arg == "--batch-target-time") + { + bench.set_batch_target_time(value); + } else if (prop_arg == "--throttle-threshold") { bench.set_throttle_threshold(static_cast(value) / 100.0f); diff --git a/nvbench/state.cuh b/nvbench/state.cuh index 3de00465..89316b8d 100644 --- a/nvbench/state.cuh +++ b/nvbench/state.cuh @@ -202,6 +202,11 @@ struct state void set_skip_batched(bool v) { m_skip_batched = v; } /// @} + /// Target accumulated GPU time for batched measurements. @{ + [[nodiscard]] nvbench::float64_t get_batch_target_time() const { return m_batch_target_time; } + void set_batch_target_time(nvbench::float64_t batch_target_time); + /// @} + /// If true, the benchmark does not use the blocking_kernel. This is intended /// for use with external profiling tools. @{ [[nodiscard]] bool get_disable_blocking_kernel() const { return m_disable_blocking_kernel; } @@ -211,8 +216,8 @@ struct state /// If a warmup run finishes in less than `skip_time`, the measurement will /// be skipped. /// Extremely fast kernels (< 5000 ns) often timeout before they can - /// accumulate `min_time` measurements, and are often uninteresting. Setting - /// this value can help improve performance by skipping time consuming + /// accumulate enough measurement time, and are often uninteresting. Setting + /// this value can help improve performance by skipping time-consuming /// measurement that don't provide much information. /// Default value is -1., which disables the feature. /// @{ @@ -361,6 +366,7 @@ private: nvbench::int64_t m_cold_warmup_runs; nvbench::float64_t m_cold_max_warmup_walltime; + nvbench::float64_t m_batch_target_time; nvbench::float64_t m_skip_time; nvbench::float64_t m_timeout; diff --git a/nvbench/state.cxx b/nvbench/state.cxx index a81fe693..23166e7f 100644 --- a/nvbench/state.cxx +++ b/nvbench/state.cxx @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -48,6 +49,7 @@ state::state(const benchmark_base &bench) , m_min_samples{bench.get_min_samples()} , m_cold_warmup_runs{bench.get_cold_warmup_runs()} , m_cold_max_warmup_walltime{bench.get_cold_max_warmup_walltime()} + , m_batch_target_time{bench.get_batch_target_time()} , m_skip_time{bench.get_skip_time()} , m_timeout{bench.get_timeout()} , m_throttle_threshold{bench.get_throttle_threshold()} @@ -72,6 +74,7 @@ state::state(const benchmark_base &bench, , m_min_samples{bench.get_min_samples()} , m_cold_warmup_runs{bench.get_cold_warmup_runs()} , m_cold_max_warmup_walltime{bench.get_cold_max_warmup_walltime()} + , m_batch_target_time{bench.get_batch_target_time()} , m_skip_time{bench.get_skip_time()} , m_timeout{bench.get_timeout()} , m_throttle_threshold{bench.get_throttle_threshold()} @@ -79,6 +82,12 @@ state::state(const benchmark_base &bench, , m_cuda_stream{std::nullopt} {} +void state::set_batch_target_time(nvbench::float64_t batch_target_time) +{ + nvbench::detail::validate_batch_target_time(batch_target_time); + m_batch_target_time = batch_target_time; +} + nvbench::int64_t state::get_int64(const std::string &axis_name) const { return m_axis_values.get_int64(axis_name); diff --git a/python/cuda/bench/__init__.pyi b/python/cuda/bench/__init__.pyi index 0e1ad7d1..c0bc037f 100644 --- a/python/cuda/bench/__init__.pyi +++ b/python/cuda/bench/__init__.pyi @@ -65,6 +65,7 @@ class Benchmark: def set_throttle_recovery_delay(self, delay_seconds: SupportsFloat) -> Self: ... def set_throttle_threshold(self, threshold: SupportsFloat) -> Self: ... def set_timeout(self, duration_seconds: SupportsFloat) -> Self: ... + def set_batch_target_time(self, duration_seconds: SupportsFloat) -> Self: ... def set_stopping_criterion(self, criterion: str) -> Self: ... def set_criterion_param_float64(self, name: str, value: SupportsFloat) -> Self: ... def set_criterion_param_int64(self, name: str, value: SupportsInt) -> Self: ... @@ -125,6 +126,8 @@ class State: def set_run_once(self, run_once_flag: bool) -> None: ... def get_timeout(self) -> float: ... def set_timeout(self, duration: SupportsFloat) -> None: ... + def get_batch_target_time(self) -> float: ... + def set_batch_target_time(self, duration_seconds: SupportsFloat) -> None: ... def get_blocking_kernel_timeout(self) -> float: ... def set_blocking_kernel_timeout(self, duration: SupportsFloat) -> None: ... @overload @@ -201,6 +204,12 @@ class _OptionDecorators: ) -> Callable[[_F], _F]: ... def timeout(self, duration_seconds: SupportsFloat) -> Callable[[_F], _F]: ... def set_timeout(self, duration_seconds: SupportsFloat) -> Callable[[_F], _F]: ... + def batch_target_time( + self, duration_seconds: SupportsFloat + ) -> Callable[[_F], _F]: ... + def set_batch_target_time( + self, duration_seconds: SupportsFloat + ) -> Callable[[_F], _F]: ... def stopping_criterion(self, criterion: str) -> Callable[[_F], _F]: ... def set_stopping_criterion(self, criterion: str) -> Callable[[_F], _F]: ... def criterion_param_float64( diff --git a/python/cuda/bench/_decorators.py b/python/cuda/bench/_decorators.py index a45f0e92..bee2fbc1 100644 --- a/python/cuda/bench/_decorators.py +++ b/python/cuda/bench/_decorators.py @@ -240,6 +240,16 @@ def set_timeout(self, duration_seconds: float) -> Callable[[_F], _F]: lambda benchmark: benchmark.set_timeout(duration_seconds) ) + def batch_target_time(self, duration_seconds: float) -> Callable[[_F], _F]: + """Set the target accumulated GPU time for batched measurements.""" + return self.set_batch_target_time(duration_seconds) + + def set_batch_target_time(self, duration_seconds: float) -> Callable[[_F], _F]: + """Set the target accumulated GPU time for batched measurements.""" + return _append_benchmark_action( + lambda benchmark: benchmark.set_batch_target_time(duration_seconds) + ) + def stopping_criterion(self, criterion: str) -> Callable[[_F], _F]: """Set the benchmark stopping criterion.""" return self.set_stopping_criterion(criterion) diff --git a/python/src/py_nvbench.cpp b/python/src/py_nvbench.cpp index 2f307977..5d138b15 100644 --- a/python/src/py_nvbench.cpp +++ b/python/src/py_nvbench.cpp @@ -705,6 +705,21 @@ Set benchmark run duration timeout value, in seconds py::return_value_policy::reference, py::arg("duration_seconds")); + // method Benchmark.set_batch_target_time + auto method_set_batch_target_time_impl = [](nvbench::benchmark_base &self, + nvbench::float64_t duration_seconds) { + self.set_batch_target_time(duration_seconds); + return std::ref(self); + }; + static constexpr const char *method_set_batch_target_time_doc = R"XXXX( +Set target accumulated GPU time for batched measurements, in seconds +)XXXX"; + py_benchmark_cls.def("set_batch_target_time", + method_set_batch_target_time_impl, + method_set_batch_target_time_doc, + py::return_value_policy::reference, + py::arg("duration_seconds")); + // method Benchmark.set_throttle_threshold auto method_set_throttle_threshold_impl = [](nvbench::benchmark_base &self, nvbench::float32_t threshold) { @@ -887,6 +902,8 @@ void def_class_State(py::module_ m) // nvbench::state::get_skip_time // nvbench::state::set_timeout // nvbench::state::get_timeout + // nvbench::state::set_batch_target_time + // nvbench::state::get_batch_target_time // nvbench::state::set_throttle_threshold // nvbench::state::get_throttle_threshold // nvbench::state::set_throttle_recovery_delay @@ -1242,6 +1259,21 @@ Use argument True to disable use of blocking kernel by NVBench" method_set_timeout_doc, py::arg("duration_seconds")); + // method State.get_batch_target_time + static constexpr const char *method_get_batch_target_time_doc = + R"XXXX(Get target accumulated GPU time for batched measurements, in seconds)XXXX"; + pystate_cls.def("get_batch_target_time", + &nvbench::state::get_batch_target_time, + method_get_batch_target_time_doc); + + // method State.set_batch_target_time + static constexpr const char *method_set_batch_target_time_doc = + R"XXXX(Set target accumulated GPU time for batched measurements, in seconds)XXXX"; + pystate_cls.def("set_batch_target_time", + &nvbench::state::set_batch_target_time, + method_set_batch_target_time_doc, + py::arg("duration_seconds")); + // method State.get_blocking_kernel_timeout static constexpr const char *method_get_blocking_kernel_timeout_doc = R"XXXX(Get time-out value for execution of blocking kernel, in seconds)XXXX"; diff --git a/python/test/test_cuda_bench.py b/python/test/test_cuda_bench.py index 64300c38..b354c6dc 100644 --- a/python/test/test_cuda_bench.py +++ b/python/test/test_cuda_bench.py @@ -99,6 +99,26 @@ def cold_warmup_state_probe(state: bench.State): state.exec(lambda launch: None) + def batch_target_state_probe(state: bench.State): + observed["benchmark_batch_target_time"] = state.get_batch_target_time() + + state.set_batch_target_time(0.125) + observed["state_batch_target_time"] = state.get_batch_target_time() + + for duration_seconds in [0.0, -1.0, float("inf"), float("nan")]: + with pytest.raises(ValueError, match="finite and positive"): + state.set_batch_target_time(duration_seconds) + + state.exec(lambda launch: None) + + batch_target_benchmark = bench.register(batch_target_state_probe) + batch_target_benchmark.set_is_cpu_only(True) + batch_target_benchmark.set_batch_target_time(0.75) + + for duration_seconds in [0.0, -1.0, float("inf"), float("nan")]: + with pytest.raises(ValueError, match="finite and positive"): + batch_target_benchmark.set_batch_target_time(duration_seconds) + @bench.register() @bench.option.set_is_cpu_only(True) def external_stream_state_probe(state: bench.State): @@ -175,6 +195,8 @@ def __cuda_stream__(self): "benchmark_walltime": 0.5, "state_runs": 3, "state_walltime": 0.125, + "benchmark_batch_target_time": 0.75, + "state_batch_target_time": 0.125, "external_stream_handle": external_stream_handle, } @@ -221,6 +243,8 @@ def test_decorator_docstrings(): obj_has_docstring_check(bench.option.set_throttle_threshold) obj_has_docstring_check(bench.option.timeout) obj_has_docstring_check(bench.option.set_timeout) + obj_has_docstring_check(bench.option.batch_target_time) + obj_has_docstring_check(bench.option.set_batch_target_time) obj_has_docstring_check(bench.option.stopping_criterion) obj_has_docstring_check(bench.option.set_stopping_criterion) obj_has_docstring_check(bench.option.criterion_param_float64) @@ -260,6 +284,10 @@ def set_cold_max_warmup_walltime(self, duration_seconds): self.calls.append(("cold_max_warmup_walltime", duration_seconds)) return self + def set_batch_target_time(self, duration_seconds): + self.calls.append(("batch_target_time", duration_seconds)) + return self + fake_benchmark = FakeBenchmark() registered_functions = [] @@ -274,6 +302,7 @@ def fake_register(fn): @bench.option.min_samples(11) @bench.option.cold_warmup_runs(7) @bench.option.cold_max_warmup_walltime(0.25) + @bench.option.batch_target_time(0.75) def decorated(state: bench.State): pass @@ -283,6 +312,7 @@ def decorated(state: bench.State): ("min_samples", 11), ("cold_warmup_runs", 7), ("cold_max_warmup_walltime", 0.25), + ("batch_target_time", 0.75), ] assert callable(decorated) @@ -394,6 +424,8 @@ def test_State_doc(): obj_has_docstring_check(cl.set_cold_warmup_runs) obj_has_docstring_check(cl.get_cold_max_warmup_walltime) obj_has_docstring_check(cl.set_cold_max_warmup_walltime) + obj_has_docstring_check(cl.get_batch_target_time) + obj_has_docstring_check(cl.set_batch_target_time) obj_has_docstring_check(cl.skip) @@ -424,3 +456,4 @@ def test_Benchmark_doc(): obj_has_docstring_check(cl.add_string_axis) obj_has_docstring_check(cl.set_cold_warmup_runs) obj_has_docstring_check(cl.set_cold_max_warmup_walltime) + obj_has_docstring_check(cl.set_batch_target_time) diff --git a/testing/option_parser.cu b/testing/option_parser.cu index 9634e90c..aa386e65 100644 --- a/testing/option_parser.cu +++ b/testing/option_parser.cu @@ -24,8 +24,12 @@ #include #include +#include +#include #include #include +#include +#include #include #if __has_include() @@ -156,6 +160,30 @@ struct temp_tree return states_to_string(parser_to_states(parser)); } +void assert_parse_error_contains(std::vector args, + std::initializer_list snippets) +{ + try + { + nvbench::option_parser parser; + parser.parse(std::move(args)); + } + catch (const std::runtime_error &ex) + { + const std::string message = ex.what(); + for (const auto snippet : snippets) + { + ASSERT_MSG(message.find(std::string{snippet}) != std::string::npos, + "Expected error message to contain `{}`. Message:\n{}", + snippet, + message); + } + return; + } + + ASSERT_MSG(false, "Expected parser error.", ""); +} + } // namespace void test_empty() @@ -1265,6 +1293,85 @@ void test_timeout() ASSERT(std::abs(states[0].get_timeout() - 12345e2) < 1.); } +void test_batch_target_time() +{ + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "DummyBench"}); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(std::abs(states[0].get_batch_target_time() - 0.5) < 1e-6); + } + + { + nvbench::option_parser parser; + parser.parse({"--benchmark", "DummyBench", "--batch-target-time", "1.25"}); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(std::abs(states[0].get_batch_target_time() - 1.25) < 1e-6); + } + + { + nvbench::option_parser parser; + parser.parse({"--batch-target-time", "2.5", "--benchmark", "DummyBench"}); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(std::abs(states[0].get_batch_target_time() - 2.5) < 1e-6); + } + + { + nvbench::option_parser parser; + parser.parse({ + "--batch-target-time", + "2.5", + "--benchmark", + "DummyBench", + "--batch-target-time", + "1.0", + "--benchmark", + "TestBench", + }); + + const auto &benches = parser.get_benchmarks(); + ASSERT(benches.size() == 2); + ASSERT(benches[0] != nullptr); + ASSERT(benches[1] != nullptr); + + const auto dummy_states = nvbench::detail::state_generator::create(*benches[0]); + ASSERT(dummy_states.size() == 1); + ASSERT(std::abs(dummy_states[0].get_batch_target_time() - 1.0) < 1e-6); + + const auto test_states = nvbench::detail::state_generator::create(*benches[1]); + ASSERT(!test_states.empty()); + for (const auto &state : test_states) + { + ASSERT(std::abs(state.get_batch_target_time() - 2.5) < 1e-6); + } + } + + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--benchmark", "DummyBench", "--batch-target-time", "0"})); + } + + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--benchmark", "DummyBench", "--batch-target-time", "-1"})); + } + + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--benchmark", "DummyBench", "--batch-target-time", "inf"})); + } + { + nvbench::option_parser parser; + ASSERT_THROWS_ANY(parser.parse({"--benchmark", "DummyBench", "--batch-target-time", "nan"})); + } +} + void test_json_stream_destinations() { { @@ -1384,6 +1491,36 @@ void test_stopping_criterion() ASSERT(criterion_params.get_float64("max-angle") == 0.42); ASSERT(criterion_params.get_float64("min-r2") == 0.6); } + { // Global criterion rejects params it does not accept: + assert_parse_error_contains( + { + "--stopping-criterion", + "entropy", + "--min-time", + "0.1", + "--benchmark", + "DummyBench", + }, + { + "--min-time is not valid for the active stopping criterion 'entropy'.", + "Current criterion 'entropy' accepts: --max-angle, --min-r2.", + }); + } + { // Per-benchmark criterion rejects params it does not accept: + assert_parse_error_contains( + { + "--benchmark", + "DummyBench", + "--stopping-criterion", + "entropy", + "--min-time", + "0.1", + }, + { + "--min-time is not valid for the active stopping criterion 'entropy'.", + "Current criterion 'entropy' accepts: --max-angle, --min-r2.", + }); + } { // Global params to default criterion should work: nvbench::option_parser parser; parser.parse({ @@ -1677,6 +1814,7 @@ try test_skip_time(); test_cold_max_warmup_walltime(); test_timeout(); + test_batch_target_time(); test_json_stream_destinations(); test_output_parent_directories_created();