From 1ecbeab5f1a4f876e9a30a8e9697990b226d2e3a Mon Sep 17 00:00:00 2001 From: Ioannis Makris Date: Wed, 4 Nov 2026 02:06:00 +0100 Subject: [PATCH 1/5] Establish repeatable plot workloads and paired benchmark comparisons Recover the completed benchmark baseline from the original local work. Source tree is byte-identical to original commit 88d459dbbad9cff1a292b3ffa87be97970d371e6, including the initial workload suite and unavailable-counter handling. Original commits are preserved in vnm_plot-benchmarks-2026-11-04.zip; this publication uses a new commit identity because direct Git push authentication was unavailable. --- benchmark/CMakeLists.txt | 21 + benchmark/README.md | 136 +++++ benchmark/include/benchmark_data_source.h | 10 +- benchmark/include/benchmark_profiler.h | 26 + benchmark/include/benchmark_window.h | 13 + benchmark/src/benchmark_window.cpp | 111 +++- benchmark/src/main.cpp | 65 ++- benchmark/tools/run_suite.py | 664 ++++++++++++++++++++++ benchmark/tools/test_phase_trace_flush.py | 5 +- benchmark/tools/test_run_suite.py | 146 +++++ benchmark/tools/validate_workloads.py | 103 ++++ tests/test_core_algo.cpp | 74 +++ 12 files changed, 1348 insertions(+), 26 deletions(-) create mode 100644 benchmark/tools/run_suite.py create mode 100644 benchmark/tools/test_run_suite.py create mode 100644 benchmark/tools/validate_workloads.py diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index d4a26dd4..17c3acc1 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -309,4 +309,25 @@ set_tests_properties(benchmark_shared_frame_pipeline PROPERTIES ENVIRONMENT "PYTHONDONTWRITEBYTECODE=1" ) +add_test( + NAME benchmark_workload_contracts + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/tools/validate_workloads.py" + --executable "$" +) +set_tests_properties(benchmark_workload_contracts PROPERTIES + TIMEOUT 90 + ENVIRONMENT "PYTHONDONTWRITEBYTECODE=1" +) + +add_test( + NAME benchmark_suite_validation + COMMAND "${Python3_EXECUTABLE}" -m unittest discover + -s "${CMAKE_CURRENT_SOURCE_DIR}/tools" -p test_run_suite.py +) +set_tests_properties(benchmark_suite_validation PROPERTIES + TIMEOUT 15 + ENVIRONMENT "PYTHONDONTWRITEBYTECODE=1" +) + endif() diff --git a/benchmark/README.md b/benchmark/README.md index f2aa398d..30c9b135 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -45,3 +45,139 @@ to measured frames. On Windows, benchmark file operations support extended-length native paths while reports keep their ordinary logical path spelling. + +## Repeatable workload suite + +Build the existing executable and its contract tests with Qt 6.12 and the +dependencies described in [Building](../docs/building.md): + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ + -DVNM_PLOT_BUILD_BENCHMARK=ON -DVNM_PLOT_BUILD_TESTS=ON +cmake --build build --config Release --target vnm_plot_benchmark +python3 benchmark/tools/run_suite.py \ + --candidate build/benchmark/vnm_plot_benchmark \ + --graphics-backend opengl --output-dir results/baseline +``` + +Use `d3d11` on Windows, `metal` on macOS, or another explicitly supported API. +`native` accepts Qt's platform default but rejects Null fallback. On a host +without raster rendering, `QT_QPA_PLATFORM=offscreen` and +`--graphics-backend null` exercise the production CPU pipeline only. Null +does not execute shaders, rasterize, measure GPU work, or validate pixels. + +The suite runs these named cases, selectable with `--cases NAME ...`: + +| Case | Purpose | +|---|---| +| `warm-line-1`, `warm-line-8`, `warm-line-32` | Settled frame cost as series count grows | +| `dense-line-8` | 100,000 visible samples per series, ten times the ordinary case | +| `warm-area-8`, `warm-dots-8` | Other production primitive paths | +| `warm-stacked-area-8` | Stack composition and reuse | +| `pan-line-8` | A quarter-width window moving across fixed history | +| `zoom-line-8` | A centered view changing from 10% to 100% of history | +| `append-line-8` | Exactly 16 new samples per series before each measured frame | +| `wrap-line-8` | The same publication schedule into a full ring, including split spans | +| `warm-line-8-no-text` | Control for text/layout changes | +| `warm-line-8-conservative` | Callbacks without a declared stable semantic identity | + +Ordinary cases contain 10,000 samples per series. The seed, dimensions, series +count, rates, data/style, warmup and measured frames are explicit in every +command/report. The dense case also increases sample rate, keeping its entire +history within the ten-second view instead of only increasing invisible data. +Preview rendering is enabled in the suite; individual runs can use +`--no-preview` to isolate the main pane. + +The suite passes `--stable-access` except in the conservative control. The +benchmark's stateless callbacks then declare their actual immutable semantics +through the public policy API, allowing production range-cache reuse. A +byte-layout key alone does not establish callback semantics. This is a +workload distinction, not a production performance improvement. The original +ad-hoc conservative default remains available without the flag. + +Panning and zooming use a 120-frame triangular cycle. Append/motion require +`--static --frames N`; warmup uses the initial data and view, and the measured +sequence does not depend on the number of warmup frames. `--static` means +prefilled data with no producer thread; `--append-per-frame` then advances it +synchronously before each render timer starts. Synthetic generation and +publication cost is reported separately as `benchmark.workload.publish_ms`. +The existing time-paced producer mode remains available for contention and +live-rate experiments, but is not used for paired performance comparisons. + +Input schedules are deterministic. Label fades still use elapsed real time, +so motion/update cases may retain different outgoing labels on differently +paced runs. They are not pixel-identical frame sequences. Settled cases are +the strongest timing controls; fixed-clock CoreAlgo tests cover fade behavior. +The sources currently expose LOD 0 only, so this suite does not cover LOD +switching, custom layers, or the Qt Quick event/synchronization loop. + +## Compare an optimization + +Build baseline and candidate with the **same benchmark code, compiler, +dependencies and configuration**. Keep both executables, then run: + +```bash +python3 benchmark/tools/run_suite.py \ + --baseline /absolute/path/to/baseline/vnm_plot_benchmark \ + --candidate /absolute/path/to/candidate/vnm_plot_benchmark \ + --graphics-backend opengl --output-dir results/comparison \ + --repetitions 5 --warmup-frames 30 --frames 240 +``` + +Each process is a replicate. The runner executes sequentially, alternates +baseline/candidate order by repetition, preserves every raw report and log, +and rejects failed frames, wrong publication/occupancy/overwrite counts, +unexpected static uploads/scans, mismatched metadata or backend fallback. +It records executable hashes and host/renderer details. Record the exact +source/dependency revisions alongside results: a binary's enclosing checkout +at run time does not prove what was compiled into it. + +`summary.md` gives medians of per-process means and p95s. `summary.json` also +contains paired changes, IQR, min/max, counter totals and inclusive scope +costs. `manifest.json` links raw reports and preserves run order. Negative +paired percentages mean lower cost. Frame samples are not pooled as independent +replicates, and spread is not presented as a confidence interval. There are +no machine-dependent performance thresholds in CTest. + +Use a second run with `--frame-timing-only` to check whether a result survives +without internal renderer/source profiling. It retains outer frame timings +and render-thread C++ allocation accounting, plus exact publication counts. +Internal counters/scopes are absent in this mode and are not inferred as zero. +Keep internal-profiling mode identical within each comparison. + +### Measurement boundaries + +- `benchmark.frame.total_ms` is offscreen frame **wall time**, including + begin/end-frame, production planning/layout/text/series preparation, + recording and `--finish`. It is not application FPS or isolated CPU time. +- `benchmark.frame.submission_ms` includes QRhi begin/end calls, which may + synchronize. `gpu_finish_ms` is the extra finish call; it is not a GPU timer. +- Internal named scopes are inclusive and aggregated by full scope name in + the raw JSON `scopes` map. Do not sum parent and child times. +- C++ allocation counters cover the measured render thread and exclude + profiler storage. They do not count all `malloc` calls, other threads, GPU + memory, or retained memory. QRhi buffer creation has separate counters. +- Setup/first submission and warmup are separate from measured frames. + Existing font/disk caches can affect cold results; the suite reports warm + frames and does not claim controlled cold-start performance. +- Timing runs disable readback. Run `--pixel-checksum` separately on a raster + backend and compare appropriate stable frames when validating a change. + Faster execution alone is not evidence that rendering stayed correct. + +The offscreen runner consumes measured value-axis width feedback, as the +shipping widget does. Reusing the initial width would cause layout misses in +otherwise settled frames; the workload contract test checks this invariant. + +## Maintained checks + +With benchmark tests enabled, CTest includes `benchmark_workload_contracts` +and `benchmark_suite_validation`. They check actual fixed/pan/zoom/publication +behavior, settled layout/range/upload reuse, frame-only mode, invalid input, +and rejection of incomplete or incomparable results. The phase-flush test +uses Null by default because it tests durable output and external termination; +native smoke/readback tests retain their raster backend requirement. + +```bash +QT_QPA_PLATFORM=offscreen ctest --test-dir build --output-on-failure \ + -R '^(benchmark_workload_contracts|benchmark_suite_validation|benchmark_shared_frame_pipeline|benchmark_phase_trace_flush)$' +``` diff --git a/benchmark/include/benchmark_data_source.h b/benchmark/include/benchmark_data_source.h index 372feb9c..f2c96cf9 100644 --- a/benchmark/include/benchmark_data_source.h +++ b/benchmark/include/benchmark_data_source.h @@ -113,7 +113,7 @@ class Benchmark_data_source : public vnm::plot::Data_source { /// Create a Data_access_policy for Bar_sample. The renderer owns the GPU-side /// sample layout (gpu_sample_t) and rebases timestamps on upload, so the /// policy only exposes value extractors plus a stable layout_key for caching. -inline vnm::plot::Data_access_policy make_bar_access_policy() { +inline vnm::plot::Data_access_policy make_bar_access_policy(bool stable_semantics = false) { vnm::plot::Data_access_policy policy; policy.get_timestamp = [](const void* sample) -> std::int64_t { @@ -130,6 +130,9 @@ inline vnm::plot::Data_access_policy make_bar_access_policy() { }; policy.layout_key = k_bar_sample_layout_key; + if (stable_semantics) { + policy.set_semantics_key(k_bar_sample_layout_key, 1); + } return policy; } @@ -137,7 +140,7 @@ inline vnm::plot::Data_access_policy make_bar_access_policy() { /// Create a Data_access_policy for Trade_sample. The renderer owns the GPU-side /// sample layout (gpu_sample_t) and rebases timestamps on upload, so the /// policy only exposes value extractors plus a stable layout_key for caching. -inline vnm::plot::Data_access_policy make_trade_access_policy() { +inline vnm::plot::Data_access_policy make_trade_access_policy(bool stable_semantics = false) { vnm::plot::Data_access_policy policy; policy.get_timestamp = [](const void* sample) -> std::int64_t { @@ -154,6 +157,9 @@ inline vnm::plot::Data_access_policy make_trade_access_policy() { }; policy.layout_key = k_trade_sample_layout_key; + if (stable_semantics) { + policy.set_semantics_key(k_trade_sample_layout_key, 1); + } return policy; } diff --git a/benchmark/include/benchmark_profiler.h b/benchmark/include/benchmark_profiler.h index daa99c7e..eab5e9d8 100644 --- a/benchmark/include/benchmark_profiler.h +++ b/benchmark/include/benchmark_profiler.h @@ -357,6 +357,32 @@ class Benchmark_profiler : public vnm::plot::Profiler { } ofs << "\n"; } + ofs << " },\n \"scopes\": {\n"; + // Named scope totals are inclusive. Aggregate the same full scope name + // across parents; never add parent and child durations into frame time. + std::map scopes; + const auto collect_scopes = [&](const auto& self, const Scope_stats& parent) -> void { + for (const auto& [name, child] : parent.children) { + auto& total = scopes[name]; + total.call_count += child->call_count; + total.total_ms += child->total_ms; + total.min_ms = std::min(total.min_ms, child->min_ms); + total.max_ms = std::max(total.max_ms, child->max_ms); + self(self, *child); + } + }; + collect_scopes(collect_scopes, m_root); + std::size_t scope_index = 0; + for (const auto& [name, stats] : scopes) { + ofs << " \"" << json_escape(name) << "\": {\n"; + ofs << " \"count\": " << stats.call_count << ",\n"; + ofs << " \"total_ms\": " << json_number(stats.total_ms) << ",\n"; + ofs << " \"mean_ms\": " << json_number(stats.call_count + ? stats.total_ms / static_cast(stats.call_count) : 0.0) << ",\n"; + ofs << " \"min_ms\": " << json_number(stats.min_ms) << ",\n"; + ofs << " \"max_ms\": " << json_number(stats.max_ms) << "\n }"; + ofs << (++scope_index < scopes.size() ? ",\n" : "\n"); + } ofs << " }\n}\n"; if (!ofs) { throw std::runtime_error("cannot write raw benchmark report: " + output_path.string()); diff --git a/benchmark/include/benchmark_window.h b/benchmark/include/benchmark_window.h index 8d70e027..95caba57 100644 --- a/benchmark/include/benchmark_window.h +++ b/benchmark/include/benchmark_window.h @@ -82,6 +82,18 @@ struct Benchmark_config { // skips the producer thread. bool static_data = false; std::size_t static_sample_count = 10'000; + // Deterministic offscreen workloads start from the static prefill. A fixed + // publication per measured frame avoids faster builds seeing less data. + std::size_t append_per_frame = 0; + std::string view_motion = "fixed"; // fixed|pan|zoom, 120-frame cycle + bool preview_enabled = true; + // Keep outer frame timing/allocation accounting, without renderer/source + // profiling. Use alongside diagnostic runs to expose instrumentation cost. + bool internal_profiling = true; + // The synthetic callbacks are immutable. Opt into their explicit semantic + // identity to exercise normal range-cache reuse, retaining the legacy + // conservative callable path as a separate benchmark control. + bool stable_access = false; // Line thickness in pixels (default 1.5 for animated runs; bumped via // --line-px when comparing visuals so segments are easy to inspect). double line_width_px = 1.5; @@ -207,6 +219,7 @@ class Benchmark_rhi_offscreen_runner void fill_static_data(); bool initialize_rhi(std::string& error_message); bool render_frame(std::string& error_message, bool measured); + void append_frame_samples(); void record_final_statistics(double measured_seconds); void record_phase(const char* phase, std::size_t frame = 0) const; diff --git a/benchmark/src/benchmark_window.cpp b/benchmark/src/benchmark_window.cpp index 83045ccc..3aa8f28b 100644 --- a/benchmark/src/benchmark_window.cpp +++ b/benchmark/src/benchmark_window.cpp @@ -135,13 +135,17 @@ double steady_clock_resolution_ns() return std::isfinite(resolution) ? resolution : 0.0; } -void ensure_interval_observations(Benchmark_profiler& profiler) +void ensure_interval_observations(Benchmark_profiler& profiler, bool internal_profiling) { profiler.ensure_observation("benchmark.producer.lock_wait_ns"); - profiler.ensure_observation("renderer.series_window.monotonicity_scan_count"); - profiler.ensure_observation("renderer.series_window.monotonicity_scan_samples"); - profiler.ensure_observation("renderer.frame.gpu_buffer_allocation_bytes"); - profiler.ensure_observation("renderer.frame.gpu_buffer_allocation_count"); + if (internal_profiling) { + profiler.ensure_observation("renderer.series_window.monotonicity_scan_count"); + profiler.ensure_observation("renderer.series_window.monotonicity_scan_samples"); + profiler.ensure_observation("renderer.auto_range.query_count"); + profiler.ensure_observation("renderer.auto_range.range_scan_count"); + profiler.ensure_observation("renderer.frame.gpu_buffer_allocation_bytes"); + profiler.ensure_observation("renderer.frame.gpu_buffer_allocation_count"); + } profiler.ensure_observation("benchmark.frame.cpu_allocation_bytes"); profiler.ensure_observation("benchmark.frame.cpu_allocation_count"); } @@ -465,11 +469,11 @@ void Benchmark_rhi_window::setup_series() if (m_config.data_type == "Trades") { series->set_data_source_ref(*m_trade_sources[index]); - series->access = make_trade_access_policy(); + series->access = make_trade_access_policy(m_config.stable_access); } else { series->set_data_source_ref(*m_bar_sources[index]); - series->access = make_bar_access_policy(); + series->access = make_bar_access_policy(m_config.stable_access); } if (style == "Dots") { @@ -589,7 +593,7 @@ void Benchmark_rhi_window::record_final_statistics() m_profiler.record_observation( "benchmark.memory.process_high_water_bytes", static_cast(process_memory_high_water_bytes())); - ensure_interval_observations(m_profiler); + ensure_interval_observations(m_profiler, m_config.internal_profiling); } Benchmark_rhi_offscreen_runner::Benchmark_rhi_offscreen_runner(const Benchmark_config& config) @@ -615,14 +619,15 @@ Benchmark_rhi_offscreen_runner::~Benchmark_rhi_offscreen_runner() void Benchmark_rhi_offscreen_runner::setup_data_source() { + auto* profiler = m_config.internal_profiling ? &m_profiler : nullptr; if (m_config.data_type == "Trades") { m_trade_buffers.reserve(m_config.series_count); m_trade_sources.reserve(m_config.series_count); for (std::size_t index = 0; index < m_config.series_count; ++index) { auto buffer = std::make_unique>(m_config.ring_capacity); - buffer->set_profiler(&m_profiler); + buffer->set_profiler(profiler); auto source = std::make_unique>(*buffer); - source->set_profiler(&m_profiler); + source->set_profiler(profiler); m_trade_buffers.push_back(std::move(buffer)); m_trade_sources.push_back(std::move(source)); } @@ -632,9 +637,9 @@ void Benchmark_rhi_offscreen_runner::setup_data_source() m_bar_sources.reserve(m_config.series_count); for (std::size_t index = 0; index < m_config.series_count; ++index) { auto buffer = std::make_unique>(m_config.ring_capacity); - buffer->set_profiler(&m_profiler); + buffer->set_profiler(profiler); auto source = std::make_unique>(*buffer); - source->set_profiler(&m_profiler); + source->set_profiler(profiler); m_bar_buffers.push_back(std::move(buffer)); m_bar_sources.push_back(std::move(source)); } @@ -646,7 +651,8 @@ void Benchmark_rhi_offscreen_runner::setup_rendering() m_frame_renderer->initialize(); m_snapshot->adjusted_font_px = k_adjusted_font_px; m_snapshot->base_label_height_px = k_base_label_height_px; - m_snapshot->adjusted_preview_height = k_adjusted_preview_height; + m_snapshot->adjusted_preview_height = m_config.preview_enabled + ? k_adjusted_preview_height : 0.0; m_snapshot->vbar_width_pixels = k_vbar_width_pixels; m_snapshot->config.dark_mode = true; @@ -661,10 +667,13 @@ void Benchmark_rhi_offscreen_runner::setup_rendering() m_snapshot->config.point_diameter_px = m_config.point_diameter_px; m_snapshot->config.font_size_px = k_adjusted_font_px; m_snapshot->config.base_label_height_px = k_base_label_height_px; - m_snapshot->config.preview_height_px = k_adjusted_preview_height; + m_snapshot->config.preview_height_px = m_snapshot->adjusted_preview_height; + m_snapshot->config.preview_visibility = m_config.preview_enabled ? 1.0 : 0.0; m_snapshot->config.format_timestamp = format_benchmark_timestamp; - m_snapshot->config.profiler = - std::shared_ptr(&m_profiler, [](vnm::plot::Profiler*) {}); + if (m_config.internal_profiling) { + m_snapshot->config.profiler = + std::shared_ptr(&m_profiler, [](vnm::plot::Profiler*) {}); + } } void Benchmark_rhi_offscreen_runner::setup_series() @@ -681,11 +690,11 @@ void Benchmark_rhi_offscreen_runner::setup_series() if (m_config.data_type == "Trades") { series->set_data_source_ref(*m_trade_sources[index]); - series->access = make_trade_access_policy(); + series->access = make_trade_access_policy(m_config.stable_access); } else { series->set_data_source_ref(*m_bar_sources[index]); - series->access = make_bar_access_policy(); + series->access = make_bar_access_policy(m_config.stable_access); } if (style == "Dots") { @@ -1039,7 +1048,8 @@ bool Benchmark_rhi_offscreen_runner::render_frame( ? static_cast(m_trade_sources.front().get()) : static_cast(m_bar_sources.front().get()); { - VNM_PLOT_PROFILE_SCOPE(&m_profiler, "benchmark.update_time_range"); + VNM_PLOT_PROFILE_SCOPE( + m_snapshot->config.profiler.get(), "benchmark.update_time_range"); update_time_range_from_source( source, m_config.data_type, @@ -1047,11 +1057,33 @@ bool Benchmark_rhi_offscreen_runner::render_frame( m_snapshot->data_cfg.t_max, m_snapshot->data_cfg.t_available_min); m_snapshot->data_cfg.t_available_max = m_snapshot->data_cfg.t_max; + if (m_config.view_motion != "fixed") { + // Warmup settles at the initial view. Measured inputs depend only + // on the frame index, never machine speed or the warmup count. + const std::size_t step = measured ? (m_measured_frame_count + 1) % 120 : 0; + const long double phase = static_cast( + step <= 60 ? step : 120 - step) / 60.0L; + const auto first = m_snapshot->data_cfg.t_available_min; + const auto last = m_snapshot->data_cfg.t_available_max; + const long double span = static_cast(last) - first; + const long double fraction = m_config.view_motion == "pan" + ? 0.25L : 0.10L + 0.90L * phase; + const long double offset = m_config.view_motion == "pan" + ? (1.0L - fraction) * phase : (1.0L - fraction) * 0.5L; + m_snapshot->data_cfg.t_min = first + static_cast(span * offset); + m_snapshot->data_cfg.t_max = std::max( + m_snapshot->data_cfg.t_min + 1, + first + static_cast(span * (offset + fraction))); + } } - m_frame_renderer->render(*m_snapshot, m_rhi.get(), m_render_target.get(), cb); + const auto result = m_frame_renderer->render( + *m_snapshot, m_rhi.get(), m_render_target.get(), cb); + // Plot_widget consumes this feedback too. Keeping the initial width here + // makes every otherwise static frame miss the settled layout cache. + m_snapshot->vbar_width_pixels = result.feedback.measured_vbar_width; m_last_prepare_ms = m_profiler.last_series_prepare_ms(); - if (measured) { + if (measured && m_config.internal_profiling) { m_profiler.record_observation("benchmark.planning.time_ms", m_last_prepare_ms); } @@ -1147,12 +1179,44 @@ bool Benchmark_rhi_offscreen_runner::render_frame( "benchmark.frame.cpu_allocation_bytes", static_cast(allocations.bytes)); m_profiler.record_observation("benchmark.frame.total_ms", frame_ms); + m_profiler.record_observation( + "benchmark.view.t_min_ns", static_cast(m_snapshot->data_cfg.t_min)); + m_profiler.record_observation( + "benchmark.view.t_max_ns", static_cast(m_snapshot->data_cfg.t_max)); + m_profiler.record_observation( + "benchmark.view.vbar_width_px", m_snapshot->vbar_width_pixels); m_profiler.record_counter("benchmark.frame.output_count"); ++m_measured_frame_count; } return true; } +void Benchmark_rhi_offscreen_runner::append_frame_samples() +{ + if (m_config.append_per_frame == 0) { + return; + } + const auto started = std::chrono::steady_clock::now(); + for (std::size_t i = 0; i < m_config.append_per_frame; ++i) { + if (m_config.data_type == "Trades") { + const auto sample = m_generator.next_trade(); + for (auto& buffer : m_trade_buffers) { + buffer->push(sample); + } + } + else { + const auto sample = m_generator.next_bar(); + for (auto& buffer : m_bar_buffers) { + buffer->push(sample); + } + } + } + m_samples_generated.fetch_add(m_config.append_per_frame); + m_profiler.record_observation("benchmark.workload.publish_ms", + std::chrono::duration( + std::chrono::steady_clock::now() - started).count()); +} + bool Benchmark_rhi_offscreen_runner::run(std::string& error_message) { m_phase_trace_started = std::chrono::steady_clock::now(); @@ -1253,6 +1317,9 @@ bool Benchmark_rhi_offscreen_runner::run(std::string& error_message) const auto duration = std::chrono::duration(m_config.duration_seconds); const auto render_measured_frame = [&](std::size_t frame) { + // Synthetic generation/publication is deliberately outside the render + // timer and allocation scope, and has its own reported cost. + append_frame_samples(); if (render_frame(error_message, true)) { return true; } @@ -1317,7 +1384,7 @@ void Benchmark_rhi_offscreen_runner::record_final_statistics(double measured_sec m_profiler.record_observation( "benchmark.memory.process_high_water_bytes", static_cast(process_memory_high_water_bytes())); - ensure_interval_observations(m_profiler); + ensure_interval_observations(m_profiler, m_config.internal_profiling); } } // namespace vnm::benchmark diff --git a/benchmark/src/main.cpp b/benchmark/src/main.cpp index 762f3b3c..0be57c25 100644 --- a/benchmark/src/main.cpp +++ b/benchmark/src/main.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -94,6 +95,12 @@ void print_usage(const char* program_name) << " --render-style