From 9121c52d57124023ddf0402ef20fdc4b13c05128 Mon Sep 17 00:00:00 2001 From: Tomasz Rudzki Date: Tue, 13 Oct 2026 13:16:31 +0200 Subject: [PATCH 1/3] Process OBR audio without allocating ObrImpl::Process() allocated and freed heap memory on every call (#28): a temporary AudioBuffer for each processing group's output, the two vectors PeakLimiter::Process() filled for the block, and, with loudspeaker layout or object elements, Eigen temporaries in AmbisonicEncoder::ProcessPlanarAudioData() for every output frame (the product assigned to a frame built a temporary vector, and with a moving source each frame also built a new encoding matrix and an interpolated column per source), the spherical harmonic and Legendre vectors of each source, and the source index lists of each processing group. Process() runs on the render thread of a live host. - AmbisonicEncoder keeps its working storage as members sized in the constructor (the start and target column of each source, the per-frame encoding matrix and interpolated column, the spherical harmonic coefficients and the Legendre values) and writes the per-frame products with noalias(). Its documentation states that the output buffer must not be the input buffer, which every caller already ensures. The block's static encoding matrix is built in encoding_matrix_, which held a copy of it before. - AssociatedLegendrePolynomialsGenerator::Generate(x, &values_out) writes into a caller's vector; the returning overload calls it. - ProcessingGroup fills its world-locked and head-locked source index lists into members whose capacity UpdateAmbisonicEncoder() reserves, and UpdateAmbisonicEncoder() counts the sources instead of building their index list. - ObrImpl keeps the per-group output buffer as a member, allocated in the constructor. - PeakLimiter::Process() makes one pass per frame (the maximum sample across channels, the envelope, the gain applied to every channel) instead of filling two vectors. The arithmetic and its order are unchanged, and each frame reads its input before writing its output, so ObrImpl's in-place call gives the same result. Output is bit-identical to main for the scenarios in the pull request. --- .../ambisonic_encoder/ambisonic_encoder.cc | 137 ++++++++++-------- .../obr/ambisonic_encoder/ambisonic_encoder.h | 13 ++ ...sociated_legendre_polynomials_generator.cc | 11 +- ...ssociated_legendre_polynomials_generator.h | 9 ++ .../obr/obr/peak_limiter/peak_limiter.cc | 30 ++-- .../obr/obr_capi/obr/obr/renderer/obr_impl.cc | 14 +- .../obr/obr_capi/obr/obr/renderer/obr_impl.h | 4 + .../obr/obr/renderer/processing_group.cc | 62 ++++---- .../obr/obr/renderer/processing_group.h | 20 ++- 9 files changed, 169 insertions(+), 131 deletions(-) diff --git a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.cc b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.cc index 853e50a..a852411 100644 --- a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.cc +++ b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.cc @@ -38,6 +38,25 @@ AmbisonicEncoder::AmbisonicEncoder(size_t number_of_input_channels, encoding_matrix_ = Eigen::MatrixXf::Zero(static_cast(number_of_output_channels_), static_cast(number_of_input_channels_)); + + // Size the working storage of `ProcessPlanarAudioData()`. + associated_legendre_polynomials_.reserve(alp_generator_.GetNumValues()); + sh_coeffs_.assign(number_of_output_channels_, 0.0f); + column_start_.assign( + number_of_input_channels_, + Eigen::VectorXf::Zero(static_cast(number_of_output_channels_))); + column_end_.assign( + number_of_input_channels_, + Eigen::VectorXf::Zero(static_cast(number_of_output_channels_))); + needs_interpolation_.assign(number_of_input_channels_, false); + encoding_frame_ = + Eigen::MatrixXf::Zero(static_cast(number_of_output_channels_), + static_cast(number_of_input_channels_)); + interpolated_column_ = + Eigen::VectorXf::Zero(static_cast(number_of_output_channels_)); + + // `SetSource()` adds at most one entry per input channel. + sources_.reserve(number_of_input_channels_); } void AmbisonicEncoder::SetSource(size_t input_channel, float gain, @@ -131,17 +150,10 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, // Compute a per-frame encoding matrix (output x input) by linearly // interpolating each source's parameters across the block. This avoids - // clicks when object positions change quickly. - Eigen::MatrixXf last_encoding = - Eigen::MatrixXf::Zero(static_cast(number_of_output_channels_), - static_cast(number_of_input_channels_)); - - // Precompute start and target encoding columns for each input channel. - std::vector column_start(number_of_input_channels_); - std::vector column_end(number_of_input_channels_); - - // Track which input channels actually need per-frame interpolation. - std::vector needs_interpolation(number_of_input_channels_, false); + // clicks when object positions change quickly. The start and target + // encoding columns of each input channel go into `column_start_` and + // `column_end_`, and `needs_interpolation_` tracks which input channels + // actually need per-frame interpolation. // Helper lambda to compare two sets of source parameters for equality. auto params_equal = [&](const SingleSourceParams& a, @@ -169,10 +181,9 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, // For each input channel, compute the encoding matrix column at the start. for (size_t in_ch = 0; in_ch < number_of_input_channels_; ++in_ch) { - column_start[in_ch] = - Eigen::VectorXf::Zero(static_cast(number_of_output_channels_)); - column_end[in_ch] = - Eigen::VectorXf::Zero(static_cast(number_of_output_channels_)); + column_start_[in_ch].setZero(); + column_end_[in_ch].setZero(); + needs_interpolation_[in_ch] = false; auto it = sources_.find(in_ch); if (it == sources_.end()) { @@ -189,31 +200,31 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, } // Compute spherical harmonic coefficients for this direction. - std::vector sh_coeffs(number_of_output_channels_); - GetShCoeffs(p.azimuth, p.elevation, ambisonic_order_, sh_coeffs); + std::fill(sh_coeffs_.begin(), sh_coeffs_.end(), 0.0f); + GetShCoeffs(p.azimuth, p.elevation, ambisonic_order_, sh_coeffs_); // Apply SH damping per-degree if needed. Use normalized_head_radius // = 0.1f (i.e. damping applies for distance < 0.1). - ApplyWithinHeadSHCoefficientsDamping(p.distance, 0.1f, sh_coeffs); + ApplyWithinHeadSHCoefficientsDamping(p.distance, 0.1f, sh_coeffs_); // Scale by gain and write into the Eigen vector. for (size_t out_ch = 0; out_ch < number_of_output_channels_; ++out_ch) { - col(static_cast(out_ch)) = sh_coeffs.at(out_ch) * p.gain; + col(static_cast(out_ch)) = sh_coeffs_.at(out_ch) * p.gain; } }; // Always compute the target column. - fill_column(source_properties.target, column_end[in_ch]); + fill_column(source_properties.target, column_end_[in_ch]); // Decide whether this source needs interpolation across frames. if (!params_equal(source_properties.current, source_properties.target)) { // Only compute the start/current column if parameters differ. - fill_column(source_properties.current, column_start[in_ch]); - needs_interpolation[in_ch] = true; + fill_column(source_properties.current, column_start_[in_ch]); + needs_interpolation_[in_ch] = true; } else { // Parameters equal: reuse the target column as the start. - column_start[in_ch] = column_end[in_ch]; - needs_interpolation[in_ch] = false; + column_start_[in_ch] = column_end_[in_ch]; + needs_interpolation_[in_ch] = false; } } @@ -221,7 +232,7 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, // static encoding matrix and reuse it for all frames. bool any_needs_interpolation = false; for (size_t i = 0; i < number_of_input_channels_; ++i) { - if (needs_interpolation[i]) { + if (needs_interpolation_[i]) { any_needs_interpolation = true; break; } @@ -230,28 +241,26 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, if (!any_needs_interpolation) { // Interpolation disabled: build one static encoding matrix for the // whole block from the (already computed) target columns and reuse it - // for every frame. This avoids per-frame interpolation and copies. - Eigen::MatrixXf static_encoding( - static_cast(number_of_output_channels_), - static_cast(number_of_input_channels_)); - static_encoding.setZero(); + // for every frame. This avoids per-frame interpolation and copies. It is + // built in `encoding_matrix_`, which keeps it after the block. + encoding_matrix_.setZero(); for (size_t in_ch = 0; in_ch < number_of_input_channels_; ++in_ch) { - if (column_end[in_ch].isZero(0)) { + if (column_end_[in_ch].isZero(0)) { continue; } for (int out_ch = 0; out_ch < static_cast(number_of_output_channels_); ++out_ch) { - static_encoding(out_ch, static_cast(in_ch)) = - column_end[in_ch](out_ch); + encoding_matrix_(out_ch, static_cast(in_ch)) = + column_end_[in_ch](out_ch); } } + // The output buffer does not alias the matrix or the input, so the + // product is written in place rather than through a temporary. for (size_t frame = 0; frame < num_frames; ++frame) { - encoded_buffer.col(static_cast(frame)) = - static_encoding * unencoded_buffer.col(static_cast(frame)); + encoded_buffer.col(static_cast(frame)).noalias() = + encoding_matrix_ * unencoded_buffer.col(static_cast(frame)); } - - last_encoding = static_encoding; } else { // Some channels need interpolation: perform per-frame interpolation so // parameters ramp smoothly across the block. @@ -262,45 +271,50 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, alpha = static_cast(frame) / static_cast(num_frames - 1); } - Eigen::MatrixXf encoding_frame( - static_cast(number_of_output_channels_), - static_cast(number_of_input_channels_)); - encoding_frame.setZero(); + encoding_frame_.setZero(); - // Build encoding_frame column-by-column for each input/source by + // Build encoding_frame_ column-by-column for each input/source by // interpolating the precomputed start/end columns. for (size_t in_ch = 0; in_ch < number_of_input_channels_; ++in_ch) { // If both start and end are zeros, skip. - if (column_start[in_ch].isZero(0) && column_end[in_ch].isZero(0)) { + if (column_start_[in_ch].isZero(0) && column_end_[in_ch].isZero(0)) { continue; } // Interpolate column values directly in coefficient space only if // this channel needs interpolation. Otherwise, use the target column // directly to avoid per-frame arithmetic. - Eigen::VectorXf col; - if (needs_interpolation[in_ch]) { - col = column_start[in_ch] + - alpha * (column_end[in_ch] - column_start[in_ch]); + if (needs_interpolation_[in_ch]) { + interpolated_column_ = + column_start_[in_ch] + + alpha * (column_end_[in_ch] - column_start_[in_ch]); } else { - col = column_end[in_ch]; + interpolated_column_ = column_end_[in_ch]; } // Copy into the encoding frame column. for (int out_ch = 0; out_ch < static_cast(number_of_output_channels_); ++out_ch) { - encoding_frame(out_ch, static_cast(in_ch)) = col(out_ch); + encoding_frame_(out_ch, static_cast(in_ch)) = + interpolated_column_(out_ch); } } // Multiply the per-frame encoding matrix with the single-frame input // column to produce the encoded output for this frame index. // Note: use .col() with the actual frame index (frame) mapping into the - // row-major mapped buffers. - encoded_buffer.col(static_cast(frame)) = - encoding_frame * unencoded_buffer.col(static_cast(frame)); + // row-major mapped buffers. The output buffer does not alias the + // matrix or the input, so the product is written in place. + encoded_buffer.col(static_cast(frame)).noalias() = + encoding_frame_ * unencoded_buffer.col(static_cast(frame)); + } - last_encoding = encoding_frame; // keep for later update + // Keep the last frame's matrix (zero for an empty block) so code that + // expects a static encoding matrix still has a reasonable value. + if (num_frames > 0) { + encoding_matrix_ = encoding_frame_; + } else { + encoding_matrix_.setZero(); } } @@ -310,10 +324,6 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, kv.second.current = kv.second.target; } - // Update the stored encoding matrix to the last frame's matrix so code that - // expects a static encoding matrix still has a reasonable value. - encoding_matrix_ = last_encoding; - // Audio has now been rendered; subsequent `SetSource()` updates ramp from // the previously audible parameters. has_processed_ = true; @@ -325,8 +335,8 @@ void AmbisonicEncoder::GetShCoeffs(float azimuth, float elevation, const float azimuth_rad = azimuth * kRadiansFromDegrees; const float elevation_rad = elevation * kRadiansFromDegrees; - std::vector associated_legendre_polynomials_temp_ = - alp_generator_.Generate(std::sin(elevation_rad)); + alp_generator_.Generate(std::sin(elevation_rad), + &associated_legendre_polynomials_); // Compute the actual spherical harmonics using the generated polynomials. for (int degree = 0; degree <= ambisonic_order; degree++) { for (int order = -degree; order <= degree; order++) { @@ -339,11 +349,10 @@ void AmbisonicEncoder::GetShCoeffs(float azimuth, float elevation, const float angle = static_cast(order) * azimuth_rad; const float last_term = (order >= 0) ? std::cos(angle) : std::sin(-angle); - coeffs.at(row) = - Sn3dNormalization(degree, order) * - associated_legendre_polynomials_temp_[alp_generator_.GetIndex( - degree, std::abs(order))] * - last_term; + coeffs.at(row) = Sn3dNormalization(degree, order) * + associated_legendre_polynomials_[alp_generator_.GetIndex( + degree, std::abs(order))] * + last_term; } } } diff --git a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.h b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.h index 077b1ab..4295518 100644 --- a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.h +++ b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.h @@ -63,6 +63,9 @@ class AmbisonicEncoder { void RemoveSource(size_t input_channel); /*!\brief Processing callback for planar audio data (AudioBuffer). + * + * The output is written in place frame by frame, so `output_buffer` must + * not be `input_buffer`. * * \param input_buffer Input buffer of samples. * \param output_buffer Output buffer of processed samples. @@ -137,6 +140,16 @@ class AmbisonicEncoder { AssociatedLegendrePolynomialsGenerator alp_generator_; Eigen::MatrixXf encoding_matrix_; + + // Working storage of `ProcessPlanarAudioData()` and `GetShCoeffs()`, sized + // in the constructor so that processing does not allocate. + std::vector associated_legendre_polynomials_; + std::vector sh_coeffs_; + std::vector column_start_; + std::vector column_end_; + std::vector needs_interpolation_; + Eigen::MatrixXf encoding_frame_; + Eigen::VectorXf interpolated_column_; }; } // namespace obr diff --git a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.cc b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.cc index a4445ae..73d65e9 100644 --- a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.cc +++ b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.cc @@ -31,6 +31,16 @@ AssociatedLegendrePolynomialsGenerator::AssociatedLegendrePolynomialsGenerator( std::vector AssociatedLegendrePolynomialsGenerator::Generate( float x) const { std::vector values(GetNumValues()); + Generate(x, &values); + return values; +} + +void AssociatedLegendrePolynomialsGenerator::Generate( + float x, std::vector* values_out) const { + ABSL_DCHECK_NE(values_out, nullptr); + // Assigning a size within the capacity does not allocate. + values_out->assign(GetNumValues(), 0.0f); + std::vector& values = *values_out; // Bases for the recurrence relations. values[GetIndex(0, 0)] = ComputeValue(0, 0, x, values); @@ -78,7 +88,6 @@ std::vector AssociatedLegendrePolynomialsGenerator::Generate( } } } - return values; } size_t AssociatedLegendrePolynomialsGenerator::GetNumValues() const { diff --git a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.h b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.h index 0248200..77376b8 100644 --- a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.h +++ b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.h @@ -41,6 +41,15 @@ class AssociatedLegendrePolynomialsGenerator { */ std::vector Generate(float x) const; + /*!\brief Generates the associated Legendre polynomials for the given `x` + * into `values_out`, which is resized to `GetNumValues()` and does + * not allocate when its capacity is at least that. + * + * \param x Abscissa (the polynomials' variable). + * \param values_out Output vector of computed sequence values. + */ + void Generate(float x, std::vector* values_out) const; + /*!\brief Gets the produced number of associated Legendre polynomials. * * \return Number of associated Legendre polynomials this generator diff --git a/src/renderer/obr/obr_capi/obr/obr/peak_limiter/peak_limiter.cc b/src/renderer/obr/obr_capi/obr/obr/peak_limiter/peak_limiter.cc index a37fb86..282677f 100644 --- a/src/renderer/obr/obr_capi/obr/obr/peak_limiter/peak_limiter.cc +++ b/src/renderer/obr/obr_capi/obr/obr/peak_limiter/peak_limiter.cc @@ -15,7 +15,6 @@ #include #include #include -#include #include "absl/log/absl_check.h" #include "obr/audio_buffer/audio_buffer.h" @@ -36,30 +35,27 @@ void PeakLimiter::Process(const AudioBuffer& input, AudioBuffer* output) { const size_t num_channels = input.num_channels(); const size_t num_frames = input.num_frames(); - // Get a vector of maximum sample values across all channels. - std::vector max_samples(num_frames, 0.0); - for (const auto& channel : input) { - for (size_t frame = 0; frame < num_frames; ++frame) { - max_samples[frame] = std::max(max_samples[frame], abs(channel[frame])); + // One pass per frame, so the limiter needs no per-call storage: the + // maximum sample value across all channels, the limiter envelope at that + // frame, then the envelope applied to every channel. Each frame reads its + // input before writing its output, so `input` and `output` may be the + // same buffer. + for (size_t frame = 0; frame < num_frames; ++frame) { + float max_sample = 0.0f; + for (size_t channel = 0; channel < num_channels; ++channel) { + max_sample = std::max(max_sample, abs(input[channel][frame])); } - } - // Get the limiter envelope. - std::vector limiter_env(num_frames, 0.0); - for (size_t frame = 0; frame < num_frames; ++frame) { - double max_req_gain = GetMaximumRequiredGain(max_samples[frame]); + const double max_req_gain = GetMaximumRequiredGain(max_sample); if (max_req_gain < env_) { env_ = max_req_gain; } else { env_ = release_time_constant_ * (env_ - max_req_gain) + max_req_gain; } - limiter_env[frame] = static_cast(env_); - } + const float gain = static_cast(env_); - // Apply the limiter envelope to the input buffer. - for (size_t channel = 0; channel < num_channels; ++channel) { - for (size_t frame = 0; frame < num_frames; ++frame) { - (*output)[channel][frame] = input[channel][frame] * limiter_env[frame]; + for (size_t channel = 0; channel < num_channels; ++channel) { + (*output)[channel][frame] = input[channel][frame] * gain; } } } diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc index a745e4d..abf9b75 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc @@ -41,7 +41,8 @@ ObrImpl::ObrImpl(int buffer_size_per_channel, int sampling_rate) head_tracking_enabled_(false), limiter_enabled_(true), world_rotation_(WorldRotation()), - fft_manager_(buffer_size_per_channel_) { + fft_manager_(buffer_size_per_channel_), + group_output_(kNumBinauralChannels, buffer_size_per_channel_) { ABSL_CHECK_GT(buffer_size_per_channel, 0) << "Buffer size per channel must be greater than 0."; ABSL_CHECK_GT(sampling_rate_, 0) << "Sampling rate must be greater than 0."; @@ -203,18 +204,15 @@ void ObrImpl::Process(const AudioBuffer& input_buffer, return; } - // Create a temporary buffer for each group's output. - AudioBuffer group_output(kNumBinauralChannels, buffer_size_per_channel_); - // Process each processing group. for (auto& group : processing_groups_) { - group_output.Clear(); + group_output_.Clear(); group.Process(input_buffer, audio_elements_, head_tracking_enabled_, - world_rotation_, &group_output); + world_rotation_, &group_output_); // Add this group's binaural output to the main output buffer. - (*output_buffer)[0] += group_output[0]; - (*output_buffer)[1] += group_output[1]; + (*output_buffer)[0] += group_output_[0]; + (*output_buffer)[1] += group_output_[1]; } // Peak limit the output if enabled. diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h index dd00ded..02ce94c 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h @@ -255,6 +255,10 @@ class ObrImpl { // TODO(b/468218887): Provide an option to disable peak limiting. // Peak limiter (applied to final output). std::unique_ptr peak_limiter_; + + // Each processing group's binaural output in `Process()`, allocated in the + // constructor so that processing does not allocate. + AudioBuffer group_output_; }; } // namespace obr diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.cc b/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.cc index da3ea3a..abf140c 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.cc +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.cc @@ -123,8 +123,9 @@ void ProcessingGroup::Process( ambisonic_mix_bed_head_locked_.Clear(); // PASS 1: Process world-locked elements (head_locked = false) - const std::vector world_locked_indices = - GetAmbisonicEncoderSourceChannelIndices(audio_elements, false); + GetAmbisonicEncoderSourceChannelIndices(audio_elements, false, + &world_locked_indices_); + const std::vector& world_locked_indices = world_locked_indices_; if (!world_locked_indices.empty() && ambisonic_encoder_ != nullptr) { // Copy selected channels from input buffer to encoder input buffer. @@ -151,8 +152,9 @@ void ProcessingGroup::Process( } // PASS 2: Process head-locked elements (head_locked = true) - const std::vector head_locked_indices = - GetAmbisonicEncoderSourceChannelIndices(audio_elements, true); + GetAmbisonicEncoderSourceChannelIndices(audio_elements, true, + &head_locked_indices_); + const std::vector& head_locked_indices = head_locked_indices_; if (!head_locked_indices.empty() && ambisonic_encoder_ != nullptr) { // Copy selected channels from input buffer to encoder input buffer. @@ -199,10 +201,15 @@ void ProcessingGroup::Process( absl::Status ProcessingGroup::UpdateAmbisonicEncoder( std::vector& audio_elements) { - // Get encoder source indices to determine if we need an encoder. - const auto indices = GetAmbisonicEncoderSourceChannelIndices(audio_elements); + // Count the encoder sources to determine if we need an encoder. + const size_t num_sources = + GetNumberOfAmbisonicEncoderSourceChannels(audio_elements); - if (indices.empty()) { + // `Process()` fills the index lists within this capacity. + world_locked_indices_.reserve(num_sources); + head_locked_indices_.reserve(num_sources); + + if (num_sources == 0) { // No sources to encode, so no encoder needed. ambisonic_encoder_.reset(); ambisonic_encoder_input_buffer_ = AudioBuffer(); @@ -211,11 +218,11 @@ absl::Status ProcessingGroup::UpdateAmbisonicEncoder( // Initialize or reinitialize encoder if source count changed. if (ambisonic_encoder_ == nullptr || - ambisonic_encoder_input_buffer_.num_channels() != indices.size()) { + ambisonic_encoder_input_buffer_.num_channels() != num_sources) { ambisonic_encoder_input_buffer_ = - AudioBuffer(indices.size(), buffer_size_per_channel_); - ambisonic_encoder_ = std::make_unique( - indices.size(), key_.ambisonic_order); + AudioBuffer(num_sources, buffer_size_per_channel_); + ambisonic_encoder_ = + std::make_unique(num_sources, key_.ambisonic_order); } // Update Ambisonic encoder settings. @@ -243,38 +250,25 @@ absl::Status ProcessingGroup::UpdateAmbisonicEncoder( return absl::OkStatus(); } -std::vector ProcessingGroup::GetAmbisonicEncoderSourceChannelIndices( +size_t ProcessingGroup::GetNumberOfAmbisonicEncoderSourceChannels( const std::vector& audio_elements) const { - std::vector source_channel_indices; - // Reserve capacity based on estimated number of channels to avoid - // reallocations. - size_t estimated_channels = 0; + size_t num_channels = 0; for (size_t ae_index : audio_element_indices_) { const auto& audio_element = audio_elements[ae_index]; if (IsLoudspeakerLayoutType(audio_element.GetType()) || IsObjectType(audio_element.GetType())) { - estimated_channels += audio_element.GetNumberOfInputChannels(); + num_channels += audio_element.GetNumberOfInputChannels(); } } - source_channel_indices.reserve(estimated_channels); - - for (size_t ae_index : audio_element_indices_) { - const auto& audio_element = audio_elements[ae_index]; - if (IsLoudspeakerLayoutType(audio_element.GetType()) || - IsObjectType(audio_element.GetType())) { - for (size_t i = 0; i < audio_element.GetNumberOfInputChannels(); ++i) { - source_channel_indices.push_back(audio_element.GetFirstChannelIndex() + - i); - } - } - } - return source_channel_indices; + return num_channels; } -std::vector ProcessingGroup::GetAmbisonicEncoderSourceChannelIndices( +void ProcessingGroup::GetAmbisonicEncoderSourceChannelIndices( const std::vector& audio_elements, - bool filter_head_locked) const { - std::vector source_channel_indices; + bool filter_head_locked, std::vector* indices) const { + ABSL_CHECK_NE(indices, nullptr); + std::vector& source_channel_indices = *indices; + source_channel_indices.clear(); for (size_t ae_index : audio_element_indices_) { const auto& audio_element = audio_elements[ae_index]; @@ -292,8 +286,6 @@ std::vector ProcessingGroup::GetAmbisonicEncoderSourceChannelIndices( } } } - - return source_channel_indices; } } // namespace obr diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.h b/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.h index cb57912..e6630de 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.h +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.h @@ -146,12 +146,12 @@ class ProcessingGroup { BinauralFilterProfile GetFilterProfile() const { return key_.filter_profile; } private: - /*!\brief Get source channel indices for the Ambisonic encoder. + /*!\brief Get the number of source channels for the Ambisonic encoder. * * \param audio_elements Reference to all audio elements. - * \return Vector of input channel indices for all sources. + * \return Number of input channels of all sources. */ - std::vector GetAmbisonicEncoderSourceChannelIndices( + size_t GetNumberOfAmbisonicEncoderSourceChannels( const std::vector& audio_elements) const; /*!\brief Get source channel indices for the Ambisonic encoder, filtered by @@ -161,11 +161,13 @@ class ProcessingGroup { * \param filter_head_locked If true, return indices of head-locked elements; * if false, return indices of world-locked * elements. - * \return Vector of input channel indices matching the filter. + * \param indices Output vector of input channel indices matching the + * filter. It does not allocate when its capacity holds all source + * channels, which `UpdateAmbisonicEncoder()` reserves. */ - std::vector GetAmbisonicEncoderSourceChannelIndices( + void GetAmbisonicEncoderSourceChannelIndices( const std::vector& audio_elements, - bool filter_head_locked) const; + bool filter_head_locked, std::vector* indices) const; ProcessingGroupKey key_; @@ -186,6 +188,12 @@ class ProcessingGroup { std::unique_ptr ambisonic_encoder_; AudioBuffer ambisonic_encoder_input_buffer_; + // Encoder source channel indices of the world-locked and head-locked + // elements, filled by `Process()` within the capacity reserved by + // `UpdateAmbisonicEncoder()`. + std::vector world_locked_indices_; + std::vector head_locked_indices_; + // Ambisonic rotator for this group. std::unique_ptr ambisonic_rotator_; From 3f13f1f06865b118b44c0680908d5fabb28c097a Mon Sep 17 00:00:00 2001 From: Tomasz Rudzki Date: Tue, 13 Oct 2026 13:20:10 +0200 Subject: [PATCH 2/3] Render obr_process() through buffers ObrImpl keeps obr_process() copied the planar input into a vector of vectors, then into a new input AudioBuffer, built a new output AudioBuffer, called Process() and copied the output back, so every call through the C API allocated and freed several buffers (#28). It also took ObrImpl's mutex twice per call: once to read the input channel count, and again in Process(). ObrImpl::ProcessPlanar(input, output, num_frames) copies flat planar arrays into an input buffer ObrImpl keeps, renders it as Process() does and copies the two binaural channels out, under mutex_ taken once. AddAudioElement() and RemoveLastAudioElement() size the input buffer in the same critical section that changes the element list, and the output buffer is sized in the constructor, so ProcessPlanar() checks that the channel counts match instead of resizing. Process() checks its arguments, takes the lock and runs the shared body, ProcessLocked(), so its callers are unchanged. obr_process() calls ProcessPlanar() and keeps no buffers of its own. ObrImplTest.TestProcessPlanarMatchesProcess renders the same input through Process() on one renderer and ProcessPlanar() on another with the same elements, and requires identical output with one element, after a second is added and after it is removed. --- .../obr/obr_capi/obr/obr/renderer/obr_impl.cc | 42 +++++++++++++++- .../obr/obr_capi/obr/obr/renderer/obr_impl.h | 36 ++++++++++++- .../obr/obr/renderer/tests/obr_impl_test.cc | 50 +++++++++++++++++++ src/renderer/obr/obr_capi/obr_capi.cpp | 28 ++--------- 4 files changed, 129 insertions(+), 27 deletions(-) diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc index abf9b75..f61456b 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc @@ -12,6 +12,7 @@ #include "obr/renderer/obr_impl.h" +#include #include #include #include @@ -42,7 +43,9 @@ ObrImpl::ObrImpl(int buffer_size_per_channel, int sampling_rate) limiter_enabled_(true), world_rotation_(WorldRotation()), fft_manager_(buffer_size_per_channel_), - group_output_(kNumBinauralChannels, buffer_size_per_channel_) { + group_output_(kNumBinauralChannels, buffer_size_per_channel_), + planar_input_(0, buffer_size_per_channel_), + planar_output_(kNumBinauralChannels, buffer_size_per_channel_) { ABSL_CHECK_GT(buffer_size_per_channel, 0) << "Buffer size per channel must be greater than 0."; ABSL_CHECK_GT(sampling_rate_, 0) << "Sampling rate must be greater than 0."; @@ -147,7 +150,35 @@ void ObrImpl::Process(const AudioBuffer& input_buffer, // Acquire the lock for the entire processing duration. absl::MutexLock lock(&mutex_); + ProcessLocked(input_buffer, output_buffer); +} + +void ObrImpl::ProcessPlanar(const float* input, float* output, + size_t num_frames) { + ABSL_CHECK_NE(input, nullptr); + ABSL_CHECK_NE(output, nullptr); + ABSL_CHECK_EQ(num_frames, buffer_size_per_channel_); + + absl::MutexLock lock(&mutex_); + + // `AddAudioElement()` and `RemoveLastAudioElement()` size the input buffer + // for the input channels in the same critical section that changes them. + ABSL_CHECK_EQ(planar_input_.num_channels(), GetNumberOfInputChannelsLocked()); + + for (size_t ch = 0; ch < planar_input_.num_channels(); ++ch) { + std::copy_n(input + ch * num_frames, num_frames, planar_input_[ch].begin()); + } + ProcessLocked(planar_input_, &planar_output_); + + for (size_t ch = 0; ch < planar_output_.num_channels(); ++ch) { + std::copy_n(planar_output_[ch].begin(), num_frames, + output + ch * num_frames); + } +} + +void ObrImpl::ProcessLocked(const AudioBuffer& input_buffer, + AudioBuffer* output_buffer) { // Clear the output buffer. output_buffer->Clear(); @@ -244,6 +275,13 @@ size_t ObrImpl::GetNumberOfInputChannelsLocked() const { return number_of_input_channels; } +void ObrImpl::ResizePlanarInputLocked() { + const size_t num_channels = GetNumberOfInputChannelsLocked(); + if (planar_input_.num_channels() != num_channels) { + planar_input_ = AudioBuffer(num_channels, buffer_size_per_channel_); + } +} + size_t ObrImpl::GetNumberOfInputChannels() { absl::MutexLock lock(&mutex_); return GetNumberOfInputChannelsLocked(); @@ -289,6 +327,7 @@ absl::Status ObrImpl::AddAudioElement(const AudioElementType type, // All checks passed, add the Audio Element config. audio_elements_.push_back(audio_element_config); + ResizePlanarInputLocked(); ABSL_LOG(INFO) << "Added audio element: " << GetAudioElementTypeStr(audio_elements_.back().GetType()) << "."; @@ -316,6 +355,7 @@ absl::Status ObrImpl::RemoveLastAudioElement() { << GetAudioElementTypeStr(audio_elements_.back().GetType()) << "."; audio_elements_.pop_back(); + ResizePlanarInputLocked(); if (audio_elements_.empty()) { ABSL_LOG(INFO) << "No audio elements left."; diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h index 02ce94c..b71ddb8 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h @@ -52,6 +52,22 @@ class ObrImpl { */ void Process(const AudioBuffer& input_buffer, AudioBuffer* output_buffer); + /*!\brief Processes audio held in flat planar arrays. + * + * Copies `input` into an input buffer the renderer keeps, processes it as + * `Process()` does, and copies the binaural result into `output`. The + * renderer sizes its buffers when an audio element is added or removed, so + * this call does not allocate. It takes the same lock as `Process()`, once. + * + * \param input `GetNumberOfInputChannels()` channels of `num_frames` + * samples, one channel after the other. + * \param output Receives the two binaural channels of `num_frames` samples, + * one channel after the other. + * \param num_frames Samples per channel; must equal + * `GetBufferSizePerChannel()`. + */ + void ProcessPlanar(const float* input, float* output, size_t num_frames); + /*!\brief Returns the buffer size per channel. * * \return Buffer size per channel. @@ -231,6 +247,16 @@ class ObrImpl { */ size_t GetNumberOfInputChannelsLocked() const; + /*!\brief The body of `Process()`, run while holding mutex_. + */ + void ProcessLocked(const AudioBuffer& input_buffer, + AudioBuffer* output_buffer); + + /*!\brief Sizes `planar_input_` for the current number of input channels. + * Must be called while holding mutex_. + */ + void ResizePlanarInputLocked(); + const int buffer_size_per_channel_; const int sampling_rate_; @@ -238,7 +264,9 @@ class ObrImpl { bool limiter_enabled_; WorldRotation world_rotation_; - // Mutex to protect data accessed in different threads. + // Mutex to protect data accessed in different threads, including the + // group output and planar buffers. `Process()` and `ProcessPlanar()` hold + // it for the whole call. mutable absl::Mutex mutex_; std::vector audio_elements_; @@ -259,6 +287,12 @@ class ObrImpl { // Each processing group's binaural output in `Process()`, allocated in the // constructor so that processing does not allocate. AudioBuffer group_output_; + + // The input and output of `ProcessPlanar()`. The input is sized for the + // input channels when an audio element is added or removed, the output in + // the constructor. Accessed under `mutex_`. + AudioBuffer planar_input_; + AudioBuffer planar_output_; }; } // namespace obr diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc b/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc index 8d6878b..15b5a8d 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc @@ -1074,5 +1074,55 @@ TEST(ObrImplTest, TestPassthroughOnlyAppliesPeakLimiter) { EXPECT_LT(peak_right, 1.1f); } +// Renders the same input through `Process()` on one renderer and through +// `ProcessPlanar()` on another with the same elements, and requires identical +// output: with one element, after a second is added and after it is removed. +TEST(ObrImplTest, TestProcessPlanarMatchesProcess) { + const size_t kNumFrames = 256; + ObrImpl buffered(kNumFrames, kSamplingRate); + ObrImpl planar(kNumFrames, kSamplingRate); + + auto expect_same_output = [&](int seed) { + const size_t num_channels = buffered.GetNumberOfInputChannels(); + ASSERT_EQ(planar.GetNumberOfInputChannels(), num_channels); + AudioBuffer input_buffer(num_channels, kNumFrames); + std::vector input(num_channels * kNumFrames); + for (size_t ch = 0; ch < num_channels; ++ch) { + GenerateWhiteNoise(0.1f, &input_buffer[ch], static_cast(seed + ch)); + for (size_t frame = 0; frame < kNumFrames; ++frame) { + input[ch * kNumFrames + frame] = input_buffer[ch][frame]; + } + } + AudioBuffer output_buffer(2, kNumFrames); + std::vector output(2 * kNumFrames, 1.0f); + for (int block = 0; block < 3; ++block) { + buffered.Process(input_buffer, &output_buffer); + planar.ProcessPlanar(input.data(), output.data(), kNumFrames); + for (size_t ch = 0; ch < 2; ++ch) { + for (size_t frame = 0; frame < kNumFrames; ++frame) { + ASSERT_EQ(output[ch * kNumFrames + frame], output_buffer[ch][frame]) + << "block " << block << ", channel " << ch << ", frame " << frame; + } + } + } + }; + + for (ObrImpl* renderer : {&buffered, &planar}) { + EXPECT_THAT(renderer->AddAudioElement(AudioElementType::k3OA), IsOk()); + } + expect_same_output(10); + + for (ObrImpl* renderer : {&buffered, &planar}) { + EXPECT_THAT(renderer->AddAudioElement(AudioElementType::kLayout7_1_4_ch), + IsOk()); + } + expect_same_output(20); + + for (ObrImpl* renderer : {&buffered, &planar}) { + EXPECT_THAT(renderer->RemoveLastAudioElement(), IsOk()); + } + expect_same_output(30); +} + } // namespace } // namespace obr diff --git a/src/renderer/obr/obr_capi/obr_capi.cpp b/src/renderer/obr/obr_capi/obr_capi.cpp index 9738021..432bcb8 100644 --- a/src/renderer/obr/obr_capi/obr_capi.cpp +++ b/src/renderer/obr/obr_capi/obr_capi.cpp @@ -21,9 +21,6 @@ #include #include -#include -#include -#include #include "obr/renderer/obr_impl.h" using namespace obr; @@ -83,30 +80,11 @@ int obr_add_audio_element(obr_handle handle, ObrAudioElementType type, int obr_process(obr_handle handle, const float *input, float *output, int num_samples) { - int input_nch = 0; - int output_nch = 2; - size_t buffer_size = num_samples; - if (!handle || !input || !output) return 0; - input_nch = (int)cast_(handle)->GetNumberOfInputChannels(); - - std::vector> input_buffer_float( - input_nch, std::vector(buffer_size)); - - for (size_t ch = 0; ch < input_nch; ch++) - for (size_t smp = 0; smp < buffer_size; smp++) - input_buffer_float[ch][smp] = input[ch * buffer_size + smp]; - - AudioBuffer input_buffer(input_nch, buffer_size); - input_buffer = input_buffer_float; - AudioBuffer output_buffer(output_nch, buffer_size); - - cast_(handle)->Process(input_buffer, &output_buffer); - - for (size_t ch = 0; ch < output_buffer.num_channels(); ch++) - for (size_t smp = 0; smp < output_buffer.num_frames(); smp++) - output[ch * output_buffer.num_frames() + smp] = output_buffer[ch][smp]; + // The renderer keeps the buffers it processes in and takes its lock once; + // this function keeps no buffers and takes no lock of its own. + cast_(handle)->ProcessPlanar(input, output, static_cast(num_samples)); return num_samples; } From 34bcc46bcf44402f2ed67efa2a441abeb98375b5 Mon Sep 17 00:00:00 2001 From: Tomasz Rudzki Date: Fri, 16 Oct 2026 10:59:02 +0200 Subject: [PATCH 3/3] Compare ProcessPlanar() output per channel with EXPECT_THAT TestProcessPlanarMatchesProcess compared the output of ProcessPlanar() with the output of Process() with one ASSERT_EQ per sample. It now compares each channel of each block with one EXPECT_THAT(actual, Pointwise(Eq(), expected)), a non-fatal check like the rest of the file uses. A mismatch is reported once per channel, with the index and both values of the first differing sample, and the test goes on to the next channel and block. The ASSERT_EQ on the input channel count stays, because the input buffers are sized from it. --- .../obr_capi/obr/obr/renderer/tests/obr_impl_test.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc b/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc index 15b5a8d..84a450b 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc @@ -29,6 +29,8 @@ namespace obr { namespace { using ::absl_testing::IsOk; +using ::testing::Eq; +using ::testing::Pointwise; constexpr int kBufferSizePerChannel = 44; constexpr int kSamplingRate = 48000; @@ -1099,10 +1101,12 @@ TEST(ObrImplTest, TestProcessPlanarMatchesProcess) { buffered.Process(input_buffer, &output_buffer); planar.ProcessPlanar(input.data(), output.data(), kNumFrames); for (size_t ch = 0; ch < 2; ++ch) { - for (size_t frame = 0; frame < kNumFrames; ++frame) { - ASSERT_EQ(output[ch * kNumFrames + frame], output_buffer[ch][frame]) - << "block " << block << ", channel " << ch << ", frame " << frame; - } + const std::vector expected(output_buffer[ch].begin(), + output_buffer[ch].end()); + const std::vector actual(output.begin() + ch * kNumFrames, + output.begin() + (ch + 1) * kNumFrames); + EXPECT_THAT(actual, Pointwise(Eq(), expected)) + << "block " << block << ", channel " << ch; } } };