diff --git a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.cc b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.cc index 853e50a..a852411 100644 --- a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.cc +++ b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.cc @@ -38,6 +38,25 @@ AmbisonicEncoder::AmbisonicEncoder(size_t number_of_input_channels, encoding_matrix_ = Eigen::MatrixXf::Zero(static_cast(number_of_output_channels_), static_cast(number_of_input_channels_)); + + // Size the working storage of `ProcessPlanarAudioData()`. + associated_legendre_polynomials_.reserve(alp_generator_.GetNumValues()); + sh_coeffs_.assign(number_of_output_channels_, 0.0f); + column_start_.assign( + number_of_input_channels_, + Eigen::VectorXf::Zero(static_cast(number_of_output_channels_))); + column_end_.assign( + number_of_input_channels_, + Eigen::VectorXf::Zero(static_cast(number_of_output_channels_))); + needs_interpolation_.assign(number_of_input_channels_, false); + encoding_frame_ = + Eigen::MatrixXf::Zero(static_cast(number_of_output_channels_), + static_cast(number_of_input_channels_)); + interpolated_column_ = + Eigen::VectorXf::Zero(static_cast(number_of_output_channels_)); + + // `SetSource()` adds at most one entry per input channel. + sources_.reserve(number_of_input_channels_); } void AmbisonicEncoder::SetSource(size_t input_channel, float gain, @@ -131,17 +150,10 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, // Compute a per-frame encoding matrix (output x input) by linearly // interpolating each source's parameters across the block. This avoids - // clicks when object positions change quickly. - Eigen::MatrixXf last_encoding = - Eigen::MatrixXf::Zero(static_cast(number_of_output_channels_), - static_cast(number_of_input_channels_)); - - // Precompute start and target encoding columns for each input channel. - std::vector column_start(number_of_input_channels_); - std::vector column_end(number_of_input_channels_); - - // Track which input channels actually need per-frame interpolation. - std::vector needs_interpolation(number_of_input_channels_, false); + // clicks when object positions change quickly. The start and target + // encoding columns of each input channel go into `column_start_` and + // `column_end_`, and `needs_interpolation_` tracks which input channels + // actually need per-frame interpolation. // Helper lambda to compare two sets of source parameters for equality. auto params_equal = [&](const SingleSourceParams& a, @@ -169,10 +181,9 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, // For each input channel, compute the encoding matrix column at the start. for (size_t in_ch = 0; in_ch < number_of_input_channels_; ++in_ch) { - column_start[in_ch] = - Eigen::VectorXf::Zero(static_cast(number_of_output_channels_)); - column_end[in_ch] = - Eigen::VectorXf::Zero(static_cast(number_of_output_channels_)); + column_start_[in_ch].setZero(); + column_end_[in_ch].setZero(); + needs_interpolation_[in_ch] = false; auto it = sources_.find(in_ch); if (it == sources_.end()) { @@ -189,31 +200,31 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, } // Compute spherical harmonic coefficients for this direction. - std::vector sh_coeffs(number_of_output_channels_); - GetShCoeffs(p.azimuth, p.elevation, ambisonic_order_, sh_coeffs); + std::fill(sh_coeffs_.begin(), sh_coeffs_.end(), 0.0f); + GetShCoeffs(p.azimuth, p.elevation, ambisonic_order_, sh_coeffs_); // Apply SH damping per-degree if needed. Use normalized_head_radius // = 0.1f (i.e. damping applies for distance < 0.1). - ApplyWithinHeadSHCoefficientsDamping(p.distance, 0.1f, sh_coeffs); + ApplyWithinHeadSHCoefficientsDamping(p.distance, 0.1f, sh_coeffs_); // Scale by gain and write into the Eigen vector. for (size_t out_ch = 0; out_ch < number_of_output_channels_; ++out_ch) { - col(static_cast(out_ch)) = sh_coeffs.at(out_ch) * p.gain; + col(static_cast(out_ch)) = sh_coeffs_.at(out_ch) * p.gain; } }; // Always compute the target column. - fill_column(source_properties.target, column_end[in_ch]); + fill_column(source_properties.target, column_end_[in_ch]); // Decide whether this source needs interpolation across frames. if (!params_equal(source_properties.current, source_properties.target)) { // Only compute the start/current column if parameters differ. - fill_column(source_properties.current, column_start[in_ch]); - needs_interpolation[in_ch] = true; + fill_column(source_properties.current, column_start_[in_ch]); + needs_interpolation_[in_ch] = true; } else { // Parameters equal: reuse the target column as the start. - column_start[in_ch] = column_end[in_ch]; - needs_interpolation[in_ch] = false; + column_start_[in_ch] = column_end_[in_ch]; + needs_interpolation_[in_ch] = false; } } @@ -221,7 +232,7 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, // static encoding matrix and reuse it for all frames. bool any_needs_interpolation = false; for (size_t i = 0; i < number_of_input_channels_; ++i) { - if (needs_interpolation[i]) { + if (needs_interpolation_[i]) { any_needs_interpolation = true; break; } @@ -230,28 +241,26 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, if (!any_needs_interpolation) { // Interpolation disabled: build one static encoding matrix for the // whole block from the (already computed) target columns and reuse it - // for every frame. This avoids per-frame interpolation and copies. - Eigen::MatrixXf static_encoding( - static_cast(number_of_output_channels_), - static_cast(number_of_input_channels_)); - static_encoding.setZero(); + // for every frame. This avoids per-frame interpolation and copies. It is + // built in `encoding_matrix_`, which keeps it after the block. + encoding_matrix_.setZero(); for (size_t in_ch = 0; in_ch < number_of_input_channels_; ++in_ch) { - if (column_end[in_ch].isZero(0)) { + if (column_end_[in_ch].isZero(0)) { continue; } for (int out_ch = 0; out_ch < static_cast(number_of_output_channels_); ++out_ch) { - static_encoding(out_ch, static_cast(in_ch)) = - column_end[in_ch](out_ch); + encoding_matrix_(out_ch, static_cast(in_ch)) = + column_end_[in_ch](out_ch); } } + // The output buffer does not alias the matrix or the input, so the + // product is written in place rather than through a temporary. for (size_t frame = 0; frame < num_frames; ++frame) { - encoded_buffer.col(static_cast(frame)) = - static_encoding * unencoded_buffer.col(static_cast(frame)); + encoded_buffer.col(static_cast(frame)).noalias() = + encoding_matrix_ * unencoded_buffer.col(static_cast(frame)); } - - last_encoding = static_encoding; } else { // Some channels need interpolation: perform per-frame interpolation so // parameters ramp smoothly across the block. @@ -262,45 +271,50 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, alpha = static_cast(frame) / static_cast(num_frames - 1); } - Eigen::MatrixXf encoding_frame( - static_cast(number_of_output_channels_), - static_cast(number_of_input_channels_)); - encoding_frame.setZero(); + encoding_frame_.setZero(); - // Build encoding_frame column-by-column for each input/source by + // Build encoding_frame_ column-by-column for each input/source by // interpolating the precomputed start/end columns. for (size_t in_ch = 0; in_ch < number_of_input_channels_; ++in_ch) { // If both start and end are zeros, skip. - if (column_start[in_ch].isZero(0) && column_end[in_ch].isZero(0)) { + if (column_start_[in_ch].isZero(0) && column_end_[in_ch].isZero(0)) { continue; } // Interpolate column values directly in coefficient space only if // this channel needs interpolation. Otherwise, use the target column // directly to avoid per-frame arithmetic. - Eigen::VectorXf col; - if (needs_interpolation[in_ch]) { - col = column_start[in_ch] + - alpha * (column_end[in_ch] - column_start[in_ch]); + if (needs_interpolation_[in_ch]) { + interpolated_column_ = + column_start_[in_ch] + + alpha * (column_end_[in_ch] - column_start_[in_ch]); } else { - col = column_end[in_ch]; + interpolated_column_ = column_end_[in_ch]; } // Copy into the encoding frame column. for (int out_ch = 0; out_ch < static_cast(number_of_output_channels_); ++out_ch) { - encoding_frame(out_ch, static_cast(in_ch)) = col(out_ch); + encoding_frame_(out_ch, static_cast(in_ch)) = + interpolated_column_(out_ch); } } // Multiply the per-frame encoding matrix with the single-frame input // column to produce the encoded output for this frame index. // Note: use .col() with the actual frame index (frame) mapping into the - // row-major mapped buffers. - encoded_buffer.col(static_cast(frame)) = - encoding_frame * unencoded_buffer.col(static_cast(frame)); + // row-major mapped buffers. The output buffer does not alias the + // matrix or the input, so the product is written in place. + encoded_buffer.col(static_cast(frame)).noalias() = + encoding_frame_ * unencoded_buffer.col(static_cast(frame)); + } - last_encoding = encoding_frame; // keep for later update + // Keep the last frame's matrix (zero for an empty block) so code that + // expects a static encoding matrix still has a reasonable value. + if (num_frames > 0) { + encoding_matrix_ = encoding_frame_; + } else { + encoding_matrix_.setZero(); } } @@ -310,10 +324,6 @@ void AmbisonicEncoder::ProcessPlanarAudioData(const AudioBuffer& input_buffer, kv.second.current = kv.second.target; } - // Update the stored encoding matrix to the last frame's matrix so code that - // expects a static encoding matrix still has a reasonable value. - encoding_matrix_ = last_encoding; - // Audio has now been rendered; subsequent `SetSource()` updates ramp from // the previously audible parameters. has_processed_ = true; @@ -325,8 +335,8 @@ void AmbisonicEncoder::GetShCoeffs(float azimuth, float elevation, const float azimuth_rad = azimuth * kRadiansFromDegrees; const float elevation_rad = elevation * kRadiansFromDegrees; - std::vector associated_legendre_polynomials_temp_ = - alp_generator_.Generate(std::sin(elevation_rad)); + alp_generator_.Generate(std::sin(elevation_rad), + &associated_legendre_polynomials_); // Compute the actual spherical harmonics using the generated polynomials. for (int degree = 0; degree <= ambisonic_order; degree++) { for (int order = -degree; order <= degree; order++) { @@ -339,11 +349,10 @@ void AmbisonicEncoder::GetShCoeffs(float azimuth, float elevation, const float angle = static_cast(order) * azimuth_rad; const float last_term = (order >= 0) ? std::cos(angle) : std::sin(-angle); - coeffs.at(row) = - Sn3dNormalization(degree, order) * - associated_legendre_polynomials_temp_[alp_generator_.GetIndex( - degree, std::abs(order))] * - last_term; + coeffs.at(row) = Sn3dNormalization(degree, order) * + associated_legendre_polynomials_[alp_generator_.GetIndex( + degree, std::abs(order))] * + last_term; } } } diff --git a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.h b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.h index 077b1ab..4295518 100644 --- a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.h +++ b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/ambisonic_encoder.h @@ -63,6 +63,9 @@ class AmbisonicEncoder { void RemoveSource(size_t input_channel); /*!\brief Processing callback for planar audio data (AudioBuffer). + * + * The output is written in place frame by frame, so `output_buffer` must + * not be `input_buffer`. * * \param input_buffer Input buffer of samples. * \param output_buffer Output buffer of processed samples. @@ -137,6 +140,16 @@ class AmbisonicEncoder { AssociatedLegendrePolynomialsGenerator alp_generator_; Eigen::MatrixXf encoding_matrix_; + + // Working storage of `ProcessPlanarAudioData()` and `GetShCoeffs()`, sized + // in the constructor so that processing does not allocate. + std::vector associated_legendre_polynomials_; + std::vector sh_coeffs_; + std::vector column_start_; + std::vector column_end_; + std::vector needs_interpolation_; + Eigen::MatrixXf encoding_frame_; + Eigen::VectorXf interpolated_column_; }; } // namespace obr diff --git a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.cc b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.cc index a4445ae..73d65e9 100644 --- a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.cc +++ b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.cc @@ -31,6 +31,16 @@ AssociatedLegendrePolynomialsGenerator::AssociatedLegendrePolynomialsGenerator( std::vector AssociatedLegendrePolynomialsGenerator::Generate( float x) const { std::vector values(GetNumValues()); + Generate(x, &values); + return values; +} + +void AssociatedLegendrePolynomialsGenerator::Generate( + float x, std::vector* values_out) const { + ABSL_DCHECK_NE(values_out, nullptr); + // Assigning a size within the capacity does not allocate. + values_out->assign(GetNumValues(), 0.0f); + std::vector& values = *values_out; // Bases for the recurrence relations. values[GetIndex(0, 0)] = ComputeValue(0, 0, x, values); @@ -78,7 +88,6 @@ std::vector AssociatedLegendrePolynomialsGenerator::Generate( } } } - return values; } size_t AssociatedLegendrePolynomialsGenerator::GetNumValues() const { diff --git a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.h b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.h index 0248200..77376b8 100644 --- a/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.h +++ b/src/renderer/obr/obr_capi/obr/obr/ambisonic_encoder/associated_legendre_polynomials_generator.h @@ -41,6 +41,15 @@ class AssociatedLegendrePolynomialsGenerator { */ std::vector Generate(float x) const; + /*!\brief Generates the associated Legendre polynomials for the given `x` + * into `values_out`, which is resized to `GetNumValues()` and does + * not allocate when its capacity is at least that. + * + * \param x Abscissa (the polynomials' variable). + * \param values_out Output vector of computed sequence values. + */ + void Generate(float x, std::vector* values_out) const; + /*!\brief Gets the produced number of associated Legendre polynomials. * * \return Number of associated Legendre polynomials this generator diff --git a/src/renderer/obr/obr_capi/obr/obr/peak_limiter/peak_limiter.cc b/src/renderer/obr/obr_capi/obr/obr/peak_limiter/peak_limiter.cc index a37fb86..282677f 100644 --- a/src/renderer/obr/obr_capi/obr/obr/peak_limiter/peak_limiter.cc +++ b/src/renderer/obr/obr_capi/obr/obr/peak_limiter/peak_limiter.cc @@ -15,7 +15,6 @@ #include #include #include -#include #include "absl/log/absl_check.h" #include "obr/audio_buffer/audio_buffer.h" @@ -36,30 +35,27 @@ void PeakLimiter::Process(const AudioBuffer& input, AudioBuffer* output) { const size_t num_channels = input.num_channels(); const size_t num_frames = input.num_frames(); - // Get a vector of maximum sample values across all channels. - std::vector max_samples(num_frames, 0.0); - for (const auto& channel : input) { - for (size_t frame = 0; frame < num_frames; ++frame) { - max_samples[frame] = std::max(max_samples[frame], abs(channel[frame])); + // One pass per frame, so the limiter needs no per-call storage: the + // maximum sample value across all channels, the limiter envelope at that + // frame, then the envelope applied to every channel. Each frame reads its + // input before writing its output, so `input` and `output` may be the + // same buffer. + for (size_t frame = 0; frame < num_frames; ++frame) { + float max_sample = 0.0f; + for (size_t channel = 0; channel < num_channels; ++channel) { + max_sample = std::max(max_sample, abs(input[channel][frame])); } - } - // Get the limiter envelope. - std::vector limiter_env(num_frames, 0.0); - for (size_t frame = 0; frame < num_frames; ++frame) { - double max_req_gain = GetMaximumRequiredGain(max_samples[frame]); + const double max_req_gain = GetMaximumRequiredGain(max_sample); if (max_req_gain < env_) { env_ = max_req_gain; } else { env_ = release_time_constant_ * (env_ - max_req_gain) + max_req_gain; } - limiter_env[frame] = static_cast(env_); - } + const float gain = static_cast(env_); - // Apply the limiter envelope to the input buffer. - for (size_t channel = 0; channel < num_channels; ++channel) { - for (size_t frame = 0; frame < num_frames; ++frame) { - (*output)[channel][frame] = input[channel][frame] * limiter_env[frame]; + for (size_t channel = 0; channel < num_channels; ++channel) { + (*output)[channel][frame] = input[channel][frame] * gain; } } } diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc index a745e4d..f61456b 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.cc @@ -12,6 +12,7 @@ #include "obr/renderer/obr_impl.h" +#include #include #include #include @@ -41,7 +42,10 @@ ObrImpl::ObrImpl(int buffer_size_per_channel, int sampling_rate) head_tracking_enabled_(false), limiter_enabled_(true), world_rotation_(WorldRotation()), - fft_manager_(buffer_size_per_channel_) { + fft_manager_(buffer_size_per_channel_), + group_output_(kNumBinauralChannels, buffer_size_per_channel_), + planar_input_(0, buffer_size_per_channel_), + planar_output_(kNumBinauralChannels, buffer_size_per_channel_) { ABSL_CHECK_GT(buffer_size_per_channel, 0) << "Buffer size per channel must be greater than 0."; ABSL_CHECK_GT(sampling_rate_, 0) << "Sampling rate must be greater than 0."; @@ -146,7 +150,35 @@ void ObrImpl::Process(const AudioBuffer& input_buffer, // Acquire the lock for the entire processing duration. absl::MutexLock lock(&mutex_); + ProcessLocked(input_buffer, output_buffer); +} + +void ObrImpl::ProcessPlanar(const float* input, float* output, + size_t num_frames) { + ABSL_CHECK_NE(input, nullptr); + ABSL_CHECK_NE(output, nullptr); + ABSL_CHECK_EQ(num_frames, buffer_size_per_channel_); + + absl::MutexLock lock(&mutex_); + + // `AddAudioElement()` and `RemoveLastAudioElement()` size the input buffer + // for the input channels in the same critical section that changes them. + ABSL_CHECK_EQ(planar_input_.num_channels(), GetNumberOfInputChannelsLocked()); + + for (size_t ch = 0; ch < planar_input_.num_channels(); ++ch) { + std::copy_n(input + ch * num_frames, num_frames, planar_input_[ch].begin()); + } + + ProcessLocked(planar_input_, &planar_output_); + for (size_t ch = 0; ch < planar_output_.num_channels(); ++ch) { + std::copy_n(planar_output_[ch].begin(), num_frames, + output + ch * num_frames); + } +} + +void ObrImpl::ProcessLocked(const AudioBuffer& input_buffer, + AudioBuffer* output_buffer) { // Clear the output buffer. output_buffer->Clear(); @@ -203,18 +235,15 @@ void ObrImpl::Process(const AudioBuffer& input_buffer, return; } - // Create a temporary buffer for each group's output. - AudioBuffer group_output(kNumBinauralChannels, buffer_size_per_channel_); - // Process each processing group. for (auto& group : processing_groups_) { - group_output.Clear(); + group_output_.Clear(); group.Process(input_buffer, audio_elements_, head_tracking_enabled_, - world_rotation_, &group_output); + world_rotation_, &group_output_); // Add this group's binaural output to the main output buffer. - (*output_buffer)[0] += group_output[0]; - (*output_buffer)[1] += group_output[1]; + (*output_buffer)[0] += group_output_[0]; + (*output_buffer)[1] += group_output_[1]; } // Peak limit the output if enabled. @@ -246,6 +275,13 @@ size_t ObrImpl::GetNumberOfInputChannelsLocked() const { return number_of_input_channels; } +void ObrImpl::ResizePlanarInputLocked() { + const size_t num_channels = GetNumberOfInputChannelsLocked(); + if (planar_input_.num_channels() != num_channels) { + planar_input_ = AudioBuffer(num_channels, buffer_size_per_channel_); + } +} + size_t ObrImpl::GetNumberOfInputChannels() { absl::MutexLock lock(&mutex_); return GetNumberOfInputChannelsLocked(); @@ -291,6 +327,7 @@ absl::Status ObrImpl::AddAudioElement(const AudioElementType type, // All checks passed, add the Audio Element config. audio_elements_.push_back(audio_element_config); + ResizePlanarInputLocked(); ABSL_LOG(INFO) << "Added audio element: " << GetAudioElementTypeStr(audio_elements_.back().GetType()) << "."; @@ -318,6 +355,7 @@ absl::Status ObrImpl::RemoveLastAudioElement() { << GetAudioElementTypeStr(audio_elements_.back().GetType()) << "."; audio_elements_.pop_back(); + ResizePlanarInputLocked(); if (audio_elements_.empty()) { ABSL_LOG(INFO) << "No audio elements left."; diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h index dd00ded..b71ddb8 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/obr_impl.h @@ -52,6 +52,22 @@ class ObrImpl { */ void Process(const AudioBuffer& input_buffer, AudioBuffer* output_buffer); + /*!\brief Processes audio held in flat planar arrays. + * + * Copies `input` into an input buffer the renderer keeps, processes it as + * `Process()` does, and copies the binaural result into `output`. The + * renderer sizes its buffers when an audio element is added or removed, so + * this call does not allocate. It takes the same lock as `Process()`, once. + * + * \param input `GetNumberOfInputChannels()` channels of `num_frames` + * samples, one channel after the other. + * \param output Receives the two binaural channels of `num_frames` samples, + * one channel after the other. + * \param num_frames Samples per channel; must equal + * `GetBufferSizePerChannel()`. + */ + void ProcessPlanar(const float* input, float* output, size_t num_frames); + /*!\brief Returns the buffer size per channel. * * \return Buffer size per channel. @@ -231,6 +247,16 @@ class ObrImpl { */ size_t GetNumberOfInputChannelsLocked() const; + /*!\brief The body of `Process()`, run while holding mutex_. + */ + void ProcessLocked(const AudioBuffer& input_buffer, + AudioBuffer* output_buffer); + + /*!\brief Sizes `planar_input_` for the current number of input channels. + * Must be called while holding mutex_. + */ + void ResizePlanarInputLocked(); + const int buffer_size_per_channel_; const int sampling_rate_; @@ -238,7 +264,9 @@ class ObrImpl { bool limiter_enabled_; WorldRotation world_rotation_; - // Mutex to protect data accessed in different threads. + // Mutex to protect data accessed in different threads, including the + // group output and planar buffers. `Process()` and `ProcessPlanar()` hold + // it for the whole call. mutable absl::Mutex mutex_; std::vector audio_elements_; @@ -255,6 +283,16 @@ class ObrImpl { // TODO(b/468218887): Provide an option to disable peak limiting. // Peak limiter (applied to final output). std::unique_ptr peak_limiter_; + + // Each processing group's binaural output in `Process()`, allocated in the + // constructor so that processing does not allocate. + AudioBuffer group_output_; + + // The input and output of `ProcessPlanar()`. The input is sized for the + // input channels when an audio element is added or removed, the output in + // the constructor. Accessed under `mutex_`. + AudioBuffer planar_input_; + AudioBuffer planar_output_; }; } // namespace obr diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.cc b/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.cc index da3ea3a..abf140c 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.cc +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.cc @@ -123,8 +123,9 @@ void ProcessingGroup::Process( ambisonic_mix_bed_head_locked_.Clear(); // PASS 1: Process world-locked elements (head_locked = false) - const std::vector world_locked_indices = - GetAmbisonicEncoderSourceChannelIndices(audio_elements, false); + GetAmbisonicEncoderSourceChannelIndices(audio_elements, false, + &world_locked_indices_); + const std::vector& world_locked_indices = world_locked_indices_; if (!world_locked_indices.empty() && ambisonic_encoder_ != nullptr) { // Copy selected channels from input buffer to encoder input buffer. @@ -151,8 +152,9 @@ void ProcessingGroup::Process( } // PASS 2: Process head-locked elements (head_locked = true) - const std::vector head_locked_indices = - GetAmbisonicEncoderSourceChannelIndices(audio_elements, true); + GetAmbisonicEncoderSourceChannelIndices(audio_elements, true, + &head_locked_indices_); + const std::vector& head_locked_indices = head_locked_indices_; if (!head_locked_indices.empty() && ambisonic_encoder_ != nullptr) { // Copy selected channels from input buffer to encoder input buffer. @@ -199,10 +201,15 @@ void ProcessingGroup::Process( absl::Status ProcessingGroup::UpdateAmbisonicEncoder( std::vector& audio_elements) { - // Get encoder source indices to determine if we need an encoder. - const auto indices = GetAmbisonicEncoderSourceChannelIndices(audio_elements); + // Count the encoder sources to determine if we need an encoder. + const size_t num_sources = + GetNumberOfAmbisonicEncoderSourceChannels(audio_elements); - if (indices.empty()) { + // `Process()` fills the index lists within this capacity. + world_locked_indices_.reserve(num_sources); + head_locked_indices_.reserve(num_sources); + + if (num_sources == 0) { // No sources to encode, so no encoder needed. ambisonic_encoder_.reset(); ambisonic_encoder_input_buffer_ = AudioBuffer(); @@ -211,11 +218,11 @@ absl::Status ProcessingGroup::UpdateAmbisonicEncoder( // Initialize or reinitialize encoder if source count changed. if (ambisonic_encoder_ == nullptr || - ambisonic_encoder_input_buffer_.num_channels() != indices.size()) { + ambisonic_encoder_input_buffer_.num_channels() != num_sources) { ambisonic_encoder_input_buffer_ = - AudioBuffer(indices.size(), buffer_size_per_channel_); - ambisonic_encoder_ = std::make_unique( - indices.size(), key_.ambisonic_order); + AudioBuffer(num_sources, buffer_size_per_channel_); + ambisonic_encoder_ = + std::make_unique(num_sources, key_.ambisonic_order); } // Update Ambisonic encoder settings. @@ -243,38 +250,25 @@ absl::Status ProcessingGroup::UpdateAmbisonicEncoder( return absl::OkStatus(); } -std::vector ProcessingGroup::GetAmbisonicEncoderSourceChannelIndices( +size_t ProcessingGroup::GetNumberOfAmbisonicEncoderSourceChannels( const std::vector& audio_elements) const { - std::vector source_channel_indices; - // Reserve capacity based on estimated number of channels to avoid - // reallocations. - size_t estimated_channels = 0; + size_t num_channels = 0; for (size_t ae_index : audio_element_indices_) { const auto& audio_element = audio_elements[ae_index]; if (IsLoudspeakerLayoutType(audio_element.GetType()) || IsObjectType(audio_element.GetType())) { - estimated_channels += audio_element.GetNumberOfInputChannels(); + num_channels += audio_element.GetNumberOfInputChannels(); } } - source_channel_indices.reserve(estimated_channels); - - for (size_t ae_index : audio_element_indices_) { - const auto& audio_element = audio_elements[ae_index]; - if (IsLoudspeakerLayoutType(audio_element.GetType()) || - IsObjectType(audio_element.GetType())) { - for (size_t i = 0; i < audio_element.GetNumberOfInputChannels(); ++i) { - source_channel_indices.push_back(audio_element.GetFirstChannelIndex() + - i); - } - } - } - return source_channel_indices; + return num_channels; } -std::vector ProcessingGroup::GetAmbisonicEncoderSourceChannelIndices( +void ProcessingGroup::GetAmbisonicEncoderSourceChannelIndices( const std::vector& audio_elements, - bool filter_head_locked) const { - std::vector source_channel_indices; + bool filter_head_locked, std::vector* indices) const { + ABSL_CHECK_NE(indices, nullptr); + std::vector& source_channel_indices = *indices; + source_channel_indices.clear(); for (size_t ae_index : audio_element_indices_) { const auto& audio_element = audio_elements[ae_index]; @@ -292,8 +286,6 @@ std::vector ProcessingGroup::GetAmbisonicEncoderSourceChannelIndices( } } } - - return source_channel_indices; } } // namespace obr diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.h b/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.h index cb57912..e6630de 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.h +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/processing_group.h @@ -146,12 +146,12 @@ class ProcessingGroup { BinauralFilterProfile GetFilterProfile() const { return key_.filter_profile; } private: - /*!\brief Get source channel indices for the Ambisonic encoder. + /*!\brief Get the number of source channels for the Ambisonic encoder. * * \param audio_elements Reference to all audio elements. - * \return Vector of input channel indices for all sources. + * \return Number of input channels of all sources. */ - std::vector GetAmbisonicEncoderSourceChannelIndices( + size_t GetNumberOfAmbisonicEncoderSourceChannels( const std::vector& audio_elements) const; /*!\brief Get source channel indices for the Ambisonic encoder, filtered by @@ -161,11 +161,13 @@ class ProcessingGroup { * \param filter_head_locked If true, return indices of head-locked elements; * if false, return indices of world-locked * elements. - * \return Vector of input channel indices matching the filter. + * \param indices Output vector of input channel indices matching the + * filter. It does not allocate when its capacity holds all source + * channels, which `UpdateAmbisonicEncoder()` reserves. */ - std::vector GetAmbisonicEncoderSourceChannelIndices( + void GetAmbisonicEncoderSourceChannelIndices( const std::vector& audio_elements, - bool filter_head_locked) const; + bool filter_head_locked, std::vector* indices) const; ProcessingGroupKey key_; @@ -186,6 +188,12 @@ class ProcessingGroup { std::unique_ptr ambisonic_encoder_; AudioBuffer ambisonic_encoder_input_buffer_; + // Encoder source channel indices of the world-locked and head-locked + // elements, filled by `Process()` within the capacity reserved by + // `UpdateAmbisonicEncoder()`. + std::vector world_locked_indices_; + std::vector head_locked_indices_; + // Ambisonic rotator for this group. std::unique_ptr ambisonic_rotator_; diff --git a/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc b/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc index 8d6878b..84a450b 100644 --- a/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc +++ b/src/renderer/obr/obr_capi/obr/obr/renderer/tests/obr_impl_test.cc @@ -29,6 +29,8 @@ namespace obr { namespace { using ::absl_testing::IsOk; +using ::testing::Eq; +using ::testing::Pointwise; constexpr int kBufferSizePerChannel = 44; constexpr int kSamplingRate = 48000; @@ -1074,5 +1076,57 @@ TEST(ObrImplTest, TestPassthroughOnlyAppliesPeakLimiter) { EXPECT_LT(peak_right, 1.1f); } +// Renders the same input through `Process()` on one renderer and through +// `ProcessPlanar()` on another with the same elements, and requires identical +// output: with one element, after a second is added and after it is removed. +TEST(ObrImplTest, TestProcessPlanarMatchesProcess) { + const size_t kNumFrames = 256; + ObrImpl buffered(kNumFrames, kSamplingRate); + ObrImpl planar(kNumFrames, kSamplingRate); + + auto expect_same_output = [&](int seed) { + const size_t num_channels = buffered.GetNumberOfInputChannels(); + ASSERT_EQ(planar.GetNumberOfInputChannels(), num_channels); + AudioBuffer input_buffer(num_channels, kNumFrames); + std::vector input(num_channels * kNumFrames); + for (size_t ch = 0; ch < num_channels; ++ch) { + GenerateWhiteNoise(0.1f, &input_buffer[ch], static_cast(seed + ch)); + for (size_t frame = 0; frame < kNumFrames; ++frame) { + input[ch * kNumFrames + frame] = input_buffer[ch][frame]; + } + } + AudioBuffer output_buffer(2, kNumFrames); + std::vector output(2 * kNumFrames, 1.0f); + for (int block = 0; block < 3; ++block) { + buffered.Process(input_buffer, &output_buffer); + planar.ProcessPlanar(input.data(), output.data(), kNumFrames); + for (size_t ch = 0; ch < 2; ++ch) { + const std::vector expected(output_buffer[ch].begin(), + output_buffer[ch].end()); + const std::vector actual(output.begin() + ch * kNumFrames, + output.begin() + (ch + 1) * kNumFrames); + EXPECT_THAT(actual, Pointwise(Eq(), expected)) + << "block " << block << ", channel " << ch; + } + } + }; + + for (ObrImpl* renderer : {&buffered, &planar}) { + EXPECT_THAT(renderer->AddAudioElement(AudioElementType::k3OA), IsOk()); + } + expect_same_output(10); + + for (ObrImpl* renderer : {&buffered, &planar}) { + EXPECT_THAT(renderer->AddAudioElement(AudioElementType::kLayout7_1_4_ch), + IsOk()); + } + expect_same_output(20); + + for (ObrImpl* renderer : {&buffered, &planar}) { + EXPECT_THAT(renderer->RemoveLastAudioElement(), IsOk()); + } + expect_same_output(30); +} + } // namespace } // namespace obr diff --git a/src/renderer/obr/obr_capi/obr_capi.cpp b/src/renderer/obr/obr_capi/obr_capi.cpp index 9738021..432bcb8 100644 --- a/src/renderer/obr/obr_capi/obr_capi.cpp +++ b/src/renderer/obr/obr_capi/obr_capi.cpp @@ -21,9 +21,6 @@ #include #include -#include -#include -#include #include "obr/renderer/obr_impl.h" using namespace obr; @@ -83,30 +80,11 @@ int obr_add_audio_element(obr_handle handle, ObrAudioElementType type, int obr_process(obr_handle handle, const float *input, float *output, int num_samples) { - int input_nch = 0; - int output_nch = 2; - size_t buffer_size = num_samples; - if (!handle || !input || !output) return 0; - input_nch = (int)cast_(handle)->GetNumberOfInputChannels(); - - std::vector> input_buffer_float( - input_nch, std::vector(buffer_size)); - - for (size_t ch = 0; ch < input_nch; ch++) - for (size_t smp = 0; smp < buffer_size; smp++) - input_buffer_float[ch][smp] = input[ch * buffer_size + smp]; - - AudioBuffer input_buffer(input_nch, buffer_size); - input_buffer = input_buffer_float; - AudioBuffer output_buffer(output_nch, buffer_size); - - cast_(handle)->Process(input_buffer, &output_buffer); - - for (size_t ch = 0; ch < output_buffer.num_channels(); ch++) - for (size_t smp = 0; smp < output_buffer.num_frames(); smp++) - output[ch * output_buffer.num_frames() + smp] = output_buffer[ch][smp]; + // The renderer keeps the buffers it processes in and takes its lock once; + // this function keeps no buffers and takes no lock of its own. + cast_(handle)->ProcessPlanar(input, output, static_cast(num_samples)); return num_samples; }