From 15024ac6f86ce879cd44ec70b7a60f73fd38b35f Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Wed, 28 Oct 2026 14:06:52 -0400 Subject: [PATCH 1/2] perf: tdigest grows its buffers as they fill The constructor reserved room for 2k + fudge centroids and four times as many buffered values (about 10 KB for k = 100 with doubles), also when deserializing. Code that holds many small t-digests paid for all of it. Grow the buffer geometrically up to its limit on update, and the centroids before a compress or merge needs them. Results are unchanged. Co-Authored-By: Claude Opus 5.5 --- tdigest/include/tdigest.hpp | 2 ++ tdigest/include/tdigest_impl.hpp | 17 +++++++++-- .../test/tdigest_custom_allocator_test.cpp | 29 +++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/tdigest/include/tdigest.hpp b/tdigest/include/tdigest.hpp index 095752e94..1603e5db0 100644 --- a/tdigest/include/tdigest.hpp +++ b/tdigest/include/tdigest.hpp @@ -299,6 +299,8 @@ class tdigest { bool is_single_value() const; uint8_t get_preamble_longs() const; void merge(vector_centroid& buffer, W weight); + // grows centroids_ to hold num centroids, up to centroids_capacity_, before a merge needs them + void reserve_centroids(size_t num); // for deserialize tdigest(bool reverse_merge, uint16_t k, T min, T max, vector_centroid&& centroids, uint64_t total_weight_, vector_t&& buffer); diff --git a/tdigest/include/tdigest_impl.hpp b/tdigest/include/tdigest_impl.hpp index 958c8c408..6addf32c4 100644 --- a/tdigest/include/tdigest_impl.hpp +++ b/tdigest/include/tdigest_impl.hpp @@ -62,6 +62,9 @@ void tdigest::update(T value) { if (std::isnan(value)) return; if (std::isinf(value)) return; if (buffer_.size() == centroids_capacity_ * BUFFER_MULTIPLIER) compress(); + if (buffer_.size() == buffer_.capacity()) { + buffer_.reserve(std::min(std::max(2 * buffer_.size(), 8), centroids_capacity_ * BUFFER_MULTIPLIER)); + } buffer_.push_back(value); min_ = std::min(min_, value); max_ = std::max(max_, value); @@ -73,6 +76,7 @@ void tdigest::merge(const tdigest& other) { // Extreme centroid means are not the true min/max when those centroids have weight > 1. min_ = std::min(min_, other.min_); max_ = std::max(max_, other.max_); + reserve_centroids(buffer_.size() + centroids_.size() + other.buffer_.size() + other.centroids_.size()); vector_centroid tmp(buffer_.get_allocator()); tmp.reserve(buffer_.size() + centroids_.size() + other.buffer_.size() + other.centroids_.size()); for (const T value: buffer_) tmp.push_back(centroid(value, 1)); @@ -84,6 +88,7 @@ void tdigest::merge(const tdigest& other) { template void tdigest::compress() { if (buffer_.size() == 0) return; + reserve_centroids(buffer_.size() + centroids_.size()); vector_centroid tmp(buffer_.get_allocator()); tmp.reserve(buffer_.size() + centroids_.size()); for (const T value: buffer_) tmp.push_back(centroid(value, 1)); @@ -711,8 +716,16 @@ buffer_(std::move(buffer)) if (k < 10) throw std::invalid_argument("k must be at least 10"); const size_t fudge = k < 30 ? 30 : 10; centroids_capacity_ = 2 * k_ + fudge; - centroids_.reserve(centroids_capacity_); - buffer_.reserve(centroids_capacity_ * BUFFER_MULTIPLIER); + // centroids_ and buffer_ grow as they fill (see update() and reserve_centroids()) instead of being reserved + // here: many t-digests (e.g. per-group aggregation states) hold few values +} + +template +void tdigest::reserve_centroids(size_t num) { + num = std::min(num, centroids_capacity_); + if (centroids_.capacity() < num) { + centroids_.reserve(std::max(num, std::min(2 * centroids_.capacity(), centroids_capacity_))); + } } template diff --git a/tdigest/test/tdigest_custom_allocator_test.cpp b/tdigest/test/tdigest_custom_allocator_test.cpp index b0687d689..06c5e53be 100644 --- a/tdigest/test/tdigest_custom_allocator_test.cpp +++ b/tdigest/test/tdigest_custom_allocator_test.cpp @@ -40,4 +40,33 @@ TEST_CASE("tdigest custom allocator", "[tdigest]") { REQUIRE(test_allocator_net_allocations == 0); } +TEST_CASE("tdigest custom allocator: buffers grow as they fill", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + tdigest_d empty(100, alloc_d(0)); + REQUIRE(test_allocator_total_bytes == 0); // nothing reserved up front + + tdigest_d small(100, alloc_d(0)); + for (int i = 0; i < 3; ++i) small.update(static_cast(i)); + // a few values take a small buffer, not room for 4 * (2k + fudge) values + REQUIRE(test_allocator_total_bytes <= static_cast(8 * sizeof(double))); + REQUIRE(small.get_total_weight() == 3); + REQUIRE(small.get_quantile(0.5) == 1.0); + + // a digest built through the growing buffers gives the same result as before + tdigest_d grown(100, alloc_d(0)); + tdigest reference(100); + for (int i = 0; i < 100000; ++i) { + grown.update(static_cast(i % 7919)); + reference.update(static_cast(i % 7919)); + } + for (double rank: {0.01, 0.25, 0.5, 0.75, 0.99}) { + REQUIRE(grown.get_quantile(rank) == reference.get_quantile(rank)); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + } /* namespace datasketches */ From f965cc5e355024221087026fade30789a45854d1 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 12 Nov 2026 11:28:29 -0800 Subject: [PATCH 2/2] t-digest: add reserve(), allocation-failure tests and docs for growing storage reserve() allocates the full storage for centroids and buffered values right after construction, for callers who want allocation failures there or don't want updates to pay for growing the storage. merge() now allocates before it widens min and max, so a failed merge leaves the t-Digest unchanged. The new tests use an allocator that fails on purpose. A failure at any allocation of reserve(), a stream of updates or a merge leaves the t-Digest exactly as it was, and it stays usable. Co-Authored-By: Claude Opus 5.5 --- tdigest/include/tdigest.hpp | 19 +- tdigest/include/tdigest_impl.hpp | 13 +- tdigest/test/CMakeLists.txt | 1 + tdigest/test/tdigest_allocation_test.cpp | 245 +++++++++++++++++++++++ 4 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 tdigest/test/tdigest_allocation_test.cpp diff --git a/tdigest/include/tdigest.hpp b/tdigest/include/tdigest.hpp index 1603e5db0..291c2eb75 100644 --- a/tdigest/include/tdigest.hpp +++ b/tdigest/include/tdigest.hpp @@ -59,6 +59,13 @@ using tdigest_double = tdigest; * and the following implementation in Java: * https://github.com/tdunning/t-digest * This implementation is similar to MergingDigest in the above Java implementation + * + * Memory allocation: a new t-Digest does not allocate its storage for centroids and buffered + * values up front. The storage grows as values arrive, up to its full size. t-Digests that stay + * empty or hold few values use little memory. As a result, update() and merge() can throw + * std::bad_alloc while growing this storage, rather than the constructor. If that happens, the + * t-Digest is unchanged and can still be used. Call reserve() right after construction to allocate + * the full storage up front. */ template class tdigest { @@ -100,7 +107,8 @@ class tdigest { }; /** - * Constructor + * Constructor. + * Storage for centroids and buffered values is allocated as it fills, see reserve(). * @param k affects the size of the sketch and its estimation error * @param allocator used to allocate memory */ @@ -119,6 +127,15 @@ class tdigest { */ void merge(const tdigest& other); + /** + * Allocates the full storage for centroids and buffered values now, rather than as it fills. + * Call this right after construction if allocation failures should happen here, + * or if updates should not pay for growing the storage. + * Does nothing for storage that is already allocated in full. + * @throw std::bad_alloc if an allocation fails, in which case the contents of the t-Digest are unchanged + */ + void reserve(); + /** * Process buffered values and merge centroids if needed */ diff --git a/tdigest/include/tdigest_impl.hpp b/tdigest/include/tdigest_impl.hpp index 6addf32c4..1a13f0cfe 100644 --- a/tdigest/include/tdigest_impl.hpp +++ b/tdigest/include/tdigest_impl.hpp @@ -73,12 +73,13 @@ void tdigest::update(T value) { template void tdigest::merge(const tdigest& other) { if (other.is_empty()) return; - // Extreme centroid means are not the true min/max when those centroids have weight > 1. - min_ = std::min(min_, other.min_); - max_ = std::max(max_, other.max_); + // allocate before changing anything, so that a failure leaves this t-Digest unchanged reserve_centroids(buffer_.size() + centroids_.size() + other.buffer_.size() + other.centroids_.size()); vector_centroid tmp(buffer_.get_allocator()); tmp.reserve(buffer_.size() + centroids_.size() + other.buffer_.size() + other.centroids_.size()); + // Extreme centroid means are not the true min/max when those centroids have weight > 1. + min_ = std::min(min_, other.min_); + max_ = std::max(max_, other.max_); for (const T value: buffer_) tmp.push_back(centroid(value, 1)); for (const T value: other.buffer_) tmp.push_back(centroid(value, 1)); std::copy(other.centroids_.begin(), other.centroids_.end(), std::back_inserter(tmp)); @@ -720,6 +721,12 @@ buffer_(std::move(buffer)) // here: many t-digests (e.g. per-group aggregation states) hold few values } +template +void tdigest::reserve() { + centroids_.reserve(centroids_capacity_); + buffer_.reserve(centroids_capacity_ * BUFFER_MULTIPLIER); +} + template void tdigest::reserve_centroids(size_t num) { num = std::min(num, centroids_capacity_); diff --git a/tdigest/test/CMakeLists.txt b/tdigest/test/CMakeLists.txt index 8dcfb4f03..adc28fb97 100644 --- a/tdigest/test/CMakeLists.txt +++ b/tdigest/test/CMakeLists.txt @@ -40,6 +40,7 @@ target_sources(tdigest_test tdigest_test.cpp tdigest_custom_allocator_test.cpp tdigest_iterator_test.cpp + tdigest_allocation_test.cpp ) if (SERDE_COMPAT) diff --git a/tdigest/test/tdigest_allocation_test.cpp b/tdigest/test/tdigest_allocation_test.cpp new file mode 100644 index 000000000..45097852f --- /dev/null +++ b/tdigest/test/tdigest_allocation_test.cpp @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "tdigest.hpp" +#include "test_allocator.hpp" + +namespace datasketches { + +namespace { + +// Counts allocations, and throws std::bad_alloc on the allocation numbered fail_at (counting from 0). +struct allocation_control { + long count = 0; + long fail_at = -1; + void arm(long at) { count = 0; fail_at = at; } + void disarm() { fail_at = -1; } +}; + +template +struct failing_allocator: test_allocator { + allocation_control* control; + template struct rebind { using other = failing_allocator; }; + explicit failing_allocator(allocation_control& c): test_allocator(0), control(&c) {} + template + failing_allocator(const failing_allocator& other): test_allocator(0), control(other.control) {} + T* allocate(std::size_t n, const T* hint = nullptr) { + if (control->fail_at >= 0 && control->count++ == control->fail_at) throw std::bad_alloc(); + return test_allocator::allocate(n, hint); + } +}; + +template +bool operator==(const failing_allocator& a, const failing_allocator& b) { return a.control == b.control; } +template +bool operator!=(const failing_allocator& a, const failing_allocator& b) { return !(a == b); } + +using tdigest_test_alloc = tdigest>; +using tdigest_failing = tdigest>; + +const uint16_t K = 100; + +double value(int i) { + // from 100000 on, values extend past both ends of the earlier ones (sources in the merge tests) + if (i >= 100000) return (i % 2 == 0 ? 1.0 : -1.0) * i; + // not monotonic, so that merges interleave + return static_cast((i * 7919) % 10007); +} + +tdigest_failing make_tdigest(int first, int count, allocation_control& control) { + tdigest_failing td(K, failing_allocator(control)); + for (int i = first; i < first + count; ++i) td.update(value(i)); + return td; +} + +// the complete state, including the buffered values +std::vector state(const tdigest_failing& td) { + const auto bytes = td.serialize(0, true); + return std::vector(bytes.begin(), bytes.end()); +} + +// the t-Digest must still work: query it, update it, merge into it, and round-trip it +void check_usable(tdigest_failing& td, allocation_control& control) { + control.disarm(); + const uint64_t weight = td.get_total_weight(); + td.update(-1); + REQUIRE(td.get_total_weight() == weight + 1); + REQUIRE(td.get_min_value() == -1); + REQUIRE(td.get_rank(-1) <= 0.5); + td.merge(make_tdigest(0, 500, control)); + REQUIRE(td.get_total_weight() == weight + 501); + const auto bytes = td.serialize(0, true); + const auto copy = tdigest_failing::deserialize(bytes.data(), bytes.size(), failing_allocator(control)); + REQUIRE(copy.get_total_weight() == td.get_total_weight()); +} + +} // namespace + +TEST_CASE("tdigest allocation: reserve allocates the full storage up front", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + tdigest_test_alloc td(K, test_allocator(0)); + const long long empty_bytes = test_allocator_total_bytes; + + td.reserve(); + const long long reserved_bytes = test_allocator_total_bytes; + REQUIRE(reserved_bytes > empty_bytes); + + td.reserve(); // already allocated + REQUIRE(test_allocator_total_bytes == reserved_bytes); + + // compression allocates a temporary vector, but the storage never grows + for (int i = 0; i < 100000; ++i) td.update(value(i)); + REQUIRE(test_allocator_total_bytes == reserved_bytes); + REQUIRE(td.get_total_weight() == 100000); + + // the same results as without reserve() + tdigest_test_alloc unreserved(K, test_allocator(0)); + for (int i = 0; i < 100000; ++i) unreserved.update(value(i)); + REQUIRE(td.serialize(0, true) == unreserved.serialize(0, true)); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("tdigest allocation: reserve after updates does not change the contents", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + tdigest_test_alloc td(K, test_allocator(0)); + for (int i = 0; i < 1000; ++i) td.update(value(i)); + const auto bytes = td.serialize(0, true); + td.reserve(); + REQUIRE(td.serialize(0, true) == bytes); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("tdigest allocation: failed reserve", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + for (long fail_at: {0, 1}) { // centroids, then the buffer + tdigest_failing td = make_tdigest(0, 10, control); + const auto before = state(td); + control.arm(fail_at); + REQUIRE_THROWS_AS(td.reserve(), std::bad_alloc); + REQUIRE(state(td) == before); + check_usable(td, control); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("tdigest allocation: failed first update leaves the t-Digest unchanged", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + tdigest_failing td(K, failing_allocator(control)); + const auto before = state(td); + control.arm(0); + REQUIRE_THROWS_AS(td.update(1), std::bad_alloc); + REQUIRE(td.is_empty()); + REQUIRE(state(td) == before); + + control.disarm(); + td.update(2); + REQUIRE(td.get_total_weight() == 1); + REQUIRE(td.get_min_value() == 2); // nothing left over from the failed update + REQUIRE(td.get_max_value() == 2); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +// Fail each allocation made by a stream of updates in turn. The update that fails must leave the +// t-Digest exactly as the updates before it left it, so it must equal one built from those updates. +TEST_CASE("tdigest allocation: failure at each allocation of the updates", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + const int n = 5000; + for (long fail_at = 0; ; ++fail_at) { + tdigest_failing td(K, failing_allocator(control)); + control.arm(fail_at); + int updated = 0; + try { + for (; updated < n; ++updated) td.update(value(updated)); + control.disarm(); + REQUIRE(fail_at > 0); + break; + } catch (const std::bad_alloc&) { + control.disarm(); + CAPTURE(fail_at, updated); + REQUIRE(state(td) == state(make_tdigest(0, updated, control))); + check_usable(td, control); + } + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +// Fail each allocation of target.merge(source) in turn: a failed merge leaves both unchanged. +TEST_CASE("tdigest allocation: failure at each allocation of a merge", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + long failures_tested = 0; + for (int target_n: {0, 5, 1000, 20000}) { + for (int source_n: {5, 1000, 20000}) { + CAPTURE(target_n, source_n); + const tdigest_failing source = make_tdigest(100000, source_n, control); + const auto source_before = state(source); + for (long fail_at = 0; ; ++fail_at) { + tdigest_failing target = make_tdigest(0, target_n, control); + const auto target_before = state(target); + control.arm(fail_at); + try { + target.merge(source); + control.disarm(); + REQUIRE(target.get_total_weight() == static_cast(target_n + source_n)); + break; + } catch (const std::bad_alloc&) { + ++failures_tested; + REQUIRE(state(target) == target_before); + REQUIRE(state(source) == source_before); + check_usable(target, control); + } + } + } + } + REQUIRE(failures_tested > 0); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +} /* namespace datasketches */