diff --git a/tdigest/include/tdigest.hpp b/tdigest/include/tdigest.hpp index 095752e9..291c2eb7 100644 --- a/tdigest/include/tdigest.hpp +++ b/tdigest/include/tdigest.hpp @@ -59,6 +59,13 @@ using tdigest_double = tdigest; * and the following implementation in Java: * https://github.com/tdunning/t-digest * This implementation is similar to MergingDigest in the above Java implementation + * + * Memory allocation: a new t-Digest does not allocate its storage for centroids and buffered + * values up front. The storage grows as values arrive, up to its full size. t-Digests that stay + * empty or hold few values use little memory. As a result, update() and merge() can throw + * std::bad_alloc while growing this storage, rather than the constructor. If that happens, the + * t-Digest is unchanged and can still be used. Call reserve() right after construction to allocate + * the full storage up front. */ template class tdigest { @@ -100,7 +107,8 @@ class tdigest { }; /** - * Constructor + * Constructor. + * Storage for centroids and buffered values is allocated as it fills, see reserve(). * @param k affects the size of the sketch and its estimation error * @param allocator used to allocate memory */ @@ -119,6 +127,15 @@ class tdigest { */ void merge(const tdigest& other); + /** + * Allocates the full storage for centroids and buffered values now, rather than as it fills. + * Call this right after construction if allocation failures should happen here, + * or if updates should not pay for growing the storage. + * Does nothing for storage that is already allocated in full. + * @throw std::bad_alloc if an allocation fails, in which case the contents of the t-Digest are unchanged + */ + void reserve(); + /** * Process buffered values and merge centroids if needed */ @@ -299,6 +316,8 @@ class tdigest { bool is_single_value() const; uint8_t get_preamble_longs() const; void merge(vector_centroid& buffer, W weight); + // grows centroids_ to hold num centroids, up to centroids_capacity_, before a merge needs them + void reserve_centroids(size_t num); // for deserialize tdigest(bool reverse_merge, uint16_t k, T min, T max, vector_centroid&& centroids, uint64_t total_weight_, vector_t&& buffer); diff --git a/tdigest/include/tdigest_impl.hpp b/tdigest/include/tdigest_impl.hpp index ab9da04c..1f129f04 100644 --- a/tdigest/include/tdigest_impl.hpp +++ b/tdigest/include/tdigest_impl.hpp @@ -62,6 +62,9 @@ void tdigest::update(T value) { if (std::isnan(value)) return; if (std::isinf(value)) return; if (buffer_.size() == centroids_capacity_ * BUFFER_MULTIPLIER) compress(); + if (buffer_.size() == buffer_.capacity()) { + buffer_.reserve(std::min(std::max(2 * buffer_.size(), 8), centroids_capacity_ * BUFFER_MULTIPLIER)); + } buffer_.push_back(value); min_ = std::min(min_, value); max_ = std::max(max_, value); @@ -70,11 +73,13 @@ void tdigest::update(T value) { template void tdigest::merge(const tdigest& other) { if (other.is_empty()) return; + // allocate before changing anything, so that a failure leaves this t-Digest unchanged + reserve_centroids(buffer_.size() + centroids_.size() + other.buffer_.size() + other.centroids_.size()); + vector_centroid tmp(buffer_.get_allocator()); + tmp.reserve(buffer_.size() + centroids_.size() + other.buffer_.size() + other.centroids_.size()); // Extreme centroid means are not the true min/max when those centroids have weight > 1. min_ = std::min(min_, other.min_); max_ = std::max(max_, other.max_); - vector_centroid tmp(buffer_.get_allocator()); - tmp.reserve(buffer_.size() + centroids_.size() + other.buffer_.size() + other.centroids_.size()); for (const T value: buffer_) tmp.push_back(centroid(value, 1)); for (const T value: other.buffer_) tmp.push_back(centroid(value, 1)); std::copy(other.centroids_.begin(), other.centroids_.end(), std::back_inserter(tmp)); @@ -84,6 +89,7 @@ void tdigest::merge(const tdigest& other) { template void tdigest::compress() { if (buffer_.size() == 0) return; + reserve_centroids(buffer_.size() + centroids_.size()); vector_centroid tmp(buffer_.get_allocator()); tmp.reserve(buffer_.size() + centroids_.size()); for (const T value: buffer_) tmp.push_back(centroid(value, 1)); @@ -711,10 +717,24 @@ buffer_(std::move(buffer)) if (k < 10) throw std::invalid_argument("k must be at least 10"); const size_t fudge = k < 30 ? 30 : 10; centroids_capacity_ = 2 * k_ + fudge; + // centroids_ and buffer_ grow as they fill (see update() and reserve_centroids()) instead of being reserved + // here: many t-digests (e.g. per-group aggregation states) hold few values +} + +template +void tdigest::reserve() { centroids_.reserve(centroids_capacity_); buffer_.reserve(centroids_capacity_ * BUFFER_MULTIPLIER); } +template +void tdigest::reserve_centroids(size_t num) { + num = std::min(num, centroids_capacity_); + if (centroids_.capacity() < num) { + centroids_.reserve(std::max(num, std::min(2 * centroids_.capacity(), centroids_capacity_))); + } +} + template void tdigest::check_split_points(const T* values, uint32_t size) { for (uint32_t i = 0; i < size ; i++) { diff --git a/tdigest/test/CMakeLists.txt b/tdigest/test/CMakeLists.txt index 8dcfb4f0..adc28fb9 100644 --- a/tdigest/test/CMakeLists.txt +++ b/tdigest/test/CMakeLists.txt @@ -40,6 +40,7 @@ target_sources(tdigest_test tdigest_test.cpp tdigest_custom_allocator_test.cpp tdigest_iterator_test.cpp + tdigest_allocation_test.cpp ) if (SERDE_COMPAT) diff --git a/tdigest/test/tdigest_allocation_test.cpp b/tdigest/test/tdigest_allocation_test.cpp new file mode 100644 index 00000000..45097852 --- /dev/null +++ b/tdigest/test/tdigest_allocation_test.cpp @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "tdigest.hpp" +#include "test_allocator.hpp" + +namespace datasketches { + +namespace { + +// Counts allocations, and throws std::bad_alloc on the allocation numbered fail_at (counting from 0). +struct allocation_control { + long count = 0; + long fail_at = -1; + void arm(long at) { count = 0; fail_at = at; } + void disarm() { fail_at = -1; } +}; + +template +struct failing_allocator: test_allocator { + allocation_control* control; + template struct rebind { using other = failing_allocator; }; + explicit failing_allocator(allocation_control& c): test_allocator(0), control(&c) {} + template + failing_allocator(const failing_allocator& other): test_allocator(0), control(other.control) {} + T* allocate(std::size_t n, const T* hint = nullptr) { + if (control->fail_at >= 0 && control->count++ == control->fail_at) throw std::bad_alloc(); + return test_allocator::allocate(n, hint); + } +}; + +template +bool operator==(const failing_allocator& a, const failing_allocator& b) { return a.control == b.control; } +template +bool operator!=(const failing_allocator& a, const failing_allocator& b) { return !(a == b); } + +using tdigest_test_alloc = tdigest>; +using tdigest_failing = tdigest>; + +const uint16_t K = 100; + +double value(int i) { + // from 100000 on, values extend past both ends of the earlier ones (sources in the merge tests) + if (i >= 100000) return (i % 2 == 0 ? 1.0 : -1.0) * i; + // not monotonic, so that merges interleave + return static_cast((i * 7919) % 10007); +} + +tdigest_failing make_tdigest(int first, int count, allocation_control& control) { + tdigest_failing td(K, failing_allocator(control)); + for (int i = first; i < first + count; ++i) td.update(value(i)); + return td; +} + +// the complete state, including the buffered values +std::vector state(const tdigest_failing& td) { + const auto bytes = td.serialize(0, true); + return std::vector(bytes.begin(), bytes.end()); +} + +// the t-Digest must still work: query it, update it, merge into it, and round-trip it +void check_usable(tdigest_failing& td, allocation_control& control) { + control.disarm(); + const uint64_t weight = td.get_total_weight(); + td.update(-1); + REQUIRE(td.get_total_weight() == weight + 1); + REQUIRE(td.get_min_value() == -1); + REQUIRE(td.get_rank(-1) <= 0.5); + td.merge(make_tdigest(0, 500, control)); + REQUIRE(td.get_total_weight() == weight + 501); + const auto bytes = td.serialize(0, true); + const auto copy = tdigest_failing::deserialize(bytes.data(), bytes.size(), failing_allocator(control)); + REQUIRE(copy.get_total_weight() == td.get_total_weight()); +} + +} // namespace + +TEST_CASE("tdigest allocation: reserve allocates the full storage up front", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + tdigest_test_alloc td(K, test_allocator(0)); + const long long empty_bytes = test_allocator_total_bytes; + + td.reserve(); + const long long reserved_bytes = test_allocator_total_bytes; + REQUIRE(reserved_bytes > empty_bytes); + + td.reserve(); // already allocated + REQUIRE(test_allocator_total_bytes == reserved_bytes); + + // compression allocates a temporary vector, but the storage never grows + for (int i = 0; i < 100000; ++i) td.update(value(i)); + REQUIRE(test_allocator_total_bytes == reserved_bytes); + REQUIRE(td.get_total_weight() == 100000); + + // the same results as without reserve() + tdigest_test_alloc unreserved(K, test_allocator(0)); + for (int i = 0; i < 100000; ++i) unreserved.update(value(i)); + REQUIRE(td.serialize(0, true) == unreserved.serialize(0, true)); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("tdigest allocation: reserve after updates does not change the contents", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + tdigest_test_alloc td(K, test_allocator(0)); + for (int i = 0; i < 1000; ++i) td.update(value(i)); + const auto bytes = td.serialize(0, true); + td.reserve(); + REQUIRE(td.serialize(0, true) == bytes); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("tdigest allocation: failed reserve", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + for (long fail_at: {0, 1}) { // centroids, then the buffer + tdigest_failing td = make_tdigest(0, 10, control); + const auto before = state(td); + control.arm(fail_at); + REQUIRE_THROWS_AS(td.reserve(), std::bad_alloc); + REQUIRE(state(td) == before); + check_usable(td, control); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("tdigest allocation: failed first update leaves the t-Digest unchanged", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + tdigest_failing td(K, failing_allocator(control)); + const auto before = state(td); + control.arm(0); + REQUIRE_THROWS_AS(td.update(1), std::bad_alloc); + REQUIRE(td.is_empty()); + REQUIRE(state(td) == before); + + control.disarm(); + td.update(2); + REQUIRE(td.get_total_weight() == 1); + REQUIRE(td.get_min_value() == 2); // nothing left over from the failed update + REQUIRE(td.get_max_value() == 2); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +// Fail each allocation made by a stream of updates in turn. The update that fails must leave the +// t-Digest exactly as the updates before it left it, so it must equal one built from those updates. +TEST_CASE("tdigest allocation: failure at each allocation of the updates", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + const int n = 5000; + for (long fail_at = 0; ; ++fail_at) { + tdigest_failing td(K, failing_allocator(control)); + control.arm(fail_at); + int updated = 0; + try { + for (; updated < n; ++updated) td.update(value(updated)); + control.disarm(); + REQUIRE(fail_at > 0); + break; + } catch (const std::bad_alloc&) { + control.disarm(); + CAPTURE(fail_at, updated); + REQUIRE(state(td) == state(make_tdigest(0, updated, control))); + check_usable(td, control); + } + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +// Fail each allocation of target.merge(source) in turn: a failed merge leaves both unchanged. +TEST_CASE("tdigest allocation: failure at each allocation of a merge", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + long failures_tested = 0; + for (int target_n: {0, 5, 1000, 20000}) { + for (int source_n: {5, 1000, 20000}) { + CAPTURE(target_n, source_n); + const tdigest_failing source = make_tdigest(100000, source_n, control); + const auto source_before = state(source); + for (long fail_at = 0; ; ++fail_at) { + tdigest_failing target = make_tdigest(0, target_n, control); + const auto target_before = state(target); + control.arm(fail_at); + try { + target.merge(source); + control.disarm(); + REQUIRE(target.get_total_weight() == static_cast(target_n + source_n)); + break; + } catch (const std::bad_alloc&) { + ++failures_tested; + REQUIRE(state(target) == target_before); + REQUIRE(state(source) == source_before); + check_usable(target, control); + } + } + } + } + REQUIRE(failures_tested > 0); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +} /* namespace datasketches */ diff --git a/tdigest/test/tdigest_custom_allocator_test.cpp b/tdigest/test/tdigest_custom_allocator_test.cpp index b0687d68..06c5e53b 100644 --- a/tdigest/test/tdigest_custom_allocator_test.cpp +++ b/tdigest/test/tdigest_custom_allocator_test.cpp @@ -40,4 +40,33 @@ TEST_CASE("tdigest custom allocator", "[tdigest]") { REQUIRE(test_allocator_net_allocations == 0); } +TEST_CASE("tdigest custom allocator: buffers grow as they fill", "[tdigest]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + tdigest_d empty(100, alloc_d(0)); + REQUIRE(test_allocator_total_bytes == 0); // nothing reserved up front + + tdigest_d small(100, alloc_d(0)); + for (int i = 0; i < 3; ++i) small.update(static_cast(i)); + // a few values take a small buffer, not room for 4 * (2k + fudge) values + REQUIRE(test_allocator_total_bytes <= static_cast(8 * sizeof(double))); + REQUIRE(small.get_total_weight() == 3); + REQUIRE(small.get_quantile(0.5) == 1.0); + + // a digest built through the growing buffers gives the same result as before + tdigest_d grown(100, alloc_d(0)); + tdigest reference(100); + for (int i = 0; i < 100000; ++i) { + grown.update(static_cast(i % 7919)); + reference.update(static_cast(i % 7919)); + } + for (double rank: {0.01, 0.25, 0.5, 0.75, 0.99}) { + REQUIRE(grown.get_quantile(rank) == reference.get_quantile(rank)); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + } /* namespace datasketches */