diff --git a/kll/include/kll_sketch.hpp b/kll/include/kll_sketch.hpp index 274d5bec..467b0514 100644 --- a/kll/include/kll_sketch.hpp +++ b/kll/include/kll_sketch.hpp @@ -158,6 +158,13 @@ namespace kll_constants { *
  • Then qlo ≤ q ≤ qhi, with 99% confidence.
  • * * + *

    Memory allocation: a new sketch does not allocate storage for items until it first needs it, + * on the first update or the first merge of a non-empty sketch. Sketches that stay empty, or that + * are only ever merged into, use little memory. As a result, the first update or merge can throw + * std::bad_alloc while allocating this storage, rather than the constructor. If that happens, the + * sketch is unchanged and can still be used. Call reserve() right after construction to allocate + * the storage up front. + * * author Kevin Lang * author Alexander Saydakov * author Lee Rhodes @@ -183,7 +190,8 @@ class kll_sketch { using quantile_return_type = typename quantiles_sorted_view::quantile_return_type; /** - * Constructor + * Constructor. + * Storage for items is allocated on first use, see reserve(). * @param k affects the size of the sketch and its estimation error * @param comparator strict weak ordering function (see C++ named requirements: Compare) * @param allocator used by this sketch to allocate memory @@ -232,6 +240,7 @@ class kll_sketch { * Updates this sketch with the given data item. * If cross-language portability is required, callers should ensure that * the input string uses a compatible encoding (valid UTF-8). + * The first update allocates storage for items, unless reserve() or a merge already did. * @param item from a stream of items */ template @@ -242,11 +251,22 @@ class kll_sketch { * If sketches contain strings, callers are responsible for ensuring that * both sketches were built using compatible string encodings. * If other is this sketch, this throws std::invalid_argument. + * The first merge of a non-empty sketch allocates storage for items, + * unless reserve() or an update already did. * @param other sketch to merge into this one */ template void merge(FwdSk&& other); + /** + * Allocates storage for items now, rather than on the first update or merge. + * Call this right after construction if allocation failures should happen here, + * or if the first update should not pay for the allocation. + * Does nothing if the storage is already allocated. + * @throw std::bad_alloc if the allocation fails, in which case the sketch is unchanged + */ + void reserve(); + /** * Returns true if this sketch is empty. * @return empty flag @@ -522,6 +542,7 @@ class kll_sketch { /** * Gets the sorted view of this sketch + * An empty sketch returns an empty view without allocating item storage. * @return the sorted view of this sketch */ quantiles_sorted_view get_sorted_view() const; @@ -559,7 +580,7 @@ class kll_sketch { bool is_level_zero_sorted_; uint64_t n_; vector_u32 levels_; - T* items_; + T* items_; // nullptr until the sketch first receives items, see allocate_items() uint32_t items_size_; optional min_item_; optional max_item_; @@ -573,6 +594,13 @@ class kll_sketch { // common update code inline void update_min_max(const T& item); + // items_ is allocated lazily, and these are the only places that deal with it being nullptr: + // allocates items_ (items_size_ items) unless it is allocated already + inline void allocate_items(); + // copy-constructs the items of other, if other has any, into this newly constructed sketch + template void copy_items(const O& other); + // &items_[index], or nullptr without item storage, which makes every item range empty + inline T* item_ptr(uint32_t index) const; inline uint32_t internal_update(); // The following code is only valid in the special case of exactly reaching capacity while updating. diff --git a/kll/include/kll_sketch_impl.hpp b/kll/include/kll_sketch_impl.hpp index e1ed85ec..acd6f0df 100644 --- a/kll/include/kll_sketch_impl.hpp +++ b/kll/include/kll_sketch_impl.hpp @@ -55,7 +55,8 @@ sorted_view_(nullptr) + std::to_string(kll_constants::MAX_K) + ": " + std::to_string(k)); } levels_[0] = levels_[1] = k; - items_ = allocator_.allocate(items_size_); + // items_ is allocated by the first update or merge: many sketches (e.g. per-group aggregation states) + // stay empty or are only ever merged into } template @@ -75,8 +76,7 @@ min_item_(other.min_item_), max_item_(other.max_item_), sorted_view_(nullptr) { - items_ = allocator_.allocate(items_size_); - for (auto i = levels_[0]; i < levels_[num_levels_]; ++i) new (&items_[i]) T(other.items_[i]); + copy_items(other); } template @@ -171,8 +171,7 @@ sorted_view_(nullptr) std::is_constructible::value, "Type converting constructor requires new type to be constructible from existing type" ); - items_ = allocator_.allocate(items_size_); - for (auto i = levels_[0]; i < levels_[num_levels_]; ++i) new (&items_[i]) T(other.items_[i]); + copy_items(other); check_sorting(); } @@ -180,6 +179,7 @@ template template void kll_sketch::update(FwdT&& item) { if (!check_update_item(item)) { return; } + allocate_items(); // first, so that a failure leaves the sketch unchanged update_min_max(static_cast(item)); // min and max are always copies const uint32_t index = internal_update(); new (&items_[index]) T(std::forward(item)); @@ -197,8 +197,32 @@ void kll_sketch::update_min_max(const T& item) { } } +template +void kll_sketch::reserve() { + allocate_items(); +} + +template +void kll_sketch::allocate_items() { + if (items_ == nullptr) items_ = allocator_.allocate(items_size_); +} + +template +template +void kll_sketch::copy_items(const O& other) { + if (other.items_ == nullptr) return; + allocate_items(); + for (auto i = levels_[0]; i < levels_[num_levels_]; ++i) new (&items_[i]) T(other.items_[i]); +} + +template +T* kll_sketch::item_ptr(uint32_t index) const { + return items_ == nullptr ? nullptr : items_ + index; +} + template uint32_t kll_sketch::internal_update() { + allocate_items(); if (levels_[0] == 0) { compress_while_updating(); } n_++; is_level_zero_sorted_ = false; @@ -213,6 +237,7 @@ void kll_sketch::merge(FwdSk&& other) { if (m_ != other.m_) { throw std::invalid_argument("incompatible M: " + std::to_string(m_) + " and " + std::to_string(other.m_)); } + allocate_items(); // the higher levels are merged into items_ even if other's level zero is empty if (is_empty()) { min_item_.emplace(conditional_forward(*other.min_item_)); max_item_.emplace(conditional_forward(*other.max_item_)); @@ -757,7 +782,7 @@ void kll_sketch::add_empty_top_level_to_completely_full_sketch() { template void kll_sketch::sort_level_zero() { if (!is_level_zero_sorted_) { - std::sort(items_ + levels_[0], items_ + levels_[1], comparator_); + std::sort(item_ptr(levels_[0]), item_ptr(levels_[1]), comparator_); is_level_zero_sorted_ = true; } } @@ -766,8 +791,8 @@ template void kll_sketch::check_sorting() const { // not checking level 0 for (uint8_t level = 1; level < num_levels_; ++level) { - const auto from = items_ + levels_[level]; - const auto to = items_ + levels_[level + 1]; + const auto from = item_ptr(levels_[level]); + const auto to = item_ptr(levels_[level + 1]); if (!std::is_sorted(from, to, comparator_)) { throw std::logic_error("levels must be sorted"); } @@ -779,8 +804,8 @@ quantiles_sorted_view kll_sketch::get_sorted_view() const { const_cast(this)->sort_level_zero(); // allow this side effect quantiles_sorted_view view(get_num_retained(), comparator_, allocator_); for (uint8_t level = 0; level < num_levels_; ++level) { - const auto from = items_ + levels_[level]; - const auto to = items_ + levels_[level + 1]; // exclusive + const auto from = item_ptr(levels_[level]); + const auto to = item_ptr(levels_[level + 1]); // exclusive view.add(from, to, 1ULL << level); } view.convert_to_cummulative(); diff --git a/kll/test/CMakeLists.txt b/kll/test/CMakeLists.txt index 1e6f6f9f..72b57e62 100644 --- a/kll/test/CMakeLists.txt +++ b/kll/test/CMakeLists.txt @@ -39,6 +39,7 @@ target_sources(kll_test PRIVATE kll_sketch_test.cpp kll_sketch_custom_type_test.cpp + kll_sketch_allocation_test.cpp kll_sketch_validation.cpp kolmogorov_smirnov_test.cpp ) diff --git a/kll/test/kll_sketch_allocation_test.cpp b/kll/test/kll_sketch_allocation_test.cpp new file mode 100644 index 00000000..a098b5a9 --- /dev/null +++ b/kll/test/kll_sketch_allocation_test.cpp @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include +#include + +namespace datasketches { + +// Counts allocations, and throws std::bad_alloc on the allocation numbered fail_at (counting from 0). +struct allocation_control { + long count = 0; + long fail_at = -1; + void arm(long at) { count = 0; fail_at = at; } + void disarm() { fail_at = -1; } +}; + +template +struct failing_allocator: test_allocator { + allocation_control* control; + template struct rebind { using other = failing_allocator; }; + explicit failing_allocator(allocation_control& c): test_allocator(0), control(&c) {} + template + failing_allocator(const failing_allocator& other): test_allocator(0), control(other.control) {} + T* allocate(std::size_t n, const T* hint = nullptr) { + if (control->fail_at >= 0 && control->count++ == control->fail_at) throw std::bad_alloc(); + return test_allocator::allocate(n, hint); + } +}; + +template +bool operator==(const failing_allocator& a, const failing_allocator& b) { return a.control == b.control; } +template +bool operator!=(const failing_allocator& a, const failing_allocator& b) { return !(a == b); } + +using kll_double_test_sketch = kll_sketch, test_allocator>; +using kll_string_failing_sketch = kll_sketch, failing_allocator>; + +static std::string item(int i) { + // long enough to be allocated outside the string object + return std::string(32, 'x') + std::to_string(i); +} + +static void check_empty(const kll_string_failing_sketch& sketch, const kll_string_failing_sketch& fresh) { + REQUIRE(sketch.is_empty()); + REQUIRE(sketch.get_n() == 0); + REQUIRE(sketch.get_num_retained() == 0); + REQUIRE(sketch.begin() == sketch.end()); + REQUIRE(sketch.serialize() == fresh.serialize()); +} + +// the sketch must still work: query it, update it, and round-trip it +static void check_usable(kll_string_failing_sketch& sketch, allocation_control& control) { + control.disarm(); + const uint64_t n = sketch.get_n(); + sketch.update(item(-1)); + REQUIRE(sketch.get_n() == n + 1); + REQUIRE(sketch.get_min_item() <= item(-1)); + REQUIRE(sketch.get_quantile(0.5) >= sketch.get_min_item()); + const auto bytes = sketch.serialize(); + const auto copy = kll_string_failing_sketch::deserialize(bytes.data(), bytes.size(), serde(), + std::less(), failing_allocator(control)); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); +} + +TEST_CASE("kll sketch allocation: reserve allocates item storage up front", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + kll_double_test_sketch sketch(200, std::less(), test_allocator(0)); + const long long empty_bytes = test_allocator_total_bytes; + + sketch.reserve(); + const long long reserved_bytes = test_allocator_total_bytes; + REQUIRE(reserved_bytes == empty_bytes + static_cast(200 * sizeof(double))); + + sketch.reserve(); // already allocated + REQUIRE(test_allocator_total_bytes == reserved_bytes); + + for (int i = 0; i < 200; ++i) sketch.update(i); + REQUIRE(test_allocator_total_bytes == reserved_bytes); // the updates up to k allocate nothing + REQUIRE(sketch.get_n() == 200); + REQUIRE(sketch.get_min_item() == 0); + REQUIRE(sketch.get_max_item() == 199); + + kll_double_test_sketch target(200, std::less(), test_allocator(0)); + target.reserve(); + target.merge(sketch); + REQUIRE(target.get_n() == 200); + REQUIRE(target.get_min_item() == 0); + REQUIRE(target.get_max_item() == 199); + REQUIRE(target.get_quantile(0.5) == sketch.get_quantile(0.5)); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("kll sketch allocation: reserve after updates does nothing", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + kll_double_test_sketch sketch(8, std::less(), test_allocator(0)); + for (int i = 0; i < 1000; ++i) sketch.update(i); + const auto bytes = sketch.serialize(); + const long long allocated = test_allocator_total_bytes; + sketch.reserve(); + REQUIRE(test_allocator_total_bytes == allocated); + REQUIRE(sketch.serialize() == bytes); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("kll sketch allocation: failed reserve", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + const failing_allocator alloc(control); + const kll_string_failing_sketch fresh(200, std::less(), alloc); + kll_string_failing_sketch sketch(200, std::less(), alloc); + control.arm(0); + REQUIRE_THROWS_AS(sketch.reserve(), std::bad_alloc); + check_empty(sketch, fresh); + check_usable(sketch, control); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("kll sketch allocation: failed first update leaves the sketch unchanged", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + const failing_allocator alloc(control); + const kll_string_failing_sketch fresh(200, std::less(), alloc); + kll_string_failing_sketch sketch(200, std::less(), alloc); + control.arm(0); + REQUIRE_THROWS_AS(sketch.update(item(1)), std::bad_alloc); + check_empty(sketch, fresh); + + control.disarm(); + sketch.update(item(2)); + REQUIRE(sketch.get_n() == 1); + REQUIRE(sketch.get_min_item() == item(2)); // nothing left over from the failed update + REQUIRE(sketch.get_max_item() == item(2)); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("kll sketch allocation: failed first merge leaves both sketches unchanged", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + const failing_allocator alloc(control); + const kll_string_failing_sketch fresh(200, std::less(), alloc); + kll_string_failing_sketch source(200, std::less(), alloc); + for (int i = 0; i < 1000; ++i) source.update(item(i)); // estimation mode, several levels + const auto source_bytes = source.serialize(); + + kll_string_failing_sketch target(200, std::less(), alloc); + control.arm(0); + REQUIRE_THROWS_AS(target.merge(source), std::bad_alloc); + check_empty(target, fresh); + REQUIRE(source.serialize() == source_bytes); + + control.arm(0); + REQUIRE_THROWS_AS(target.merge(std::move(source)), std::bad_alloc); + check_empty(target, fresh); + REQUIRE(source.serialize() == source_bytes); // nothing was moved out + + control.disarm(); + target.merge(source); + REQUIRE(target.get_n() == source.get_n()); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +// Fail each allocation made by a stream of updates in turn: the sketch must stay valid and usable. +TEST_CASE("kll sketch allocation: failure at each allocation of the updates", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + const failing_allocator alloc(control); + const kll_string_failing_sketch fresh(8, std::less(), alloc); + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + kll_string_failing_sketch sketch(8, std::less(), alloc); + control.arm(fail_at); + try { + for (int i = 0; i < 2000; ++i) sketch.update(item(i)); + completed = true; + REQUIRE(fail_at > 0); + } catch (const std::bad_alloc&) { + if (fail_at == 0) check_empty(sketch, fresh); + check_usable(sketch, control); + } + control.disarm(); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +} /* namespace datasketches */ diff --git a/kll/test/kll_sketch_custom_type_test.cpp b/kll/test/kll_sketch_custom_type_test.cpp index f9d6f3cd..3161557c 100644 --- a/kll/test/kll_sketch_custom_type_test.cpp +++ b/kll/test/kll_sketch_custom_type_test.cpp @@ -153,4 +153,109 @@ TEST_CASE("kll sketch custom type", "[kll_sketch]") { REQUIRE(test_allocator_net_allocations == 0); } +TEST_CASE("kll sketch custom type: items are allocated on first use", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + kll_test_type_sketch sketch(200, test_type_less(), 0); + // only the levels: an empty sketch has no item storage + const long long empty_bytes = test_allocator_total_bytes; + REQUIRE(empty_bytes < static_cast(200 * sizeof(test_type))); + + kll_test_type_sketch copy(sketch); + kll_test_type_sketch converted(sketch, test_type_less(), alloc(0)); + REQUIRE(test_allocator_total_bytes == 3 * empty_bytes); + + kll_test_type_sketch target(200, test_type_less(), 0); + target.merge(sketch); // merging an empty sketch allocates nothing either + REQUIRE(test_allocator_total_bytes == 4 * empty_bytes); + REQUIRE(target.is_empty()); + + sketch.update(1); + REQUIRE(test_allocator_total_bytes == static_cast(4 * empty_bytes + 200 * sizeof(test_type))); + REQUIRE(sketch.get_n() == 1); + REQUIRE(sketch.get_min_item().get_value() == 1); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("kll sketch custom type: merge into a sketch without items", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + // estimation mode sizes, some of which leave level zero empty after a compaction + for (int n = 1; n < 2000; n += 37) { + kll_test_type_sketch source(8, test_type_less(), 0); + for (int i = 0; i < n; ++i) source.update(i); + kll_test_type_sketch target(8, test_type_less(), 0); + target.merge(source); + REQUIRE(target.get_n() == source.get_n()); + REQUIRE(target.get_min_item().get_value() == 0); + REQUIRE(target.get_max_item().get_value() == n - 1); + kll_test_type_sketch moved_target(8, test_type_less(), 0); + moved_target.merge(std::move(source)); + REQUIRE(moved_target.get_n() == target.get_n()); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +TEST_CASE("kll sketch custom type: empty sorted view", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + kll_test_type_sketch empty(200, test_type_less(), 0); + kll_test_type_sketch copy(empty); + kll_test_type_sketch converted(empty, test_type_less(), alloc(0)); + kll_test_type_sketch moved(std::move(copy)); + + kll_test_type_sketch assigned(200, test_type_less(), 0); + assigned.update(99); + REQUIRE(assigned.get_quantile(0).get_value() == 99); // populate the cached view + assigned = empty; + + kll_test_type_sketch move_assigned(200, test_type_less(), 0); + move_assigned.update(99); + REQUIRE(move_assigned.get_quantile(0).get_value() == 99); + move_assigned = kll_test_type_sketch(empty); + + const auto bytes = empty.serialize(0, test_type_serde()); + auto from_bytes = kll_test_type_sketch::deserialize(bytes.data(), bytes.size(), + test_type_serde(), test_type_less(), alloc(0)); + std::stringstream stream; + empty.serialize(stream, test_type_serde()); + auto from_stream = kll_test_type_sketch::deserialize(stream, + test_type_serde(), test_type_less(), alloc(0)); + + for (auto* sketch: {&empty, &converted, &moved, &assigned, &move_assigned, &from_bytes, &from_stream}) { + REQUIRE(sketch->is_empty()); + REQUIRE(sketch->begin() == sketch->end()); + const auto bytes_before = test_allocator_total_bytes; + const auto allocations_before = test_allocator_net_allocations; + for (int i = 0; i < 2; ++i) { + const auto view = sketch->get_sorted_view(); + REQUIRE(view.size() == 0); + REQUIRE(view.begin() == view.end()); + REQUIRE_THROWS_AS(view.get_quantile(0.5), std::runtime_error); + REQUIRE(test_allocator_total_bytes == bytes_before); + REQUIRE(test_allocator_net_allocations == allocations_before); + } + + // Reading an empty view must not interfere with allocation or sorting on the first updates. + sketch->update(2); + sketch->update(1); + REQUIRE(sketch->get_n() == 2); + const auto view = sketch->get_sorted_view(); + REQUIRE(view.size() == 2); + REQUIRE(view.get_quantile(0).get_value() == 1); + REQUIRE(view.get_quantile(1).get_value() == 2); + REQUIRE(sketch->get_quantile(0).get_value() == 1); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + } /* namespace datasketches */ diff --git a/kll/test/kll_sketch_test.cpp b/kll/test/kll_sketch_test.cpp index 05eea991..b40d729c 100644 --- a/kll/test/kll_sketch_test.cpp +++ b/kll/test/kll_sketch_test.cpp @@ -75,6 +75,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") { unused(pair); // to suppress "unused" warning FAIL("should be no iterations over an empty sketch"); } + const auto empty_bytes = test_allocator_total_bytes; + REQUIRE(sketch.get_sorted_view().size() == 0); + REQUIRE(test_allocator_total_bytes == empty_bytes); } SECTION("get bad quantile") { @@ -115,6 +118,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") { kll_float_sketch sketch(200, std::less(), 0); sketch.update(std::numeric_limits::quiet_NaN()); REQUIRE(sketch.is_empty()); + REQUIRE(sketch.get_sorted_view().size() == 0); sketch.update(0); sketch.update(std::numeric_limits::quiet_NaN());