From aed46aa751ac9a8b6306fc029e4394a7b7df4f7d Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 12 Nov 2026 11:15:57 -0800 Subject: [PATCH] KLL: keep the sketch valid when a merge fails to allocate merge_higher_levels() moved the items of this sketch into a work buffer, freed the item storage, and only then allocated the new storage. If that allocation failed, items_ was left pointing at freed memory. The capacity after compression depends only on the level sizes, so a new helper, kll_helper::predict_compress(), computes it before any item moves. The merge now allocates the new item storage and reserves the levels first, and checks that general_compress() matches the prediction. The new test fails each allocation of a merge in turn, for several target and source sizes, and checks that the target stays usable and nothing leaks. Co-Authored-By: Claude Opus 5.5 --- kll/include/kll_helper.hpp | 10 ++ kll/include/kll_helper_impl.hpp | 35 ++++++ kll/include/kll_sketch_impl.hpp | 32 +++-- kll/test/CMakeLists.txt | 1 + kll/test/kll_sketch_merge_failure_test.cpp | 131 +++++++++++++++++++++ 5 files changed, 202 insertions(+), 7 deletions(-) create mode 100644 kll/test/kll_sketch_merge_failure_test.cpp diff --git a/kll/include/kll_helper.hpp b/kll/include/kll_helper.hpp index ff7a5fe09..e0d82c27c 100644 --- a/kll/include/kll_helper.hpp +++ b/kll/include/kll_helper.hpp @@ -89,6 +89,16 @@ class kll_helper { * sorted afterwards. * Level zero is not required to be sorted before, and may not be sorted afterwards. */ + /* + * Returns the result that general_compress() will have for the given level boundaries, + * computed from the level sizes alone, without touching any items. + * The number of levels and items after compression does not depend on the item values, + * so callers can allocate the storage for the result before they move any items. + * level_sizes must have room for the final number of levels plus one. + */ + static inline compress_result predict_compress(uint16_t k, uint8_t m, uint8_t num_levels_in, + const uint32_t* in_levels, uint32_t* level_sizes); + template static compress_result general_compress(uint16_t k, uint8_t m, uint8_t num_levels_in, T* items, uint32_t* in_levels, uint32_t* out_levels, bool is_level_zero_sorted); diff --git a/kll/include/kll_helper_impl.hpp b/kll/include/kll_helper_impl.hpp index 31534d9ab..7f958677d 100644 --- a/kll/include/kll_helper_impl.hpp +++ b/kll/include/kll_helper_impl.hpp @@ -185,6 +185,41 @@ void kll_helper::merge_sorted_arrays(const T* buf_a, uint32_t start_a, uint32_t if (a != lim_a || b != lim_b) throw std::logic_error("inconsistent state"); } +// Mirrors the level loop of general_compress() on the level sizes only. +kll_helper::compress_result kll_helper::predict_compress(uint16_t k, uint8_t m, uint8_t num_levels_in, + const uint32_t* in_levels, uint32_t* level_sizes) +{ + if (num_levels_in == 0) { throw std::invalid_argument("num_levels_in == 0"); } + for (uint8_t level = 0; level < num_levels_in; ++level) { + level_sizes[level] = in_levels[level + 1] - in_levels[level]; + } + uint8_t current_num_levels = num_levels_in; + uint32_t current_item_count = in_levels[num_levels_in] - in_levels[0]; + uint32_t target_item_count = compute_total_capacity(k, m, current_num_levels); + uint8_t current_level = 0; + while (true) { + if (current_level == (current_num_levels - 1)) level_sizes[current_level + 1] = 0; + const uint32_t raw_pop = level_sizes[current_level]; + if ((current_item_count >= target_item_count) && (raw_pop >= level_capacity(k, current_num_levels, current_level, m))) { + // compaction: half of the even part moves up a level, and the other half is discarded + const uint32_t half_adj_pop = (raw_pop - (raw_pop & 1)) / 2; + current_item_count -= half_adj_pop; + level_sizes[current_level + 1] += half_adj_pop; + if (current_level == (current_num_levels - 1)) { + current_num_levels++; + target_item_count += level_capacity(k, current_num_levels, 0, m); + } + } + if (current_level == (current_num_levels - 1)) break; + current_level++; + } + compress_result result; + result.final_num_levels = current_num_levels; + result.final_capacity = target_item_count; + result.final_num_items = current_item_count; + return result; +} + /* * Here is what we do for each level: * If it does not need to be compacted, then simply copy it over. diff --git a/kll/include/kll_sketch_impl.hpp b/kll/include/kll_sketch_impl.hpp index e1ed85ec3..abea93b4b 100644 --- a/kll/include/kll_sketch_impl.hpp +++ b/kll/include/kll_sketch_impl.hpp @@ -801,26 +801,44 @@ void kll_sketch::merge_higher_levels(O&& other, uint64_t final_n) { const uint8_t provisional_num_levels = std::max(num_levels_, other.num_levels_); + // Allocate everything the merge needs before any item is moved out of this sketch, + // so that an allocation failure leaves this sketch valid. + // The level sizes after compression depend only on the level sizes before it. + worklevels[0] = 0; + worklevels[1] = safe_level_size(0); // the level zero items of other were already inserted into this + for (uint8_t lvl = 1; lvl < provisional_num_levels; lvl++) { + worklevels[lvl + 1] = worklevels[lvl] + safe_level_size(lvl) + other.safe_level_size(lvl); + } + const kll_helper::compress_result predicted = kll_helper::predict_compress(k_, m_, provisional_num_levels, + worklevels.data(), outlevels.data()); + // ub can sometimes be much bigger + if (predicted.final_num_levels > ub) throw std::logic_error("merge error"); + auto new_items_deleter = [&predicted, &alloc](T* ptr) { alloc.deallocate(ptr, predicted.final_capacity); }; // holds no items + std::unique_ptr new_items( + predicted.final_capacity != items_size_ ? allocator_.allocate(predicted.final_capacity) : nullptr, new_items_deleter); + const size_t new_levels_size = predicted.final_num_levels + 1; + levels_.reserve(new_levels_size); + populate_work_arrays(std::forward(other), workbuf.get(), worklevels.data(), provisional_num_levels); const kll_helper::compress_result result = kll_helper::general_compress(k_, m_, provisional_num_levels, workbuf.get(), worklevels.data(), outlevels.data(), is_level_zero_sorted_); - - // ub can sometimes be much bigger - if (result.final_num_levels > ub) throw std::logic_error("merge error"); + if (result.final_num_levels != predicted.final_num_levels || result.final_capacity != predicted.final_capacity + || result.final_num_items != predicted.final_num_items) { + throw std::logic_error("merge error: compression does not match the prediction"); + } // now we need to transfer the results back into "this" sketch - if (result.final_capacity != items_size_) { + if (new_items) { allocator_.deallocate(items_, items_size_); items_size_ = result.final_capacity; - items_ = allocator_.allocate(items_size_); + items_ = new_items.release(); } const uint32_t free_space_at_bottom = result.final_capacity - result.final_num_items; kll_helper::move_construct(workbuf.get(), outlevels[0], outlevels[0] + result.final_num_items, items_, free_space_at_bottom, true); - const size_t new_levels_size = result.final_num_levels + 1; if (levels_.size() < new_levels_size) { - levels_.resize(new_levels_size); + levels_.resize(new_levels_size); // within the reserved capacity } const uint32_t offset = free_space_at_bottom - outlevels[0]; for (uint8_t lvl = 0; lvl < levels_.size(); lvl++) { // includes the "extra" index diff --git a/kll/test/CMakeLists.txt b/kll/test/CMakeLists.txt index 1e6f6f9f3..6fb7e7153 100644 --- a/kll/test/CMakeLists.txt +++ b/kll/test/CMakeLists.txt @@ -41,6 +41,7 @@ target_sources(kll_test kll_sketch_custom_type_test.cpp kll_sketch_validation.cpp kolmogorov_smirnov_test.cpp + kll_sketch_merge_failure_test.cpp ) if (SERDE_COMPAT) diff --git a/kll/test/kll_sketch_merge_failure_test.cpp b/kll/test/kll_sketch_merge_failure_test.cpp new file mode 100644 index 000000000..ba51d90dc --- /dev/null +++ b/kll/test/kll_sketch_merge_failure_test.cpp @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include +#include + +namespace datasketches { + +namespace { + +// Counts allocations, and throws std::bad_alloc on the allocation numbered fail_at (counting from 0). +struct allocation_control { + long count = 0; + long fail_at = -1; + void arm(long at) { count = 0; fail_at = at; } + void disarm() { fail_at = -1; } +}; + +template +struct failing_allocator: test_allocator { + allocation_control* control; + template struct rebind { using other = failing_allocator; }; + explicit failing_allocator(allocation_control& c): test_allocator(0), control(&c) {} + template + failing_allocator(const failing_allocator& other): test_allocator(0), control(other.control) {} + T* allocate(std::size_t n, const T* hint = nullptr) { + if (control->fail_at >= 0 && control->count++ == control->fail_at) throw std::bad_alloc(); + return test_allocator::allocate(n, hint); + } +}; + +template +bool operator==(const failing_allocator& a, const failing_allocator& b) { return a.control == b.control; } +template +bool operator!=(const failing_allocator& a, const failing_allocator& b) { return !(a == b); } + +using sketch_type = kll_sketch, failing_allocator>; + +std::string item(int i) { + // long enough to be allocated outside the string object + return std::string(32, 'x') + std::to_string(i); +} + +sketch_type make_sketch(uint16_t k, int first, int count, allocation_control& control) { + sketch_type sketch(k, std::less(), failing_allocator(control)); + for (int i = first; i < first + count; ++i) sketch.update(item(i)); + return sketch; +} + +// the sketch must still work: query it, update it, merge into it, and round-trip it +void check_usable(sketch_type& sketch, allocation_control& control) { + control.disarm(); + const uint64_t n = sketch.get_n(); + sketch.update(item(-1)); + REQUIRE(sketch.get_n() == n + 1); + REQUIRE(sketch.get_min_item() <= item(-1)); + REQUIRE(sketch.get_quantile(0.5) >= sketch.get_min_item()); + sketch.merge(make_sketch(sketch.get_k(), 0, 500, control)); + REQUIRE(sketch.get_n() == n + 501); + const auto bytes = sketch.serialize(); + const auto copy = sketch_type::deserialize(bytes.data(), bytes.size(), serde(), + std::less(), failing_allocator(control)); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); +} + +// Fails each allocation of target.merge(source) in turn, until the merge completes. +// Returns the number of allocation failures that were tested. +template +long check_merge_failures(uint16_t k, int target_n, int source_n) { + allocation_control control; + const sketch_type original_source = make_sketch(k, 100000, source_n, control); + const auto source_bytes = original_source.serialize(); + for (long fail_at = 0; ; ++fail_at) { + sketch_type target = make_sketch(k, 0, target_n, control); + sketch_type source(original_source); + const uint64_t target_n_before = target.get_n(); + control.arm(fail_at); + try { + if (move_source) target.merge(std::move(source)); else target.merge(source); + control.disarm(); + REQUIRE(target.get_n() == target_n_before + source.get_n()); + return fail_at; + } catch (const std::bad_alloc&) { + check_usable(target, control); + if (!move_source) REQUIRE(source.serialize() == source_bytes); + } + } +} + +} // namespace + +TEST_CASE("kll sketch: merge failure at each allocation", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + // target sizes: empty, exact mode, estimation mode with several levels + // source sizes: exact mode, estimation mode with fewer, the same and more levels than the target + long failures_tested = 0; + for (int target_n: {0, 5, 1000}) { + for (int source_n: {5, 100, 1000, 20000}) { + CAPTURE(target_n, source_n); + failures_tested += check_merge_failures(8, target_n, source_n); + failures_tested += check_merge_failures(8, target_n, source_n); + } + } + REQUIRE(failures_tested > 0); + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +} /* namespace datasketches */