From 7121fb3a9d97747d1bf4e8a2123182fddafb78d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Stojko?= Date: Thu, 12 Nov 2026 13:36:05 +0000 Subject: [PATCH] Return the exact min and max at ranks 0 and 1 in KLL sketches kll_sketch keeps the exact min and max, but get_quantile(0) and get_quantile(1) read only the retained items, and compaction can drop the min and max from them. Since 6.0.0, datasketches-java adds them back in its sorted views (IncludeMinMax, apache/datasketches-java#547). Port that step for KLL. A missing min or max is added to the sorted view as an entry of weight 1, taken from its neighbor, so the total weight does not change. The levels are sorted, so get_sorted_view() finds out if the min is retained by comparing it with the first item of each level. If not, it adds the min with add_min() before the levels, so no entry has to move, and add() does not merge the first level into the lone min. convert_to_cummulative(max_item) takes the weight of the min from the next entry and appends a missing max. The view reserves room for both, but only for a non-empty sketch. The quantiles and REQ sketches do not change. For non-arithmetic types, the view and get_quantile() refer to the items of the sketch, now also to its min and max. Document that they must not be used after the sketch is updated, merged, assigned to, moved or destroyed. The new tests check the KLL quantiles at ranks 0 and 1 in estimation mode, for floats and strings, and after deserialize and merge, plus the view with the min, the max, both or neither missing, and with empty ranges. Co-authored-by: Isaac --- common/include/quantiles_sorted_view.hpp | 11 +- common/include/quantiles_sorted_view_impl.hpp | 22 ++- common/test/quantiles_sorted_view_test.cpp | 128 ++++++++++++++++++ kll/include/kll_sketch.hpp | 8 +- kll/include/kll_sketch_impl.hpp | 17 ++- kll/test/kll_sketch_test.cpp | 90 ++++++++++++ 6 files changed, 271 insertions(+), 5 deletions(-) diff --git a/common/include/quantiles_sorted_view.hpp b/common/include/quantiles_sorted_view.hpp index c3ab6af0b..0f51b7d21 100755 --- a/common/include/quantiles_sorted_view.hpp +++ b/common/include/quantiles_sorted_view.hpp @@ -28,7 +28,9 @@ namespace datasketches { /** - * Sorted view for quantiles sketches (REQ, KLL and Quantiles) + * Sorted view for quantiles sketches (REQ, KLL and Quantiles). + * For non-arithmetic types, the view refers to the items of the sketch it came from, + * so it must not be used after that sketch is updated, merged, assigned to, moved or destroyed. */ template< typename T, @@ -52,6 +54,12 @@ class quantiles_sorted_view { /// @private void convert_to_cummulative(); + /// @private + void add_min(const T& min_item); + + /// @private + void convert_to_cummulative(const T& max_item); + class const_iterator; /** @@ -155,6 +163,7 @@ class quantiles_sorted_view { private: Comparator comparator_; uint64_t total_weight_; + bool min_added_; Container entries_; static inline const T& deref_helper(const T* t) { return *t; } diff --git a/common/include/quantiles_sorted_view_impl.hpp b/common/include/quantiles_sorted_view_impl.hpp index 0e02aaba6..b8d29db20 100755 --- a/common/include/quantiles_sorted_view_impl.hpp +++ b/common/include/quantiles_sorted_view_impl.hpp @@ -30,9 +30,11 @@ template quantiles_sorted_view::quantiles_sorted_view(uint32_t num, const C& comparator, const A& allocator): comparator_(comparator), total_weight_(0), +min_added_(false), entries_(allocator) { - entries_.reserve(num); + if (num > 0) // the if exists to optimise for empty sketches + entries_.reserve(num + 2); // room for the min and max } template @@ -41,7 +43,7 @@ void quantiles_sorted_view::add(Iterator first, Iterator last, uint64_t if (first == last) return; const size_t size_before = entries_.size(); for (auto it = first; it != last; ++it) entries_.emplace_back(ref_helper(*it), weight); - if (size_before > 0) { + if (size_before > (min_added_ ? 1 : 0)) { // an added min is smaller than all items, so it needs no merge Container tmp(entries_.get_allocator()); tmp.reserve(entries_.capacity()); std::merge( @@ -61,6 +63,22 @@ void quantiles_sorted_view::convert_to_cummulative() { } } +template +void quantiles_sorted_view::add_min(const T& min_item) { + entries_.emplace_back(ref_helper(min_item), 1); + min_added_ = true; +} + +template +void quantiles_sorted_view::convert_to_cummulative(const T& max_item) { + if (min_added_) --entries_[1].second; + if (comparator_(deref_helper(entries_.back().first), max_item)) { + --entries_.back().second; + entries_.push_back(Entry(ref_helper(max_item), 1)); + } + convert_to_cummulative(); +} + template double quantiles_sorted_view::get_rank(const T& item, bool inclusive) const { if (entries_.empty()) throw std::runtime_error("operation is undefined for an empty sketch"); diff --git a/common/test/quantiles_sorted_view_test.cpp b/common/test/quantiles_sorted_view_test.cpp index 609408ddf..27bc81f2f 100644 --- a/common/test/quantiles_sorted_view_test.cpp +++ b/common/test/quantiles_sorted_view_test.cpp @@ -456,4 +456,132 @@ TEST_CASE("set 4", "sorted view") { REQUIRE(view.get_quantile(1, false) == 40); } +TEST_CASE("add empty items", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(4, std::less(), std::allocator()); + std::vector empty; + std::vector l0 {1, 3}; + std::vector l2 {2, 5}; + view.add(empty.begin(), empty.end(), 1); + view.add(l0.begin(), l0.end(), 1); + view.add(empty.begin(), empty.end(), 2); + view.add(l2.begin(), l2.end(), 4); + view.convert_to_cummulative(); + REQUIRE(view.size() == 4); + + const std::vector> expected {{1, 1}, {2, 5}, {3, 6}, {5, 10}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); +} + +TEST_CASE("include min and max", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(4, std::less(), std::allocator()); + std::vector l1 {2, 4, 6, 7}; + view.add_min(1); + view.add(l1.begin(), l1.end(), 2); + view.convert_to_cummulative(8); + REQUIRE(view.size() == 6); + + const std::vector> expected {{1, 1}, {2, 2}, {4, 4}, {6, 6}, {7, 7}, {8, 8}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); + + REQUIRE(view.get_quantile(0, true) == 1); + REQUIRE(view.get_quantile(0, false) == 1); + REQUIRE(view.get_quantile(1, true) == 8); + REQUIRE(view.get_quantile(1, false) == 8); + + REQUIRE(view.get_rank(1, true) == 0.125); + REQUIRE(view.get_rank(8, false) == 0.875); +} + +TEST_CASE("include min and max already retained", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(4, std::less(), std::allocator()); + std::vector l1 {2, 4, 6, 7}; + view.add(l1.begin(), l1.end(), 2); + view.convert_to_cummulative(7); + REQUIRE(view.size() == 4); + + const std::vector> expected {{2, 2}, {4, 4}, {6, 6}, {7, 8}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); +} + +TEST_CASE("include min and max with one retained item", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(1, std::less(), std::allocator()); + std::vector l2 {5}; + view.add_min(1); + view.add(l2.begin(), l2.end(), 4); + view.convert_to_cummulative(9); + REQUIRE(view.size() == 3); + + const std::vector> expected {{1, 1}, {5, 3}, {9, 4}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); + + REQUIRE(view.get_quantile(0) == 1); + REQUIRE(view.get_quantile(0.5) == 5); + REQUIRE(view.get_quantile(1) == 9); +} + +TEST_CASE("include only max", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(3, std::less(), std::allocator()); + std::vector l1 {2, 4}; + std::vector l0 {6}; + view.add(l1.begin(), l1.end(), 2); + view.add(l0.begin(), l0.end(), 1); + view.convert_to_cummulative(8); + REQUIRE(view.size() == 4); + + // 6 gives its only weight to the max + const std::vector> expected {{2, 2}, {4, 4}, {6, 4}, {8, 5}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); +} + +TEST_CASE("include only min", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(3, std::less(), std::allocator()); + std::vector l1 {4, 6}; + std::vector l0 {2}; + view.add_min(1); + view.add(l1.begin(), l1.end(), 2); + view.add(l0.begin(), l0.end(), 1); + view.convert_to_cummulative(6); + REQUIRE(view.size() == 4); + + // 2 gives its only weight to the min + const std::vector> expected {{1, 1}, {2, 1}, {4, 3}, {6, 5}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); +} + } /* namespace datasketches */ diff --git a/kll/include/kll_sketch.hpp b/kll/include/kll_sketch.hpp index 467b05144..a90889408 100644 --- a/kll/include/kll_sketch.hpp +++ b/kll/include/kll_sketch.hpp @@ -329,6 +329,9 @@ class kll_sketch { * *

If the sketch is empty this throws std::runtime_error. * + *

For non-arithmetic types, the returned reference refers to an item of this sketch, so it + * must not be used after this sketch is updated, merged, assigned to, moved or destroyed. + * * @param rank of an item in the hypothetical sorted stream. * @param inclusive if true, the given rank is considered inclusive (includes weight of an item) * @@ -541,8 +544,10 @@ class kll_sketch { const_iterator end() const; /** - * Gets the sorted view of this sketch + * Gets the sorted view of this sketch. * An empty sketch returns an empty view without allocating item storage. + * For non-arithmetic types, the view refers to the items of this sketch, so it must not be used + * after this sketch is updated, merged, assigned to, moved or destroyed. * @return the sorted view of this sketch */ quantiles_sorted_view get_sorted_view() const; @@ -619,6 +624,7 @@ class kll_sketch { void assert_correct_total_weight() const; uint32_t safe_level_size(uint8_t level) const; uint32_t get_num_retained_above_level_zero() const; + bool is_min_retained() const; static void check_k(uint16_t k); static void check_m(uint8_t m); diff --git a/kll/include/kll_sketch_impl.hpp b/kll/include/kll_sketch_impl.hpp index 6a7cdcbfe..bb87d1579 100644 --- a/kll/include/kll_sketch_impl.hpp +++ b/kll/include/kll_sketch_impl.hpp @@ -810,12 +810,14 @@ template quantiles_sorted_view kll_sketch::get_sorted_view() const { const_cast(this)->sort_level_zero(); // allow this side effect quantiles_sorted_view view(get_num_retained(), comparator_, allocator_); + if (is_empty()) return view; + if (!is_min_retained()) view.add_min(*min_item_); for (uint8_t level = 0; level < num_levels_; ++level) { const auto from = item_ptr(levels_[level]); const auto to = item_ptr(levels_[level + 1]); // exclusive view.add(from, to, 1ULL << level); } - view.convert_to_cummulative(); + view.convert_to_cummulative(*max_item_); return view; } @@ -926,6 +928,19 @@ uint32_t kll_sketch::get_num_retained_above_level_zero() const { return levels_[num_levels_] - levels_[1]; } +template +bool kll_sketch::is_min_retained() const { + for (uint8_t level = 0; level < num_levels_; ++level) { + const bool level_is_nonempty = levels_[level] < levels_[level + 1]; + if (level_is_nonempty) { // the if exists to skip a potentially expensive comparator run + // min_item_ is the smallest item of the stream, so min_item_ <= item, + // and !(min_item_ < item) means that item == min_item_ + if (!comparator_(*min_item_, items_[levels_[level]])) return true; + } + } + return false; +} + template void kll_sketch::check_k(uint16_t k) { if (k < kll_constants::MIN_K) { diff --git a/kll/test/kll_sketch_test.cpp b/kll/test/kll_sketch_test.cpp index 6f0444191..75bf779c0 100644 --- a/kll/test/kll_sketch_test.cpp +++ b/kll/test/kll_sketch_test.cpp @@ -206,6 +206,96 @@ TEST_CASE("kll sketch", "[kll_sketch]") { REQUIRE(total_weight == sketch.get_n()); } + SECTION("estimation mode, exact min and max at ranks 0 and 1") { + const int n = 1000; + int num_min_dropped = 0; + int num_max_dropped = 0; + for (int t = 0; t < 1000; ++t) { + kll_float_sketch sketch(8, std::less(), 0); + // min and max go in first, so compaction often drops them + sketch.update(0.0f); + sketch.update(static_cast(n - 1)); + for (int i = 1; i < n - 1; ++i) sketch.update(static_cast(i)); + bool min_retained = false; + bool max_retained = false; + for (auto pair: sketch) { + if (pair.first == 0) min_retained = true; + if (pair.first == n - 1) max_retained = true; + } + REQUIRE(sketch.get_quantile(0) == 0); + REQUIRE(sketch.get_quantile(0, false) == 0); + REQUIRE(sketch.get_quantile(1) == n - 1); + REQUIRE(sketch.get_quantile(1, false) == n - 1); + if (!min_retained) { + ++num_min_dropped; + REQUIRE(sketch.get_rank(0.0f) == 1.0 / n); + } + if (!max_retained) { + ++num_max_dropped; + REQUIRE(sketch.get_rank(static_cast(n - 1), false) == (n - 1.0) / n); + } + } + REQUIRE(num_min_dropped > 0); + REQUIRE(num_max_dropped > 0); + } + + SECTION("estimation mode, exact min and max at ranks 0 and 1, strings") { + int num_dropped = 0; + for (int t = 0; t < 1000; ++t) { + kll_string_sketch sketch(8, std::less(), 0); + sketch.update(std::string("a")); + sketch.update(std::string("z")); + for (int i = 0; i < 1000; ++i) sketch.update("m" + std::to_string(i)); + bool min_retained = false; + bool max_retained = false; + for (auto pair: sketch) { + if (pair.first == "a") min_retained = true; + if (pair.first == "z") max_retained = true; + } + if (!min_retained || !max_retained) ++num_dropped; + REQUIRE(sketch.get_quantile(0) == "a"); + REQUIRE(sketch.get_quantile(0, false) == "a"); + REQUIRE(sketch.get_quantile(1) == "z"); + REQUIRE(sketch.get_quantile(1, false) == "z"); + } + REQUIRE(num_dropped > 0); + } + + SECTION("estimation mode, exact min and max at ranks 0 and 1 after deserialize and merge") { + const int n = 1000; + int num_dropped = 0; + for (int t = 0; t < 1000; ++t) { + kll_float_sketch sketch1(8, std::less(), 0); + kll_float_sketch sketch2(8, std::less(), 0); + // sketch1 gets the min first and sketch2 gets the max first, so compaction often drops them + sketch1.update(0.0f); + for (int i = 1; i < n; ++i) sketch1.update(static_cast(i)); + sketch2.update(static_cast(2 * n - 1)); + for (int i = n; i < 2 * n - 1; ++i) sketch2.update(static_cast(i)); + + auto bytes = sketch1.serialize(); + auto sketch = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde(), std::less(), 0); + REQUIRE(sketch.get_quantile(0) == 0); + REQUIRE(sketch.get_quantile(0, false) == 0); + REQUIRE(sketch.get_quantile(1) == n - 1); + REQUIRE(sketch.get_quantile(1, false) == n - 1); + + sketch.merge(sketch2); + bool min_retained = false; + bool max_retained = false; + for (auto pair: sketch) { + if (pair.first == 0) min_retained = true; + if (pair.first == 2 * n - 1) max_retained = true; + } + if (!min_retained || !max_retained) ++num_dropped; + REQUIRE(sketch.get_quantile(0) == 0); + REQUIRE(sketch.get_quantile(0, false) == 0); + REQUIRE(sketch.get_quantile(1) == 2 * n - 1); + REQUIRE(sketch.get_quantile(1, false) == 2 * n - 1); + } + REQUIRE(num_dropped > 0); + } + SECTION("consistency between get_rank and get_PMF/CDF") { kll_float_sketch sketch(200, std::less(), 0); const int n = 200;