diff --git a/common/include/quantiles_sorted_view.hpp b/common/include/quantiles_sorted_view.hpp index c3ab6af0..0f51b7d2 100755 --- a/common/include/quantiles_sorted_view.hpp +++ b/common/include/quantiles_sorted_view.hpp @@ -28,7 +28,9 @@ namespace datasketches { /** - * Sorted view for quantiles sketches (REQ, KLL and Quantiles) + * Sorted view for quantiles sketches (REQ, KLL and Quantiles). + * For non-arithmetic types, the view refers to the items of the sketch it came from, + * so it must not be used after that sketch is updated, merged, assigned to, moved or destroyed. */ template< typename T, @@ -52,6 +54,12 @@ class quantiles_sorted_view { /// @private void convert_to_cummulative(); + /// @private + void add_min(const T& min_item); + + /// @private + void convert_to_cummulative(const T& max_item); + class const_iterator; /** @@ -155,6 +163,7 @@ class quantiles_sorted_view { private: Comparator comparator_; uint64_t total_weight_; + bool min_added_; Container entries_; static inline const T& deref_helper(const T* t) { return *t; } diff --git a/common/include/quantiles_sorted_view_impl.hpp b/common/include/quantiles_sorted_view_impl.hpp index 0e02aaba..b8d29db2 100755 --- a/common/include/quantiles_sorted_view_impl.hpp +++ b/common/include/quantiles_sorted_view_impl.hpp @@ -30,9 +30,11 @@ template quantiles_sorted_view::quantiles_sorted_view(uint32_t num, const C& comparator, const A& allocator): comparator_(comparator), total_weight_(0), +min_added_(false), entries_(allocator) { - entries_.reserve(num); + if (num > 0) // the if exists to optimise for empty sketches + entries_.reserve(num + 2); // room for the min and max } template @@ -41,7 +43,7 @@ void quantiles_sorted_view::add(Iterator first, Iterator last, uint64_t if (first == last) return; const size_t size_before = entries_.size(); for (auto it = first; it != last; ++it) entries_.emplace_back(ref_helper(*it), weight); - if (size_before > 0) { + if (size_before > (min_added_ ? 1 : 0)) { // an added min is smaller than all items, so it needs no merge Container tmp(entries_.get_allocator()); tmp.reserve(entries_.capacity()); std::merge( @@ -61,6 +63,22 @@ void quantiles_sorted_view::convert_to_cummulative() { } } +template +void quantiles_sorted_view::add_min(const T& min_item) { + entries_.emplace_back(ref_helper(min_item), 1); + min_added_ = true; +} + +template +void quantiles_sorted_view::convert_to_cummulative(const T& max_item) { + if (min_added_) --entries_[1].second; + if (comparator_(deref_helper(entries_.back().first), max_item)) { + --entries_.back().second; + entries_.push_back(Entry(ref_helper(max_item), 1)); + } + convert_to_cummulative(); +} + template double quantiles_sorted_view::get_rank(const T& item, bool inclusive) const { if (entries_.empty()) throw std::runtime_error("operation is undefined for an empty sketch"); diff --git a/common/test/quantiles_sorted_view_test.cpp b/common/test/quantiles_sorted_view_test.cpp index 609408dd..27bc81f2 100644 --- a/common/test/quantiles_sorted_view_test.cpp +++ b/common/test/quantiles_sorted_view_test.cpp @@ -456,4 +456,132 @@ TEST_CASE("set 4", "sorted view") { REQUIRE(view.get_quantile(1, false) == 40); } +TEST_CASE("add empty items", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(4, std::less(), std::allocator()); + std::vector empty; + std::vector l0 {1, 3}; + std::vector l2 {2, 5}; + view.add(empty.begin(), empty.end(), 1); + view.add(l0.begin(), l0.end(), 1); + view.add(empty.begin(), empty.end(), 2); + view.add(l2.begin(), l2.end(), 4); + view.convert_to_cummulative(); + REQUIRE(view.size() == 4); + + const std::vector> expected {{1, 1}, {2, 5}, {3, 6}, {5, 10}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); +} + +TEST_CASE("include min and max", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(4, std::less(), std::allocator()); + std::vector l1 {2, 4, 6, 7}; + view.add_min(1); + view.add(l1.begin(), l1.end(), 2); + view.convert_to_cummulative(8); + REQUIRE(view.size() == 6); + + const std::vector> expected {{1, 1}, {2, 2}, {4, 4}, {6, 6}, {7, 7}, {8, 8}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); + + REQUIRE(view.get_quantile(0, true) == 1); + REQUIRE(view.get_quantile(0, false) == 1); + REQUIRE(view.get_quantile(1, true) == 8); + REQUIRE(view.get_quantile(1, false) == 8); + + REQUIRE(view.get_rank(1, true) == 0.125); + REQUIRE(view.get_rank(8, false) == 0.875); +} + +TEST_CASE("include min and max already retained", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(4, std::less(), std::allocator()); + std::vector l1 {2, 4, 6, 7}; + view.add(l1.begin(), l1.end(), 2); + view.convert_to_cummulative(7); + REQUIRE(view.size() == 4); + + const std::vector> expected {{2, 2}, {4, 4}, {6, 6}, {7, 8}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); +} + +TEST_CASE("include min and max with one retained item", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(1, std::less(), std::allocator()); + std::vector l2 {5}; + view.add_min(1); + view.add(l2.begin(), l2.end(), 4); + view.convert_to_cummulative(9); + REQUIRE(view.size() == 3); + + const std::vector> expected {{1, 1}, {5, 3}, {9, 4}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); + + REQUIRE(view.get_quantile(0) == 1); + REQUIRE(view.get_quantile(0.5) == 5); + REQUIRE(view.get_quantile(1) == 9); +} + +TEST_CASE("include only max", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(3, std::less(), std::allocator()); + std::vector l1 {2, 4}; + std::vector l0 {6}; + view.add(l1.begin(), l1.end(), 2); + view.add(l0.begin(), l0.end(), 1); + view.convert_to_cummulative(8); + REQUIRE(view.size() == 4); + + // 6 gives its only weight to the max + const std::vector> expected {{2, 2}, {4, 4}, {6, 4}, {8, 5}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); +} + +TEST_CASE("include only min", "sorted view") { + auto view = quantiles_sorted_view, std::allocator>(3, std::less(), std::allocator()); + std::vector l1 {4, 6}; + std::vector l0 {2}; + view.add_min(1); + view.add(l1.begin(), l1.end(), 2); + view.add(l0.begin(), l0.end(), 1); + view.convert_to_cummulative(6); + REQUIRE(view.size() == 4); + + // 2 gives its only weight to the min + const std::vector> expected {{1, 1}, {2, 1}, {4, 3}, {6, 5}}; + auto it = view.begin(); + for (const auto& entry: expected) { + REQUIRE(it->first == entry.first); + REQUIRE(it->second == entry.second); + ++it; + } + REQUIRE(it == view.end()); +} + } /* namespace datasketches */ diff --git a/kll/include/kll_sketch.hpp b/kll/include/kll_sketch.hpp index 467b0514..a9088940 100644 --- a/kll/include/kll_sketch.hpp +++ b/kll/include/kll_sketch.hpp @@ -329,6 +329,9 @@ class kll_sketch { * *

If the sketch is empty this throws std::runtime_error. * + *

For non-arithmetic types, the returned reference refers to an item of this sketch, so it + * must not be used after this sketch is updated, merged, assigned to, moved or destroyed. + * * @param rank of an item in the hypothetical sorted stream. * @param inclusive if true, the given rank is considered inclusive (includes weight of an item) * @@ -541,8 +544,10 @@ class kll_sketch { const_iterator end() const; /** - * Gets the sorted view of this sketch + * Gets the sorted view of this sketch. * An empty sketch returns an empty view without allocating item storage. + * For non-arithmetic types, the view refers to the items of this sketch, so it must not be used + * after this sketch is updated, merged, assigned to, moved or destroyed. * @return the sorted view of this sketch */ quantiles_sorted_view get_sorted_view() const; @@ -619,6 +624,7 @@ class kll_sketch { void assert_correct_total_weight() const; uint32_t safe_level_size(uint8_t level) const; uint32_t get_num_retained_above_level_zero() const; + bool is_min_retained() const; static void check_k(uint16_t k); static void check_m(uint8_t m); diff --git a/kll/include/kll_sketch_impl.hpp b/kll/include/kll_sketch_impl.hpp index 6a7cdcbf..bb87d157 100644 --- a/kll/include/kll_sketch_impl.hpp +++ b/kll/include/kll_sketch_impl.hpp @@ -810,12 +810,14 @@ template quantiles_sorted_view kll_sketch::get_sorted_view() const { const_cast(this)->sort_level_zero(); // allow this side effect quantiles_sorted_view view(get_num_retained(), comparator_, allocator_); + if (is_empty()) return view; + if (!is_min_retained()) view.add_min(*min_item_); for (uint8_t level = 0; level < num_levels_; ++level) { const auto from = item_ptr(levels_[level]); const auto to = item_ptr(levels_[level + 1]); // exclusive view.add(from, to, 1ULL << level); } - view.convert_to_cummulative(); + view.convert_to_cummulative(*max_item_); return view; } @@ -926,6 +928,19 @@ uint32_t kll_sketch::get_num_retained_above_level_zero() const { return levels_[num_levels_] - levels_[1]; } +template +bool kll_sketch::is_min_retained() const { + for (uint8_t level = 0; level < num_levels_; ++level) { + const bool level_is_nonempty = levels_[level] < levels_[level + 1]; + if (level_is_nonempty) { // the if exists to skip a potentially expensive comparator run + // min_item_ is the smallest item of the stream, so min_item_ <= item, + // and !(min_item_ < item) means that item == min_item_ + if (!comparator_(*min_item_, items_[levels_[level]])) return true; + } + } + return false; +} + template void kll_sketch::check_k(uint16_t k) { if (k < kll_constants::MIN_K) { diff --git a/kll/test/kll_sketch_test.cpp b/kll/test/kll_sketch_test.cpp index 6f044419..75bf779c 100644 --- a/kll/test/kll_sketch_test.cpp +++ b/kll/test/kll_sketch_test.cpp @@ -206,6 +206,96 @@ TEST_CASE("kll sketch", "[kll_sketch]") { REQUIRE(total_weight == sketch.get_n()); } + SECTION("estimation mode, exact min and max at ranks 0 and 1") { + const int n = 1000; + int num_min_dropped = 0; + int num_max_dropped = 0; + for (int t = 0; t < 1000; ++t) { + kll_float_sketch sketch(8, std::less(), 0); + // min and max go in first, so compaction often drops them + sketch.update(0.0f); + sketch.update(static_cast(n - 1)); + for (int i = 1; i < n - 1; ++i) sketch.update(static_cast(i)); + bool min_retained = false; + bool max_retained = false; + for (auto pair: sketch) { + if (pair.first == 0) min_retained = true; + if (pair.first == n - 1) max_retained = true; + } + REQUIRE(sketch.get_quantile(0) == 0); + REQUIRE(sketch.get_quantile(0, false) == 0); + REQUIRE(sketch.get_quantile(1) == n - 1); + REQUIRE(sketch.get_quantile(1, false) == n - 1); + if (!min_retained) { + ++num_min_dropped; + REQUIRE(sketch.get_rank(0.0f) == 1.0 / n); + } + if (!max_retained) { + ++num_max_dropped; + REQUIRE(sketch.get_rank(static_cast(n - 1), false) == (n - 1.0) / n); + } + } + REQUIRE(num_min_dropped > 0); + REQUIRE(num_max_dropped > 0); + } + + SECTION("estimation mode, exact min and max at ranks 0 and 1, strings") { + int num_dropped = 0; + for (int t = 0; t < 1000; ++t) { + kll_string_sketch sketch(8, std::less(), 0); + sketch.update(std::string("a")); + sketch.update(std::string("z")); + for (int i = 0; i < 1000; ++i) sketch.update("m" + std::to_string(i)); + bool min_retained = false; + bool max_retained = false; + for (auto pair: sketch) { + if (pair.first == "a") min_retained = true; + if (pair.first == "z") max_retained = true; + } + if (!min_retained || !max_retained) ++num_dropped; + REQUIRE(sketch.get_quantile(0) == "a"); + REQUIRE(sketch.get_quantile(0, false) == "a"); + REQUIRE(sketch.get_quantile(1) == "z"); + REQUIRE(sketch.get_quantile(1, false) == "z"); + } + REQUIRE(num_dropped > 0); + } + + SECTION("estimation mode, exact min and max at ranks 0 and 1 after deserialize and merge") { + const int n = 1000; + int num_dropped = 0; + for (int t = 0; t < 1000; ++t) { + kll_float_sketch sketch1(8, std::less(), 0); + kll_float_sketch sketch2(8, std::less(), 0); + // sketch1 gets the min first and sketch2 gets the max first, so compaction often drops them + sketch1.update(0.0f); + for (int i = 1; i < n; ++i) sketch1.update(static_cast(i)); + sketch2.update(static_cast(2 * n - 1)); + for (int i = n; i < 2 * n - 1; ++i) sketch2.update(static_cast(i)); + + auto bytes = sketch1.serialize(); + auto sketch = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde(), std::less(), 0); + REQUIRE(sketch.get_quantile(0) == 0); + REQUIRE(sketch.get_quantile(0, false) == 0); + REQUIRE(sketch.get_quantile(1) == n - 1); + REQUIRE(sketch.get_quantile(1, false) == n - 1); + + sketch.merge(sketch2); + bool min_retained = false; + bool max_retained = false; + for (auto pair: sketch) { + if (pair.first == 0) min_retained = true; + if (pair.first == 2 * n - 1) max_retained = true; + } + if (!min_retained || !max_retained) ++num_dropped; + REQUIRE(sketch.get_quantile(0) == 0); + REQUIRE(sketch.get_quantile(0, false) == 0); + REQUIRE(sketch.get_quantile(1) == 2 * n - 1); + REQUIRE(sketch.get_quantile(1, false) == 2 * n - 1); + } + REQUIRE(num_dropped > 0); + } + SECTION("consistency between get_rank and get_PMF/CDF") { kll_float_sketch sketch(200, std::less(), 0); const int n = 200;