Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion common/include/quantiles_sorted_view.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@
namespace datasketches {

/**
* Sorted view for quantiles sketches (REQ, KLL and Quantiles)
* Sorted view for quantiles sketches (REQ, KLL and Quantiles).
* For non-arithmetic types, the view refers to the items of the sketch it came from,
* so it must not be used after that sketch is updated, merged, assigned to, moved or destroyed.
*/
template<
typename T,
Expand All @@ -52,6 +54,12 @@ class quantiles_sorted_view {
/// @private
void convert_to_cummulative();

/// @private
void add_min(const T& min_item);

/// @private
void convert_to_cummulative(const T& max_item);

class const_iterator;

/**
Expand Down Expand Up @@ -155,6 +163,7 @@ class quantiles_sorted_view {
private:
Comparator comparator_;
uint64_t total_weight_;
bool min_added_;
Container entries_;

static inline const T& deref_helper(const T* t) { return *t; }
Expand Down
22 changes: 20 additions & 2 deletions common/include/quantiles_sorted_view_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,11 @@ template<typename T, typename C, typename A>
quantiles_sorted_view<T, C, A>::quantiles_sorted_view(uint32_t num, const C& comparator, const A& allocator):
comparator_(comparator),
total_weight_(0),
min_added_(false),
entries_(allocator)
{
entries_.reserve(num);
if (num > 0) // the if exists to optimise for empty sketches
entries_.reserve(num + 2); // room for the min and max
}

template<typename T, typename C, typename A>
Expand All @@ -41,7 +43,7 @@ void quantiles_sorted_view<T, C, A>::add(Iterator first, Iterator last, uint64_t
if (first == last) return;
const size_t size_before = entries_.size();
for (auto it = first; it != last; ++it) entries_.emplace_back(ref_helper(*it), weight);
if (size_before > 0) {
if (size_before > (min_added_ ? 1 : 0)) { // an added min is smaller than all items, so it needs no merge
Container tmp(entries_.get_allocator());
tmp.reserve(entries_.capacity());
std::merge(
Expand All @@ -61,6 +63,22 @@ void quantiles_sorted_view<T, C, A>::convert_to_cummulative() {
}
}

template<typename T, typename C, typename A>
void quantiles_sorted_view<T, C, A>::add_min(const T& min_item) {
entries_.emplace_back(ref_helper(min_item), 1);
min_added_ = true;
}

template<typename T, typename C, typename A>
void quantiles_sorted_view<T, C, A>::convert_to_cummulative(const T& max_item) {
if (min_added_) --entries_[1].second;
if (comparator_(deref_helper(entries_.back().first), max_item)) {
--entries_.back().second;
entries_.push_back(Entry(ref_helper(max_item), 1));
}
convert_to_cummulative();
}

template<typename T, typename C, typename A>
double quantiles_sorted_view<T, C, A>::get_rank(const T& item, bool inclusive) const {
if (entries_.empty()) throw std::runtime_error("operation is undefined for an empty sketch");
Expand Down
128 changes: 128 additions & 0 deletions common/test/quantiles_sorted_view_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -456,4 +456,132 @@ TEST_CASE("set 4", "sorted view") {
REQUIRE(view.get_quantile(1, false) == 40);
}

TEST_CASE("add empty items", "sorted view") {
auto view = quantiles_sorted_view<float, std::less<float>, std::allocator<float>>(4, std::less<float>(), std::allocator<float>());
std::vector<float> empty;
std::vector<float> l0 {1, 3};
std::vector<float> l2 {2, 5};
view.add(empty.begin(), empty.end(), 1);
view.add(l0.begin(), l0.end(), 1);
view.add(empty.begin(), empty.end(), 2);
view.add(l2.begin(), l2.end(), 4);
view.convert_to_cummulative();
REQUIRE(view.size() == 4);

const std::vector<std::pair<float, uint64_t>> expected {{1, 1}, {2, 5}, {3, 6}, {5, 10}};
auto it = view.begin();
for (const auto& entry: expected) {
REQUIRE(it->first == entry.first);
REQUIRE(it->second == entry.second);
++it;
}
REQUIRE(it == view.end());
}

TEST_CASE("include min and max", "sorted view") {
auto view = quantiles_sorted_view<float, std::less<float>, std::allocator<float>>(4, std::less<float>(), std::allocator<float>());
std::vector<float> l1 {2, 4, 6, 7};
view.add_min(1);
view.add(l1.begin(), l1.end(), 2);
view.convert_to_cummulative(8);
REQUIRE(view.size() == 6);

const std::vector<std::pair<float, uint64_t>> expected {{1, 1}, {2, 2}, {4, 4}, {6, 6}, {7, 7}, {8, 8}};
auto it = view.begin();
for (const auto& entry: expected) {
REQUIRE(it->first == entry.first);
REQUIRE(it->second == entry.second);
++it;
}
REQUIRE(it == view.end());

REQUIRE(view.get_quantile(0, true) == 1);
REQUIRE(view.get_quantile(0, false) == 1);
REQUIRE(view.get_quantile(1, true) == 8);
REQUIRE(view.get_quantile(1, false) == 8);

REQUIRE(view.get_rank(1, true) == 0.125);
REQUIRE(view.get_rank(8, false) == 0.875);
}

TEST_CASE("include min and max already retained", "sorted view") {
auto view = quantiles_sorted_view<float, std::less<float>, std::allocator<float>>(4, std::less<float>(), std::allocator<float>());
std::vector<float> l1 {2, 4, 6, 7};
view.add(l1.begin(), l1.end(), 2);
view.convert_to_cummulative(7);
REQUIRE(view.size() == 4);

const std::vector<std::pair<float, uint64_t>> expected {{2, 2}, {4, 4}, {6, 6}, {7, 8}};
auto it = view.begin();
for (const auto& entry: expected) {
REQUIRE(it->first == entry.first);
REQUIRE(it->second == entry.second);
++it;
}
REQUIRE(it == view.end());
}

TEST_CASE("include min and max with one retained item", "sorted view") {
auto view = quantiles_sorted_view<float, std::less<float>, std::allocator<float>>(1, std::less<float>(), std::allocator<float>());
std::vector<float> l2 {5};
view.add_min(1);
view.add(l2.begin(), l2.end(), 4);
view.convert_to_cummulative(9);
REQUIRE(view.size() == 3);

const std::vector<std::pair<float, uint64_t>> expected {{1, 1}, {5, 3}, {9, 4}};
auto it = view.begin();
for (const auto& entry: expected) {
REQUIRE(it->first == entry.first);
REQUIRE(it->second == entry.second);
++it;
}
REQUIRE(it == view.end());

REQUIRE(view.get_quantile(0) == 1);
REQUIRE(view.get_quantile(0.5) == 5);
REQUIRE(view.get_quantile(1) == 9);
}

TEST_CASE("include only max", "sorted view") {
auto view = quantiles_sorted_view<float, std::less<float>, std::allocator<float>>(3, std::less<float>(), std::allocator<float>());
std::vector<float> l1 {2, 4};
std::vector<float> l0 {6};
view.add(l1.begin(), l1.end(), 2);
view.add(l0.begin(), l0.end(), 1);
view.convert_to_cummulative(8);
REQUIRE(view.size() == 4);

// 6 gives its only weight to the max
const std::vector<std::pair<float, uint64_t>> expected {{2, 2}, {4, 4}, {6, 4}, {8, 5}};
auto it = view.begin();
for (const auto& entry: expected) {
REQUIRE(it->first == entry.first);
REQUIRE(it->second == entry.second);
++it;
}
REQUIRE(it == view.end());
}

TEST_CASE("include only min", "sorted view") {
auto view = quantiles_sorted_view<float, std::less<float>, std::allocator<float>>(3, std::less<float>(), std::allocator<float>());
std::vector<float> l1 {4, 6};
std::vector<float> l0 {2};
view.add_min(1);
view.add(l1.begin(), l1.end(), 2);
view.add(l0.begin(), l0.end(), 1);
view.convert_to_cummulative(6);
REQUIRE(view.size() == 4);

// 2 gives its only weight to the min
const std::vector<std::pair<float, uint64_t>> expected {{1, 1}, {2, 1}, {4, 3}, {6, 5}};
auto it = view.begin();
for (const auto& entry: expected) {
REQUIRE(it->first == entry.first);
REQUIRE(it->second == entry.second);
++it;
}
REQUIRE(it == view.end());
}

} /* namespace datasketches */
8 changes: 7 additions & 1 deletion kll/include/kll_sketch.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,9 @@ class kll_sketch {
*
* <p>If the sketch is empty this throws std::runtime_error.
*
* <p>For non-arithmetic types, the returned reference refers to an item of this sketch, so it
* must not be used after this sketch is updated, merged, assigned to, moved or destroyed.
*
* @param rank of an item in the hypothetical sorted stream.
* @param inclusive if true, the given rank is considered inclusive (includes weight of an item)
*
Expand Down Expand Up @@ -541,8 +544,10 @@ class kll_sketch {
const_iterator end() const;

/**
* Gets the sorted view of this sketch
* Gets the sorted view of this sketch.
* An empty sketch returns an empty view without allocating item storage.
* For non-arithmetic types, the view refers to the items of this sketch, so it must not be used
* after this sketch is updated, merged, assigned to, moved or destroyed.
* @return the sorted view of this sketch
*/
quantiles_sorted_view<T, C, A> get_sorted_view() const;
Expand Down Expand Up @@ -619,6 +624,7 @@ class kll_sketch {
void assert_correct_total_weight() const;
uint32_t safe_level_size(uint8_t level) const;
uint32_t get_num_retained_above_level_zero() const;
bool is_min_retained() const;

static void check_k(uint16_t k);
static void check_m(uint8_t m);
Expand Down
17 changes: 16 additions & 1 deletion kll/include/kll_sketch_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -810,12 +810,14 @@ template<typename T, typename C, typename A>
quantiles_sorted_view<T, C, A> kll_sketch<T, C, A>::get_sorted_view() const {
const_cast<kll_sketch*>(this)->sort_level_zero(); // allow this side effect
quantiles_sorted_view<T, C, A> view(get_num_retained(), comparator_, allocator_);
if (is_empty()) return view;
if (!is_min_retained()) view.add_min(*min_item_);
for (uint8_t level = 0; level < num_levels_; ++level) {
const auto from = item_ptr(levels_[level]);
const auto to = item_ptr(levels_[level + 1]); // exclusive
view.add(from, to, 1ULL << level);
}
view.convert_to_cummulative();
view.convert_to_cummulative(*max_item_);
return view;
}

Expand Down Expand Up @@ -926,6 +928,19 @@ uint32_t kll_sketch<T, C, A>::get_num_retained_above_level_zero() const {
return levels_[num_levels_] - levels_[1];
}

template<typename T, typename C, typename A>
bool kll_sketch<T, C, A>::is_min_retained() const {
for (uint8_t level = 0; level < num_levels_; ++level) {
const bool level_is_nonempty = levels_[level] < levels_[level + 1];
if (level_is_nonempty) { // the if exists to skip a potentially expensive comparator run
// min_item_ is the smallest item of the stream, so min_item_ <= item,
// and !(min_item_ < item) means that item == min_item_
if (!comparator_(*min_item_, items_[levels_[level]])) return true;
}
}
return false;
}

template<typename T, typename C, typename A>
void kll_sketch<T, C, A>::check_k(uint16_t k) {
if (k < kll_constants::MIN_K) {
Expand Down
90 changes: 90 additions & 0 deletions kll/test/kll_sketch_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,96 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(total_weight == sketch.get_n());
}

SECTION("estimation mode, exact min and max at ranks 0 and 1") {
const int n = 1000;
int num_min_dropped = 0;
int num_max_dropped = 0;
for (int t = 0; t < 1000; ++t) {
kll_float_sketch sketch(8, std::less<float>(), 0);
// min and max go in first, so compaction often drops them
sketch.update(0.0f);
sketch.update(static_cast<float>(n - 1));
for (int i = 1; i < n - 1; ++i) sketch.update(static_cast<float>(i));
bool min_retained = false;
bool max_retained = false;
for (auto pair: sketch) {
if (pair.first == 0) min_retained = true;
if (pair.first == n - 1) max_retained = true;
}
REQUIRE(sketch.get_quantile(0) == 0);
REQUIRE(sketch.get_quantile(0, false) == 0);
REQUIRE(sketch.get_quantile(1) == n - 1);
REQUIRE(sketch.get_quantile(1, false) == n - 1);
if (!min_retained) {
++num_min_dropped;
REQUIRE(sketch.get_rank(0.0f) == 1.0 / n);
}
if (!max_retained) {
++num_max_dropped;
REQUIRE(sketch.get_rank(static_cast<float>(n - 1), false) == (n - 1.0) / n);
}
}
REQUIRE(num_min_dropped > 0);
REQUIRE(num_max_dropped > 0);
}

SECTION("estimation mode, exact min and max at ranks 0 and 1, strings") {
int num_dropped = 0;
for (int t = 0; t < 1000; ++t) {
kll_string_sketch sketch(8, std::less<std::string>(), 0);
sketch.update(std::string("a"));
sketch.update(std::string("z"));
for (int i = 0; i < 1000; ++i) sketch.update("m" + std::to_string(i));
bool min_retained = false;
bool max_retained = false;
for (auto pair: sketch) {
if (pair.first == "a") min_retained = true;
if (pair.first == "z") max_retained = true;
}
if (!min_retained || !max_retained) ++num_dropped;
REQUIRE(sketch.get_quantile(0) == "a");
REQUIRE(sketch.get_quantile(0, false) == "a");
REQUIRE(sketch.get_quantile(1) == "z");
REQUIRE(sketch.get_quantile(1, false) == "z");
}
REQUIRE(num_dropped > 0);
}

SECTION("estimation mode, exact min and max at ranks 0 and 1 after deserialize and merge") {
const int n = 1000;
int num_dropped = 0;
for (int t = 0; t < 1000; ++t) {
kll_float_sketch sketch1(8, std::less<float>(), 0);
kll_float_sketch sketch2(8, std::less<float>(), 0);
// sketch1 gets the min first and sketch2 gets the max first, so compaction often drops them
sketch1.update(0.0f);
for (int i = 1; i < n; ++i) sketch1.update(static_cast<float>(i));
sketch2.update(static_cast<float>(2 * n - 1));
for (int i = n; i < 2 * n - 1; ++i) sketch2.update(static_cast<float>(i));

auto bytes = sketch1.serialize();
auto sketch = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde<float>(), std::less<float>(), 0);
REQUIRE(sketch.get_quantile(0) == 0);
REQUIRE(sketch.get_quantile(0, false) == 0);
REQUIRE(sketch.get_quantile(1) == n - 1);
REQUIRE(sketch.get_quantile(1, false) == n - 1);

sketch.merge(sketch2);
bool min_retained = false;
bool max_retained = false;
for (auto pair: sketch) {
if (pair.first == 0) min_retained = true;
if (pair.first == 2 * n - 1) max_retained = true;
}
if (!min_retained || !max_retained) ++num_dropped;
REQUIRE(sketch.get_quantile(0) == 0);
REQUIRE(sketch.get_quantile(0, false) == 0);
REQUIRE(sketch.get_quantile(1) == 2 * n - 1);
REQUIRE(sketch.get_quantile(1, false) == 2 * n - 1);
}
REQUIRE(num_dropped > 0);
}

SECTION("consistency between get_rank and get_PMF/CDF") {
kll_float_sketch sketch(200, std::less<float>(), 0);
const int n = 200;
Expand Down
Loading