diff --git a/kll/include/kll_helper_impl.hpp b/kll/include/kll_helper_impl.hpp index 7f958677..e8dc073e 100644 --- a/kll/include/kll_helper_impl.hpp +++ b/kll/include/kll_helper_impl.hpp @@ -39,8 +39,9 @@ uint8_t kll_helper::floor_of_log2_of_fraction(uint64_t numer, uint64_t denom) { if (denom > numer) { return 0; } uint8_t count = 0; while (true) { + // denom * 2 > numer, without overflowing denom + if (denom > (numer >> 1)) { return count; } denom <<= 1; - if (denom > numer) { return count; } count++; } } diff --git a/kll/include/kll_sketch.hpp b/kll/include/kll_sketch.hpp index 467b0514..f7011c74 100644 --- a/kll/include/kll_sketch.hpp +++ b/kll/include/kll_sketch.hpp @@ -39,6 +39,8 @@ namespace kll_constants { const uint16_t MIN_K = DEFAULT_M; /// max value of parameter K const uint16_t MAX_K = (1 << 16) - 1; + /// maximum weight of a weighted update, the same as in datasketches-java + const uint64_t MAX_WEIGHT = (1ULL << 61) - 1; } /** @@ -246,6 +248,20 @@ class kll_sketch { template void update(FwdT&& item); + /** + * Updates this sketch with the given data item repeated the given number of times. + * The result is equivalent to calling update(item) weight times, at a cost that grows + * with the logarithm of the weight rather than with the weight itself. + * If cross-language portability is required, callers should ensure that + * the input string uses a compatible encoding (valid UTF-8). + * The first update allocates storage for items, unless reserve() or a merge already did. + * @param item from a stream of items + * @param weight number of times the item is repeated, from 1 to kll_constants::MAX_WEIGHT (2^61 - 1) + * @throw std::invalid_argument if the weight is 0 or greater than kll_constants::MAX_WEIGHT + */ + template + void update(FwdT&& item, uint64_t weight); + /** * Merges another sketch into this one. * If sketches contain strings, callers are responsible for ensuring that @@ -592,6 +608,9 @@ class kll_sketch { std::unique_ptr items, uint32_t items_size, optional&& min_item, optional&& max_item, bool is_level_zero_sorted, const C& comparator); + // for weighted update + kll_sketch(uint16_t k, const T& item, uint64_t weight, const C& comparator, const A& allocator); + // common update code inline void update_min_max(const T& item); // items_ is allocated lazily, and these are the only places that deal with it being nullptr: diff --git a/kll/include/kll_sketch_impl.hpp b/kll/include/kll_sketch_impl.hpp index 6a7cdcbf..9639520a 100644 --- a/kll/include/kll_sketch_impl.hpp +++ b/kll/include/kll_sketch_impl.hpp @@ -193,6 +193,70 @@ void kll_sketch::update(FwdT&& item) { reset_sorted_view(); } +template +template +void kll_sketch::update(FwdT&& item, uint64_t weight) { + if (!check_update_item(item)) { return; } + if (weight == 0) throw std::invalid_argument("weight must be at least 1"); + if (weight > kll_constants::MAX_WEIGHT) { + throw std::invalid_argument("weight must be less than 2^61: " + std::to_string(weight)); + } + if (weight == 1) { + update(std::forward(item)); + return; + } + allocate_items(); // first, so that a failure leaves the sketch unchanged + reset_sorted_view(); // before any change, so that a failed copy can't leave a stale view + if (weight < levels_[0]) { // fits into level zero without compaction + update_min_max(static_cast(item)); + // construct each copy before counting it, so that a failed copy leaves a valid sketch + for (uint64_t i = 1; i < weight; ++i) { + new (&items_[levels_[0] - 1]) T(static_cast(item)); + internal_update(); + } + new (&items_[levels_[0] - 1]) T(std::forward(item)); + internal_update(); + } else { + merge(kll_sketch(k_, static_cast(item), weight, comparator_, allocator_)); + } +} + +// An item at level h carries weight 2^h, so one copy of the item at each level +// whose bit is set in the weight is an exact sketch of that many copies. +// The weight is less than 2^61, so this needs at most 61 levels. +template +kll_sketch::kll_sketch(uint16_t k, const T& item, uint64_t weight, const C& comparator, const A& allocator): +comparator_(comparator), +allocator_(allocator), +k_(k), +m_(kll_constants::DEFAULT_M), +min_k_(k), +num_levels_(static_cast(64 - count_leading_zeros_in_u64(weight))), +is_level_zero_sorted_(true), +n_(weight), +levels_(num_levels_ + 1, 0, allocator), +items_(nullptr), +items_size_(0), +min_item_(item), +max_item_(item), +sorted_view_(nullptr) +{ + for (uint8_t level = 0; level < num_levels_; ++level) { + levels_[level + 1] = levels_[level] + static_cast((weight >> level) & 1); + } + items_size_ = levels_[num_levels_]; + items_ = allocator_.allocate(items_size_); + // the destructor doesn't run if a constructor throws, so release the items here + uint32_t num_constructed = 0; + try { + for (; num_constructed < items_size_; ++num_constructed) new (&items_[num_constructed]) T(item); + } catch (...) { + for (uint32_t i = 0; i < num_constructed; ++i) items_[i].~T(); + allocator_.deallocate(items_, items_size_); + throw; + } +} + template void kll_sketch::update_min_max(const T& item) { if (is_empty()) { @@ -257,7 +321,7 @@ void kll_sketch::merge(FwdSk&& other) { const uint32_t index = internal_update(); new (&items_[index]) T(conditional_forward(other.items_[i])); } - if (other.num_levels_ >= 2) { merge_higher_levels(other, final_n); } + if (other.num_levels_ >= 2) { merge_higher_levels(std::forward(other), final_n); } n_ = final_n; if (other.is_estimation_mode()) { min_k_ = std::min(min_k_, other.min_k_); } assert_correct_total_weight(); @@ -1097,7 +1161,13 @@ void kll_sketch::reset_sorted_view() { template kll_sketch::const_iterator::const_iterator(const T* items, const uint32_t* levels, const uint8_t num_levels): items(items), levels(levels), num_levels(num_levels), index(items == nullptr ? levels[num_levels] : levels[0]), level(items == nullptr ? num_levels : 0), weight(1) -{} +{ + // start at the first non-empty level: level zero can be empty after a merge + while (level < num_levels && levels[level] == levels[level + 1]) { + ++level; + weight *= 2; + } +} template typename kll_sketch::const_iterator& kll_sketch::const_iterator::operator++() { diff --git a/kll/test/kll_sketch_allocation_test.cpp b/kll/test/kll_sketch_allocation_test.cpp index a098b5a9..a0ac1cf0 100644 --- a/kll/test/kll_sketch_allocation_test.cpp +++ b/kll/test/kll_sketch_allocation_test.cpp @@ -19,7 +19,9 @@ #include #include +#include #include +#include #include #include @@ -227,4 +229,115 @@ TEST_CASE("kll sketch allocation: failure at each allocation of the updates", "[ REQUIRE(test_allocator_net_allocations == 0); } +TEST_CASE("kll sketch allocation: failed first weighted update leaves the sketch unchanged", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + const failing_allocator alloc(control); + const kll_string_failing_sketch fresh(200, std::less(), alloc); + for (const uint64_t weight: {uint64_t(10), uint64_t(1000)}) { // copies into level zero, and a merge + kll_string_failing_sketch sketch(200, std::less(), alloc); + control.arm(0); + REQUIRE_THROWS_AS(sketch.update(item(1), weight), std::bad_alloc); + check_empty(sketch, fresh); + control.disarm(); + sketch.update(item(2), weight); + REQUIRE(sketch.get_n() == weight); + REQUIRE(sketch.get_min_item() == item(2)); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +// Fail each allocation made by a stream of weighted updates in turn: the sketch must stay valid and usable. +TEST_CASE("kll sketch allocation: failure at each allocation of the weighted updates", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + allocation_control control; + const failing_allocator alloc(control); + const kll_string_failing_sketch fresh(8, std::less(), alloc); + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + kll_string_failing_sketch sketch(8, std::less(), alloc); + control.arm(fail_at); + try { + for (int i = 0; i < 300; ++i) sketch.update(item(i), static_cast(i % 7) * 100 + 1); + completed = true; + REQUIRE(fail_at > 0); + } catch (const std::bad_alloc&) { + if (fail_at == 0) check_empty(sketch, fresh); + check_usable(sketch, control); + } + control.disarm(); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + +namespace { + +long copies_allowed = -1; // copies left before a copy throws; negative means no limit + +// An item whose copy constructor throws once copies_allowed runs out. +// Each item owns memory from test_allocator, so an item that is never destroyed shows up as a leak. +struct fragile_item { + int* value; + explicit fragile_item(int v): value(test_allocator(0).allocate(1)) { *value = v; } + fragile_item(const fragile_item& other): value(nullptr) { + if (copies_allowed == 0) throw std::runtime_error("copy failed"); + if (copies_allowed > 0) --copies_allowed; + value = test_allocator(0).allocate(1); + *value = *other.value; + } + fragile_item(fragile_item&& other) noexcept: value(other.value) { other.value = nullptr; } + fragile_item& operator=(fragile_item other) noexcept { std::swap(value, other.value); return *this; } + ~fragile_item() { if (value != nullptr) test_allocator(0).deallocate(value, 1); } +}; + +struct fragile_less { + bool operator()(const fragile_item& a, const fragile_item& b) const { return *a.value < *b.value; } +}; + +using kll_fragile_sketch = kll_sketch>; + +} // namespace + +// Fail each item copy made by a weighted update in turn: nothing may leak, +// and the sketch must stay usable, with no stale sorted view. +TEST_CASE("kll sketch allocation: failure at each item copy of a weighted update", "[kll_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + for (const uint64_t weight: {uint64_t(10), uint64_t(1000)}) { // copies into level zero, and a merge + CAPTURE(weight); + bool completed = false; + for (long fail_at = 0; !completed; ++fail_at) { + CAPTURE(fail_at); + kll_fragile_sketch sketch(200, fragile_less(), test_allocator(0)); + for (int i = 0; i < 5; ++i) sketch.update(fragile_item(i)); + const fragile_item probe(2); + const double rank_before = sketch.get_rank(probe); // caches a sorted view + REQUIRE(rank_before > 0); + copies_allowed = fail_at; + try { + sketch.update(fragile_item(100), weight); + completed = true; + } catch (const std::runtime_error&) {} + copies_allowed = -1; + REQUIRE(sketch.get_n() >= 5); + REQUIRE(sketch.get_n() <= 5 + weight); + // a copy of the sketch builds its sorted view from the current items + const kll_fragile_sketch copy(sketch); + REQUIRE(sketch.get_rank(probe) == copy.get_rank(probe)); + sketch.update(fragile_item(-1)); + REQUIRE(*sketch.get_min_item().value == -1); + } + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + } /* namespace datasketches */ diff --git a/kll/test/kll_sketch_test.cpp b/kll/test/kll_sketch_test.cpp index 6f044419..8fc63499 100644 --- a/kll/test/kll_sketch_test.cpp +++ b/kll/test/kll_sketch_test.cpp @@ -248,10 +248,13 @@ TEST_CASE("kll sketch", "[kll_sketch]") { SECTION("stream serialize deserialize empty") { kll_float_sketch sketch(200, std::less(), 0); + const long long empty_bytes = test_allocator_total_bytes; std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); sketch.serialize(s); REQUIRE(static_cast(s.tellp()) == sketch.get_serialized_size_bytes()); + const long long bytes_before = test_allocator_total_bytes; auto sketch2 = kll_float_sketch::deserialize(s, serde(), std::less(), 0); + REQUIRE(test_allocator_total_bytes - bytes_before == empty_bytes); // no item storage, like a new sketch REQUIRE(static_cast(s.tellp()) == sketch2.get_serialized_size_bytes()); REQUIRE(s.tellg() == s.tellp()); REQUIRE(sketch2.is_empty() == sketch.is_empty()); @@ -266,8 +269,11 @@ TEST_CASE("kll sketch", "[kll_sketch]") { SECTION("bytes serialize deserialize empty") { kll_float_sketch sketch(200, std::less(), 0); + const long long empty_bytes = test_allocator_total_bytes; auto bytes = sketch.serialize(); + const long long bytes_before = test_allocator_total_bytes; auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde(), std::less(), 0); + REQUIRE(test_allocator_total_bytes - bytes_before == empty_bytes); // no item storage, like a new sketch REQUIRE(bytes.size() == sketch.get_serialized_size_bytes()); REQUIRE(sketch2.is_empty() == sketch.is_empty()); REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode()); @@ -447,6 +453,9 @@ TEST_CASE("kll sketch", "[kll_sketch]") { REQUIRE(kll_helper::floor_of_log2_of_fraction(6, 2) == 1); REQUIRE(kll_helper::floor_of_log2_of_fraction(7, 2) == 1); REQUIRE(kll_helper::floor_of_log2_of_fraction(8, 2) == 2); + REQUIRE(kll_helper::floor_of_log2_of_fraction(1ULL << 63, 1) == 63); + REQUIRE(kll_helper::floor_of_log2_of_fraction(std::numeric_limits::max(), 1) == 63); + REQUIRE(kll_helper::floor_of_log2_of_fraction(std::numeric_limits::max(), 3) == 62); } SECTION("out of order split points, float") { @@ -845,11 +854,182 @@ TEST_CASE("kll sketch", "[kll_sketch]") { REQUIRE(sb.get_n() == 3); } + SECTION("weighted update: zero weight") { + kll_float_sketch sketch(200, std::less(), 0); + REQUIRE_THROWS_AS(sketch.update(1.0f, 0), std::invalid_argument); + REQUIRE(sketch.is_empty()); + } + + SECTION("weighted update: NaN") { + kll_float_sketch sketch(200, std::less(), 0); + sketch.update(std::numeric_limits::quiet_NaN(), 1000); + REQUIRE(sketch.is_empty()); + } + + SECTION("weighted update: same as repeated updates in exact mode") { + kll_float_sketch weighted(200, std::less(), 0); + kll_float_sketch repeated(200, std::less(), 0); + for (int i = 1; i <= 15; ++i) { + weighted.update(static_cast(i), i); + for (int j = 0; j < i; ++j) repeated.update(static_cast(i)); + } + REQUIRE(weighted.get_n() == repeated.get_n()); + REQUIRE(weighted.get_num_retained() == repeated.get_num_retained()); + REQUIRE_FALSE(weighted.is_estimation_mode()); + for (int i = 0; i <= 16; ++i) { + REQUIRE(weighted.get_rank(static_cast(i)) == repeated.get_rank(static_cast(i))); + REQUIRE(weighted.get_rank(static_cast(i), false) == repeated.get_rank(static_cast(i), false)); + } + } + + SECTION("weighted update: one item with large weight") { + kll_float_sketch sketch(200, std::less(), 0); + const uint64_t weight = (1ULL << 40) + 12345; + sketch.update(7.0f, weight); + REQUIRE(sketch.get_n() == weight); + REQUIRE(sketch.get_num_retained() == 7); // number of bits set in the weight + REQUIRE(sketch.get_min_item() == 7.0f); + REQUIRE(sketch.get_max_item() == 7.0f); + REQUIRE(sketch.get_rank(7.0f) == 1); + REQUIRE(sketch.get_rank(7.0f, false) == 0); + REQUIRE(sketch.get_quantile(0.5) == 7.0f); + + uint64_t total_weight = 0; + for (auto pair: sketch) total_weight += pair.second; + REQUIRE(total_weight == weight); + } + + SECTION("weighted update: maximum weight") { + kll_float_sketch sketch(200, std::less(), 0); + sketch.update(1.0f); + sketch.update(2.0f, kll_constants::MAX_WEIGHT); + REQUIRE(sketch.get_n() == kll_constants::MAX_WEIGHT + 1); + REQUIRE(sketch.get_min_item() == 1.0f); + REQUIRE(sketch.get_max_item() == 2.0f); + REQUIRE(sketch.get_quantile(0.5) == 2.0f); + + uint64_t total_weight = 0; + for (auto pair: sketch) total_weight += pair.second; + REQUIRE(total_weight == sketch.get_n()); + + auto bytes = sketch.serialize(); + auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde(), std::less(), 0); + REQUIRE(sketch2.get_n() == sketch.get_n()); + REQUIRE(sketch2.get_num_retained() == sketch.get_num_retained()); + } + + SECTION("weighted update: weight above the maximum throws") { + kll_float_sketch sketch(200, std::less(), 0); + sketch.update(1.0f, 5); + const auto bytes = sketch.serialize(); + REQUIRE_THROWS_AS(sketch.update(2.0f, kll_constants::MAX_WEIGHT + 1), std::invalid_argument); + REQUIRE_THROWS_AS(sketch.update(2.0f, std::numeric_limits::max()), std::invalid_argument); + REQUIRE(sketch.serialize() == bytes); + } + + SECTION("weighted update: weight 1 is a plain update") { + kll_float_sketch weighted(8, std::less(), 0); + kll_float_sketch plain(8, std::less(), 0); + random_utils::random_bit.seed(7); + for (int i = 0; i < 1000; ++i) weighted.update(static_cast(i), 1); + random_utils::random_bit.seed(7); + for (int i = 0; i < 1000; ++i) plain.update(static_cast(i)); + REQUIRE(weighted.serialize() == plain.serialize()); + } + + SECTION("weighted update: first update of a new sketch, both paths") { + // small weight: copies into level zero of the storage that this update allocates + kll_float_sketch small(200, std::less(), 0); + small.update(3.0f, 10); + REQUIRE(small.get_n() == 10); + REQUIRE(small.get_num_retained() == 10); + REQUIRE(small.get_min_item() == 3.0f); + // large weight: merges an exact sketch of the item + kll_float_sketch large(200, std::less(), 0); + large.update(3.0f, 1000); + REQUIRE(large.get_n() == 1000); + REQUIRE(large.get_max_item() == 3.0f); + uint64_t total_weight = 0; + for (auto pair: large) total_weight += pair.second; + REQUIRE(total_weight == 1000); + } + + SECTION("weighted update: large weights into a full sketch") { + kll_float_sketch sketch(200, std::less(), 0); + for (int i = 0; i < 1000; ++i) sketch.update(static_cast(i)); + sketch.update(-1.0f, 1000); + sketch.update(2000.0f, 3000); + REQUIRE(sketch.get_n() == 5000); + REQUIRE(sketch.get_min_item() == -1.0f); + REQUIRE(sketch.get_max_item() == 2000.0f); + REQUIRE(sketch.get_rank(-1.0f) == Approx(0.2).margin(RANK_EPS_FOR_K_200)); + REQUIRE(sketch.get_rank(2000.0f, false) == Approx(0.4).margin(RANK_EPS_FOR_K_200)); + } + + SECTION("weighted update: estimation mode") { + kll_float_sketch sketch(200, std::less(), 0); + const int n = 10000; + std::vector weights(n); + uint64_t total = 0; + for (int i = 0; i < n; ++i) { + weights[i] = static_cast(i % 13) * 1000 + 1; + total += weights[i]; + sketch.update(static_cast(i), weights[i]); + } + REQUIRE(sketch.get_n() == total); + REQUIRE(sketch.is_estimation_mode()); + REQUIRE(sketch.get_min_item() == 0); + REQUIRE(sketch.get_max_item() == n - 1); + + uint64_t weight_below = 0; + for (int i = 0; i < n; ++i) { + const double true_rank = static_cast(weight_below) / total; + REQUIRE(sketch.get_rank(static_cast(i), false) == Approx(true_rank).margin(RANK_EPS_FOR_K_200)); + weight_below += weights[i]; + } + + auto bytes = sketch.serialize(); + auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), serde(), std::less(), 0); + REQUIRE(sketch2.get_n() == sketch.get_n()); + REQUIRE(sketch2.get_num_retained() == sketch.get_num_retained()); + REQUIRE(sketch2.get_quantile(0.5) == sketch.get_quantile(0.5)); + } + + SECTION("weighted update: strings") { + kll_string_sketch sketch(200, std::less(), 0); + sketch.update(std::string("b"), 3); + const std::string a("a"); + sketch.update(a, 1000); + sketch.update(std::string("c"), 1ULL << 20); + REQUIRE(sketch.get_n() == 3 + 1000 + (1ULL << 20)); + REQUIRE(sketch.get_min_item() == "a"); + REQUIRE(sketch.get_max_item() == "c"); + REQUIRE(sketch.get_rank("a") == Approx(1000.0 / sketch.get_n()).margin(RANK_EPS_FOR_K_200)); + } + // cleanup REQUIRE(test_allocator_total_bytes == 0); } +TEST_CASE("kll sketch: iterator weights when level zero is empty", "[kll_sketch]") { + // merges can leave level zero empty; the iterator must still give each item the weight of its level + bool found_empty_level_zero = false; + for (int trial = 0; trial < 50; ++trial) { + kll_float_sketch target(8, std::less(), 0); + for (int j = 0; j < 1 + trial % 5; ++j) { + kll_float_sketch sketch(8, std::less(), 0); + for (int i = 0; i < 10 + (trial * 37 + j * 11) % 300; ++i) sketch.update(static_cast(i)); + target.merge(sketch); + } + uint64_t total_weight = 0; + for (const auto pair: target) total_weight += pair.second; + REQUIRE(total_weight == target.get_n()); + if (target.to_string(true, false).find(" 0: 8, 0\n") != std::string::npos) found_empty_level_zero = true; + } + REQUIRE(found_empty_level_zero); +} + TEST_CASE("kll sketch: deserialize rejects inconsistent levels", "[kll_sketch]") { using vector_bytes = kll_sketch::vector_bytes; const size_t k_offset = 4;