Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 34 additions & 7 deletions count/include/count_min.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,18 @@ namespace datasketches {
* [1] - http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf
* The template type W is the type of the vector that contains the weights of the objects inserted into the sketch,
* not the type of the input items themselves.
*
* New sketches use serialization version 2, as specified in the Count-Min format specification in
* <a href="https://github.com/apache/datasketches-tck/blob/main/docs/count-min-format.md">datasketches-tck</a>.
* Version 2 hashes items with XXH64 and derives the row seeds from the seed, so its images give the same
* results in every language and with every C++ standard library. Other languages read only images with
* W = int64_t (signed 64-bit counters).
*
* Sketches deserialized from version 1 images keep the version 1 hashing, and serialize as version 1.
* Version 1 row seeds depend on the C++ standard library, so version 1 images are reliable only with the
* standard library that wrote them. Sketches of different versions can't be merged.
*
* Empty items (an empty string, or a size of 0) are ignored: an update does nothing, and every query returns 0.
* @author Charlie Dickens
*/
template <typename W,
Expand All @@ -45,13 +57,17 @@ class count_min_sketch{

/**
* Creates an instance of the sketch given parameters _num_hashes, _num_buckets and hash seed, `seed`.
* @param num_hashes number of hash functions in the sketch. Equivalently the number of rows in the array
* @param num_buckets number of buckets that hash functions map into. Equivalently the number of columns in the array
* The sketch uses serialization version 2.
* @param num_hashes number of hash functions in the sketch. Equivalently the number of rows in the array.
* It must be in [1, 127].
* @param num_buckets number of buckets that hash functions map into. Equivalently the number of columns in the array.
* It must be at least 3, and num_hashes * num_buckets must be less than 2^30.
* @param seed for hash function
* @param allocator to acquire and release memory
* @throw std::invalid_argument if a parameter is out of range
*
* The items inserted into the sketch can be arbitrary type, so long as they are hashable via murmurhash.
* Only update and estimate methods are added for uint64_t and string types.
* Items are hashed as bytes: the generic methods take a pointer and a size, and there are
* specific methods for 64-bit integers (8 little-endian bytes) and strings (their bytes, which should be UTF-8).
*/
count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());

Expand All @@ -60,6 +76,11 @@ class count_min_sketch{
*/
uint8_t get_num_hashes() const;

/**
* @return the serialization version: 2 for new sketches, or 1 for a sketch deserialized from a version 1 image
*/
uint8_t get_serial_version() const;

/**
* @return configured _num_buckets of this sketch
*/
Expand Down Expand Up @@ -100,7 +121,7 @@ class count_min_sketch{
* @param confidence the desired confidence with which estimates should be correct.
* For example, with 95% confidence, frequency estimates satisfy the `relative_error` guarantee.
* @return the number of hash functions that are required in
* order to achieve the specified confidence of the sketch.
* order to achieve the specified confidence of the sketch, from 1 to 127.
* confidence = 1 - delta, with delta denoting the sketch failure probability in the literature.
* [1] - Section 3 ``Data Structure'', page 6.
*/
Expand Down Expand Up @@ -377,6 +398,7 @@ class count_min_sketch{

private:
Allocator _allocator;
uint8_t _serial_version; // determines the row seeds and the hashing
uint8_t _num_hashes;
uint32_t _num_buckets;
std::vector<W, Allocator> _sketch_array; // the array stored by the sketch
Expand All @@ -388,17 +410,22 @@ class count_min_sketch{
static const uint8_t PREAMBLE_LONGS_SHORT = 2; // Empty -> need second byte for sketch parameters
static const uint8_t PREAMBLE_LONGS_FULL = 3; // Not empty -> need (at least) third byte for total weight.
static const uint8_t SERIAL_VERSION_1 = 1;
static const uint8_t SERIAL_VERSION_2 = 2;
static const uint8_t MAX_NUM_HASHES = 127; // stored as a signed byte in Java and Go
static const uint8_t FAMILY_ID = 18;
static const uint8_t NULL_8 = 0;
static const uint32_t NULL_32 = 0;

// used by deserialize to keep the hashing of a version 1 image
count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed, uint8_t serial_version, const Allocator& allocator);

/**
* Throws an error if the header is not valid.
* @param preamble_longs
* @param serial_version
* @param flags_byte
* @param family_id
*/
static void check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id, uint8_t flags_byte);
static void check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id);

/*
* Compute the hash locations for an input item
Expand Down
103 changes: 68 additions & 35 deletions count/include/count_min_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,33 @@

#include "MurmurHash3.h"
#include "count_min.hpp"
#include "xxhash64.h"
#include "memory_operations.hpp"

namespace datasketches {

template<typename W, typename A>
count_min_sketch<W,A>::count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed, const A& allocator):
count_min_sketch(num_hashes, num_buckets, seed, SERIAL_VERSION_2, allocator)
{}

template<typename W, typename A>
count_min_sketch<W,A>::count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed, uint8_t serial_version,
const A& allocator):
_allocator(allocator),
_serial_version(serial_version),
_num_hashes(num_hashes),
_num_buckets(num_buckets),
_sketch_array((static_cast<uint64_t>(num_hashes) * num_buckets < 1<<30) ? static_cast<size_t>(num_hashes) * num_buckets : 0, 0, _allocator),
_seed(seed),
_total_weight(0),
hash_seeds(_allocator) {
// version 1 images written by C++ may have up to 255 rows
const uint8_t max_num_hashes = serial_version == SERIAL_VERSION_1 ? UINT8_MAX : MAX_NUM_HASHES;
if (num_hashes < 1 || num_hashes > max_num_hashes) {
throw std::invalid_argument("The number of hashes must be in [1, " + std::to_string(max_num_hashes) + "], actual "
+ std::to_string(num_hashes));
}
if (num_buckets < 3) {
throw std::invalid_argument("Using fewer than 3 buckets incurs relative error greater than 1.");
}
Expand All @@ -51,15 +65,27 @@ hash_seeds(_allocator) {
"Try reducing either the number of buckets or the number of hash functions.");
}

std::default_random_engine rng(_seed);
std::uniform_int_distribution<uint64_t> extra_hash_seeds(0, std::numeric_limits<uint64_t>::max());
hash_seeds.reserve(num_hashes);

for (uint64_t i=0; i < num_hashes; ++i) {
hash_seeds.push_back(extra_hash_seeds(rng) + _seed); // Adds the global seed to all hash functions.
if (serial_version == SERIAL_VERSION_1) {
// legacy row seeds: these depend on the C++ standard library
std::default_random_engine rng(_seed);
std::uniform_int_distribution<uint64_t> extra_hash_seeds(0, std::numeric_limits<uint64_t>::max());
for (uint64_t i=0; i < num_hashes; ++i) {
hash_seeds.push_back(extra_hash_seeds(rng) + _seed); // Adds the global seed to all hash functions.
}
} else {
// rowSeed[i] = XXH64(LE64(i), seed = seed), see the Count-Min format specification in datasketches-tck
for (uint64_t i=0; i < num_hashes; ++i) {
hash_seeds.push_back(XXHash64::hash(&i, sizeof(i), _seed));
}
}
}

template<typename W, typename A>
uint8_t count_min_sketch<W,A>::get_serial_version() const {
return _serial_version;
}

template<typename W, typename A>
uint8_t count_min_sketch<W,A>::get_num_hashes() const {
return _num_hashes;
Expand Down Expand Up @@ -107,7 +133,7 @@ uint8_t count_min_sketch<W,A>::suggest_num_hashes(double confidence) {
if (confidence < 0. || confidence > 1.0) {
throw std::invalid_argument("Confidence must be between 0 and 1.0 (inclusive).");
}
return std::min<uint8_t>(ceil(log(1.0 / (1.0 - confidence))), UINT8_MAX);
return static_cast<uint8_t>(std::max(1.0, std::min(ceil(log(1.0 / (1.0 - confidence))), static_cast<double>(MAX_NUM_HASHES))));
}

template<typename W, typename A>
Expand All @@ -131,10 +157,14 @@ void count_min_sketch<W,A>::foreach_hash_location(const void* item, size_t size,

uint64_t hash_seed_index = 0;
for (const auto &it: hash_seeds) {
HashState hashes;
MurmurHash3_x64_128(item, size, it, hashes); // ? BEWARE OVERFLOW.
uint64_t hash = hashes.h1;
bucket_index = hash % _num_buckets;
if (_serial_version == SERIAL_VERSION_1) {
HashState hashes;
MurmurHash3_x64_128(item, size, it, hashes); // ? BEWARE OVERFLOW.
bucket_index = hashes.h1 % _num_buckets;
} else {
// the shift clears the top bit, so that languages with signed 64-bit integers get the same bucket
bucket_index = (XXHash64::hash(item, size, it) >> 1) % _num_buckets;
}
callback((hash_seed_index * _num_buckets) + bucket_index);
hash_seed_index += 1;
}
Expand All @@ -157,6 +187,7 @@ W count_min_sketch<W,A>::get_estimate(const void* item, size_t size) const {
/*
* Returns the estimated frequency of the item
*/
if (size == 0) { return 0; } // empty items are not inserted into the sketch
W estimate = std::numeric_limits<W>::max();
foreach_hash_location(item, size, [this, &estimate](uint64_t h) {
estimate = std::min(estimate, _sketch_array[h]);
Expand Down Expand Up @@ -186,6 +217,7 @@ void count_min_sketch<W,A>::update(const void* item, size_t size, W weight) {
* Gets the item's hash locations and then increments the sketch in those
* locations by the weight.
*/
if (size == 0) { return; } // empty items are ignored
_total_weight += weight >= 0 ? weight : -weight;
foreach_hash_location(item, size, [this, weight](uint64_t h) {
_sketch_array[h] += weight;
Expand All @@ -206,6 +238,7 @@ W count_min_sketch<W,A>::get_upper_bound(const std::string& item) const {

template<typename W, typename A>
W count_min_sketch<W,A>::get_upper_bound(const void* item, size_t size) const {
if (size == 0) { return 0; } // empty items are not inserted into the sketch
return static_cast<W>(get_estimate(item, size) + get_relative_error() * get_total_weight());
}

Expand All @@ -223,6 +256,7 @@ W count_min_sketch<W,A>::get_lower_bound(const std::string& item) const {

template<typename W, typename A>
W count_min_sketch<W,A>::get_lower_bound(const void* item, size_t size) const {
if (size == 0) { return 0; } // empty items are not inserted into the sketch
return get_estimate(item, size);
}

Expand All @@ -238,6 +272,12 @@ void count_min_sketch<W,A>::merge(const count_min_sketch &other_sketch) {
(get_num_buckets() == other_sketch.get_num_buckets()) &&
(get_seed() == other_sketch.get_seed());
if (!acceptable_config) { throw std::invalid_argument( "Incompatible sketch configuration." ); }
if (get_serial_version() != other_sketch.get_serial_version()) {
// the two versions hash items into different buckets
throw std::invalid_argument("Cannot merge sketches with different serial versions: "
+ std::to_string(get_serial_version()) + " and "
+ std::to_string(other_sketch.get_serial_version()));
}

// Merge step - iterate over the other vector and add the weights to this sketch
auto it = _sketch_array.begin(); // This is a std::vector iterator.
Expand Down Expand Up @@ -282,7 +322,7 @@ size_t count_min_sketch<W,A>::serialize_to(Sink&& sink) const {
// Long 0
//const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_SHORT : PREAMBLE_LONGS_FULL;
const uint8_t preamble_longs = PREAMBLE_LONGS_SHORT;
const uint8_t ser_ver = SERIAL_VERSION_1;
const uint8_t ser_ver = _serial_version;
const uint8_t family_id = FAMILY_ID;
const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
const uint32_t unused32 = NULL_32;
Expand Down Expand Up @@ -327,7 +367,7 @@ auto count_min_sketch<W,A>::deserialize(std::istream& is, uint64_t seed, const A
const auto flags_byte = read<uint8_t>(is);
read<uint32_t>(is); // 4 unused bytes

check_header_validity(preamble_longs, serial_version, family_id, flags_byte);
check_header_validity(preamble_longs, serial_version, family_id);

// Sketch parameters
const auto nbuckets = read<uint32_t>(is);
Expand All @@ -339,7 +379,7 @@ auto count_min_sketch<W,A>::deserialize(std::istream& is, uint64_t seed, const A
throw std::invalid_argument("Incompatible seed hashes: " + std::to_string(seed_hash) + ", "
+ std::to_string(compute_seed_hash(seed)));
}
count_min_sketch c(nhashes, nbuckets, seed, allocator);
count_min_sketch c(nhashes, nbuckets, seed, serial_version, allocator);
const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
if (is_empty == 1) { return c; } // sketch is empty, no need to read further.

Expand Down Expand Up @@ -389,7 +429,7 @@ auto count_min_sketch<W,A>::deserialize(const void* bytes, size_t size, uint64_t
ptr += copy_from_mem(ptr, flags_byte);
ptr += sizeof(uint32_t);

check_header_validity(preamble_longs, serial_version, family_id, flags_byte);
check_header_validity(preamble_longs, serial_version, family_id);

// Second 8 bytes are the sketch parameters with a final, unused byte.
uint32_t nbuckets;
Expand All @@ -404,7 +444,7 @@ auto count_min_sketch<W,A>::deserialize(const void* bytes, size_t size, uint64_t
throw std::invalid_argument("Incompatible seed hashes: " + std::to_string(seed_hash) + ", "
+ std::to_string(compute_seed_hash(seed)));
}
count_min_sketch c(nhashes, nbuckets, seed, allocator);
count_min_sketch c(nhashes, nbuckets, seed, serial_version, allocator);
const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
if (is_empty) { return c; } // sketch is empty, no need to read further.

Expand Down Expand Up @@ -451,27 +491,20 @@ string<A> count_min_sketch<W,A>::to_string() const {
}

template<typename W, typename A>
void count_min_sketch<W,A>::check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id, uint8_t flags_byte) {
const bool empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;

const uint8_t sw = (empty ? 1 : 0) + (2 * serial_version) + (4 * family_id) + (32 * (preamble_longs & 0x3F));
bool valid = true;

switch (sw) { // exhaustive list and description of all valid cases
case 138 : break; // !empty, ser_ver==1, family==18, preLongs=2;
case 139 : break; // empty, ser_ver==1, family==18, preLongs=2;
//case 170 : break; // !empty, ser_ver==1, family==18, preLongs=3;
default : // all other case values are invalid
valid = false;
void count_min_sketch<W,A>::check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id) {
// each field is checked on its own, so that no combination of wrong values can pass
if (preamble_longs != PREAMBLE_LONGS_SHORT) {
throw std::invalid_argument("Possible sketch corruption. Preamble longs mismatch: expected "
+ std::to_string(PREAMBLE_LONGS_SHORT) + ", actual " + std::to_string(preamble_longs));
}

if (!valid) {
std::ostringstream os;
os << "Possible sketch corruption. Inconsistent state: "
<< "preamble_longs = " << static_cast<uint32_t>(preamble_longs)
<< ", empty = " << (empty ? "true" : "false")
<< ", serialization_version = " << static_cast<uint32_t>(serial_version);
throw std::invalid_argument(os.str());
if (serial_version != SERIAL_VERSION_1 && serial_version != SERIAL_VERSION_2) {
throw std::invalid_argument("Possible sketch corruption. Serial version mismatch: expected "
+ std::to_string(SERIAL_VERSION_1) + " or " + std::to_string(SERIAL_VERSION_2)
+ ", actual " + std::to_string(serial_version));
}
if (family_id != FAMILY_ID) {
throw std::invalid_argument("Possible sketch corruption. Family ID mismatch: expected "
+ std::to_string(FAMILY_ID) + ", actual " + std::to_string(family_id));
}
}

Expand Down
15 changes: 15 additions & 0 deletions count/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,19 @@ target_sources(count_min_test
PRIVATE
count_min_test.cpp
count_min_allocation_test.cpp
count_min_v2_test.cpp
)

if (SERDE_COMPAT)
target_sources(count_min_test
PRIVATE
count_min_deserialize_from_java_test.cpp
)
endif()

if (GENERATE)
target_sources(count_min_test
PRIVATE
count_min_serialize_for_java.cpp
)
endif()
Loading
Loading