diff --git a/count/include/count_min.hpp b/count/include/count_min.hpp
index f78dce1a..a319c0fd 100644
--- a/count/include/count_min.hpp
+++ b/count/include/count_min.hpp
@@ -32,6 +32,18 @@ namespace datasketches {
* [1] - http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf
* The template type W is the type of the vector that contains the weights of the objects inserted into the sketch,
* not the type of the input items themselves.
+ *
+ * New sketches use serialization version 2, as specified in the Count-Min format specification in
+ * datasketches-tck.
+ * Version 2 hashes items with XXH64 and derives the row seeds from the seed, so its images give the same
+ * results in every language and with every C++ standard library. Other languages read only images with
+ * W = int64_t (signed 64-bit counters).
+ *
+ * Sketches deserialized from version 1 images keep the version 1 hashing, and serialize as version 1.
+ * Version 1 row seeds depend on the C++ standard library, so version 1 images are reliable only with the
+ * standard library that wrote them. Sketches of different versions can't be merged.
+ *
+ * Empty items (an empty string, or a size of 0) are ignored: an update does nothing, and every query returns 0.
* @author Charlie Dickens
*/
template _sketch_array; // the array stored by the sketch
@@ -388,17 +410,22 @@ class count_min_sketch{
static const uint8_t PREAMBLE_LONGS_SHORT = 2; // Empty -> need second byte for sketch parameters
static const uint8_t PREAMBLE_LONGS_FULL = 3; // Not empty -> need (at least) third byte for total weight.
static const uint8_t SERIAL_VERSION_1 = 1;
+ static const uint8_t SERIAL_VERSION_2 = 2;
+ static const uint8_t MAX_NUM_HASHES = 127; // stored as a signed byte in Java and Go
static const uint8_t FAMILY_ID = 18;
static const uint8_t NULL_8 = 0;
static const uint32_t NULL_32 = 0;
+ // used by deserialize to keep the hashing of a version 1 image
+ count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed, uint8_t serial_version, const Allocator& allocator);
+
/**
* Throws an error if the header is not valid.
* @param preamble_longs
* @param serial_version
- * @param flags_byte
+ * @param family_id
*/
- static void check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id, uint8_t flags_byte);
+ static void check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id);
/*
* Compute the hash locations for an input item
diff --git a/count/include/count_min_impl.hpp b/count/include/count_min_impl.hpp
index c1c053e3..689f3df9 100644
--- a/count/include/count_min_impl.hpp
+++ b/count/include/count_min_impl.hpp
@@ -27,19 +27,33 @@
#include "MurmurHash3.h"
#include "count_min.hpp"
+#include "xxhash64.h"
#include "memory_operations.hpp"
namespace datasketches {
template
count_min_sketch::count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed, const A& allocator):
+count_min_sketch(num_hashes, num_buckets, seed, SERIAL_VERSION_2, allocator)
+{}
+
+template
+count_min_sketch::count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed, uint8_t serial_version,
+ const A& allocator):
_allocator(allocator),
+_serial_version(serial_version),
_num_hashes(num_hashes),
_num_buckets(num_buckets),
_sketch_array((static_cast(num_hashes) * num_buckets < 1<<30) ? static_cast(num_hashes) * num_buckets : 0, 0, _allocator),
_seed(seed),
_total_weight(0),
hash_seeds(_allocator) {
+ // version 1 images written by C++ may have up to 255 rows
+ const uint8_t max_num_hashes = serial_version == SERIAL_VERSION_1 ? UINT8_MAX : MAX_NUM_HASHES;
+ if (num_hashes < 1 || num_hashes > max_num_hashes) {
+ throw std::invalid_argument("The number of hashes must be in [1, " + std::to_string(max_num_hashes) + "], actual "
+ + std::to_string(num_hashes));
+ }
if (num_buckets < 3) {
throw std::invalid_argument("Using fewer than 3 buckets incurs relative error greater than 1.");
}
@@ -51,15 +65,27 @@ hash_seeds(_allocator) {
"Try reducing either the number of buckets or the number of hash functions.");
}
- std::default_random_engine rng(_seed);
- std::uniform_int_distribution extra_hash_seeds(0, std::numeric_limits::max());
hash_seeds.reserve(num_hashes);
-
- for (uint64_t i=0; i < num_hashes; ++i) {
- hash_seeds.push_back(extra_hash_seeds(rng) + _seed); // Adds the global seed to all hash functions.
+ if (serial_version == SERIAL_VERSION_1) {
+ // legacy row seeds: these depend on the C++ standard library
+ std::default_random_engine rng(_seed);
+ std::uniform_int_distribution extra_hash_seeds(0, std::numeric_limits::max());
+ for (uint64_t i=0; i < num_hashes; ++i) {
+ hash_seeds.push_back(extra_hash_seeds(rng) + _seed); // Adds the global seed to all hash functions.
+ }
+ } else {
+ // rowSeed[i] = XXH64(LE64(i), seed = seed), see the Count-Min format specification in datasketches-tck
+ for (uint64_t i=0; i < num_hashes; ++i) {
+ hash_seeds.push_back(XXHash64::hash(&i, sizeof(i), _seed));
+ }
}
}
+template
+uint8_t count_min_sketch::get_serial_version() const {
+ return _serial_version;
+}
+
template
uint8_t count_min_sketch::get_num_hashes() const {
return _num_hashes;
@@ -107,7 +133,7 @@ uint8_t count_min_sketch::suggest_num_hashes(double confidence) {
if (confidence < 0. || confidence > 1.0) {
throw std::invalid_argument("Confidence must be between 0 and 1.0 (inclusive).");
}
- return std::min(ceil(log(1.0 / (1.0 - confidence))), UINT8_MAX);
+ return static_cast(std::max(1.0, std::min(ceil(log(1.0 / (1.0 - confidence))), static_cast(MAX_NUM_HASHES))));
}
template
@@ -131,10 +157,14 @@ void count_min_sketch::foreach_hash_location(const void* item, size_t size,
uint64_t hash_seed_index = 0;
for (const auto &it: hash_seeds) {
- HashState hashes;
- MurmurHash3_x64_128(item, size, it, hashes); // ? BEWARE OVERFLOW.
- uint64_t hash = hashes.h1;
- bucket_index = hash % _num_buckets;
+ if (_serial_version == SERIAL_VERSION_1) {
+ HashState hashes;
+ MurmurHash3_x64_128(item, size, it, hashes); // ? BEWARE OVERFLOW.
+ bucket_index = hashes.h1 % _num_buckets;
+ } else {
+ // the shift clears the top bit, so that languages with signed 64-bit integers get the same bucket
+ bucket_index = (XXHash64::hash(item, size, it) >> 1) % _num_buckets;
+ }
callback((hash_seed_index * _num_buckets) + bucket_index);
hash_seed_index += 1;
}
@@ -157,6 +187,7 @@ W count_min_sketch::get_estimate(const void* item, size_t size) const {
/*
* Returns the estimated frequency of the item
*/
+ if (size == 0) { return 0; } // empty items are not inserted into the sketch
W estimate = std::numeric_limits::max();
foreach_hash_location(item, size, [this, &estimate](uint64_t h) {
estimate = std::min(estimate, _sketch_array[h]);
@@ -186,6 +217,7 @@ void count_min_sketch::update(const void* item, size_t size, W weight) {
* Gets the item's hash locations and then increments the sketch in those
* locations by the weight.
*/
+ if (size == 0) { return; } // empty items are ignored
_total_weight += weight >= 0 ? weight : -weight;
foreach_hash_location(item, size, [this, weight](uint64_t h) {
_sketch_array[h] += weight;
@@ -206,6 +238,7 @@ W count_min_sketch::get_upper_bound(const std::string& item) const {
template
W count_min_sketch::get_upper_bound(const void* item, size_t size) const {
+ if (size == 0) { return 0; } // empty items are not inserted into the sketch
return static_cast(get_estimate(item, size) + get_relative_error() * get_total_weight());
}
@@ -223,6 +256,7 @@ W count_min_sketch::get_lower_bound(const std::string& item) const {
template
W count_min_sketch::get_lower_bound(const void* item, size_t size) const {
+ if (size == 0) { return 0; } // empty items are not inserted into the sketch
return get_estimate(item, size);
}
@@ -238,6 +272,12 @@ void count_min_sketch::merge(const count_min_sketch &other_sketch) {
(get_num_buckets() == other_sketch.get_num_buckets()) &&
(get_seed() == other_sketch.get_seed());
if (!acceptable_config) { throw std::invalid_argument( "Incompatible sketch configuration." ); }
+ if (get_serial_version() != other_sketch.get_serial_version()) {
+ // the two versions hash items into different buckets
+ throw std::invalid_argument("Cannot merge sketches with different serial versions: "
+ + std::to_string(get_serial_version()) + " and "
+ + std::to_string(other_sketch.get_serial_version()));
+ }
// Merge step - iterate over the other vector and add the weights to this sketch
auto it = _sketch_array.begin(); // This is a std::vector iterator.
@@ -282,7 +322,7 @@ size_t count_min_sketch::serialize_to(Sink&& sink) const {
// Long 0
//const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_SHORT : PREAMBLE_LONGS_FULL;
const uint8_t preamble_longs = PREAMBLE_LONGS_SHORT;
- const uint8_t ser_ver = SERIAL_VERSION_1;
+ const uint8_t ser_ver = _serial_version;
const uint8_t family_id = FAMILY_ID;
const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0);
const uint32_t unused32 = NULL_32;
@@ -327,7 +367,7 @@ auto count_min_sketch::deserialize(std::istream& is, uint64_t seed, const A
const auto flags_byte = read(is);
read(is); // 4 unused bytes
- check_header_validity(preamble_longs, serial_version, family_id, flags_byte);
+ check_header_validity(preamble_longs, serial_version, family_id);
// Sketch parameters
const auto nbuckets = read(is);
@@ -339,7 +379,7 @@ auto count_min_sketch::deserialize(std::istream& is, uint64_t seed, const A
throw std::invalid_argument("Incompatible seed hashes: " + std::to_string(seed_hash) + ", "
+ std::to_string(compute_seed_hash(seed)));
}
- count_min_sketch c(nhashes, nbuckets, seed, allocator);
+ count_min_sketch c(nhashes, nbuckets, seed, serial_version, allocator);
const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
if (is_empty == 1) { return c; } // sketch is empty, no need to read further.
@@ -389,7 +429,7 @@ auto count_min_sketch::deserialize(const void* bytes, size_t size, uint64_t
ptr += copy_from_mem(ptr, flags_byte);
ptr += sizeof(uint32_t);
- check_header_validity(preamble_longs, serial_version, family_id, flags_byte);
+ check_header_validity(preamble_longs, serial_version, family_id);
// Second 8 bytes are the sketch parameters with a final, unused byte.
uint32_t nbuckets;
@@ -404,7 +444,7 @@ auto count_min_sketch::deserialize(const void* bytes, size_t size, uint64_t
throw std::invalid_argument("Incompatible seed hashes: " + std::to_string(seed_hash) + ", "
+ std::to_string(compute_seed_hash(seed)));
}
- count_min_sketch c(nhashes, nbuckets, seed, allocator);
+ count_min_sketch c(nhashes, nbuckets, seed, serial_version, allocator);
const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
if (is_empty) { return c; } // sketch is empty, no need to read further.
@@ -451,27 +491,20 @@ string count_min_sketch::to_string() const {
}
template
-void count_min_sketch::check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id, uint8_t flags_byte) {
- const bool empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0;
-
- const uint8_t sw = (empty ? 1 : 0) + (2 * serial_version) + (4 * family_id) + (32 * (preamble_longs & 0x3F));
- bool valid = true;
-
- switch (sw) { // exhaustive list and description of all valid cases
- case 138 : break; // !empty, ser_ver==1, family==18, preLongs=2;
- case 139 : break; // empty, ser_ver==1, family==18, preLongs=2;
- //case 170 : break; // !empty, ser_ver==1, family==18, preLongs=3;
- default : // all other case values are invalid
- valid = false;
+void count_min_sketch::check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id) {
+ // each field is checked on its own, so that no combination of wrong values can pass
+ if (preamble_longs != PREAMBLE_LONGS_SHORT) {
+ throw std::invalid_argument("Possible sketch corruption. Preamble longs mismatch: expected "
+ + std::to_string(PREAMBLE_LONGS_SHORT) + ", actual " + std::to_string(preamble_longs));
}
-
- if (!valid) {
- std::ostringstream os;
- os << "Possible sketch corruption. Inconsistent state: "
- << "preamble_longs = " << static_cast(preamble_longs)
- << ", empty = " << (empty ? "true" : "false")
- << ", serialization_version = " << static_cast(serial_version);
- throw std::invalid_argument(os.str());
+ if (serial_version != SERIAL_VERSION_1 && serial_version != SERIAL_VERSION_2) {
+ throw std::invalid_argument("Possible sketch corruption. Serial version mismatch: expected "
+ + std::to_string(SERIAL_VERSION_1) + " or " + std::to_string(SERIAL_VERSION_2)
+ + ", actual " + std::to_string(serial_version));
+ }
+ if (family_id != FAMILY_ID) {
+ throw std::invalid_argument("Possible sketch corruption. Family ID mismatch: expected "
+ + std::to_string(FAMILY_ID) + ", actual " + std::to_string(family_id));
}
}
diff --git a/count/test/CMakeLists.txt b/count/test/CMakeLists.txt
index cfcfb266..d7bb7af4 100644
--- a/count/test/CMakeLists.txt
+++ b/count/test/CMakeLists.txt
@@ -39,4 +39,19 @@ target_sources(count_min_test
PRIVATE
count_min_test.cpp
count_min_allocation_test.cpp
+ count_min_v2_test.cpp
)
+
+if (SERDE_COMPAT)
+target_sources(count_min_test
+ PRIVATE
+ count_min_deserialize_from_java_test.cpp
+)
+endif()
+
+if (GENERATE)
+target_sources(count_min_test
+ PRIVATE
+ count_min_serialize_for_java.cpp
+)
+endif()
diff --git a/count/test/count_min_deserialize_from_java_test.cpp b/count/test/count_min_deserialize_from_java_test.cpp
new file mode 100644
index 00000000..800f9f5a
--- /dev/null
+++ b/count/test/count_min_deserialize_from_java_test.cpp
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include
+#include
+#include
+#include
+
+#include "count_min_v2_cases.hpp"
+
+namespace datasketches {
+
+// assume the binary sketches for this test have been generated by datasketches-java code
+// in the subdirectory called "serialization_test_data/java_generated_files" in the root directory of this project
+static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../serialization_test_data/java_generated_files/";
+
+// the checks of section 8.2 of the Count-Min format specification in datasketches-tck
+TEST_CASE("count min v2 java", "[serde_compat]") {
+ for (const auto& c: count_min_v2_cases) {
+ CAPTURE(c.name);
+ std::ifstream is;
+ is.exceptions(std::ios::failbit | std::ios::badbit);
+ is.open(testBinaryInputPath + count_min_v2_file_name(c, "java"), std::ios::binary);
+ const std::vector bytes((std::istreambuf_iterator(is)), std::istreambuf_iterator());
+ const auto sketch = count_min_sketch::deserialize(bytes.data(), bytes.size(), c.seed);
+ REQUIRE(sketch.get_serial_version() == 2);
+ REQUIRE(sketch.get_num_hashes() == 3);
+ REQUIRE(sketch.get_num_buckets() == 1000);
+ REQUIRE(sketch.is_empty() == (c.n == 0));
+ REQUIRE(sketch.get_total_weight() == c.n * (c.n + 1) / 2 + c.n);
+ for (int64_t i = 0; i < c.n; ++i) {
+ REQUIRE(sketch.get_estimate(i) >= i + 1);
+ REQUIRE(sketch.get_estimate(i) <= sketch.get_upper_bound(i));
+ REQUIRE(sketch.get_estimate(std::to_string(i)) >= 1);
+ REQUIRE(sketch.get_estimate(std::to_string(i)) <= sketch.get_upper_bound(std::to_string(i)));
+ }
+ // version 2 is deterministic: the same case is byte-identical in every language
+ const auto cpp_bytes = make_count_min_v2_case(c).serialize();
+ REQUIRE(std::vector(cpp_bytes.begin(), cpp_bytes.end()) == bytes);
+ }
+}
+
+} /* namespace datasketches */
diff --git a/count/test/count_min_serialize_for_java.cpp b/count/test/count_min_serialize_for_java.cpp
new file mode 100644
index 00000000..8cb5ebd5
--- /dev/null
+++ b/count/test/count_min_serialize_for_java.cpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include
+#include
+
+#include "count_min_v2_cases.hpp"
+
+namespace datasketches {
+
+TEST_CASE("count min v2 generate", "[serialize_for_java]") {
+ for (const auto& c: count_min_v2_cases) {
+ std::ofstream os(count_min_v2_file_name(c, "cpp"), std::ios::binary);
+ make_count_min_v2_case(c).serialize(os);
+ }
+}
+
+} /* namespace datasketches */
diff --git a/count/test/count_min_test.cpp b/count/test/count_min_test.cpp
index 3e811f9c..31737b09 100644
--- a/count/test/count_min_test.cpp
+++ b/count/test/count_min_test.cpp
@@ -21,7 +21,6 @@
#include
#include
#include
-#include
#include "count_min.hpp"
#include "common_defs.hpp"
@@ -107,7 +106,7 @@ TEST_CASE("CM one update: uint64_t") {
}
TEST_CASE("CM frequency cancellation") {
- count_min_sketch c(1, 5);
+ count_min_sketch c(1, 6); // "x" and "y" are in different buckets
c.update("x");
c.update("y", -1);
REQUIRE(c.get_total_weight() == 2);
@@ -215,8 +214,6 @@ TEST_CASE("CountMin sketch: serialize-deserialize empty", "[cm_sketch]") {
for (auto di: d) {
REQUIRE(di == 0);
}
- std::ofstream os("count_min_empty_cpp.sk", std::ios::binary);
- c.serialize(os);
}
TEST_CASE("CountMin sketch: serialize-deserialize non-empty", "[cm_sketch]") {
@@ -242,9 +239,6 @@ TEST_CASE("CountMin sketch: serialize-deserialize non-empty", "[cm_sketch]") {
++c_it;
++d_it;
}
-
- std::ofstream os("count_min_non_empty_cpp.sk", std::ios::binary);
- c.serialize(os);
}
TEST_CASE("CountMin sketch: bytes serialize-deserialize empty", "[cm_sketch]") {
diff --git a/count/test/count_min_v2_cases.hpp b/count/test/count_min_v2_cases.hpp
new file mode 100644
index 00000000..4ab7f131
--- /dev/null
+++ b/count/test/count_min_v2_cases.hpp
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COUNT_MIN_V2_CASES_HPP_
+#define COUNT_MIN_V2_CASES_HPP_
+
+// The cross-language test files of the Count-Min format specification in datasketches-tck,
+// section 8: count_min_v2__.sk, with numHashes 3 and numBuckets 1000.
+
+#include
+#include
+
+#include "count_min.hpp"
+
+namespace datasketches {
+
+struct count_min_v2_case {
+ const char* name;
+ uint64_t seed;
+ int64_t n;
+};
+
+static const count_min_v2_case count_min_v2_cases[] = {
+ {"n0", 9001, 0},
+ {"n1", 9001, 1},
+ {"n10", 9001, 10},
+ {"n100", 9001, 100},
+ {"n1000", 9001, 1000},
+ {"n10000", 9001, 10000},
+ {"seed42_n1000", 42, 1000}
+};
+
+// for each i from 0 to n - 1: the 64-bit integer i with weight i + 1, and the decimal string of i with weight 1
+inline count_min_sketch make_count_min_v2_case(const count_min_v2_case& c) {
+ count_min_sketch sketch(3, 1000, c.seed);
+ for (int64_t i = 0; i < c.n; ++i) {
+ sketch.update(i, i + 1);
+ sketch.update(std::to_string(i), 1);
+ }
+ return sketch;
+}
+
+inline std::string count_min_v2_file_name(const count_min_v2_case& c, const std::string& language) {
+ return "count_min_v2_" + std::string(c.name) + "_" + language + ".sk";
+}
+
+} /* namespace datasketches */
+
+#endif
diff --git a/count/test/count_min_v2_test.cpp b/count/test/count_min_v2_test.cpp
new file mode 100644
index 00000000..7a8a87d8
--- /dev/null
+++ b/count/test/count_min_v2_test.cpp
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Tests of serialization version 2, against the Count-Min format specification in datasketches-tck
+// (docs/count-min-format.md). Section numbers refer to that specification.
+
+#include
+#include
+#include
+#include
+#include
+
+#include "count_min.hpp"
+#include "xxhash64.h"
+
+namespace datasketches {
+
+namespace {
+
+using sketch_type = count_min_sketch;
+
+// indices of the non-zero counters
+std::vector nonzero_counters(const sketch_type& sketch) {
+ std::vector indices;
+ size_t index = 0;
+ for (auto it = sketch.begin(); it != sketch.end(); ++it, ++index) {
+ if (*it != 0) indices.push_back(index);
+ }
+ return indices;
+}
+
+template
+std::vector counters_of(uint64_t seed, uint32_t num_buckets, const Item& item) {
+ sketch_type sketch(3, num_buckets, seed);
+ sketch.update(item, 1);
+ return nonzero_counters(sketch);
+}
+
+std::vector rows(size_t b0, size_t b1, size_t b2, size_t num_buckets = 1000) {
+ return {b0, num_buckets + b1, 2 * num_buckets + b2};
+}
+
+std::vector to_vector(const sketch_type::vector_bytes& bytes) {
+ return std::vector(bytes.begin(), bytes.end());
+}
+
+} // namespace
+
+TEST_CASE("count min v2: XXH64 test vectors (7.1)", "[count_min]") {
+ REQUIRE(XXHash64::hash("", 0, 0) == 0xEF46DB3751D8E999ULL);
+ REQUIRE(XXHash64::hash("a", 1, 0) == 0xD24EC4F1A98C6E5BULL);
+ REQUIRE(XXHash64::hash("abc", 3, 0) == 0x44BC2CF5AD770999ULL);
+}
+
+TEST_CASE("count min v2: row seed test vectors (7.2)", "[count_min]") {
+ // rowSeed[i] = XXH64(LE64(i), seed = baseSeed)
+ auto row_seed = [](uint64_t i, uint64_t base_seed) { return XXHash64::hash(&i, sizeof(i), base_seed); };
+ REQUIRE(row_seed(0, 9001) == 0xD71D64972584A288ULL);
+ REQUIRE(row_seed(1, 9001) == 0xEFE8454A6499F311ULL);
+ REQUIRE(row_seed(2, 9001) == 0x23819BE21B1B1D9EULL);
+ REQUIRE(row_seed(0, 42) == 0xB71B47EBDA15746CULL);
+ REQUIRE(row_seed(1, 42) == 0x9ED50FD59358D232ULL);
+ REQUIRE(row_seed(2, 42) == 0xD19FE30002B9A3CDULL);
+}
+
+TEST_CASE("count min v2: bucket test vectors (7.3)", "[count_min]") {
+ REQUIRE(counters_of(9001, 1000, int64_t(1)) == rows(185, 898, 4));
+ REQUIRE(counters_of(9001, 1000, uint64_t(1)) == rows(185, 898, 4)); // the same 8 bytes
+ REQUIRE(counters_of(9001, 1000, int64_t(9)) == rows(490, 217, 440));
+ REQUIRE(counters_of(9001, 1000, int64_t(-1)) == rows(256, 641, 308));
+ REQUIRE(counters_of(9001, 1000, uint64_t(0x8000000000000000ULL)) == rows(951, 0, 298));
+ REQUIRE(counters_of(9001, 1000, std::string("apple")) == rows(130, 747, 144));
+ REQUIRE(counters_of(9001, 1000, std::string("\xc3\xa9t\xc3\xa9")) == rows(806, 491, 163)); // "été" in UTF-8
+ REQUIRE(counters_of(42, 1000, int64_t(1)) == rows(468, 459, 696));
+ REQUIRE(counters_of(42, 1000, int64_t(9)) == rows(841, 428, 322));
+}
+
+TEST_CASE("count min v2: image test vectors (7.4)", "[count_min]") {
+ sketch_type sketch(2, 9, 9001);
+ REQUIRE(sketch.get_serial_version() == 2);
+ const std::vector empty_image = {
+ 0x02, 0x02, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x93, 0x00
+ };
+ REQUIRE(to_vector(sketch.serialize()) == empty_image);
+
+ sketch.update(int64_t(1), 10);
+ sketch.update(int64_t(9), 810);
+ sketch.update(std::string("apple"), 5);
+ std::vector image(168, 0);
+ const uint8_t header[] = {
+ 0x02, 0x02, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x93, 0x00
+ };
+ std::memcpy(image.data(), header, sizeof(header));
+ auto put = [&image](size_t offset, int64_t value) { std::memcpy(image.data() + offset, &value, sizeof(value)); };
+ put(16, 825); // total weight
+ put(24, 810); // row 0, bucket 0: 9
+ put(56, 5); // row 0, bucket 4: "apple"
+ put(72, 10); // row 0, bucket 6: 1
+ put(104, 5); // row 1, bucket 1: "apple"
+ put(112, 810); // row 1, bucket 2: 9
+ put(152, 10); // row 1, bucket 7: 1
+ const auto bytes = sketch.serialize();
+ REQUIRE(to_vector(bytes) == image);
+
+ const auto read = sketch_type::deserialize(bytes.data(), bytes.size(), 9001);
+ REQUIRE(read.get_serial_version() == 2);
+ REQUIRE(read.get_estimate(int64_t(1)) == 10);
+ REQUIRE(read.get_estimate(int64_t(9)) == 810);
+ REQUIRE(read.get_estimate(std::string("apple")) == 5);
+}
+
+// 4.4: an empty string, an empty byte array and a null are ignored, and every query returns 0
+TEST_CASE("count min v2: empty items are ignored (4.4)", "[count_min]") {
+ sketch_type sketch(3, 1000);
+ sketch.update(nullptr, 0, 7);
+ sketch.update("x", 0, 7);
+ sketch.update(std::string(), 7);
+ REQUIRE(sketch.is_empty());
+ REQUIRE(sketch.get_total_weight() == 0);
+
+ // on a non-empty sketch, so that a missing check on the upper bound would show
+ sketch.update(int64_t(1), 1000);
+ const auto bytes = to_vector(sketch.serialize());
+ sketch.update(nullptr, 0, 7);
+ sketch.update("x", 0, 7);
+ sketch.update(std::string(), 7);
+ REQUIRE(to_vector(sketch.serialize()) == bytes);
+ REQUIRE(sketch.get_estimate(nullptr, 0) == 0);
+ REQUIRE(sketch.get_lower_bound(nullptr, 0) == 0);
+ REQUIRE(sketch.get_upper_bound(nullptr, 0) == 0);
+ REQUIRE(sketch.get_estimate("x", 0) == 0);
+ REQUIRE(sketch.get_upper_bound("x", 0) == 0);
+ REQUIRE(sketch.get_estimate(std::string()) == 0);
+ REQUIRE(sketch.get_lower_bound(std::string()) == 0);
+ REQUIRE(sketch.get_upper_bound(std::string()) == 0);
+ REQUIRE(sketch.get_upper_bound(int64_t(2)) > 0); // a non-empty item that was never added still gets eps * weight
+}
+
+// 1: numHashes is 1 to 127
+TEST_CASE("count min v2: number of hashes", "[count_min]") {
+ REQUIRE_THROWS_AS(sketch_type(0, 1000), std::invalid_argument);
+ REQUIRE_THROWS_AS(sketch_type(128, 1000), std::invalid_argument);
+ REQUIRE_THROWS_AS(sketch_type(255, 1000), std::invalid_argument);
+ REQUIRE(sketch_type(1, 1000).get_num_hashes() == 1);
+ REQUIRE(sketch_type(127, 1000).get_num_hashes() == 127);
+ REQUIRE(sketch_type::suggest_num_hashes(1.0) == 127);
+ REQUIRE(sketch_type::suggest_num_hashes(0.0) == 1);
+
+ // the same limits apply to version 2 images
+ auto image = to_vector(sketch_type(3, 1000).serialize());
+ image[12] = 0;
+ REQUIRE_THROWS_AS(sketch_type::deserialize(image.data(), image.size()), std::invalid_argument);
+ image[12] = 128;
+ REQUIRE_THROWS_AS(sketch_type::deserialize(image.data(), image.size()), std::invalid_argument);
+}
+
+// 6: readers accept version 1 images, keep their hashing and write them back as version 1
+TEST_CASE("count min v2: version 1 images", "[count_min]") {
+ // written by the C++ library before version 2: numHashes 2, numBuckets 5, seed 9001,
+ // update(1, 10), update(9, 810), update("apple", 5)
+ const std::vector v1_image = {
+ 0x02, 0x01, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x05, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x93, 0x00,
+ 0x39, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x34, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ };
+ auto v1 = sketch_type::deserialize(v1_image.data(), v1_image.size(), 9001);
+ REQUIRE(v1.get_serial_version() == 1);
+ REQUIRE(v1.get_total_weight() == 825);
+ REQUIRE(to_vector(v1.serialize()) == v1_image); // written back as version 1
+
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+ s.write(reinterpret_cast(v1_image.data()), v1_image.size());
+ REQUIRE(sketch_type::deserialize(s, 9001).get_serial_version() == 1);
+
+ // Version 1 row seeds depend on the standard library, so the estimates of the items above
+ // are checked only with the standard library that wrote the image. A version 1 sketch is
+ // consistent with itself everywhere.
+ v1.update(std::string("pear"), 3);
+ REQUIRE(v1.get_estimate(std::string("pear")) >= 3);
+ REQUIRE(v1.get_total_weight() == 828);
+ const auto v1_bytes = v1.serialize();
+ REQUIRE(v1_bytes[1] == 1);
+
+ // version 1 and version 2 sketches hash differently, so they can't be merged
+ sketch_type v2(2, 5, 9001);
+ v2.update(int64_t(1), 1);
+ REQUIRE_THROWS_AS(v1.merge(v2), std::invalid_argument);
+ REQUIRE_THROWS_AS(v2.merge(v1), std::invalid_argument);
+ auto other_v1 = sketch_type::deserialize(v1_image.data(), v1_image.size(), 9001);
+ v1.merge(other_v1);
+ REQUIRE(v1.get_total_weight() == 828 + 825);
+ REQUIRE(v1.get_serial_version() == 1);
+
+ // version 1 images written by C++ may have up to 255 rows; 0 is never valid
+ std::vector v1_empty = {
+ 0x02, 0x01, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 200, 0xcc, 0x93, 0x00
+ };
+ const auto wide = sketch_type::deserialize(v1_empty.data(), v1_empty.size(), 9001);
+ REQUIRE(wide.get_serial_version() == 1);
+ REQUIRE(wide.get_num_hashes() == 200);
+ v1_empty[12] = 0;
+ REQUIRE_THROWS_AS(sketch_type::deserialize(v1_empty.data(), v1_empty.size(), 9001), std::invalid_argument);
+}
+
+TEST_CASE("count min v2: other serial versions are rejected", "[count_min]") {
+ for (uint8_t version: {0, 3, 255}) {
+ auto image = to_vector(sketch_type(3, 1000).serialize());
+ image[1] = version;
+ REQUIRE_THROWS_AS(sketch_type::deserialize(image.data(), image.size()), std::invalid_argument);
+ }
+}
+
+TEST_CASE("count min v2: each header field is checked", "[count_min]") {
+ // {preamble longs, serial version, family ID}: each has a wrong field, and the first three
+ // used to pass, because the old check only looked at a weighted sum of the fields
+ const uint8_t headers[][3] = {{2, 0, 19}, {2, 3, 17}, {10, 2, 18}, {3, 2, 18}, {2, 2, 17}};
+ sketch_type sketch(3, 1000);
+ sketch.update(static_cast(1));
+ for (const auto& header: headers) {
+ auto image = to_vector(sketch.serialize());
+ image[0] = header[0];
+ image[1] = header[1];
+ image[2] = header[2];
+ REQUIRE_THROWS_AS(sketch_type::deserialize(image.data(), image.size()), std::invalid_argument);
+ std::stringstream is(std::string(image.begin(), image.end()));
+ REQUIRE_THROWS_AS(sketch_type::deserialize(is), std::invalid_argument);
+ }
+}
+
+TEST_CASE("count min v2: copies keep the serial version", "[count_min]") {
+ const std::vector v1_empty = {
+ 0x02, 0x01, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x93, 0x00
+ };
+ const auto v1 = sketch_type::deserialize(v1_empty.data(), v1_empty.size());
+ sketch_type copy(v1);
+ REQUIRE(copy.get_serial_version() == 1);
+ sketch_type assigned(3, 10);
+ assigned = v1;
+ REQUIRE(assigned.get_serial_version() == 1);
+ sketch_type moved(std::move(copy));
+ REQUIRE(moved.get_serial_version() == 1);
+ REQUIRE(moved.serialize()[1] == 1);
+}
+
+} /* namespace datasketches */