From 353dba53a0b4f5a9590b0f5e75b90a9869ea7f53 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 12 Nov 2026 13:44:56 -0800 Subject: [PATCH 1/2] Count-Min: write serialization version 2, keep reading version 1 Implements version 2 of the Count-Min format specification in datasketches-tck (docs/count-min-format.md), so that images give the same results in every language and with every C++ standard library. - Row seeds are XXH64(LE64(i), seed) instead of values drawn from std::default_random_engine, which differs between standard libraries. - Items are hashed with XXH64, and bucket = (hash >> 1) mod numBuckets. - New sketches write serial version 2. Version 1 images are still read; such sketches keep the version 1 hashing and are written back as version 1. Sketches of different versions can't be merged. - numHashes must be in [1, 127]; version 1 images may have up to 255. suggest_num_hashes() returns at most 127. - Empty items are ignored: an update with size 0 does nothing, and every query returns 0, as the string overloads already did. - New get_serial_version(). Tests check the specification's test vectors, the empty-item rule, the numHashes range, version 1 images, and the cross-language files of section 8 (generated with GENERATE, and compared byte for byte with Java's with SERDE_COMPAT). The tests no longer write the version 1 file names count_min_empty_cpp.sk and count_min_non_empty_cpp.sk. Co-Authored-By: Claude Opus 5.5 --- count/include/count_min.hpp | 37 ++- count/include/count_min_impl.hpp | 68 ++++- count/test/CMakeLists.txt | 15 ++ .../count_min_deserialize_from_java_test.cpp | 59 ++++ count/test/count_min_serialize_for_java.cpp | 34 +++ count/test/count_min_test.cpp | 8 +- count/test/count_min_v2_cases.hpp | 65 +++++ count/test/count_min_v2_test.cpp | 254 ++++++++++++++++++ 8 files changed, 515 insertions(+), 25 deletions(-) create mode 100644 count/test/count_min_deserialize_from_java_test.cpp create mode 100644 count/test/count_min_serialize_for_java.cpp create mode 100644 count/test/count_min_v2_cases.hpp create mode 100644 count/test/count_min_v2_test.cpp diff --git a/count/include/count_min.hpp b/count/include/count_min.hpp index f78dce1a5..3b94493a5 100644 --- a/count/include/count_min.hpp +++ b/count/include/count_min.hpp @@ -32,6 +32,18 @@ namespace datasketches { * [1] - http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf * The template type W is the type of the vector that contains the weights of the objects inserted into the sketch, * not the type of the input items themselves. + * + * New sketches use serialization version 2, as specified in the Count-Min format specification in + * datasketches-tck. + * Version 2 hashes items with XXH64 and derives the row seeds from the seed, so its images give the same + * results in every language and with every C++ standard library. Other languages read only images with + * W = int64_t (signed 64-bit counters). + * + * Sketches deserialized from version 1 images keep the version 1 hashing, and serialize as version 1. + * Version 1 row seeds depend on the C++ standard library, so version 1 images are reliable only with the + * standard library that wrote them. Sketches of different versions can't be merged. + * + * Empty items (an empty string, or a size of 0) are ignored: an update does nothing, and every query returns 0. * @author Charlie Dickens */ template _sketch_array; // the array stored by the sketch @@ -388,10 +410,15 @@ class count_min_sketch{ static const uint8_t PREAMBLE_LONGS_SHORT = 2; // Empty -> need second byte for sketch parameters static const uint8_t PREAMBLE_LONGS_FULL = 3; // Not empty -> need (at least) third byte for total weight. static const uint8_t SERIAL_VERSION_1 = 1; + static const uint8_t SERIAL_VERSION_2 = 2; + static const uint8_t MAX_NUM_HASHES = 127; // stored as a signed byte in Java and Go static const uint8_t FAMILY_ID = 18; static const uint8_t NULL_8 = 0; static const uint32_t NULL_32 = 0; + // used by deserialize to keep the hashing of a version 1 image + count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed, uint8_t serial_version, const Allocator& allocator); + /** * Throws an error if the header is not valid. * @param preamble_longs diff --git a/count/include/count_min_impl.hpp b/count/include/count_min_impl.hpp index c1c053e3d..23123c292 100644 --- a/count/include/count_min_impl.hpp +++ b/count/include/count_min_impl.hpp @@ -27,19 +27,33 @@ #include "MurmurHash3.h" #include "count_min.hpp" +#include "xxhash64.h" #include "memory_operations.hpp" namespace datasketches { template count_min_sketch::count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed, const A& allocator): +count_min_sketch(num_hashes, num_buckets, seed, SERIAL_VERSION_2, allocator) +{} + +template +count_min_sketch::count_min_sketch(uint8_t num_hashes, uint32_t num_buckets, uint64_t seed, uint8_t serial_version, + const A& allocator): _allocator(allocator), +_serial_version(serial_version), _num_hashes(num_hashes), _num_buckets(num_buckets), _sketch_array((static_cast(num_hashes) * num_buckets < 1<<30) ? static_cast(num_hashes) * num_buckets : 0, 0, _allocator), _seed(seed), _total_weight(0), hash_seeds(_allocator) { + // version 1 images written by C++ may have up to 255 rows + const uint8_t max_num_hashes = serial_version == SERIAL_VERSION_1 ? UINT8_MAX : MAX_NUM_HASHES; + if (num_hashes < 1 || num_hashes > max_num_hashes) { + throw std::invalid_argument("The number of hashes must be in [1, " + std::to_string(max_num_hashes) + "], actual " + + std::to_string(num_hashes)); + } if (num_buckets < 3) { throw std::invalid_argument("Using fewer than 3 buckets incurs relative error greater than 1."); } @@ -51,15 +65,27 @@ hash_seeds(_allocator) { "Try reducing either the number of buckets or the number of hash functions."); } - std::default_random_engine rng(_seed); - std::uniform_int_distribution extra_hash_seeds(0, std::numeric_limits::max()); hash_seeds.reserve(num_hashes); - - for (uint64_t i=0; i < num_hashes; ++i) { - hash_seeds.push_back(extra_hash_seeds(rng) + _seed); // Adds the global seed to all hash functions. + if (serial_version == SERIAL_VERSION_1) { + // legacy row seeds: these depend on the C++ standard library + std::default_random_engine rng(_seed); + std::uniform_int_distribution extra_hash_seeds(0, std::numeric_limits::max()); + for (uint64_t i=0; i < num_hashes; ++i) { + hash_seeds.push_back(extra_hash_seeds(rng) + _seed); // Adds the global seed to all hash functions. + } + } else { + // rowSeed[i] = XXH64(LE64(i), seed = seed), see the Count-Min format specification in datasketches-tck + for (uint64_t i=0; i < num_hashes; ++i) { + hash_seeds.push_back(XXHash64::hash(&i, sizeof(i), _seed)); + } } } +template +uint8_t count_min_sketch::get_serial_version() const { + return _serial_version; +} + template uint8_t count_min_sketch::get_num_hashes() const { return _num_hashes; @@ -107,7 +133,7 @@ uint8_t count_min_sketch::suggest_num_hashes(double confidence) { if (confidence < 0. || confidence > 1.0) { throw std::invalid_argument("Confidence must be between 0 and 1.0 (inclusive)."); } - return std::min(ceil(log(1.0 / (1.0 - confidence))), UINT8_MAX); + return static_cast(std::max(1.0, std::min(ceil(log(1.0 / (1.0 - confidence))), static_cast(MAX_NUM_HASHES)))); } template @@ -131,10 +157,14 @@ void count_min_sketch::foreach_hash_location(const void* item, size_t size, uint64_t hash_seed_index = 0; for (const auto &it: hash_seeds) { - HashState hashes; - MurmurHash3_x64_128(item, size, it, hashes); // ? BEWARE OVERFLOW. - uint64_t hash = hashes.h1; - bucket_index = hash % _num_buckets; + if (_serial_version == SERIAL_VERSION_1) { + HashState hashes; + MurmurHash3_x64_128(item, size, it, hashes); // ? BEWARE OVERFLOW. + bucket_index = hashes.h1 % _num_buckets; + } else { + // the shift clears the top bit, so that languages with signed 64-bit integers get the same bucket + bucket_index = (XXHash64::hash(item, size, it) >> 1) % _num_buckets; + } callback((hash_seed_index * _num_buckets) + bucket_index); hash_seed_index += 1; } @@ -157,6 +187,7 @@ W count_min_sketch::get_estimate(const void* item, size_t size) const { /* * Returns the estimated frequency of the item */ + if (size == 0) { return 0; } // empty items are not inserted into the sketch W estimate = std::numeric_limits::max(); foreach_hash_location(item, size, [this, &estimate](uint64_t h) { estimate = std::min(estimate, _sketch_array[h]); @@ -186,6 +217,7 @@ void count_min_sketch::update(const void* item, size_t size, W weight) { * Gets the item's hash locations and then increments the sketch in those * locations by the weight. */ + if (size == 0) { return; } // empty items are ignored _total_weight += weight >= 0 ? weight : -weight; foreach_hash_location(item, size, [this, weight](uint64_t h) { _sketch_array[h] += weight; @@ -206,6 +238,7 @@ W count_min_sketch::get_upper_bound(const std::string& item) const { template W count_min_sketch::get_upper_bound(const void* item, size_t size) const { + if (size == 0) { return 0; } // empty items are not inserted into the sketch return static_cast(get_estimate(item, size) + get_relative_error() * get_total_weight()); } @@ -223,6 +256,7 @@ W count_min_sketch::get_lower_bound(const std::string& item) const { template W count_min_sketch::get_lower_bound(const void* item, size_t size) const { + if (size == 0) { return 0; } // empty items are not inserted into the sketch return get_estimate(item, size); } @@ -238,6 +272,12 @@ void count_min_sketch::merge(const count_min_sketch &other_sketch) { (get_num_buckets() == other_sketch.get_num_buckets()) && (get_seed() == other_sketch.get_seed()); if (!acceptable_config) { throw std::invalid_argument( "Incompatible sketch configuration." ); } + if (get_serial_version() != other_sketch.get_serial_version()) { + // the two versions hash items into different buckets + throw std::invalid_argument("Cannot merge sketches with different serial versions: " + + std::to_string(get_serial_version()) + " and " + + std::to_string(other_sketch.get_serial_version())); + } // Merge step - iterate over the other vector and add the weights to this sketch auto it = _sketch_array.begin(); // This is a std::vector iterator. @@ -282,7 +322,7 @@ size_t count_min_sketch::serialize_to(Sink&& sink) const { // Long 0 //const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_SHORT : PREAMBLE_LONGS_FULL; const uint8_t preamble_longs = PREAMBLE_LONGS_SHORT; - const uint8_t ser_ver = SERIAL_VERSION_1; + const uint8_t ser_ver = _serial_version; const uint8_t family_id = FAMILY_ID; const uint8_t flags_byte = (is_empty() ? 1 << flags::IS_EMPTY : 0); const uint32_t unused32 = NULL_32; @@ -339,7 +379,7 @@ auto count_min_sketch::deserialize(std::istream& is, uint64_t seed, const A throw std::invalid_argument("Incompatible seed hashes: " + std::to_string(seed_hash) + ", " + std::to_string(compute_seed_hash(seed))); } - count_min_sketch c(nhashes, nbuckets, seed, allocator); + count_min_sketch c(nhashes, nbuckets, seed, serial_version, allocator); const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0; if (is_empty == 1) { return c; } // sketch is empty, no need to read further. @@ -404,7 +444,7 @@ auto count_min_sketch::deserialize(const void* bytes, size_t size, uint64_t throw std::invalid_argument("Incompatible seed hashes: " + std::to_string(seed_hash) + ", " + std::to_string(compute_seed_hash(seed))); } - count_min_sketch c(nhashes, nbuckets, seed, allocator); + count_min_sketch c(nhashes, nbuckets, seed, serial_version, allocator); const bool is_empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0; if (is_empty) { return c; } // sketch is empty, no need to read further. @@ -460,6 +500,8 @@ void count_min_sketch::check_header_validity(uint8_t preamble_longs, uint8_ switch (sw) { // exhaustive list and description of all valid cases case 138 : break; // !empty, ser_ver==1, family==18, preLongs=2; case 139 : break; // empty, ser_ver==1, family==18, preLongs=2; + case 140 : break; // !empty, ser_ver==2, family==18, preLongs=2; + case 141 : break; // empty, ser_ver==2, family==18, preLongs=2; //case 170 : break; // !empty, ser_ver==1, family==18, preLongs=3; default : // all other case values are invalid valid = false; diff --git a/count/test/CMakeLists.txt b/count/test/CMakeLists.txt index cfcfb266a..d7bb7af43 100644 --- a/count/test/CMakeLists.txt +++ b/count/test/CMakeLists.txt @@ -39,4 +39,19 @@ target_sources(count_min_test PRIVATE count_min_test.cpp count_min_allocation_test.cpp + count_min_v2_test.cpp ) + +if (SERDE_COMPAT) +target_sources(count_min_test + PRIVATE + count_min_deserialize_from_java_test.cpp +) +endif() + +if (GENERATE) +target_sources(count_min_test + PRIVATE + count_min_serialize_for_java.cpp +) +endif() diff --git a/count/test/count_min_deserialize_from_java_test.cpp b/count/test/count_min_deserialize_from_java_test.cpp new file mode 100644 index 000000000..800f9f5a1 --- /dev/null +++ b/count/test/count_min_deserialize_from_java_test.cpp @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include "count_min_v2_cases.hpp" + +namespace datasketches { + +// assume the binary sketches for this test have been generated by datasketches-java code +// in the subdirectory called "serialization_test_data/java_generated_files" in the root directory of this project +static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../serialization_test_data/java_generated_files/"; + +// the checks of section 8.2 of the Count-Min format specification in datasketches-tck +TEST_CASE("count min v2 java", "[serde_compat]") { + for (const auto& c: count_min_v2_cases) { + CAPTURE(c.name); + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(testBinaryInputPath + count_min_v2_file_name(c, "java"), std::ios::binary); + const std::vector bytes((std::istreambuf_iterator(is)), std::istreambuf_iterator()); + const auto sketch = count_min_sketch::deserialize(bytes.data(), bytes.size(), c.seed); + REQUIRE(sketch.get_serial_version() == 2); + REQUIRE(sketch.get_num_hashes() == 3); + REQUIRE(sketch.get_num_buckets() == 1000); + REQUIRE(sketch.is_empty() == (c.n == 0)); + REQUIRE(sketch.get_total_weight() == c.n * (c.n + 1) / 2 + c.n); + for (int64_t i = 0; i < c.n; ++i) { + REQUIRE(sketch.get_estimate(i) >= i + 1); + REQUIRE(sketch.get_estimate(i) <= sketch.get_upper_bound(i)); + REQUIRE(sketch.get_estimate(std::to_string(i)) >= 1); + REQUIRE(sketch.get_estimate(std::to_string(i)) <= sketch.get_upper_bound(std::to_string(i))); + } + // version 2 is deterministic: the same case is byte-identical in every language + const auto cpp_bytes = make_count_min_v2_case(c).serialize(); + REQUIRE(std::vector(cpp_bytes.begin(), cpp_bytes.end()) == bytes); + } +} + +} /* namespace datasketches */ diff --git a/count/test/count_min_serialize_for_java.cpp b/count/test/count_min_serialize_for_java.cpp new file mode 100644 index 000000000..8cb5ebd55 --- /dev/null +++ b/count/test/count_min_serialize_for_java.cpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "count_min_v2_cases.hpp" + +namespace datasketches { + +TEST_CASE("count min v2 generate", "[serialize_for_java]") { + for (const auto& c: count_min_v2_cases) { + std::ofstream os(count_min_v2_file_name(c, "cpp"), std::ios::binary); + make_count_min_v2_case(c).serialize(os); + } +} + +} /* namespace datasketches */ diff --git a/count/test/count_min_test.cpp b/count/test/count_min_test.cpp index 3e811f9cc..31737b09b 100644 --- a/count/test/count_min_test.cpp +++ b/count/test/count_min_test.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include "count_min.hpp" #include "common_defs.hpp" @@ -107,7 +106,7 @@ TEST_CASE("CM one update: uint64_t") { } TEST_CASE("CM frequency cancellation") { - count_min_sketch c(1, 5); + count_min_sketch c(1, 6); // "x" and "y" are in different buckets c.update("x"); c.update("y", -1); REQUIRE(c.get_total_weight() == 2); @@ -215,8 +214,6 @@ TEST_CASE("CountMin sketch: serialize-deserialize empty", "[cm_sketch]") { for (auto di: d) { REQUIRE(di == 0); } - std::ofstream os("count_min_empty_cpp.sk", std::ios::binary); - c.serialize(os); } TEST_CASE("CountMin sketch: serialize-deserialize non-empty", "[cm_sketch]") { @@ -242,9 +239,6 @@ TEST_CASE("CountMin sketch: serialize-deserialize non-empty", "[cm_sketch]") { ++c_it; ++d_it; } - - std::ofstream os("count_min_non_empty_cpp.sk", std::ios::binary); - c.serialize(os); } TEST_CASE("CountMin sketch: bytes serialize-deserialize empty", "[cm_sketch]") { diff --git a/count/test/count_min_v2_cases.hpp b/count/test/count_min_v2_cases.hpp new file mode 100644 index 000000000..4ab7f1313 --- /dev/null +++ b/count/test/count_min_v2_cases.hpp @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef COUNT_MIN_V2_CASES_HPP_ +#define COUNT_MIN_V2_CASES_HPP_ + +// The cross-language test files of the Count-Min format specification in datasketches-tck, +// section 8: count_min_v2__.sk, with numHashes 3 and numBuckets 1000. + +#include +#include + +#include "count_min.hpp" + +namespace datasketches { + +struct count_min_v2_case { + const char* name; + uint64_t seed; + int64_t n; +}; + +static const count_min_v2_case count_min_v2_cases[] = { + {"n0", 9001, 0}, + {"n1", 9001, 1}, + {"n10", 9001, 10}, + {"n100", 9001, 100}, + {"n1000", 9001, 1000}, + {"n10000", 9001, 10000}, + {"seed42_n1000", 42, 1000} +}; + +// for each i from 0 to n - 1: the 64-bit integer i with weight i + 1, and the decimal string of i with weight 1 +inline count_min_sketch make_count_min_v2_case(const count_min_v2_case& c) { + count_min_sketch sketch(3, 1000, c.seed); + for (int64_t i = 0; i < c.n; ++i) { + sketch.update(i, i + 1); + sketch.update(std::to_string(i), 1); + } + return sketch; +} + +inline std::string count_min_v2_file_name(const count_min_v2_case& c, const std::string& language) { + return "count_min_v2_" + std::string(c.name) + "_" + language + ".sk"; +} + +} /* namespace datasketches */ + +#endif diff --git a/count/test/count_min_v2_test.cpp b/count/test/count_min_v2_test.cpp new file mode 100644 index 000000000..02ddaea7a --- /dev/null +++ b/count/test/count_min_v2_test.cpp @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Tests of serialization version 2, against the Count-Min format specification in datasketches-tck +// (docs/count-min-format.md). Section numbers refer to that specification. + +#include +#include +#include +#include +#include + +#include "count_min.hpp" +#include "xxhash64.h" + +namespace datasketches { + +namespace { + +using sketch_type = count_min_sketch; + +// indices of the non-zero counters +std::vector nonzero_counters(const sketch_type& sketch) { + std::vector indices; + size_t index = 0; + for (auto it = sketch.begin(); it != sketch.end(); ++it, ++index) { + if (*it != 0) indices.push_back(index); + } + return indices; +} + +template +std::vector counters_of(uint64_t seed, uint32_t num_buckets, const Item& item) { + sketch_type sketch(3, num_buckets, seed); + sketch.update(item, 1); + return nonzero_counters(sketch); +} + +std::vector rows(size_t b0, size_t b1, size_t b2, size_t num_buckets = 1000) { + return {b0, num_buckets + b1, 2 * num_buckets + b2}; +} + +std::vector to_vector(const sketch_type::vector_bytes& bytes) { + return std::vector(bytes.begin(), bytes.end()); +} + +} // namespace + +TEST_CASE("count min v2: XXH64 test vectors (7.1)", "[count_min]") { + REQUIRE(XXHash64::hash("", 0, 0) == 0xEF46DB3751D8E999ULL); + REQUIRE(XXHash64::hash("a", 1, 0) == 0xD24EC4F1A98C6E5BULL); + REQUIRE(XXHash64::hash("abc", 3, 0) == 0x44BC2CF5AD770999ULL); +} + +TEST_CASE("count min v2: row seed test vectors (7.2)", "[count_min]") { + // rowSeed[i] = XXH64(LE64(i), seed = baseSeed) + auto row_seed = [](uint64_t i, uint64_t base_seed) { return XXHash64::hash(&i, sizeof(i), base_seed); }; + REQUIRE(row_seed(0, 9001) == 0xD71D64972584A288ULL); + REQUIRE(row_seed(1, 9001) == 0xEFE8454A6499F311ULL); + REQUIRE(row_seed(2, 9001) == 0x23819BE21B1B1D9EULL); + REQUIRE(row_seed(0, 42) == 0xB71B47EBDA15746CULL); + REQUIRE(row_seed(1, 42) == 0x9ED50FD59358D232ULL); + REQUIRE(row_seed(2, 42) == 0xD19FE30002B9A3CDULL); +} + +TEST_CASE("count min v2: bucket test vectors (7.3)", "[count_min]") { + REQUIRE(counters_of(9001, 1000, int64_t(1)) == rows(185, 898, 4)); + REQUIRE(counters_of(9001, 1000, uint64_t(1)) == rows(185, 898, 4)); // the same 8 bytes + REQUIRE(counters_of(9001, 1000, int64_t(9)) == rows(490, 217, 440)); + REQUIRE(counters_of(9001, 1000, int64_t(-1)) == rows(256, 641, 308)); + REQUIRE(counters_of(9001, 1000, uint64_t(0x8000000000000000ULL)) == rows(951, 0, 298)); + REQUIRE(counters_of(9001, 1000, std::string("apple")) == rows(130, 747, 144)); + REQUIRE(counters_of(9001, 1000, std::string("\xc3\xa9t\xc3\xa9")) == rows(806, 491, 163)); // "été" in UTF-8 + REQUIRE(counters_of(42, 1000, int64_t(1)) == rows(468, 459, 696)); + REQUIRE(counters_of(42, 1000, int64_t(9)) == rows(841, 428, 322)); +} + +TEST_CASE("count min v2: image test vectors (7.4)", "[count_min]") { + sketch_type sketch(2, 9, 9001); + REQUIRE(sketch.get_serial_version() == 2); + const std::vector empty_image = { + 0x02, 0x02, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x93, 0x00 + }; + REQUIRE(to_vector(sketch.serialize()) == empty_image); + + sketch.update(int64_t(1), 10); + sketch.update(int64_t(9), 810); + sketch.update(std::string("apple"), 5); + std::vector image(168, 0); + const uint8_t header[] = { + 0x02, 0x02, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x93, 0x00 + }; + std::memcpy(image.data(), header, sizeof(header)); + auto put = [&image](size_t offset, int64_t value) { std::memcpy(image.data() + offset, &value, sizeof(value)); }; + put(16, 825); // total weight + put(24, 810); // row 0, bucket 0: 9 + put(56, 5); // row 0, bucket 4: "apple" + put(72, 10); // row 0, bucket 6: 1 + put(104, 5); // row 1, bucket 1: "apple" + put(112, 810); // row 1, bucket 2: 9 + put(152, 10); // row 1, bucket 7: 1 + const auto bytes = sketch.serialize(); + REQUIRE(to_vector(bytes) == image); + + const auto read = sketch_type::deserialize(bytes.data(), bytes.size(), 9001); + REQUIRE(read.get_serial_version() == 2); + REQUIRE(read.get_estimate(int64_t(1)) == 10); + REQUIRE(read.get_estimate(int64_t(9)) == 810); + REQUIRE(read.get_estimate(std::string("apple")) == 5); +} + +// 4.4: an empty string, an empty byte array and a null are ignored, and every query returns 0 +TEST_CASE("count min v2: empty items are ignored (4.4)", "[count_min]") { + sketch_type sketch(3, 1000); + sketch.update(nullptr, 0, 7); + sketch.update("x", 0, 7); + sketch.update(std::string(), 7); + REQUIRE(sketch.is_empty()); + REQUIRE(sketch.get_total_weight() == 0); + + // on a non-empty sketch, so that a missing check on the upper bound would show + sketch.update(int64_t(1), 1000); + const auto bytes = to_vector(sketch.serialize()); + sketch.update(nullptr, 0, 7); + sketch.update("x", 0, 7); + sketch.update(std::string(), 7); + REQUIRE(to_vector(sketch.serialize()) == bytes); + REQUIRE(sketch.get_estimate(nullptr, 0) == 0); + REQUIRE(sketch.get_lower_bound(nullptr, 0) == 0); + REQUIRE(sketch.get_upper_bound(nullptr, 0) == 0); + REQUIRE(sketch.get_estimate("x", 0) == 0); + REQUIRE(sketch.get_upper_bound("x", 0) == 0); + REQUIRE(sketch.get_estimate(std::string()) == 0); + REQUIRE(sketch.get_lower_bound(std::string()) == 0); + REQUIRE(sketch.get_upper_bound(std::string()) == 0); + REQUIRE(sketch.get_upper_bound(int64_t(2)) > 0); // a non-empty item that was never added still gets eps * weight +} + +// 1: numHashes is 1 to 127 +TEST_CASE("count min v2: number of hashes", "[count_min]") { + REQUIRE_THROWS_AS(sketch_type(0, 1000), std::invalid_argument); + REQUIRE_THROWS_AS(sketch_type(128, 1000), std::invalid_argument); + REQUIRE_THROWS_AS(sketch_type(255, 1000), std::invalid_argument); + REQUIRE(sketch_type(1, 1000).get_num_hashes() == 1); + REQUIRE(sketch_type(127, 1000).get_num_hashes() == 127); + REQUIRE(sketch_type::suggest_num_hashes(1.0) == 127); + REQUIRE(sketch_type::suggest_num_hashes(0.0) == 1); + + // the same limits apply to version 2 images + auto image = to_vector(sketch_type(3, 1000).serialize()); + image[12] = 0; + REQUIRE_THROWS_AS(sketch_type::deserialize(image.data(), image.size()), std::invalid_argument); + image[12] = 128; + REQUIRE_THROWS_AS(sketch_type::deserialize(image.data(), image.size()), std::invalid_argument); +} + +// 6: readers accept version 1 images, keep their hashing and write them back as version 1 +TEST_CASE("count min v2: version 1 images", "[count_min]") { + // written by the C++ library before version 2: numHashes 2, numBuckets 5, seed 9001, + // update(1, 10), update(9, 810), update("apple", 5) + const std::vector v1_image = { + 0x02, 0x01, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x93, 0x00, + 0x39, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x34, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }; + auto v1 = sketch_type::deserialize(v1_image.data(), v1_image.size(), 9001); + REQUIRE(v1.get_serial_version() == 1); + REQUIRE(v1.get_total_weight() == 825); + REQUIRE(to_vector(v1.serialize()) == v1_image); // written back as version 1 + + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + s.write(reinterpret_cast(v1_image.data()), v1_image.size()); + REQUIRE(sketch_type::deserialize(s, 9001).get_serial_version() == 1); + + // Version 1 row seeds depend on the standard library, so the estimates of the items above + // are checked only with the standard library that wrote the image. A version 1 sketch is + // consistent with itself everywhere. + v1.update(std::string("pear"), 3); + REQUIRE(v1.get_estimate(std::string("pear")) >= 3); + REQUIRE(v1.get_total_weight() == 828); + const auto v1_bytes = v1.serialize(); + REQUIRE(v1_bytes[1] == 1); + + // version 1 and version 2 sketches hash differently, so they can't be merged + sketch_type v2(2, 5, 9001); + v2.update(int64_t(1), 1); + REQUIRE_THROWS_AS(v1.merge(v2), std::invalid_argument); + REQUIRE_THROWS_AS(v2.merge(v1), std::invalid_argument); + auto other_v1 = sketch_type::deserialize(v1_image.data(), v1_image.size(), 9001); + v1.merge(other_v1); + REQUIRE(v1.get_total_weight() == 828 + 825); + REQUIRE(v1.get_serial_version() == 1); + + // version 1 images written by C++ may have up to 255 rows; 0 is never valid + std::vector v1_empty = { + 0x02, 0x01, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 200, 0xcc, 0x93, 0x00 + }; + const auto wide = sketch_type::deserialize(v1_empty.data(), v1_empty.size(), 9001); + REQUIRE(wide.get_serial_version() == 1); + REQUIRE(wide.get_num_hashes() == 200); + v1_empty[12] = 0; + REQUIRE_THROWS_AS(sketch_type::deserialize(v1_empty.data(), v1_empty.size(), 9001), std::invalid_argument); +} + +TEST_CASE("count min v2: other serial versions are rejected", "[count_min]") { + for (uint8_t version: {0, 3, 255}) { + auto image = to_vector(sketch_type(3, 1000).serialize()); + image[1] = version; + REQUIRE_THROWS_AS(sketch_type::deserialize(image.data(), image.size()), std::invalid_argument); + } +} + +TEST_CASE("count min v2: copies keep the serial version", "[count_min]") { + const std::vector v1_empty = { + 0x02, 0x01, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x93, 0x00 + }; + const auto v1 = sketch_type::deserialize(v1_empty.data(), v1_empty.size()); + sketch_type copy(v1); + REQUIRE(copy.get_serial_version() == 1); + sketch_type assigned(3, 10); + assigned = v1; + REQUIRE(assigned.get_serial_version() == 1); + sketch_type moved(std::move(copy)); + REQUIRE(moved.get_serial_version() == 1); + REQUIRE(moved.serialize()[1] == 1); +} + +} /* namespace datasketches */ From a7519269ce2e3dc1ad2957a356a48d9b8624296d Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Mon, 16 Nov 2026 12:29:42 -0800 Subject: [PATCH 2/2] Count-Min: check each header field on its own The header check compared a weighted sum of the preamble longs, serial version, family ID and empty flag against a list of valid values, so some wrong combinations passed, for example serial version 0 with family ID 19. Check each field separately, as Java does. Co-Authored-By: Claude Opus 5.5 --- count/include/count_min.hpp | 4 ++-- count/include/count_min_impl.hpp | 39 ++++++++++++-------------------- count/test/count_min_v2_test.cpp | 17 ++++++++++++++ 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/count/include/count_min.hpp b/count/include/count_min.hpp index 3b94493a5..a319c0fd5 100644 --- a/count/include/count_min.hpp +++ b/count/include/count_min.hpp @@ -423,9 +423,9 @@ class count_min_sketch{ * Throws an error if the header is not valid. * @param preamble_longs * @param serial_version - * @param flags_byte + * @param family_id */ - static void check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id, uint8_t flags_byte); + static void check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id); /* * Compute the hash locations for an input item diff --git a/count/include/count_min_impl.hpp b/count/include/count_min_impl.hpp index 23123c292..689f3df9b 100644 --- a/count/include/count_min_impl.hpp +++ b/count/include/count_min_impl.hpp @@ -367,7 +367,7 @@ auto count_min_sketch::deserialize(std::istream& is, uint64_t seed, const A const auto flags_byte = read(is); read(is); // 4 unused bytes - check_header_validity(preamble_longs, serial_version, family_id, flags_byte); + check_header_validity(preamble_longs, serial_version, family_id); // Sketch parameters const auto nbuckets = read(is); @@ -429,7 +429,7 @@ auto count_min_sketch::deserialize(const void* bytes, size_t size, uint64_t ptr += copy_from_mem(ptr, flags_byte); ptr += sizeof(uint32_t); - check_header_validity(preamble_longs, serial_version, family_id, flags_byte); + check_header_validity(preamble_longs, serial_version, family_id); // Second 8 bytes are the sketch parameters with a final, unused byte. uint32_t nbuckets; @@ -491,29 +491,20 @@ string count_min_sketch::to_string() const { } template -void count_min_sketch::check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id, uint8_t flags_byte) { - const bool empty = (flags_byte & (1 << flags::IS_EMPTY)) > 0; - - const uint8_t sw = (empty ? 1 : 0) + (2 * serial_version) + (4 * family_id) + (32 * (preamble_longs & 0x3F)); - bool valid = true; - - switch (sw) { // exhaustive list and description of all valid cases - case 138 : break; // !empty, ser_ver==1, family==18, preLongs=2; - case 139 : break; // empty, ser_ver==1, family==18, preLongs=2; - case 140 : break; // !empty, ser_ver==2, family==18, preLongs=2; - case 141 : break; // empty, ser_ver==2, family==18, preLongs=2; - //case 170 : break; // !empty, ser_ver==1, family==18, preLongs=3; - default : // all other case values are invalid - valid = false; +void count_min_sketch::check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id) { + // each field is checked on its own, so that no combination of wrong values can pass + if (preamble_longs != PREAMBLE_LONGS_SHORT) { + throw std::invalid_argument("Possible sketch corruption. Preamble longs mismatch: expected " + + std::to_string(PREAMBLE_LONGS_SHORT) + ", actual " + std::to_string(preamble_longs)); } - - if (!valid) { - std::ostringstream os; - os << "Possible sketch corruption. Inconsistent state: " - << "preamble_longs = " << static_cast(preamble_longs) - << ", empty = " << (empty ? "true" : "false") - << ", serialization_version = " << static_cast(serial_version); - throw std::invalid_argument(os.str()); + if (serial_version != SERIAL_VERSION_1 && serial_version != SERIAL_VERSION_2) { + throw std::invalid_argument("Possible sketch corruption. Serial version mismatch: expected " + + std::to_string(SERIAL_VERSION_1) + " or " + std::to_string(SERIAL_VERSION_2) + + ", actual " + std::to_string(serial_version)); + } + if (family_id != FAMILY_ID) { + throw std::invalid_argument("Possible sketch corruption. Family ID mismatch: expected " + + std::to_string(FAMILY_ID) + ", actual " + std::to_string(family_id)); } } diff --git a/count/test/count_min_v2_test.cpp b/count/test/count_min_v2_test.cpp index 02ddaea7a..7a8a87d84 100644 --- a/count/test/count_min_v2_test.cpp +++ b/count/test/count_min_v2_test.cpp @@ -236,6 +236,23 @@ TEST_CASE("count min v2: other serial versions are rejected", "[count_min]") { } } +TEST_CASE("count min v2: each header field is checked", "[count_min]") { + // {preamble longs, serial version, family ID}: each has a wrong field, and the first three + // used to pass, because the old check only looked at a weighted sum of the fields + const uint8_t headers[][3] = {{2, 0, 19}, {2, 3, 17}, {10, 2, 18}, {3, 2, 18}, {2, 2, 17}}; + sketch_type sketch(3, 1000); + sketch.update(static_cast(1)); + for (const auto& header: headers) { + auto image = to_vector(sketch.serialize()); + image[0] = header[0]; + image[1] = header[1]; + image[2] = header[2]; + REQUIRE_THROWS_AS(sketch_type::deserialize(image.data(), image.size()), std::invalid_argument); + std::stringstream is(std::string(image.begin(), image.end())); + REQUIRE_THROWS_AS(sketch_type::deserialize(is), std::invalid_argument); + } +} + TEST_CASE("count min v2: copies keep the serial version", "[count_min]") { const std::vector v1_empty = { 0x02, 0x01, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x02, 0xcc, 0x93, 0x00