From 9a3c8b7660dd047a67726ef5cc7b121155a2fda1 Mon Sep 17 00:00:00 2001
From: PZ <56173356+P92Z@users.noreply.github.com>
Date: Tue, 17 Nov 2026 23:39:48 +0500
Subject: [PATCH 1/4] Remove redundant nullptr checks after allocate() in
bloom_filter
Standard-conforming allocators report failure by throwing from
allocate(), never by returning nullptr, so these checks can never fire.
One of them was also placed after std::fill_n had already written to the
buffer. Other sketches in the library don't perform such checks either.
Add a test that a failing (throwing) allocator propagates std::bad_alloc
from the constructor and both deserialize paths.
---
filters/include/bloom_filter_impl.hpp | 12 -------
filters/test/bloom_filter_allocation_test.cpp | 31 +++++++++++++++++++
2 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/filters/include/bloom_filter_impl.hpp b/filters/include/bloom_filter_impl.hpp
index 65a954bf..fb64cc84 100644
--- a/filters/include/bloom_filter_impl.hpp
+++ b/filters/include/bloom_filter_impl.hpp
@@ -73,9 +73,6 @@ bloom_filter_alloc::bloom_filter_alloc(uint64_t num_bits, uint16_t num_hashes
const uint64_t num_bytes = capacity_bits_ >> 3;
bit_array_ = AllocUint8(allocator_).allocate(num_bytes);
std::fill_n(bit_array_, num_bytes, 0);
- if (bit_array_ == nullptr) {
- throw std::bad_alloc();
- }
memory_ = nullptr;
}
@@ -172,9 +169,6 @@ bloom_filter_alloc::bloom_filter_alloc(const bloom_filter_alloc& other) :
if (is_owned_) {
const size_t num_bytes = capacity_bits_ >> 3;
bit_array_ = AllocUint8(allocator_).allocate(num_bytes);
- if (bit_array_ == nullptr) {
- throw std::bad_alloc();
- }
std::copy_n(other.bit_array_, num_bytes, bit_array_);
memory_ = nullptr;
} else {
@@ -315,9 +309,6 @@ bloom_filter_alloc bloom_filter_alloc::deserialize(std::istream& is, const
const uint64_t num_bytes = num_longs << 3;
AllocUint8 alloc(allocator);
uint8_t* bit_array = alloc.allocate(num_bytes);
- if (bit_array == nullptr) {
- throw std::bad_alloc();
- }
uint64_t num_bits_set;
try {
@@ -412,9 +403,6 @@ bloom_filter_alloc bloom_filter_alloc::internal_deserialize_or_wrap(void*
memory = nullptr;
AllocUint8 alloc(allocator);
bit_array = alloc.allocate(num_bytes);
- if (bit_array == nullptr) {
- throw std::bad_alloc();
- }
copy_from_mem(ptr, bit_array, num_bytes);
}
diff --git a/filters/test/bloom_filter_allocation_test.cpp b/filters/test/bloom_filter_allocation_test.cpp
index e8236199..e2ad009d 100644
--- a/filters/test/bloom_filter_allocation_test.cpp
+++ b/filters/test/bloom_filter_allocation_test.cpp
@@ -17,6 +17,7 @@
* under the License.
*/
+#include
#include
#include
@@ -96,4 +97,34 @@ TEST_CASE("bloom filter: truncated stream frees bit array", "[bloom_filter][test
REQUIRE(test_allocator_net_allocations == 0);
}
+// Standard-conforming allocator that always fails: allocate() throws std::bad_alloc.
+template
+struct throwing_allocator {
+ using value_type = T;
+ throwing_allocator() = default;
+ template throwing_allocator(const throwing_allocator&) {}
+ T* allocate(std::size_t) { throw std::bad_alloc(); }
+ void deallocate(T*, std::size_t) {}
+ bool operator==(const throwing_allocator&) const { return true; }
+ bool operator!=(const throwing_allocator&) const { return false; }
+};
+
+TEST_CASE("bloom filter: allocation failure propagates std::bad_alloc", "[bloom_filter]") {
+ using bloom_filter_throwing_alloc = bloom_filter_alloc>;
+ REQUIRE_THROWS_AS(bloom_filter_throwing_alloc::builder::create_by_size(1024, 3, 1234,
+ throwing_allocator()),
+ std::bad_alloc);
+
+ auto bf = bloom_filter::builder::create_by_size(1024, 3, 1234);
+ bf.update(1);
+ auto bytes = bf.serialize();
+ REQUIRE_THROWS_AS(bloom_filter_throwing_alloc::deserialize(bytes.data(), bytes.size(),
+ throwing_allocator()),
+ std::bad_alloc);
+ std::stringstream ss;
+ bf.serialize(ss);
+ REQUIRE_THROWS_AS(bloom_filter_throwing_alloc::deserialize(ss, throwing_allocator()),
+ std::bad_alloc);
+}
+
}
From 60407ee350fde950503613f6c4b6760187dcbcab Mon Sep 17 00:00:00 2001
From: PZ <56173356+P92Z@users.noreply.github.com>
Date: Tue, 17 Nov 2026 23:39:48 +0500
Subject: [PATCH 2/4] Reject non-empty bloom filter images with a zero-length
bit array
The length of the bit array is read from the serialized image, and 0
was rejected only for empty images. For a non-empty image with 0 longs,
deserialize() called allocate(0), and deserialize(), wrap() and
writable_wrap() all created a filter with a capacity of 0. update() on
that filter then divided by zero.
Reject such images as corrupt, as other invalid header fields are.
The tests cover non-empty and empty images with 0 longs, and that the
capacity of a new filter is rounded up to a multiple of 64 bits, so its
bit array is never shorter than 8 bytes.
---
filters/include/bloom_filter.hpp | 2 +-
filters/include/bloom_filter_impl.hpp | 8 ++++++
filters/test/bloom_filter_test.cpp | 39 +++++++++++++++++++++++++++
3 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/filters/include/bloom_filter.hpp b/filters/include/bloom_filter.hpp
index fd5816a1..34fd9642 100644
--- a/filters/include/bloom_filter.hpp
+++ b/filters/include/bloom_filter.hpp
@@ -621,7 +621,7 @@ class bloom_filter_alloc {
bool is_dirty_;
bool is_owned_; // if true, data is not owned by filter AND memory_ holds the entire filter not just the bit array
bool is_read_only_; // if true, filter is read-only
- uint64_t capacity_bits_;
+ uint64_t capacity_bits_; // a positive multiple of 64, so the bit array is never empty
uint64_t num_bits_set_;
uint8_t* bit_array_; // data backing bit_array_, regardless of ownership
uint8_t* memory_; // if wrapped, pointer to the start of the filter, otherwise nullptr
diff --git a/filters/include/bloom_filter_impl.hpp b/filters/include/bloom_filter_impl.hpp
index fb64cc84..e4357d5e 100644
--- a/filters/include/bloom_filter_impl.hpp
+++ b/filters/include/bloom_filter_impl.hpp
@@ -302,6 +302,10 @@ bloom_filter_alloc bloom_filter_alloc::deserialize(std::istream& is, const
if (is_empty) {
return bloom_filter_alloc(num_longs << 6, num_hashes, seed, allocator);
}
+ // a non-empty filter has at least one long, which keeps capacity_bits_ positive
+ if (num_longs == 0) {
+ throw std::invalid_argument("Possible corruption: non-empty filter with a bit array of 0 longs");
+ }
const uint64_t raw_num_bits_set = read(is);
@@ -384,6 +388,10 @@ bloom_filter_alloc bloom_filter_alloc::internal_deserialize_or_wrap(void*
} else if (is_empty) {
return bloom_filter_alloc(num_longs << 6, num_hashes, seed, allocator);
}
+ // a non-empty filter has at least one long, which keeps capacity_bits_ positive
+ if (num_longs == 0) {
+ throw std::invalid_argument("Possible corruption: non-empty filter with a bit array of 0 longs");
+ }
uint64_t raw_num_bits_set;
ptr += copy_from_mem(ptr, raw_num_bits_set);
diff --git a/filters/test/bloom_filter_test.cpp b/filters/test/bloom_filter_test.cpp
index 4ce4203b..51cdf433 100644
--- a/filters/test/bloom_filter_test.cpp
+++ b/filters/test/bloom_filter_test.cpp
@@ -443,4 +443,43 @@ TEST_CASE("bloom_filter: truncated empty stream is rejected", "[bloom_filter]")
}
}
+TEST_CASE("bloom_filter: image with zero bit array length is rejected", "[bloom_filter]") {
+ const size_t num_longs_offset = 16;
+ const size_t num_bits_set_offset = 24;
+
+ auto bf = bloom_filter::builder::create_by_size(64, 3);
+ bf.update("apple");
+ auto bytes = bf.serialize();
+ // a non-empty image whose bit array has 0 longs, with a count of bits set consistent with that
+ const uint32_t num_longs = 0;
+ const uint64_t num_bits_set = 0;
+ std::memcpy(bytes.data() + num_longs_offset, &num_longs, sizeof(num_longs));
+ std::memcpy(bytes.data() + num_bits_set_offset, &num_bits_set, sizeof(num_bits_set));
+
+ REQUIRE_THROWS_AS(bloom_filter::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
+ REQUIRE_THROWS_AS(bloom_filter::wrap(bytes.data(), bytes.size()), std::invalid_argument);
+ REQUIRE_THROWS_AS(bloom_filter::writable_wrap(bytes.data(), bytes.size()), std::invalid_argument);
+
+ const std::string serialized(reinterpret_cast(bytes.data()), bytes.size());
+ std::istringstream is(serialized, std::ios::in | std::ios::binary);
+ REQUIRE_THROWS_AS(bloom_filter::deserialize(is), std::invalid_argument);
+
+ // an empty image with 0 longs is rejected too
+ auto empty_bytes = bloom_filter::builder::create_by_size(64, 3).serialize();
+ std::memcpy(empty_bytes.data() + num_longs_offset, &num_longs, sizeof(num_longs));
+ REQUIRE_THROWS_AS(bloom_filter::deserialize(empty_bytes.data(), empty_bytes.size()), std::invalid_argument);
+ REQUIRE_THROWS_AS(bloom_filter::wrap(empty_bytes.data(), empty_bytes.size()), std::invalid_argument);
+ const std::string empty_serialized(reinterpret_cast(empty_bytes.data()), empty_bytes.size());
+ std::istringstream empty_is(empty_serialized, std::ios::in | std::ios::binary);
+ REQUIRE_THROWS_AS(bloom_filter::deserialize(empty_is), std::invalid_argument);
+}
+
+TEST_CASE("bloom_filter: capacity is rounded up to a multiple of 64 bits", "[bloom_filter]") {
+ // so the bit array of a filter is never shorter than 8 bytes
+ REQUIRE(bloom_filter::builder::create_by_size(1, 3).get_capacity() == 64);
+ REQUIRE(bloom_filter::builder::create_by_size(63, 3).get_capacity() == 64);
+ REQUIRE(bloom_filter::builder::create_by_size(64, 3).get_capacity() == 64);
+ REQUIRE(bloom_filter::builder::create_by_size(65, 3).get_capacity() == 128);
+}
+
} // namespace datasketches
From 1b1e9312d0e7ad7610c8f9dadd39903d531a7bbf Mon Sep 17 00:00:00 2001
From: PZ <56173356+P92Z@users.noreply.github.com>
Date: Tue, 17 Nov 2026 23:39:48 +0500
Subject: [PATCH 3/4] Put the test allocator for bloom_filter in an anonymous
namespace
So that it can't clash with the failing allocators that other tests
define in namespace datasketches.
---
filters/test/bloom_filter_allocation_test.cpp | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/filters/test/bloom_filter_allocation_test.cpp b/filters/test/bloom_filter_allocation_test.cpp
index e2ad009d..517188fe 100644
--- a/filters/test/bloom_filter_allocation_test.cpp
+++ b/filters/test/bloom_filter_allocation_test.cpp
@@ -97,6 +97,8 @@ TEST_CASE("bloom filter: truncated stream frees bit array", "[bloom_filter][test
REQUIRE(test_allocator_net_allocations == 0);
}
+namespace {
+
// Standard-conforming allocator that always fails: allocate() throws std::bad_alloc.
template
struct throwing_allocator {
@@ -109,6 +111,8 @@ struct throwing_allocator {
bool operator!=(const throwing_allocator&) const { return false; }
};
+} // namespace
+
TEST_CASE("bloom filter: allocation failure propagates std::bad_alloc", "[bloom_filter]") {
using bloom_filter_throwing_alloc = bloom_filter_alloc>;
REQUIRE_THROWS_AS(bloom_filter_throwing_alloc::builder::create_by_size(1024, 3, 1234,
From 1b6841d0c64688b5d90f6dc0a897dbb526369a04 Mon Sep 17 00:00:00 2001
From: PZ <56173356+P92Z@users.noreply.github.com>
Date: Wed, 18 Nov 2026 19:55:58 +0500
Subject: [PATCH 4/4] Compute the bit array size of a deserialized bloom filter
in 64 bits
num_longs is read from the image as a uint32_t, and num_longs << 3 and
num_longs << 6 were computed in 32 bits before being stored in
uint64_t. They could overflow: a non-empty image with 2^29 longs got a
bit array of 0 bytes and a capacity of 0, so it passed the check for 0
longs, called allocate(0), and update() then divided by zero. Even a
valid filter of 2^32 bits or more was read back with its capacity
modulo 2^32.
Widen num_longs before shifting, and check the size of the bit array
itself, num_bytes, instead of num_longs, so that allocate() is never
called with 0.
In the tests, give the filter type with the throwing allocator a short
name, throwing_filter, and name the parameters of the bad_alloc test.
---
filters/include/bloom_filter_impl.hpp | 32 ++++---
filters/test/bloom_filter_allocation_test.cpp | 88 +++++++++++++++++--
2 files changed, 99 insertions(+), 21 deletions(-)
diff --git a/filters/include/bloom_filter_impl.hpp b/filters/include/bloom_filter_impl.hpp
index e4357d5e..63ed0b95 100644
--- a/filters/include/bloom_filter_impl.hpp
+++ b/filters/include/bloom_filter_impl.hpp
@@ -297,20 +297,24 @@ bloom_filter_alloc bloom_filter_alloc::deserialize(std::istream& is, const
const uint32_t num_longs = read(is); // sized in java longs
read(is); // unused
if (!is.good()) throw std::runtime_error("error reading from std::istream");
+ // num_longs is 32 bits, so widen it before shifting: in 32 bits, num_longs << 3 and
+ // num_longs << 6 can overflow, for example to 0
+ const uint64_t num_bytes = static_cast(num_longs) << 3;
+ const uint64_t num_bits = static_cast(num_longs) << 6;
// if empty, stop reading
if (is_empty) {
- return bloom_filter_alloc(num_longs << 6, num_hashes, seed, allocator);
+ return bloom_filter_alloc(num_bits, num_hashes, seed, allocator);
}
- // a non-empty filter has at least one long, which keeps capacity_bits_ positive
- if (num_longs == 0) {
- throw std::invalid_argument("Possible corruption: non-empty filter with a bit array of 0 longs");
+ // a non-empty filter has a bit array: this keeps capacity_bits_ positive,
+ // and keeps allocate() from being called with 0
+ if (num_bytes == 0) {
+ throw std::invalid_argument("Possible corruption: non-empty filter with an empty bit array");
}
const uint64_t raw_num_bits_set = read(is);
// allocate memory
- const uint64_t num_bytes = num_longs << 3;
AllocUint8 alloc(allocator);
uint8_t* bit_array = alloc.allocate(num_bytes);
@@ -326,7 +330,7 @@ bloom_filter_alloc bloom_filter_alloc::deserialize(std::istream& is, const
}
// pass to constructor
- return bloom_filter_alloc(seed, num_hashes, false, true, false, num_longs << 6, num_bits_set, bit_array, nullptr, allocator);
+ return bloom_filter_alloc(seed, num_hashes, false, true, false, num_bits, num_bits_set, bit_array, nullptr, allocator);
}
template
@@ -381,22 +385,26 @@ bloom_filter_alloc bloom_filter_alloc::internal_deserialize_or_wrap(void*
uint32_t num_longs;
ptr += copy_from_mem(ptr, num_longs); // sized in java longs
ptr += sizeof(uint32_t); // unused 32 bits follow
+ // num_longs is 32 bits, so widen it before shifting: in 32 bits, num_longs << 3 and
+ // num_longs << 6 can overflow, for example to 0
+ const uint64_t num_bytes = static_cast(num_longs) << 3;
+ const uint64_t num_bits = static_cast(num_longs) << 6;
// if empty, stop reading
if (wrap && is_empty && !read_only) {
throw std::invalid_argument("Cannot wrap an empty filter for writing");
} else if (is_empty) {
- return bloom_filter_alloc(num_longs << 6, num_hashes, seed, allocator);
+ return bloom_filter_alloc(num_bits, num_hashes, seed, allocator);
}
- // a non-empty filter has at least one long, which keeps capacity_bits_ positive
- if (num_longs == 0) {
- throw std::invalid_argument("Possible corruption: non-empty filter with a bit array of 0 longs");
+ // a non-empty filter has a bit array: this keeps capacity_bits_ positive,
+ // and keeps allocate() from being called with 0
+ if (num_bytes == 0) {
+ throw std::invalid_argument("Possible corruption: non-empty filter with an empty bit array");
}
uint64_t raw_num_bits_set;
ptr += copy_from_mem(ptr, raw_num_bits_set);
- const uint64_t num_bytes = num_longs << 3;
ensure_minimum_memory(end_ptr - ptr, num_bytes);
const uint64_t num_bits_set = validate_bloom_filter_num_bits_set(
raw_num_bits_set, const_cast(ptr), num_bytes, DIRTY_BITS_VALUE);
@@ -415,7 +423,7 @@ bloom_filter_alloc bloom_filter_alloc::internal_deserialize_or_wrap(void*
}
// pass to constructor -- !wrap == is_owned_
- return bloom_filter_alloc(seed, num_hashes, false, !wrap, read_only, num_longs << 6, num_bits_set, bit_array, memory, allocator);
+ return bloom_filter_alloc(seed, num_hashes, false, !wrap, read_only, num_bits, num_bits_set, bit_array, memory, allocator);
}
template
diff --git a/filters/test/bloom_filter_allocation_test.cpp b/filters/test/bloom_filter_allocation_test.cpp
index 517188fe..af6ceb7c 100644
--- a/filters/test/bloom_filter_allocation_test.cpp
+++ b/filters/test/bloom_filter_allocation_test.cpp
@@ -17,8 +17,12 @@
* under the License.
*/
+#include
#include
#include
+#include
+#include
+#include
#include
@@ -99,36 +103,102 @@ TEST_CASE("bloom filter: truncated stream frees bit array", "[bloom_filter][test
namespace {
+// The size in bytes that throwing_allocator was last asked for.
+std::size_t last_requested_bytes = 0;
+
// Standard-conforming allocator that always fails: allocate() throws std::bad_alloc.
+// It records the requested size first, so that tests can check it without allocating it.
template
struct throwing_allocator {
using value_type = T;
throwing_allocator() = default;
template throwing_allocator(const throwing_allocator&) {}
- T* allocate(std::size_t) { throw std::bad_alloc(); }
+ T* allocate(std::size_t n) {
+ last_requested_bytes = n * sizeof(T);
+ throw std::bad_alloc();
+ }
void deallocate(T*, std::size_t) {}
bool operator==(const throwing_allocator&) const { return true; }
bool operator!=(const throwing_allocator&) const { return false; }
};
+// A bloom filter whose allocator always throws std::bad_alloc.
+using throwing_filter = bloom_filter_alloc>;
+
+// A serialized non-empty filter cut down to its 32-byte header, whose header says that its
+// bit array has num_longs longs. Its count of bits set is 0, consistent with having no bits.
+std::vector header_only_image(uint32_t num_longs) {
+ const size_t num_longs_offset = 16; // offsets in the serialized image, see serialize()
+ const size_t num_bits_set_offset = 24;
+ const size_t header_bytes = 32;
+ const uint64_t num_bits_set = 0;
+
+ auto bf = bloom_filter::builder::create_by_size(64, 3, 1234);
+ bf.update(1);
+ auto image = bf.serialize();
+ image.resize(header_bytes);
+ std::memcpy(image.data() + num_longs_offset, &num_longs, sizeof(num_longs));
+ std::memcpy(image.data() + num_bits_set_offset, &num_bits_set, sizeof(num_bits_set));
+ return image;
+}
+
} // namespace
TEST_CASE("bloom filter: allocation failure propagates std::bad_alloc", "[bloom_filter]") {
- using bloom_filter_throwing_alloc = bloom_filter_alloc>;
- REQUIRE_THROWS_AS(bloom_filter_throwing_alloc::builder::create_by_size(1024, 3, 1234,
- throwing_allocator()),
+ const throwing_allocator allocator;
+ const uint64_t num_bits = 1024;
+ const uint16_t num_hashes = 3;
+ const uint64_t seed = 1234;
+
+ REQUIRE_THROWS_AS(throwing_filter::builder::create_by_size(num_bits, num_hashes, seed, allocator),
std::bad_alloc);
- auto bf = bloom_filter::builder::create_by_size(1024, 3, 1234);
+ auto bf = bloom_filter::builder::create_by_size(num_bits, num_hashes, seed);
bf.update(1);
auto bytes = bf.serialize();
- REQUIRE_THROWS_AS(bloom_filter_throwing_alloc::deserialize(bytes.data(), bytes.size(),
- throwing_allocator()),
+ REQUIRE_THROWS_AS(throwing_filter::deserialize(bytes.data(), bytes.size(), allocator),
std::bad_alloc);
std::stringstream ss;
bf.serialize(ss);
- REQUIRE_THROWS_AS(bloom_filter_throwing_alloc::deserialize(ss, throwing_allocator()),
- std::bad_alloc);
+ REQUIRE_THROWS_AS(throwing_filter::deserialize(ss, allocator), std::bad_alloc);
+}
+
+TEST_CASE("bloom filter: the size of the bit array is computed in 64 bits", "[bloom_filter]") {
+ // The header of an image says how many 64-bit longs its bit array has. 2^29 longs is 2^32 bytes.
+ // Computed in 32 bits, that size would be 0, and deserialize() would call allocate(0).
+ const uint32_t num_longs = uint32_t(1) << 29;
+ const uint64_t expected_num_bytes = uint64_t(num_longs) << 3;
+
+ // Precondition: computed in 32 bits, the size of this bit array really overflows to 0.
+ const uint32_t num_bytes_in_32_bits = num_longs << 3;
+ REQUIRE(num_bytes_in_32_bits == 0);
+
+ // An image with this header and no bit array at all.
+ const auto image = header_only_image(num_longs);
+
+ SECTION("from memory: rejected, because the image is far too short for 2^32 bytes") {
+ // Before it allocates anything, deserialize() checks that the buffer holds num_bytes:
+ // ensure_minimum_memory() throws std::out_of_range. With a size of 0, that check passed.
+ REQUIRE_THROWS_AS(bloom_filter::deserialize(image.data(), image.size()), std::out_of_range);
+ }
+
+ SECTION("from a stream: the allocator is asked for 2^32 bytes, not 0") {
+ // A stream can't tell how much data follows, so deserialize() allocates first.
+ // throwing_allocator records the size it was asked for, then refuses: nothing is allocated.
+ const throwing_allocator allocator;
+ const std::string image_string(reinterpret_cast(image.data()), image.size());
+ std::istringstream image_stream(image_string, std::ios::in | std::ios::binary);
+ last_requested_bytes = 0;
+
+ // throwing_allocator refuses every request, so deserialize() throws std::bad_alloc here
+ // in any case, also on the old code. This only confirms that the request reached the allocator.
+ REQUIRE_THROWS_AS(throwing_filter::deserialize(image_stream, allocator), std::bad_alloc);
+
+ // What this section checks: the size of that request.
+ const std::size_t requested_bytes = last_requested_bytes;
+ REQUIRE(requested_bytes != 0); // a request for 0 bytes is the 32-bit overflow
+ REQUIRE(requested_bytes == expected_num_bytes); // the whole 2^32 bytes
+ }
}
}