diff --git a/filters/include/bloom_filter.hpp b/filters/include/bloom_filter.hpp index fd5816a1..34fd9642 100644 --- a/filters/include/bloom_filter.hpp +++ b/filters/include/bloom_filter.hpp @@ -621,7 +621,7 @@ class bloom_filter_alloc { bool is_dirty_; bool is_owned_; // if true, data is not owned by filter AND memory_ holds the entire filter not just the bit array bool is_read_only_; // if true, filter is read-only - uint64_t capacity_bits_; + uint64_t capacity_bits_; // a positive multiple of 64, so the bit array is never empty uint64_t num_bits_set_; uint8_t* bit_array_; // data backing bit_array_, regardless of ownership uint8_t* memory_; // if wrapped, pointer to the start of the filter, otherwise nullptr diff --git a/filters/include/bloom_filter_impl.hpp b/filters/include/bloom_filter_impl.hpp index 65a954bf..63ed0b95 100644 --- a/filters/include/bloom_filter_impl.hpp +++ b/filters/include/bloom_filter_impl.hpp @@ -73,9 +73,6 @@ bloom_filter_alloc::bloom_filter_alloc(uint64_t num_bits, uint16_t num_hashes const uint64_t num_bytes = capacity_bits_ >> 3; bit_array_ = AllocUint8(allocator_).allocate(num_bytes); std::fill_n(bit_array_, num_bytes, 0); - if (bit_array_ == nullptr) { - throw std::bad_alloc(); - } memory_ = nullptr; } @@ -172,9 +169,6 @@ bloom_filter_alloc::bloom_filter_alloc(const bloom_filter_alloc& other) : if (is_owned_) { const size_t num_bytes = capacity_bits_ >> 3; bit_array_ = AllocUint8(allocator_).allocate(num_bytes); - if (bit_array_ == nullptr) { - throw std::bad_alloc(); - } std::copy_n(other.bit_array_, num_bytes, bit_array_); memory_ = nullptr; } else { @@ -303,21 +297,26 @@ bloom_filter_alloc bloom_filter_alloc::deserialize(std::istream& is, const const uint32_t num_longs = read(is); // sized in java longs read(is); // unused if (!is.good()) throw std::runtime_error("error reading from std::istream"); + // num_longs is 32 bits, so widen it before shifting: in 32 bits, num_longs << 3 and + // num_longs << 6 can overflow, for example to 0 + const uint64_t num_bytes = static_cast(num_longs) << 3; + const uint64_t num_bits = static_cast(num_longs) << 6; // if empty, stop reading if (is_empty) { - return bloom_filter_alloc(num_longs << 6, num_hashes, seed, allocator); + return bloom_filter_alloc(num_bits, num_hashes, seed, allocator); + } + // a non-empty filter has a bit array: this keeps capacity_bits_ positive, + // and keeps allocate() from being called with 0 + if (num_bytes == 0) { + throw std::invalid_argument("Possible corruption: non-empty filter with an empty bit array"); } const uint64_t raw_num_bits_set = read(is); // allocate memory - const uint64_t num_bytes = num_longs << 3; AllocUint8 alloc(allocator); uint8_t* bit_array = alloc.allocate(num_bytes); - if (bit_array == nullptr) { - throw std::bad_alloc(); - } uint64_t num_bits_set; try { @@ -331,7 +330,7 @@ bloom_filter_alloc bloom_filter_alloc::deserialize(std::istream& is, const } // pass to constructor - return bloom_filter_alloc(seed, num_hashes, false, true, false, num_longs << 6, num_bits_set, bit_array, nullptr, allocator); + return bloom_filter_alloc(seed, num_hashes, false, true, false, num_bits, num_bits_set, bit_array, nullptr, allocator); } template @@ -386,18 +385,26 @@ bloom_filter_alloc bloom_filter_alloc::internal_deserialize_or_wrap(void* uint32_t num_longs; ptr += copy_from_mem(ptr, num_longs); // sized in java longs ptr += sizeof(uint32_t); // unused 32 bits follow + // num_longs is 32 bits, so widen it before shifting: in 32 bits, num_longs << 3 and + // num_longs << 6 can overflow, for example to 0 + const uint64_t num_bytes = static_cast(num_longs) << 3; + const uint64_t num_bits = static_cast(num_longs) << 6; // if empty, stop reading if (wrap && is_empty && !read_only) { throw std::invalid_argument("Cannot wrap an empty filter for writing"); } else if (is_empty) { - return bloom_filter_alloc(num_longs << 6, num_hashes, seed, allocator); + return bloom_filter_alloc(num_bits, num_hashes, seed, allocator); + } + // a non-empty filter has a bit array: this keeps capacity_bits_ positive, + // and keeps allocate() from being called with 0 + if (num_bytes == 0) { + throw std::invalid_argument("Possible corruption: non-empty filter with an empty bit array"); } uint64_t raw_num_bits_set; ptr += copy_from_mem(ptr, raw_num_bits_set); - const uint64_t num_bytes = num_longs << 3; ensure_minimum_memory(end_ptr - ptr, num_bytes); const uint64_t num_bits_set = validate_bloom_filter_num_bits_set( raw_num_bits_set, const_cast(ptr), num_bytes, DIRTY_BITS_VALUE); @@ -412,14 +419,11 @@ bloom_filter_alloc bloom_filter_alloc::internal_deserialize_or_wrap(void* memory = nullptr; AllocUint8 alloc(allocator); bit_array = alloc.allocate(num_bytes); - if (bit_array == nullptr) { - throw std::bad_alloc(); - } copy_from_mem(ptr, bit_array, num_bytes); } // pass to constructor -- !wrap == is_owned_ - return bloom_filter_alloc(seed, num_hashes, false, !wrap, read_only, num_longs << 6, num_bits_set, bit_array, memory, allocator); + return bloom_filter_alloc(seed, num_hashes, false, !wrap, read_only, num_bits, num_bits_set, bit_array, memory, allocator); } template diff --git a/filters/test/bloom_filter_allocation_test.cpp b/filters/test/bloom_filter_allocation_test.cpp index e8236199..af6ceb7c 100644 --- a/filters/test/bloom_filter_allocation_test.cpp +++ b/filters/test/bloom_filter_allocation_test.cpp @@ -17,7 +17,12 @@ * under the License. */ +#include +#include #include +#include +#include +#include #include @@ -96,4 +101,104 @@ TEST_CASE("bloom filter: truncated stream frees bit array", "[bloom_filter][test REQUIRE(test_allocator_net_allocations == 0); } +namespace { + +// The size in bytes that throwing_allocator was last asked for. +std::size_t last_requested_bytes = 0; + +// Standard-conforming allocator that always fails: allocate() throws std::bad_alloc. +// It records the requested size first, so that tests can check it without allocating it. +template +struct throwing_allocator { + using value_type = T; + throwing_allocator() = default; + template throwing_allocator(const throwing_allocator&) {} + T* allocate(std::size_t n) { + last_requested_bytes = n * sizeof(T); + throw std::bad_alloc(); + } + void deallocate(T*, std::size_t) {} + bool operator==(const throwing_allocator&) const { return true; } + bool operator!=(const throwing_allocator&) const { return false; } +}; + +// A bloom filter whose allocator always throws std::bad_alloc. +using throwing_filter = bloom_filter_alloc>; + +// A serialized non-empty filter cut down to its 32-byte header, whose header says that its +// bit array has num_longs longs. Its count of bits set is 0, consistent with having no bits. +std::vector header_only_image(uint32_t num_longs) { + const size_t num_longs_offset = 16; // offsets in the serialized image, see serialize() + const size_t num_bits_set_offset = 24; + const size_t header_bytes = 32; + const uint64_t num_bits_set = 0; + + auto bf = bloom_filter::builder::create_by_size(64, 3, 1234); + bf.update(1); + auto image = bf.serialize(); + image.resize(header_bytes); + std::memcpy(image.data() + num_longs_offset, &num_longs, sizeof(num_longs)); + std::memcpy(image.data() + num_bits_set_offset, &num_bits_set, sizeof(num_bits_set)); + return image; +} + +} // namespace + +TEST_CASE("bloom filter: allocation failure propagates std::bad_alloc", "[bloom_filter]") { + const throwing_allocator allocator; + const uint64_t num_bits = 1024; + const uint16_t num_hashes = 3; + const uint64_t seed = 1234; + + REQUIRE_THROWS_AS(throwing_filter::builder::create_by_size(num_bits, num_hashes, seed, allocator), + std::bad_alloc); + + auto bf = bloom_filter::builder::create_by_size(num_bits, num_hashes, seed); + bf.update(1); + auto bytes = bf.serialize(); + REQUIRE_THROWS_AS(throwing_filter::deserialize(bytes.data(), bytes.size(), allocator), + std::bad_alloc); + std::stringstream ss; + bf.serialize(ss); + REQUIRE_THROWS_AS(throwing_filter::deserialize(ss, allocator), std::bad_alloc); +} + +TEST_CASE("bloom filter: the size of the bit array is computed in 64 bits", "[bloom_filter]") { + // The header of an image says how many 64-bit longs its bit array has. 2^29 longs is 2^32 bytes. + // Computed in 32 bits, that size would be 0, and deserialize() would call allocate(0). + const uint32_t num_longs = uint32_t(1) << 29; + const uint64_t expected_num_bytes = uint64_t(num_longs) << 3; + + // Precondition: computed in 32 bits, the size of this bit array really overflows to 0. + const uint32_t num_bytes_in_32_bits = num_longs << 3; + REQUIRE(num_bytes_in_32_bits == 0); + + // An image with this header and no bit array at all. + const auto image = header_only_image(num_longs); + + SECTION("from memory: rejected, because the image is far too short for 2^32 bytes") { + // Before it allocates anything, deserialize() checks that the buffer holds num_bytes: + // ensure_minimum_memory() throws std::out_of_range. With a size of 0, that check passed. + REQUIRE_THROWS_AS(bloom_filter::deserialize(image.data(), image.size()), std::out_of_range); + } + + SECTION("from a stream: the allocator is asked for 2^32 bytes, not 0") { + // A stream can't tell how much data follows, so deserialize() allocates first. + // throwing_allocator records the size it was asked for, then refuses: nothing is allocated. + const throwing_allocator allocator; + const std::string image_string(reinterpret_cast(image.data()), image.size()); + std::istringstream image_stream(image_string, std::ios::in | std::ios::binary); + last_requested_bytes = 0; + + // throwing_allocator refuses every request, so deserialize() throws std::bad_alloc here + // in any case, also on the old code. This only confirms that the request reached the allocator. + REQUIRE_THROWS_AS(throwing_filter::deserialize(image_stream, allocator), std::bad_alloc); + + // What this section checks: the size of that request. + const std::size_t requested_bytes = last_requested_bytes; + REQUIRE(requested_bytes != 0); // a request for 0 bytes is the 32-bit overflow + REQUIRE(requested_bytes == expected_num_bytes); // the whole 2^32 bytes + } +} + } diff --git a/filters/test/bloom_filter_test.cpp b/filters/test/bloom_filter_test.cpp index 4ce4203b..51cdf433 100644 --- a/filters/test/bloom_filter_test.cpp +++ b/filters/test/bloom_filter_test.cpp @@ -443,4 +443,43 @@ TEST_CASE("bloom_filter: truncated empty stream is rejected", "[bloom_filter]") } } +TEST_CASE("bloom_filter: image with zero bit array length is rejected", "[bloom_filter]") { + const size_t num_longs_offset = 16; + const size_t num_bits_set_offset = 24; + + auto bf = bloom_filter::builder::create_by_size(64, 3); + bf.update("apple"); + auto bytes = bf.serialize(); + // a non-empty image whose bit array has 0 longs, with a count of bits set consistent with that + const uint32_t num_longs = 0; + const uint64_t num_bits_set = 0; + std::memcpy(bytes.data() + num_longs_offset, &num_longs, sizeof(num_longs)); + std::memcpy(bytes.data() + num_bits_set_offset, &num_bits_set, sizeof(num_bits_set)); + + REQUIRE_THROWS_AS(bloom_filter::deserialize(bytes.data(), bytes.size()), std::invalid_argument); + REQUIRE_THROWS_AS(bloom_filter::wrap(bytes.data(), bytes.size()), std::invalid_argument); + REQUIRE_THROWS_AS(bloom_filter::writable_wrap(bytes.data(), bytes.size()), std::invalid_argument); + + const std::string serialized(reinterpret_cast(bytes.data()), bytes.size()); + std::istringstream is(serialized, std::ios::in | std::ios::binary); + REQUIRE_THROWS_AS(bloom_filter::deserialize(is), std::invalid_argument); + + // an empty image with 0 longs is rejected too + auto empty_bytes = bloom_filter::builder::create_by_size(64, 3).serialize(); + std::memcpy(empty_bytes.data() + num_longs_offset, &num_longs, sizeof(num_longs)); + REQUIRE_THROWS_AS(bloom_filter::deserialize(empty_bytes.data(), empty_bytes.size()), std::invalid_argument); + REQUIRE_THROWS_AS(bloom_filter::wrap(empty_bytes.data(), empty_bytes.size()), std::invalid_argument); + const std::string empty_serialized(reinterpret_cast(empty_bytes.data()), empty_bytes.size()); + std::istringstream empty_is(empty_serialized, std::ios::in | std::ios::binary); + REQUIRE_THROWS_AS(bloom_filter::deserialize(empty_is), std::invalid_argument); +} + +TEST_CASE("bloom_filter: capacity is rounded up to a multiple of 64 bits", "[bloom_filter]") { + // so the bit array of a filter is never shorter than 8 bytes + REQUIRE(bloom_filter::builder::create_by_size(1, 3).get_capacity() == 64); + REQUIRE(bloom_filter::builder::create_by_size(63, 3).get_capacity() == 64); + REQUIRE(bloom_filter::builder::create_by_size(64, 3).get_capacity() == 64); + REQUIRE(bloom_filter::builder::create_by_size(65, 3).get_capacity() == 128); +} + } // namespace datasketches