Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion filters/include/bloom_filter.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -621,7 +621,7 @@ class bloom_filter_alloc {
bool is_dirty_;
bool is_owned_; // if true, data is not owned by filter AND memory_ holds the entire filter not just the bit array
bool is_read_only_; // if true, filter is read-only
uint64_t capacity_bits_;
uint64_t capacity_bits_; // a positive multiple of 64, so the bit array is never empty
uint64_t num_bits_set_;
uint8_t* bit_array_; // data backing bit_array_, regardless of ownership
uint8_t* memory_; // if wrapped, pointer to the start of the filter, otherwise nullptr
Expand Down
40 changes: 22 additions & 18 deletions filters/include/bloom_filter_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,6 @@ bloom_filter_alloc<A>::bloom_filter_alloc(uint64_t num_bits, uint16_t num_hashes
const uint64_t num_bytes = capacity_bits_ >> 3;
bit_array_ = AllocUint8(allocator_).allocate(num_bytes);
std::fill_n(bit_array_, num_bytes, 0);
if (bit_array_ == nullptr) {
throw std::bad_alloc();
}
memory_ = nullptr;
}

Expand Down Expand Up @@ -172,9 +169,6 @@ bloom_filter_alloc<A>::bloom_filter_alloc(const bloom_filter_alloc& other) :
if (is_owned_) {
const size_t num_bytes = capacity_bits_ >> 3;
bit_array_ = AllocUint8(allocator_).allocate(num_bytes);
if (bit_array_ == nullptr) {
throw std::bad_alloc();
}
std::copy_n(other.bit_array_, num_bytes, bit_array_);
memory_ = nullptr;
} else {
Expand Down Expand Up @@ -303,21 +297,26 @@ bloom_filter_alloc<A> bloom_filter_alloc<A>::deserialize(std::istream& is, const
const uint32_t num_longs = read<uint32_t>(is); // sized in java longs
read<uint32_t>(is); // unused
if (!is.good()) throw std::runtime_error("error reading from std::istream");
// num_longs is 32 bits, so widen it before shifting: in 32 bits, num_longs << 3 and
// num_longs << 6 can overflow, for example to 0
const uint64_t num_bytes = static_cast<uint64_t>(num_longs) << 3;
const uint64_t num_bits = static_cast<uint64_t>(num_longs) << 6;

// if empty, stop reading
if (is_empty) {
return bloom_filter_alloc<A>(num_longs << 6, num_hashes, seed, allocator);
return bloom_filter_alloc<A>(num_bits, num_hashes, seed, allocator);
}
// a non-empty filter has a bit array: this keeps capacity_bits_ positive,
// and keeps allocate() from being called with 0
if (num_bytes == 0) {
throw std::invalid_argument("Possible corruption: non-empty filter with an empty bit array");
}

const uint64_t raw_num_bits_set = read<uint64_t>(is);

// allocate memory
const uint64_t num_bytes = num_longs << 3;
AllocUint8 alloc(allocator);
uint8_t* bit_array = alloc.allocate(num_bytes);
if (bit_array == nullptr) {
throw std::bad_alloc();
}

uint64_t num_bits_set;
try {
Expand All @@ -331,7 +330,7 @@ bloom_filter_alloc<A> bloom_filter_alloc<A>::deserialize(std::istream& is, const
}

// pass to constructor
return bloom_filter_alloc<A>(seed, num_hashes, false, true, false, num_longs << 6, num_bits_set, bit_array, nullptr, allocator);
return bloom_filter_alloc<A>(seed, num_hashes, false, true, false, num_bits, num_bits_set, bit_array, nullptr, allocator);
}

template<typename A>
Expand Down Expand Up @@ -386,18 +385,26 @@ bloom_filter_alloc<A> bloom_filter_alloc<A>::internal_deserialize_or_wrap(void*
uint32_t num_longs;
ptr += copy_from_mem(ptr, num_longs); // sized in java longs
ptr += sizeof(uint32_t); // unused 32 bits follow
// num_longs is 32 bits, so widen it before shifting: in 32 bits, num_longs << 3 and
// num_longs << 6 can overflow, for example to 0
const uint64_t num_bytes = static_cast<uint64_t>(num_longs) << 3;
const uint64_t num_bits = static_cast<uint64_t>(num_longs) << 6;

// if empty, stop reading
if (wrap && is_empty && !read_only) {
throw std::invalid_argument("Cannot wrap an empty filter for writing");
} else if (is_empty) {
return bloom_filter_alloc<A>(num_longs << 6, num_hashes, seed, allocator);
return bloom_filter_alloc<A>(num_bits, num_hashes, seed, allocator);
}
// a non-empty filter has a bit array: this keeps capacity_bits_ positive,
// and keeps allocate() from being called with 0
if (num_bytes == 0) {
throw std::invalid_argument("Possible corruption: non-empty filter with an empty bit array");
}

uint64_t raw_num_bits_set;
ptr += copy_from_mem(ptr, raw_num_bits_set);

const uint64_t num_bytes = num_longs << 3;
ensure_minimum_memory(end_ptr - ptr, num_bytes);
const uint64_t num_bits_set = validate_bloom_filter_num_bits_set(
raw_num_bits_set, const_cast<uint8_t*>(ptr), num_bytes, DIRTY_BITS_VALUE);
Expand All @@ -412,14 +419,11 @@ bloom_filter_alloc<A> bloom_filter_alloc<A>::internal_deserialize_or_wrap(void*
memory = nullptr;
AllocUint8 alloc(allocator);
bit_array = alloc.allocate(num_bytes);
if (bit_array == nullptr) {
throw std::bad_alloc();
}
copy_from_mem(ptr, bit_array, num_bytes);
}

// pass to constructor -- !wrap == is_owned_
return bloom_filter_alloc<A>(seed, num_hashes, false, !wrap, read_only, num_longs << 6, num_bits_set, bit_array, memory, allocator);
return bloom_filter_alloc<A>(seed, num_hashes, false, !wrap, read_only, num_bits, num_bits_set, bit_array, memory, allocator);
}

template<typename A>
Expand Down
105 changes: 105 additions & 0 deletions filters/test/bloom_filter_allocation_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,12 @@
* under the License.
*/

#include <cstring>
#include <new>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>

#include <catch2/catch.hpp>

Expand Down Expand Up @@ -96,4 +101,104 @@ TEST_CASE("bloom filter: truncated stream frees bit array", "[bloom_filter][test
REQUIRE(test_allocator_net_allocations == 0);
}

namespace {

// The size in bytes that throwing_allocator was last asked for.
std::size_t last_requested_bytes = 0;

// Standard-conforming allocator that always fails: allocate() throws std::bad_alloc.
// It records the requested size first, so that tests can check it without allocating it.
template<typename T>
struct throwing_allocator {
using value_type = T;
throwing_allocator() = default;
template<typename U> throwing_allocator(const throwing_allocator<U>&) {}
T* allocate(std::size_t n) {
last_requested_bytes = n * sizeof(T);
throw std::bad_alloc();
}
void deallocate(T*, std::size_t) {}
bool operator==(const throwing_allocator&) const { return true; }
bool operator!=(const throwing_allocator&) const { return false; }
};

// A bloom filter whose allocator always throws std::bad_alloc.
using throwing_filter = bloom_filter_alloc<throwing_allocator<uint8_t>>;

// A serialized non-empty filter cut down to its 32-byte header, whose header says that its
// bit array has num_longs longs. Its count of bits set is 0, consistent with having no bits.
std::vector<uint8_t> header_only_image(uint32_t num_longs) {
const size_t num_longs_offset = 16; // offsets in the serialized image, see serialize()
const size_t num_bits_set_offset = 24;
const size_t header_bytes = 32;
const uint64_t num_bits_set = 0;

auto bf = bloom_filter::builder::create_by_size(64, 3, 1234);
bf.update(1);
auto image = bf.serialize();
image.resize(header_bytes);
std::memcpy(image.data() + num_longs_offset, &num_longs, sizeof(num_longs));
std::memcpy(image.data() + num_bits_set_offset, &num_bits_set, sizeof(num_bits_set));
return image;
}

} // namespace

TEST_CASE("bloom filter: allocation failure propagates std::bad_alloc", "[bloom_filter]") {
const throwing_allocator<uint8_t> allocator;
const uint64_t num_bits = 1024;
const uint16_t num_hashes = 3;
const uint64_t seed = 1234;

REQUIRE_THROWS_AS(throwing_filter::builder::create_by_size(num_bits, num_hashes, seed, allocator),
std::bad_alloc);

auto bf = bloom_filter::builder::create_by_size(num_bits, num_hashes, seed);
bf.update(1);
auto bytes = bf.serialize();
REQUIRE_THROWS_AS(throwing_filter::deserialize(bytes.data(), bytes.size(), allocator),
std::bad_alloc);
std::stringstream ss;
bf.serialize(ss);
REQUIRE_THROWS_AS(throwing_filter::deserialize(ss, allocator), std::bad_alloc);
}

TEST_CASE("bloom filter: the size of the bit array is computed in 64 bits", "[bloom_filter]") {
// The header of an image says how many 64-bit longs its bit array has. 2^29 longs is 2^32 bytes.
// Computed in 32 bits, that size would be 0, and deserialize() would call allocate(0).
const uint32_t num_longs = uint32_t(1) << 29;
const uint64_t expected_num_bytes = uint64_t(num_longs) << 3;

// Precondition: computed in 32 bits, the size of this bit array really overflows to 0.
const uint32_t num_bytes_in_32_bits = num_longs << 3;
REQUIRE(num_bytes_in_32_bits == 0);

// An image with this header and no bit array at all.
const auto image = header_only_image(num_longs);

SECTION("from memory: rejected, because the image is far too short for 2^32 bytes") {
// Before it allocates anything, deserialize() checks that the buffer holds num_bytes:
// ensure_minimum_memory() throws std::out_of_range. With a size of 0, that check passed.
REQUIRE_THROWS_AS(bloom_filter::deserialize(image.data(), image.size()), std::out_of_range);
}

SECTION("from a stream: the allocator is asked for 2^32 bytes, not 0") {
// A stream can't tell how much data follows, so deserialize() allocates first.
// throwing_allocator records the size it was asked for, then refuses: nothing is allocated.
const throwing_allocator<uint8_t> allocator;
const std::string image_string(reinterpret_cast<const char*>(image.data()), image.size());
std::istringstream image_stream(image_string, std::ios::in | std::ios::binary);
last_requested_bytes = 0;

// throwing_allocator refuses every request, so deserialize() throws std::bad_alloc here
// in any case, also on the old code. This only confirms that the request reached the allocator.
REQUIRE_THROWS_AS(throwing_filter::deserialize(image_stream, allocator), std::bad_alloc);

// What this section checks: the size of that request.
const std::size_t requested_bytes = last_requested_bytes;
REQUIRE(requested_bytes != 0); // a request for 0 bytes is the 32-bit overflow
REQUIRE(requested_bytes == expected_num_bytes); // the whole 2^32 bytes
}
}

}
39 changes: 39 additions & 0 deletions filters/test/bloom_filter_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -443,4 +443,43 @@ TEST_CASE("bloom_filter: truncated empty stream is rejected", "[bloom_filter]")
}
}

TEST_CASE("bloom_filter: image with zero bit array length is rejected", "[bloom_filter]") {
const size_t num_longs_offset = 16;
const size_t num_bits_set_offset = 24;

auto bf = bloom_filter::builder::create_by_size(64, 3);
bf.update("apple");
auto bytes = bf.serialize();
// a non-empty image whose bit array has 0 longs, with a count of bits set consistent with that
const uint32_t num_longs = 0;
const uint64_t num_bits_set = 0;
std::memcpy(bytes.data() + num_longs_offset, &num_longs, sizeof(num_longs));
std::memcpy(bytes.data() + num_bits_set_offset, &num_bits_set, sizeof(num_bits_set));

REQUIRE_THROWS_AS(bloom_filter::deserialize(bytes.data(), bytes.size()), std::invalid_argument);
REQUIRE_THROWS_AS(bloom_filter::wrap(bytes.data(), bytes.size()), std::invalid_argument);
REQUIRE_THROWS_AS(bloom_filter::writable_wrap(bytes.data(), bytes.size()), std::invalid_argument);

const std::string serialized(reinterpret_cast<const char*>(bytes.data()), bytes.size());
std::istringstream is(serialized, std::ios::in | std::ios::binary);
REQUIRE_THROWS_AS(bloom_filter::deserialize(is), std::invalid_argument);

// an empty image with 0 longs is rejected too
auto empty_bytes = bloom_filter::builder::create_by_size(64, 3).serialize();
std::memcpy(empty_bytes.data() + num_longs_offset, &num_longs, sizeof(num_longs));
REQUIRE_THROWS_AS(bloom_filter::deserialize(empty_bytes.data(), empty_bytes.size()), std::invalid_argument);
REQUIRE_THROWS_AS(bloom_filter::wrap(empty_bytes.data(), empty_bytes.size()), std::invalid_argument);
const std::string empty_serialized(reinterpret_cast<const char*>(empty_bytes.data()), empty_bytes.size());
std::istringstream empty_is(empty_serialized, std::ios::in | std::ios::binary);
REQUIRE_THROWS_AS(bloom_filter::deserialize(empty_is), std::invalid_argument);
}

TEST_CASE("bloom_filter: capacity is rounded up to a multiple of 64 bits", "[bloom_filter]") {
// so the bit array of a filter is never shorter than 8 bytes
REQUIRE(bloom_filter::builder::create_by_size(1, 3).get_capacity() == 64);
REQUIRE(bloom_filter::builder::create_by_size(63, 3).get_capacity() == 64);
REQUIRE(bloom_filter::builder::create_by_size(64, 3).get_capacity() == 64);
REQUIRE(bloom_filter::builder::create_by_size(65, 3).get_capacity() == 128);
}

} // namespace datasketches
Loading