diff --git a/src/bloom_filter.cpp b/src/bloom_filter.cpp index e89ff7c..771a7f5 100644 --- a/src/bloom_filter.cpp +++ b/src/bloom_filter.cpp @@ -1,5 +1,4 @@ -#include "datasketches_extension.hpp" -#include "datasketches_items.hpp" +#include "datasketches_arena_buffer.hpp" #include "duckdb/common/vector/flat_vector.hpp" #include "duckdb/common/vector/string_vector.hpp" @@ -79,25 +78,32 @@ namespace duckdb // ============================================================ struct DSBloomBindData : public FunctionData { + // The merge aggregate's: its filters carry their own sizes. + DSBloomBindData() : max_distinct_items(0), false_positive_rate(0), filter_bytes(0) + { + } DSBloomBindData(uint64_t max_distinct_items, double false_positive_rate) - : max_distinct_items(max_distinct_items), false_positive_rate(false_positive_rate) + : max_distinct_items(max_distinct_items), false_positive_rate(false_positive_rate), + filter_bytes(((BloomFilter::builder::suggest_num_filter_bits(max_distinct_items, false_positive_rate) + 63) / 64) * 8) { } unique_ptr Copy() const override { - return make_uniq(max_distinct_items, false_positive_rate); + return make_uniq(*this); } bool Equals(const FunctionData &other_p) const override { auto &other = other_p.Cast(); return max_distinct_items == other.max_distinct_items && false_positive_rate == other.false_positive_rate; } - BloomFilter Create() const + BloomFilter Create(const DSAllocator &allocator = DSAllocator()) const { - return BloomFilter::builder::create_by_accuracy(max_distinct_items, false_positive_rate, DS_BLOOM_SEED); + return BloomFilter::builder::create_by_accuracy(max_distinct_items, false_positive_rate, DS_BLOOM_SEED, allocator); } uint64_t max_distinct_items; double false_positive_rate; + // The size of the filter's bit array. + idx_t filter_bytes; }; // datasketch_bloom_filter(max_distinct_items, false_positive_rate, data): both constants size the filter. @@ -131,109 +137,84 @@ namespace duckdb unique_ptr DSBloomUnionBind(BindAggregateFunctionInput &) { - return nullptr; + return make_uniq(); + } + + // The constants an exported state is re-bound with. + void DSBloomConstants(const DSBloomBindData &bind, unordered_map &constants) + { + constants.emplace(0, Value::BIGINT(static_cast(bind.max_distinct_items))); + constants.emplace(1, Value::DOUBLE(bind.false_positive_rate)); } // ============================================================ // 2. State & Operations // ============================================================ - struct DSBloomState + // The aggregates' state, in the aggregate arena (see datasketches_arena_buffer.hpp): the first values, or the + // filter. Values of every type are buffered as they are (strings copied into the arena): a filter never keeps + // them, only their hashes. + template + struct DSBloomPolicy { - BloomFilter *filter = nullptr; - ~DSBloomState() + using BIND = DSBloomBindData; + using VALUE = T; + using WEIGHT = void; + using SKETCH = BloomFilter; + using ENTRY = DSBufferedEntry; + static constexpr const char *NAME = "Bloom filter"; + static constexpr uint32_t BUFFERED = 32; + + static DSAllocator ArenaMode() { - delete filter; + return DSAllocator(DSArenaMode()); } - }; - - // Shared by the build and merge aggregates. With no input rows, the result is NULL. - struct DSBloomOperationBase - { - template - static void Initialize(STATE &state) { state.filter = nullptr; } - - template - static void Destroy(STATE &state, AggregateInputData &) + static idx_t Budget(const BIND &bind) { return bind.filter_bytes; } + static ENTRY Entry(const T &value) { return ENTRY {value}; } + static BloomFilter *Create(const BIND &bind, ArenaAllocator &arena) { - delete state.filter; - state.filter = nullptr; + return arena.Make(bind.Create(ArenaMode())); } - - static bool IgnoreNull() { return true; } - - // Adding a value (or a filter) again does not change the filter. - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static BloomFilter Temporary(const BIND &bind) { return bind.Create(); } + static void Update(BloomFilter &filter, const ENTRY &entry) { DSBloomUpdate(filter, entry.value); } + // See DSBloomSerialize. + static auto Serialize(BloomFilter &filter) { - OP::template Operation(state, input, unary_input); + filter.get_bits_used(); + return filter.serialize(); } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static BloomFilter *Deserialize(const string_t &blob, ArenaAllocator &arena) { - if (!source.filter) - { - return; - } - if (!target.filter) - { - if (aggr_input_data.combine_type == AggregateCombineType::ALLOW_DESTRUCTIVE) - { - // The source is discarded after this call, so take its filter rather than copy it. - target.filter = source.filter; - const_cast(source).filter = nullptr; - } - else - { - target.filter = new BloomFilter(*source.filter); - } - return; - } - DSBloomCombine(*target.filter, *source.filter, "datasketch_bloom_filter"); + return arena.Make(DSDeserialize(blob, "Bloom filter", ArenaMode())); } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + // The copy allocates through the source's allocator, which in arena mode is the current (target's) arena. + static BloomFilter *Copy(const BloomFilter &source, ArenaAllocator &arena) { - if (state.filter) - { - target = DSBloomSerialize(finalize_data.result, *state.filter); - } - else - { - finalize_data.ReturnNull(); - } + return arena.Make(source); } - }; - - template - struct DSBloomBuildOperation : DSBloomOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &input, AggregateUnaryInput &idata) + static void Combine(BloomFilter &target, const BloomFilter &source, const BIND &, ArenaAllocator &) + { + DSBloomCombine(target, source, "datasketch_bloom_filter"); + } + // An input filter is read on the heap and freed once combined, except the first, which becomes the state's. + static void Merge(BloomFilter *&filter, const string_t &blob, const BIND &, ArenaAllocator &arena) { - if (!state.filter) + if (!filter) { - state.filter = new BloomFilter(idata.input.bind_data->template Cast().Create()); + filter = Deserialize(blob, arena); + return; } - DSBloomUpdate(*state.filter, input); + DSBloomCombine(*filter, DSBloomDeserialize(blob), "datasketch_bloom_filter"); } }; - struct DSBloomUnionOperation : DSBloomOperationBase + // Adding a value (or a filter) again does not change the filter. + template + struct DSBloomIdempotent : BASE { - template - static void Operation(STATE &state, const A_TYPE &input, AggregateUnaryInput &) + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t) { - auto filter = DSBloomDeserialize(input); - if (!state.filter) - { - state.filter = new BloomFilter(std::move(filter)); - } - else - { - DSBloomCombine(*state.filter, filter, "datasketch_bloom_filter"); - } + BASE::template Operation(state, input, unary_input); } }; @@ -342,10 +323,12 @@ namespace duckdb DSForEachHashedType([&](auto marker, const LogicalType &item_type) { using INPUT = DSCountingInput; - using OP = DSBloomBuildOperation; using T = typename INPUT::physical_type; - auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); - DSReadDataArgumentAt<2, DSBloomState, T, OP>(fun); + using STATE = DSBufferedArenaState>; + using OP = DSBloomIdempotent>; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + DSReadDataArgumentAt<2, STATE, T, OP>(fun); + DSSetBufferedStateExport(fun); DSAddAggregate(set, descs, fun, DSBloomBind, {LogicalType::BIGINT, LogicalType::DOUBLE, item_type}, {"max_distinct_items", "false_positive_rate", "data"}, "Creates a Bloom filter sized to hold max_distinct_items distinct values with the given false positive rate. " @@ -353,7 +336,10 @@ namespace duckdb {"SELECT datasketch_bloom_filter(1000, 0.01, i) FROM range(100) t(i)"}); }); // datasketch_bloom_filter(filter): combines filters. - auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + using MERGE_STATE = DSBufferedArenaState>; + using MERGE = DSBloomIdempotent>; + auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSSetBufferedStateExport>(merge); DSAddAggregate(set, descs, merge, DSBloomUnionBind, {sketch_type}, {"filter"}, "Combines Bloom filters built with the same max_distinct_items and false_positive_rate into one that " "contains the values of each", diff --git a/src/count_min_sketch.cpp b/src/count_min_sketch.cpp index 88bc25e..27a9031 100644 --- a/src/count_min_sketch.cpp +++ b/src/count_min_sketch.cpp @@ -1,5 +1,4 @@ -#include "datasketches_extension.hpp" -#include "datasketches_items.hpp" +#include "datasketches_arena_buffer.hpp" #include "duckdb/common/vector/flat_vector.hpp" #include "duckdb/common/vector/string_vector.hpp" @@ -132,139 +131,92 @@ namespace duckdb return make_uniq(static_cast(num_hashes), static_cast(num_buckets)); } + // The merge aggregate's bind data: its sketches carry their own sizes. unique_ptr DSCountMinMergeBind(BindAggregateFunctionInput &) { - return nullptr; + return make_uniq(0, 0); + } + + // The constants an exported state is re-bound with. + void DSCountMinConstants(const DSCountMinBindData &bind, unordered_map &constants) + { + constants.emplace(0, Value::INTEGER(bind.num_hashes)); + constants.emplace(1, Value::INTEGER(static_cast(bind.num_buckets))); } // ============================================================ // 2. State & Operations // ============================================================ - struct DSCountMinState - { - CountMinSketch *sketch = nullptr; - ~DSCountMinState() + // The aggregates' state, in the aggregate arena (see datasketches_arena_buffer.hpp): the first values with + // their weights, or the sketch. Values of every type are buffered as they are (strings copied into the arena): + // a sketch never keeps them, only their hashes. Counters are sums, so feeding the buffered values later gives + // the same sketch. + template + struct DSCountMinPolicy + { + using BIND = DSCountMinBindData; + using VALUE = T; + using WEIGHT = int64_t; + using SKETCH = CountMinSketch; + using ENTRY = DSBufferedEntry; + static constexpr const char *NAME = "Count-Min"; + static constexpr uint32_t BUFFERED = 32; + + static DSAllocator ArenaMode() { - delete sketch; + return DSAllocator(DSArenaMode()); } - void Create(AggregateInputData &input) + static idx_t Budget(const BIND &bind) { - if (!sketch) - { - auto &bind = input.bind_data->Cast(); - sketch = new CountMinSketch(bind.num_hashes, bind.num_buckets); - } + return static_cast(bind.num_hashes) * bind.num_buckets * sizeof(int64_t); } - }; - - // Shared by the build, weighted build and merge aggregates. With no input rows, the result is NULL. - struct DSCountMinOperationBase - { - template - static void Initialize(STATE &state) { state.sketch = nullptr; } - - template - static void Destroy(STATE &state, AggregateInputData &) + static ENTRY Entry(const T &value, int64_t weight = 1) { return ENTRY {value, weight}; } + static CountMinSketch *Create(const BIND &bind, ArenaAllocator &arena) { - delete state.sketch; - state.sketch = nullptr; + return arena.Make(bind.num_hashes, bind.num_buckets, datasketches::DEFAULT_SEED, ArenaMode()); } - - static bool IgnoreNull() { return true; } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static CountMinSketch Temporary(const BIND &bind) { return CountMinSketch(bind.num_hashes, bind.num_buckets); } + static void Update(CountMinSketch &sketch, const ENTRY &entry) { - if (!source.sketch) - { - return; - } - if (!target.sketch) - { - if (aggr_input_data.combine_type == AggregateCombineType::ALLOW_DESTRUCTIVE) - { - // The source is discarded after this call, so take its sketch rather than copy it. - target.sketch = source.sketch; - const_cast(source).sketch = nullptr; - } - else - { - target.sketch = new CountMinSketch(*source.sketch); - } - return; - } - DSCountMinMerge(*target.sketch, *source.sketch, "datasketch_count_min"); + DSCountMinUpdate(sketch, entry.value, entry.weight); } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + static auto Serialize(const CountMinSketch &sketch) { return sketch.serialize(); } + static CountMinSketch *Deserialize(const string_t &blob, ArenaAllocator &arena) { - if (state.sketch) - { - target = DSCountMinSerialize(finalize_data.result, *state.sketch); - } - else - { - finalize_data.ReturnNull(); - } + return arena.Make(DSDeserialize(blob, "Count-Min", datasketches::DEFAULT_SEED, ArenaMode())); } - }; - - // Each row counts once. - template - struct DSCountMinOperation : DSCountMinOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &input, AggregateUnaryInput &idata) - { - state.Create(idata.input); - DSCountMinUpdate(*state.sketch, input, 1); - } - - // A repeated value counts as many times as it repeats. - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + // The copy allocates through the source's allocator, which in arena mode is the current (target's) arena + // (with vcpkg-overlay/apache-datasketches/count-min-hash-seeds-allocator.patch). + static CountMinSketch *Copy(const CountMinSketch &source, ArenaAllocator &arena) { - state.Create(unary_input.input); - DSCountMinUpdate(*state.sketch, input, static_cast(count)); + return arena.Make(source); } - }; - - // Each row adds its weight. - template - struct DSCountMinWeightedOperation : DSCountMinOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &input, const W_TYPE &weight, AggregateBinaryInput &idata) + static void Combine(CountMinSketch &target, const CountMinSketch &source, const BIND &, ArenaAllocator &) { - state.Create(idata.input); - DSCountMinUpdate(*state.sketch, input, weight); + DSCountMinMerge(target, source, "datasketch_count_min"); } - }; - - struct DSCountMinMergeOperation : DSCountMinOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &input, AggregateUnaryInput &) + // An input sketch is read on the heap and freed once merged, except the first, which becomes the state's. + static void Merge(CountMinSketch *&sketch, const string_t &blob, const BIND &, ArenaAllocator &arena) { - auto sketch = DSCountMinDeserialize(input); - if (!state.sketch) - { - state.sketch = new CountMinSketch(std::move(sketch)); - } - else + if (!sketch) { - DSCountMinMerge(*state.sketch, sketch, "datasketch_count_min"); + sketch = Deserialize(blob, arena); + return; } + DSCountMinMerge(*sketch, DSCountMinDeserialize(blob), "datasketch_count_min"); } + }; + // Each row counts once; a repeated value counts as many times as it repeats. + template + struct DSCountMinOperation : DSArenaCreateOperation + { template static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) { - for (idx_t i = 0; i < count; i++) - { - Operation(state, input, unary_input); - } + DSArenaScope scope(unary_input.input.allocator); + state.UpdateWeighted(input, static_cast(count), unary_input.input.bind_data->template Cast(), + unary_input.input.allocator); } }; @@ -396,15 +348,20 @@ namespace duckdb using INPUT = DSCountingInput; using OP = DSCountMinOperation; using T = typename INPUT::physical_type; - auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); - DSReadDataArgumentAt<2, DSCountMinState, T, OP>(fun); + using STATE = DSBufferedArenaState>; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + DSReadDataArgumentAt<2, STATE, T, OP>(fun); + DSSetBufferedStateExport(fun); DSAddAggregate(set, descs, fun, DSCountMinBind, {LogicalType::INTEGER, LogicalType::INTEGER, item_type}, {"num_hashes", "num_buckets", "data"}, "Creates a Count-Min sketch that estimates how often each value occurs. An estimate exceeds the true " "count by at most e / num_buckets of the total count, with probability 1 - e^-num_hashes " "(see datasketch_count_min_suggest_num_buckets and _suggest_num_hashes)", {"SELECT datasketch_count_min(3, 128, item)" + rows}); }); - auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + using MERGE_STATE = DSBufferedArenaState>; + using MERGE = DSArenaMergeOperation; + auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSSetBufferedStateExport>(merge); DSAddAggregate(set, descs, merge, DSCountMinMergeBind, {sketch_type}, {"sketch"}, "Merges Count-Min sketches of the same size", {ex.QueryPartials("datasketch_count_min(sketch)")}); @@ -418,11 +375,12 @@ namespace duckdb DSForEachHashedType([&](auto marker, const LogicalType &item_type) { using INPUT = DSCountingInput; - using OP = DSCountMinWeightedOperation; + using OP = DSArenaWeightedOperation; using T = typename INPUT::physical_type; - auto fun = AggregateFunction::BinaryAggregate(item_type, LogicalType::BIGINT, sketch_type); - fun.SetStateDestructorCallback(AggregateFunction::StateDestroy); - DSReadDataArgumentsAt<2, 3, DSCountMinState, T, int64_t, OP>(fun); + using STATE = DSBufferedArenaState>; + auto fun = AggregateFunction::BinaryAggregate(item_type, LogicalType::BIGINT, sketch_type); + DSReadDataArgumentsAt<2, 3, STATE, T, int64_t, OP>(fun); + DSSetBufferedStateExport(fun); DSAddAggregate(set, descs, fun, DSCountMinBind, {LogicalType::INTEGER, LogicalType::INTEGER, item_type, LogicalType::BIGINT}, {"num_hashes", "num_buckets", "data", "weight"}, "Creates a Count-Min sketch of the total weight of each value: each row adds its weight (which may be " diff --git a/src/frequent_items_sketch.cpp b/src/frequent_items_sketch.cpp index 65afde3..1bf838d 100644 --- a/src/frequent_items_sketch.cpp +++ b/src/frequent_items_sketch.cpp @@ -1,4 +1,4 @@ -#include "datasketches_extension.hpp" +#include "datasketches_arena_buffer.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/types/vector.hpp" @@ -10,7 +10,9 @@ #include "duckdb/common/types/cast_helpers.hpp" // duckdb_fmt #include +#include #include +#include #include namespace duckdb @@ -254,6 +256,137 @@ namespace duckdb } }; + // ------------------------------------------------------------ + // Arena states, for numbers + // ------------------------------------------------------------ + // A sketch's items are strings, which own heap memory, so VARCHAR items and the merge aggregates keep + // DSFreqItemsState. A number's string (see DSFreqItemsToString) is at most 24 bytes, so sketches of numbers + // keep it inline in a key that owns no heap memory, in the aggregate arena (see datasketches_arena_buffer.hpp). + // A key compares, hashes and serializes as the std::string of the same bytes, so a sketch of keys holds its + // items in the same places, purges the same ones and serializes to the same bytes as one of strings. + struct DSFreqItemsKey + { + static constexpr idx_t CAPACITY = 31; + uint8_t size; + char data[CAPACITY]; + + std::string_view View() const { return std::string_view(data, size); } + bool operator==(const DSFreqItemsKey &other) const { return View() == other.View(); } + + static DSFreqItemsKey From(const std::string &text) + { + if (text.size() > CAPACITY) + { + throw InternalException("Frequent Items: the item '%s' is too long for an inline key", text); + } + DSFreqItemsKey key; + key.size = static_cast(text.size()); + memcpy(key.data, text.data(), text.size()); + return key; + } + }; + + // std::hash of a std::string equals that of its string_view. + struct DSFreqItemsKeyHash + { + size_t operator()(const DSFreqItemsKey &key) const { return std::hash()(key.View()); } + }; + + // As datasketches::serde: each item is its uint32 length, then its bytes. + struct DSFreqItemsKeySerde + { + size_t size_of_item(const DSFreqItemsKey &key) const { return sizeof(uint32_t) + key.size; } + + size_t serialize(void *ptr, size_t capacity, const DSFreqItemsKey *items, unsigned num) const + { + auto out = static_cast(ptr); + size_t written = 0; + for (unsigned i = 0; i < num; i++) + { + const uint32_t length = items[i].size; + datasketches::check_memory_size(written + sizeof(length) + length, capacity); + memcpy(out + written, &length, sizeof(length)); + memcpy(out + written + sizeof(length), items[i].data, length); + written += sizeof(length) + length; + } + return written; + } + + size_t deserialize(const void *ptr, size_t capacity, DSFreqItemsKey *items, unsigned num) const + { + auto in = static_cast(ptr); + size_t read = 0; + for (unsigned i = 0; i < num; i++) + { + uint32_t length; + datasketches::check_memory_size(read + sizeof(length), capacity); + memcpy(&length, in + read, sizeof(length)); + read += sizeof(length); + datasketches::check_memory_size(read + length, capacity); + if (length > DSFreqItemsKey::CAPACITY) + { + throw std::invalid_argument("item of " + std::to_string(length) + " bytes in a sketch of numbers"); + } + auto key = new (&items[i]) DSFreqItemsKey(); + key->size = static_cast(length); + memcpy(key->data, in + read, length); + read += length; + } + return read; + } + }; + + using DSFreqItemsArenaSketch = datasketches::frequent_items_sketch, DSAllocator>; + + // The first items with their weights, or the sketch they were fed to, in order: a sketch reserves room for + // 8 items, and DuckDB creates a state per input row when grouping reduces little. + template + struct DSFreqItemsArenaPolicy + { + using BIND = DSFreqItemsBindData; + using VALUE = T; + using WEIGHT = uint64_t; + using SKETCH = DSFreqItemsArenaSketch; + using ENTRY = DSBufferedEntry; + static constexpr const char *NAME = "Frequent Items"; + static constexpr uint32_t BUFFERED = 32; + + static DSAllocator ArenaMode() + { + return DSAllocator(DSArenaMode()); + } + static idx_t Budget(const BIND &) { return NumericLimits::Maximum(); } + static ENTRY Entry(const T &value) { return ENTRY {value, 1}; } + // The weight is checked now, as the heap state checks it. + template + static ENTRY Entry(const T &value, const W &weight) { return ENTRY {value, DSFreqItemsToWeight(weight)}; } + + static SKETCH *Create(const BIND &bind, ArenaAllocator &arena) + { + return arena.Make(bind.lg_max_k, SKETCH::LG_MIN_MAP_SIZE, std::equal_to(), ArenaMode()); + } + static SKETCH Temporary(const BIND &bind) { return SKETCH(bind.lg_max_k); } + static void Update(SKETCH &sketch, const ENTRY &entry) + { + sketch.update(DSFreqItemsKey::From(DSFreqItemsToString(entry.value)), entry.weight); + } + static auto Serialize(const SKETCH &sketch) { return sketch.serialize(0, DSFreqItemsKeySerde()); } + static SKETCH *Deserialize(const string_t &blob, ArenaAllocator &arena) + { + return arena.Make(DSDeserialize(blob, NAME, DSFreqItemsKeySerde(), std::equal_to(), ArenaMode())); + } + // The copy allocates through the source's allocator, which in arena mode is the current (target's) arena. + static SKETCH *Copy(const SKETCH &source, ArenaAllocator &arena) { return arena.Make(source); } + static void Combine(SKETCH &target, const SKETCH &source, const BIND &, ArenaAllocator &) { target.merge(source); } + }; + + // The constants an exported state is re-bound with: lg_max_k, for the overloads that take it. + void DSFreqItemsConstants(const DSFreqItemsBindData &bind, unordered_map &constants) + { + constants.emplace(0, Value::INTEGER(bind.lg_max_k)); + } + // ============================================================ // 3. Scalar Functions // ============================================================ @@ -364,14 +497,33 @@ namespace duckdb DSFreqItemsForEachItemType([&](auto item, const LogicalType &item_type) { using T = decltype(item); - auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); - DSAddAggregate(set, descs, fun, DSFreqItemsBind<2>, {item_type}, {"data"}, - "Creates a Frequent Items sketch to find heavy hitters in a data stream", - {"SELECT datasketch_frequent_items(item)" + rows}); - DSReadDataArgumentAt<1, DSFreqItemsState, T, DSFreqItemsOperation>(fun); - DSAddAggregate(set, descs, fun, DSFreqItemsBind<2>, {LogicalType::INTEGER, item_type}, {"lg_max_k", "data"}, - "Creates a Frequent Items sketch with the given log2 of max map size (lg_max_k)", - {"SELECT datasketch_frequent_items(10, item)" + rows}); }); + auto add = [&](AggregateFunction fun, AggregateFunction with_lg_max_k) + { + DSAddAggregate(set, descs, fun, DSFreqItemsBind<2>, {item_type}, {"data"}, + "Creates a Frequent Items sketch to find heavy hitters in a data stream", + {"SELECT datasketch_frequent_items(item)" + rows}); + DSAddAggregate(set, descs, with_lg_max_k, DSFreqItemsBind<2>, {LogicalType::INTEGER, item_type}, {"lg_max_k", "data"}, + "Creates a Frequent Items sketch with the given log2 of max map size (lg_max_k)", + {"SELECT datasketch_frequent_items(10, item)" + rows}); + }; + if constexpr (std::is_same_v) + { + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + auto with_lg_max_k = fun; + DSReadDataArgumentAt<1, DSFreqItemsState, T, DSFreqItemsOperation>(with_lg_max_k); + add(fun, with_lg_max_k); + } + else + { + using STATE = DSBufferedArenaState>; + using OP = DSArenaCreateOperation; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + auto with_lg_max_k = fun; + DSSetBufferedStateExport>(fun); + DSReadDataArgumentAt<1, STATE, T, OP>(with_lg_max_k); + DSSetBufferedStateExport(with_lg_max_k); + add(fun, with_lg_max_k); + } }); auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); DSAddAggregate(set, descs, merge, DSFreqItemsBind<2>, {sketch_type}, {"sketch"}, @@ -394,16 +546,35 @@ namespace duckdb auto add = [&](auto weight, const LogicalType &weight_type) { using W = decltype(weight); - using OP = DSFreqItemsWeightedOperation; - auto fun = AggregateFunction::BinaryAggregate(item_type, weight_type, sketch_type); - fun.SetStateDestructorCallback(AggregateFunction::StateDestroy); - DSAddAggregate(set, descs, fun, DSFreqItemsBind<3>, {item_type, weight_type}, {"data", "weight"}, - "Creates a weighted Frequent Items sketch to find heavy hitters by total item weight; each row adds its weight to the item's count", - {"SELECT datasketch_frequent_items_weighted(item, weight)" + rows}); - DSReadDataArgumentsAt<1, 2, DSFreqItemsState, T, W, OP>(fun); - DSAddAggregate(set, descs, fun, DSFreqItemsBind<3>, {LogicalType::INTEGER, item_type, weight_type}, {"lg_max_k", "data", "weight"}, - "Creates a weighted Frequent Items sketch with the given log2 of max map size (lg_max_k); each row adds its weight to the item's count", - {"SELECT datasketch_frequent_items_weighted(10, item, weight)" + rows}); + auto add_functions = [&](AggregateFunction fun, AggregateFunction with_lg_max_k) + { + DSAddAggregate(set, descs, fun, DSFreqItemsBind<3>, {item_type, weight_type}, {"data", "weight"}, + "Creates a weighted Frequent Items sketch to find heavy hitters by total item weight; each row adds its weight to the item's count", + {"SELECT datasketch_frequent_items_weighted(item, weight)" + rows}); + DSAddAggregate(set, descs, with_lg_max_k, DSFreqItemsBind<3>, {LogicalType::INTEGER, item_type, weight_type}, {"lg_max_k", "data", "weight"}, + "Creates a weighted Frequent Items sketch with the given log2 of max map size (lg_max_k); each row adds its weight to the item's count", + {"SELECT datasketch_frequent_items_weighted(10, item, weight)" + rows}); + }; + if constexpr (std::is_same_v) + { + using OP = DSFreqItemsWeightedOperation; + auto fun = AggregateFunction::BinaryAggregate(item_type, weight_type, sketch_type); + fun.SetStateDestructorCallback(AggregateFunction::StateDestroy); + auto with_lg_max_k = fun; + DSReadDataArgumentsAt<1, 2, DSFreqItemsState, T, W, OP>(with_lg_max_k); + add_functions(fun, with_lg_max_k); + } + else + { + using STATE = DSBufferedArenaState>; + using OP = DSArenaWeightedOperation; + auto fun = AggregateFunction::BinaryAggregate(item_type, weight_type, sketch_type); + auto with_lg_max_k = fun; + DSSetBufferedStateExport>(fun); + DSReadDataArgumentsAt<1, 2, STATE, T, W, OP>(with_lg_max_k); + DSSetBufferedStateExport(with_lg_max_k); + add_functions(fun, with_lg_max_k); + } }; add(int64_t(), LogicalType::BIGINT); add(uint64_t(), LogicalType::UBIGINT); }); diff --git a/src/include/datasketches_arena_buffer.hpp b/src/include/datasketches_arena_buffer.hpp new file mode 100644 index 0000000..1c0ca26 --- /dev/null +++ b/src/include/datasketches_arena_buffer.hpp @@ -0,0 +1,382 @@ +#pragma once + +// Arena states that buffer their first raw inputs, for the hand-written sketches (frequent items, VarOpt, EBPPS, +// Bloom filter, Count-Min). They build on the arena states of datasketches_generated_common.hpp (the operations, the +// export and import callbacks, DSArenaItems) and follow the same rules: everything a state keeps lives in the +// aggregate arena of the callback that allocated it, a state has no destructor, and Combine copies from the source +// state without taking over or growing its memory. +// +// DuckDB creates one state per input row when grouping reduces little (it skips the hash table lookups), and these +// sketches allocate a lot up front: a Bloom filter its whole bit array, a Count-Min sketch num_hashes * num_buckets +// counters, an EBPPS sketch room for K items. So a state keeps its first inputs (with their weights) as they are, and +// builds its sketch, feeding them in order, only when more arrive. + +#include "datasketches_generated_common.hpp" + +#include +#include +#include + +namespace duckdb +{ + +// A scratch arena for temporaries of arena-mode sketches (e.g. a union's result, serialized bytes): while it lives, +// arena-mode allocations go to it, and it is freed as a whole when it goes out of scope. A temporary must not +// outlive it, and nothing a state keeps may be allocated while it is the current arena (open a DSArenaScope on the +// state's arena for that). +struct DSArenaScratch +{ + DSArenaScratch() : arena(Allocator::DefaultAllocator()), scope(arena) + { + } + + ArenaAllocator arena; + DSArenaScope scope; +}; + +// The bytes fun() returns (a library's serialize(), whose vector uses the sketch's allocator), produced in a scratch +// arena and returned on the heap: serializing an arena-mode sketch leaves nothing behind in the callback's arena. +template +static std::vector DSScratchBytes(FUNC &&fun) +{ + DSArenaScratch scratch; + auto bytes = fun(); + return std::vector(bytes.begin(), bytes.end()); +} + +// One buffered input: a value and, unless WEIGHT is void, its weight. +template +struct DSBufferedEntry +{ + T value; + WEIGHT weight; +}; + +template +struct DSBufferedEntry +{ + T value; +}; + +// Strings are copied into the arena (unless inlined), so a buffered string_t never points at input or at another +// state's memory. +template +static inline idx_t DSArenaValueBytes(const T &value) +{ + if constexpr (std::is_same_v) + { + return value.IsInlined() ? 0 : value.GetSize(); + } + else + { + return 0; + } +} + +template +static inline T DSArenaCopyValue(const T &value, ArenaAllocator &arena) +{ + if constexpr (std::is_same_v) + { + if (value.IsInlined()) + { + return value; + } + auto size = value.GetSize(); + auto data = arena.Allocate(size); + memcpy(data, value.GetData(), size); + return string_t(const_char_ptr_cast(data), UnsafeNumericCast(size)); + } + else + { + return value; + } +} + +// An entry's bytes in an exported state: the weight, then the value as it is (a string as its uint32 length and +// bytes). +template +static inline void DSWriteEntry(std::vector &out, const DSBufferedEntry &entry) +{ + auto append = [&](const void *data, idx_t size) + { + auto bytes = static_cast(data); + out.insert(out.end(), bytes, bytes + size); + }; + if constexpr (!std::is_void_v) + { + append(&entry.weight, sizeof(WEIGHT)); + } + if constexpr (std::is_same_v) + { + const uint32_t size = entry.value.GetSize(); + append(&size, sizeof(size)); + append(entry.value.GetData(), size); + } + else + { + append(&entry.value, sizeof(T)); + } +} + +// Reads an entry written by DSWriteEntry, copying a string into the arena. False if the bytes end too soon. +template +static inline bool DSReadEntry(const_data_ptr_t &ptr, const_data_ptr_t end, DSBufferedEntry &entry, + ArenaAllocator &arena) +{ + auto read = [&](void *data, idx_t size) + { + if (idx_t(end - ptr) < size) + { + return false; + } + memcpy(data, ptr, size); + ptr += size; + return true; + }; + if constexpr (!std::is_void_v) + { + if (!read(&entry.weight, sizeof(WEIGHT))) + { + return false; + } + } + if constexpr (std::is_same_v) + { + uint32_t size; + if (!read(&size, sizeof(size)) || idx_t(end - ptr) < size) + { + return false; + } + entry.value = DSArenaCopyValue(string_t(const_char_ptr_cast(ptr), size), arena); + ptr += size; + return true; + } + else + { + return read(&entry.value, sizeof(T)); + } +} + +// The aggregates' state: up to POLICY::BUFFERED raw inputs, or the sketch they were fed to. POLICY (with BIND its +// bind data, SKETCH what the state keeps in the arena, ENTRY = DSBufferedEntry) provides: +// static constexpr uint32_t BUFFERED; the most inputs a state buffers +// static idx_t Budget(const BIND &); the most bytes a buffer may take (about the sketch's size) +// static SKETCH *Create(const BIND &, ArenaAllocator &); an empty sketch in the arena +// static auto Temporary(const BIND &); an empty sketch on the heap +// static void Update(S &sketch, const ENTRY &); feeds an input to a sketch (SKETCH or the temporary) +// static auto Serialize(S &sketch); the sketch's bytes (run in a scratch arena) +// static SKETCH *Deserialize(const string_t &, ArenaAllocator &); a sketch read into the arena +// static SKETCH *Copy(const SKETCH &, ArenaAllocator &); a copy in the arena +// static void Combine(SKETCH &target, const SKETCH &source, const BIND &, ArenaAllocator &); +// static void Merge(SKETCH *&sketch, const string_t &blob, const BIND &, ArenaAllocator &); merges a serialized +// sketch, creating the state's sketch if it has none +template +struct DSBufferedArenaState +{ + using BIND = typename POLICY::BIND; + using VALUE = typename POLICY::VALUE; + using WEIGHT = typename POLICY::WEIGHT; + using SKETCH = typename POLICY::SKETCH; + using ENTRY = DSBufferedEntry; + + SKETCH *sketch; + DSArenaItems buffer; + // bytes of the buffered strings that are not inlined + uint32_t string_bytes; + + void Initialize() + { + sketch = nullptr; + buffer.Initialize(); + string_bytes = 0; + } + bool IsEmpty() const + { + return !sketch && buffer.count == 0; + } + + template + void Replay(TARGET &target) const + { + for (uint32_t i = 0; i < buffer.count; i++) + { + POLICY::Update(target, buffer.items[i]); + } + } + void ClearBuffer() + { + buffer.Initialize(); + string_bytes = 0; + } + // Moves the buffered inputs into a new sketch. + SKETCH &Promote(const BIND &bind, ArenaAllocator &arena) + { + if (!sketch) + { + sketch = POLICY::Create(bind, arena); + Replay(*sketch); + ClearBuffer(); + } + return *sketch; + } + void Add(const ENTRY &entry, const BIND &bind, ArenaAllocator &arena) + { + if (!sketch && buffer.count < POLICY::BUFFERED) + { + const idx_t extra = DSArenaValueBytes(entry.value); + if ((buffer.count + 1) * sizeof(ENTRY) + string_bytes + extra <= POLICY::Budget(bind)) + { + ENTRY copy = entry; + copy.value = DSArenaCopyValue(entry.value, arena); + buffer.Append(copy, arena); + string_bytes += UnsafeNumericCast(extra); + return; + } + } + POLICY::Update(Promote(bind, arena), entry); + } + + // An input of the default weight (DSArenaCreateOperation). + template + void Update(const V &value, const BIND &bind, ArenaAllocator &arena) + { + Add(POLICY::Entry(value), bind, arena); + } + // An input with its weight (DSArenaWeightedOperation). + template + void UpdateWeighted(const V &value, const W &weight, const BIND &bind, ArenaAllocator &arena) + { + Add(POLICY::Entry(value, weight), bind, arena); + } + void Merge(const string_t &blob, const BIND &bind, ArenaAllocator &arena) + { + if (!sketch && buffer.count) + { + Promote(bind, arena); + } + POLICY::Merge(sketch, blob, bind, arena); + } + void Combine(const DSBufferedArenaState &source, const BIND &bind, ArenaAllocator &arena) + { + if (!source.sketch) + { + for (uint32_t i = 0; i < source.buffer.count; i++) + { + Add(source.buffer.items[i], bind, arena); + } + } + else if (!sketch) + { + // Copy the source's sketch, then feed it what this state buffered. + sketch = POLICY::Copy(*source.sketch, arena); + Replay(*sketch); + ClearBuffer(); + } + else + { + POLICY::Combine(*sketch, *source.sketch, bind, arena); + } + } + + // A state that still buffers its inputs is serialized through a temporary sketch on the heap, freed right away. + std::vector Serialize(const BIND &bind, ArenaAllocator &) const + { + if (sketch) + { + return DSScratchBytes([&]() { return POLICY::Serialize(*sketch); }); + } + auto temporary = POLICY::Temporary(bind); + Replay(temporary); + auto bytes = POLICY::Serialize(temporary); + return std::vector(bytes.begin(), bytes.end()); + } + + // The exported state is a tag byte, then the buffered inputs (EXPORT_BUFFER, see DSWriteEntry) or the serialized + // sketch (EXPORT_SKETCH). Buffered states stay buffered when they are imported. + static constexpr uint8_t EXPORT_BUFFER = 0; + static constexpr uint8_t EXPORT_SKETCH = 1; + std::vector Export(const BIND &, ArenaAllocator &) const + { + std::vector bytes; + if (sketch) + { + auto serialized = DSScratchBytes([&]() { return POLICY::Serialize(*sketch); }); + bytes.reserve(1 + serialized.size()); + bytes.push_back(EXPORT_SKETCH); + bytes.insert(bytes.end(), serialized.begin(), serialized.end()); + return bytes; + } + bytes.push_back(EXPORT_BUFFER); + for (uint32_t i = 0; i < buffer.count; i++) + { + DSWriteEntry(bytes, buffer.items[i]); + } + return bytes; + } + void Import(const string_t &data, ArenaAllocator &arena) + { + const auto size = data.GetSize(); + const auto bytes = const_data_ptr_cast(data.GetData()); + if (size >= 1 && bytes[0] == EXPORT_SKETCH) + { + sketch = POLICY::Deserialize(string_t(data.GetData() + 1, UnsafeNumericCast(size - 1)), arena); + return; + } + if (size < 1 || bytes[0] != EXPORT_BUFFER) + { + throw InvalidInputException("Invalid %s aggregate state", POLICY::NAME); + } + auto ptr = bytes + 1; + const auto end = bytes + size; + while (ptr < end) + { + ENTRY entry; + if (buffer.count == POLICY::BUFFERED || !DSReadEntry(ptr, end, entry, arena)) + { + throw InvalidInputException("Invalid %s aggregate state", POLICY::NAME); + } + buffer.Append(entry, arena); + string_bytes += UnsafeNumericCast(DSArenaValueBytes(entry.value)); + } + } +}; + +// Builds the sketch from values with weights (a binary aggregate). +template +struct DSArenaWeightedOperation : DSArenaOperationBase +{ + template + static void Operation(STATE &state, const A_TYPE &a_data, const B_TYPE &b_data, AggregateBinaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + state.UpdateWeighted(a_data, b_data, idata.input.bind_data->template Cast(), idata.input.allocator); + } +}; + +// The exported state is a BLOB. CONSTANTS(bind, constant_parameters) records the aggregate's constant arguments +// (e.g. K), so that re-binding the exported state supplies them. +template &)> +static AggregateStateLayout DSBufferedGetStateType(AggregateLayoutInput &input) +{ + AggregateStateLayout layout(LogicalType::BLOB, AlignValue(sizeof(STATE))); + if (input.bind_data) + { + CONSTANTS(input.bind_data->template Cast(), layout.constant_parameters); + } + return layout; +} + +template +static void DSNoConstants(const BIND_DATA_TYPE &, unordered_map &) +{ +} + +template &)> +static void DSSetBufferedStateExport(AggregateFunction &fun) +{ + fun.SetStateExportCallbacks(DSBufferedGetStateType, + DSArenaExportState, DSArenaImportState); +} + +} // namespace duckdb diff --git a/src/sampling_sketches.cpp b/src/sampling_sketches.cpp index d852c70..c704c9b 100644 --- a/src/sampling_sketches.cpp +++ b/src/sampling_sketches.cpp @@ -1,7 +1,7 @@ // The weighted sampling sketches: VarOpt (a variance-optimal sample of K items, with an estimate of the total // weight of any subset) and EBPPS (a sample whose inclusion probabilities are exactly proportional to the weights). -#include "datasketches_generated_common.hpp" +#include "datasketches_arena_buffer.hpp" #include #include @@ -50,6 +50,15 @@ namespace duckdb static constexpr const char *NAME = "VarOpt"; static constexpr const char *SQL = "varopt"; + // The library's check of an item's weight (see var_opt_sketch::update), for weights that are buffered. + static void CheckWeight(double weight) + { + if (weight <= 0.0 || std::isnan(weight) || std::isinf(weight)) + { + throw std::invalid_argument("Item weights must be positive and finite. Found: " + std::to_string(weight)); + } + } + // Merging goes through a union of at most `k` samples. static void Merge(Sketch &target, Sketch &&source, uint32_t k) { @@ -74,6 +83,15 @@ namespace duckdb static constexpr const char *NAME = "EBPPS"; static constexpr const char *SQL = "ebpps"; + // The library's check of an item's weight (see ebpps_sketch::update), for weights that are buffered. + static void CheckWeight(double weight) + { + if (weight < 0.0 || std::isnan(weight) || std::isinf(weight)) + { + throw std::invalid_argument("Item weights must be nonnegative and finite. Found: " + std::to_string(weight)); + } + } + // The merged sample has the smaller K of the two. static void Merge(Sketch &target, Sketch &&source, uint32_t) { @@ -257,6 +275,273 @@ namespace duckdb } }; + // The aggregates' state over items that own no heap memory (every item type but VARCHAR, which keeps + // DSSampleState), in the aggregate arena (see datasketches_arena_buffer.hpp): the first items with their + // weights, or the sample they were fed to, in order. An EBPPS sketch reserves room for K items up front. + template + struct DSSamplePolicyBase + { + using BIND = DSSampleBindData; + using VALUE = T; + using WEIGHT = double; + using ENTRY = DSBufferedEntry; + static constexpr const char *NAME = TRAITS::NAME; + static constexpr uint32_t BUFFERED = 32; + + static DSAllocator ArenaMode() + { + return DSAllocator(DSArenaMode()); + } + static idx_t Budget(const BIND &) { return NumericLimits::Maximum(); } + static ENTRY Entry(const T &value) { return ENTRY {value, 1.0}; } + // The weight is checked now, as the library would: a buffered item reaches a sketch only later. + static ENTRY Entry(const T &value, double weight) + { + DSSampleGuard([&]() { TRAITS::CheckWeight(weight); }); + return ENTRY {value, weight}; + } + }; + + // A VarOpt state's sample: a sketch, which takes items, until it is combined with another sample. Combining + // goes through a union of at most K samples, which the state then keeps, so that combining more samples + // updates it in place (in the arena, replacing a sketch leaves the old one behind). + template + struct DSVarOptArenaSample + { + DSVarOptSketch *sketch; + DSVarOptUnion *u; + uint32_t k; + }; + + template + struct DSVarOptArenaPolicy : DSSamplePolicyBase, T> + { + using BASE = DSSamplePolicyBase, T>; + using typename BASE::BIND; + using typename BASE::ENTRY; + using Sketch = DSVarOptSketch; + using Union = DSVarOptUnion; + using SKETCH = DSVarOptArenaSample; + + static SKETCH *Make(ArenaAllocator &arena, uint32_t k) + { + auto sample = arena.Make(); + sample->sketch = nullptr; + sample->u = nullptr; + sample->k = k; + return sample; + } + static SKETCH *Create(const BIND &bind, ArenaAllocator &arena) + { + auto sample = Make(arena, bind.k); + sample->sketch = arena.Make(bind.k, Sketch::DEFAULT_RESIZE_FACTOR, BASE::ArenaMode()); + return sample; + } + static Sketch Temporary(const BIND &bind) { return Sketch(bind.k); } + static void Update(Sketch &sketch, const ENTRY &entry) { sketch.update(entry.value, entry.weight); } + static void Update(SKETCH &sample, const ENTRY &entry) + { + if (sample.sketch) + { + sample.sketch->update(entry.value, entry.weight); + return; + } + // A union takes samples only: the item's, on the heap. + Sketch one(sample.k); + one.update(entry.value, entry.weight); + sample.u->update(std::move(one)); + } + static auto Serialize(const Sketch &sketch) { return sketch.serialize(); } + static auto Serialize(const SKETCH &sample) + { + return sample.sketch ? sample.sketch->serialize() : sample.u->get_result().serialize(); + } + static SKETCH *Deserialize(const string_t &blob, ArenaAllocator &arena) + { + auto sketch = arena.Make(DSDeserialize(blob, BASE::NAME, datasketches::serde(), BASE::ArenaMode())); + auto sample = Make(arena, sketch->get_k()); + sample->sketch = sketch; + return sample; + } + // Copies allocate through the source's allocator, which in arena mode is the current (target's) arena. + static SKETCH *Copy(const SKETCH &source, ArenaAllocator &arena) + { + auto sample = Make(arena, source.k); + if (source.sketch) + { + sample->sketch = arena.Make(*source.sketch); + } + else + { + sample->u = arena.Make(*source.u); + } + return sample; + } + static void ToUnion(SKETCH &sample, const BIND &bind, ArenaAllocator &arena) + { + if (sample.u) + { + return; + } + sample.u = arena.Make(bind.k, BASE::ArenaMode()); + if (sample.sketch) + { + sample.u->update(*sample.sketch); + sample.sketch = nullptr; + } + sample.k = bind.k; + } + static void Combine(SKETCH &target, const SKETCH &source, const BIND &bind, ArenaAllocator &arena) + { + ToUnion(target, bind, arena); + if (source.sketch) + { + target.u->update(*source.sketch); + return; + } + // The source union's result is a temporary, built in a scratch arena; the target grows in its own. + DSArenaScratch scratch; + auto result = source.u->get_result(); + DSArenaScope scope(arena); + target.u->update(std::move(result)); + } + // An input sample is read on the heap and freed once merged into the state's union. + static void Merge(SKETCH *&sample, const string_t &blob, const BIND &bind, ArenaAllocator &arena) + { + auto incoming = DSDeserialize(blob, BASE::NAME); + if (!sample) + { + sample = Make(arena, bind.k); + } + ToUnion(*sample, bind, arena); + sample->u->update(std::move(incoming)); + } + }; + + template + struct DSEbppsArenaPolicy : DSSamplePolicyBase, T> + { + using BASE = DSSamplePolicyBase, T>; + using typename BASE::BIND; + using typename BASE::ENTRY; + using SKETCH = DSEbppsSketch; + + static SKETCH *Create(const BIND &bind, ArenaAllocator &arena) + { + return arena.Make(bind.k, BASE::ArenaMode()); + } + static SKETCH Temporary(const BIND &bind) { return SKETCH(bind.k); } + static void Update(SKETCH &sketch, const ENTRY &entry) { sketch.update(entry.value, entry.weight); } + static auto Serialize(const SKETCH &sketch) { return sketch.serialize(); } + static SKETCH *Deserialize(const string_t &blob, ArenaAllocator &arena) + { + return arena.Make(DSDeserialize(blob, BASE::NAME, datasketches::serde(), BASE::ArenaMode())); + } + // Copies allocate through the source's allocator, which in arena mode is the current (target's) arena: + // also the copy merge() makes of a heavier source, which the target then keeps. + static SKETCH *Copy(const SKETCH &source, ArenaAllocator &arena) + { + return arena.Make(source); + } + static void Combine(SKETCH &target, const SKETCH &source, const BIND &, ArenaAllocator &) + { + target.merge(source); + } + // As the heap state's merge into a sketch of K: the lighter sample is merged into the heavier one, and the + // result has the smaller K. A lighter input is read on the heap and freed once merged. A heavier one + // becomes the state's sample, so it is read into the arena: merging into the state's sketch would swap + // the heap sketch in. + static void Merge(SKETCH *&sketch, const string_t &blob, const BIND &bind, ArenaAllocator &arena) + { + auto incoming = DSDeserialize(blob, BASE::NAME); + if (incoming.get_cumulative_weight() == 0.0) + { + if (!sketch) + { + sketch = Create(bind, arena); + } + return; + } + if (sketch && incoming.get_cumulative_weight() <= sketch->get_cumulative_weight()) + { + sketch->merge(std::move(incoming)); + return; + } + auto base = Deserialize(blob, arena); + if (sketch && sketch->get_cumulative_weight() > 0.0) + { + base->merge(*sketch); + } + else + { + // Merging a sample of no weight only lowers K to its K. + const uint32_t k = sketch ? sketch->get_k() : bind.k; + if (base->get_k() > k) + { + auto shrunk = arena.Make(k, BASE::ArenaMode()); + shrunk->merge(*base); + base = shrunk; + } + } + sketch = base; + } + }; + + template + struct DSSampleArena; + template + struct DSSampleArena> + { + using type = DSVarOptArenaPolicy; + }; + template + struct DSSampleArena> + { + using type = DSEbppsArenaPolicy; + }; + + // The create, merge and weighted create aggregates. + template + vector DSSampleAggregates(const LogicalType &item_type, const LogicalType &sketch_type) + { + if constexpr (std::is_same_v) + { + using STATE = DSSampleState; + using OP = DSSampleOperation; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + DSReadDataArgumentAt<1, STATE, T, OP>(fun); + + using MERGE = DSSampleMergeOperation; + auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSReadDataArgumentAt<1, STATE, string_t, MERGE>(merge); + + using WEIGHTED = DSSampleWeightedOperation; + auto weighted = AggregateFunction::BinaryAggregate(item_type, LogicalType::DOUBLE, sketch_type); + weighted.SetStateDestructorCallback(AggregateFunction::StateDestroy); + DSReadDataArgumentsAt<1, 2, STATE, T, double, WEIGHTED>(weighted); + return {fun, merge, weighted}; + } + else + { + using STATE = DSBufferedArenaState::type>; + using OP = DSArenaCreateOperation; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + DSReadDataArgumentAt<1, STATE, T, OP>(fun); + DSSetArenaStateExport(fun); + + using MERGE = DSArenaMergeOperation; + auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSReadDataArgumentAt<1, STATE, string_t, MERGE>(merge); + DSSetArenaStateExport(merge); + + using WEIGHTED = DSArenaWeightedOperation; + auto weighted = AggregateFunction::BinaryAggregate(item_type, LogicalType::DOUBLE, sketch_type); + DSReadDataArgumentsAt<1, 2, STATE, T, double, WEIGHTED>(weighted); + DSSetArenaStateExport(weighted); + return {fun, merge, weighted}; + } + } + // ============================================================ // 3. Scalar Functions // ============================================================ @@ -448,29 +733,17 @@ namespace duckdb { using T = decltype(item); using TR = TRAITS; - using STATE = DSSampleState; auto sketch_type = DSCreateSketchType(loader, "sketch_" + sql + "_" + StringUtil::Lower(item_type.ToString()), type_description + " with embedded type " + item_type.ToString()); + auto functions = DSSampleAggregates(item_type, sketch_type); - using OP = DSSampleOperation; - auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); - DSReadDataArgumentAt<1, STATE, T, OP>(fun); - DSAddAggregate(aggregates, aggregate_descs, fun, DSSampleBind, {LogicalType::INTEGER, item_type}, {"k", "data"}, + DSAddAggregate(aggregates, aggregate_descs, functions[0], DSSampleBind, {LogicalType::INTEGER, item_type}, {"k", "data"}, "Creates a sketch_" + sql + " sample of at most K of the values, each of weight 1", {"SELECT " + build + " FROM range(1000) t(i)"}); - - using MERGE = DSSampleMergeOperation; - auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); - DSReadDataArgumentAt<1, STATE, string_t, MERGE>(merge); - DSAddAggregate(aggregates, aggregate_descs, merge, DSSampleBind, {LogicalType::INTEGER, sketch_type}, {"k", "sketch"}, + DSAddAggregate(aggregates, aggregate_descs, functions[1], DSSampleBind, {LogicalType::INTEGER, sketch_type}, {"k", "sketch"}, "Merges sketch_" + sql + " samples into one sample of at most K items", {"SELECT " + name + "(16, sketch) FROM " + partials}); - - using WEIGHTED = DSSampleWeightedOperation; - auto weighted = AggregateFunction::BinaryAggregate(item_type, LogicalType::DOUBLE, sketch_type); - weighted.SetStateDestructorCallback(AggregateFunction::StateDestroy); - DSReadDataArgumentsAt<1, 2, STATE, T, double, WEIGHTED>(weighted); - DSAddAggregate(weighted_aggregates, weighted_descs, weighted, DSSampleBind, {LogicalType::INTEGER, item_type, LogicalType::DOUBLE}, + DSAddAggregate(weighted_aggregates, weighted_descs, functions[2], DSSampleBind, {LogicalType::INTEGER, item_type, LogicalType::DOUBLE}, {"k", "data", "weight"}, "Creates a sketch_" + sql + " sample of at most K of the values, sampled in proportion to their weights", {"SELECT " + name + "_weighted(16, i::DOUBLE, (i % 10 + 1)::DOUBLE) FROM range(1000) t(i)"}); diff --git a/test/sql/datasketch_spill_items.test b/test/sql/datasketch_spill_items.test new file mode 100644 index 0000000..7f1fb92 --- /dev/null +++ b/test/sql/datasketch_spill_items.test @@ -0,0 +1,200 @@ +# name: test/sql/datasketch_spill_items.test +# description: frequent items, VarOpt, EBPPS, Bloom filter and Count-Min states live in the arena and spill under memory_limit +# group: [datasketches] + +require datasketches + +require skip_reload + +statement ok +SET threads=4; + +statement ok +SET temp_directory='{TEST_DIR}/datasketch_spill_items'; + +statement ok +SET preserve_insertion_order=false; + +# Inputs (views, so they take no memory) and reference results, computed with plenty of memory: 100k groups of 40 rows (more than a state buffers) and 400k groups of +# 3 rows (fewer) +statement ok +SET memory_limit='4GB'; + +statement ok +CREATE VIEW big AS SELECT i % 100000 AS g, i AS v, (i % 5 + 1) AS w FROM range(4000000) t(i); + +statement ok +CREATE VIEW small AS SELECT i % 400000 AS g, i AS v, (i % 5 + 1) AS w FROM range(1200000) t(i); + +# (hashes of the deterministic sketches, so the reference itself stays small) +statement ok +CREATE TABLE ref AS +SELECT g, hash(datasketch_bloom_filter(1000, 0.01, v)) AS b, hash(datasketch_count_min(3, 128, v)) AS c, + hash(datasketch_count_min_weighted(3, 128, v, w)) AS cw, datasketch_frequent_items(10, v % 7) AS f, + hash(datasketch_bloom_filter(1000, 0.01, v::VARCHAR)) AS bs, hash(datasketch_count_min(3, 128, v::VARCHAR)) AS cs +FROM big GROUP BY g; + +statement ok +SET memory_limit='128MB'; + +# Bloom filters and Count-Min sketches are deterministic: the same bytes as with plenty of memory +query I +SELECT count(*) FROM ( + SELECT g, hash(datasketch_bloom_filter(1000, 0.01, v)) AS b, hash(datasketch_count_min(3, 128, v)) AS c, + hash(datasketch_count_min_weighted(3, 128, v, w)) AS cw, hash(datasketch_bloom_filter(1000, 0.01, v::VARCHAR)) AS bs, + hash(datasketch_count_min(3, 128, v::VARCHAR)) AS cs + FROM big GROUP BY g +) t JOIN ref USING (g) +WHERE t.b != ref.b OR t.c != ref.c OR t.cw != ref.cw OR t.bs != ref.bs OR t.cs != ref.cs; +---- +0 + +# Frequent items of 7 distinct values are exact +query I +SELECT count(*) FROM ( + SELECT g, datasketch_frequent_items(10, v % 7) AS f FROM big GROUP BY g +) t JOIN ref USING (g) +WHERE datasketch_frequent_items_total_weight(t.f) != 40 + OR datasketch_frequent_items_estimate(t.f, (g % 7)::BIGINT) != datasketch_frequent_items_estimate(ref.f, (g % 7)::BIGINT); +---- +0 + +# Many small groups: every sketch's exact facts +query I +SELECT count(*) FROM ( + SELECT g, + datasketch_bloom_filter(1000, 0.01, v) AS b, + datasketch_count_min(3, 128, v) AS c, + datasketch_count_min_weighted(3, 128, v, w) AS cw, + datasketch_frequent_items(10, v) AS f, + datasketch_frequent_items_weighted(v, w) AS fw, + datasketch_varopt(16, v) AS vo, + datasketch_varopt_weighted(16, v, w::DOUBLE) AS vow, + datasketch_ebpps(16, v) AS e, + datasketch_ebpps_weighted(16, v, w::DOUBLE) AS ew, + sum(w) AS total + FROM small GROUP BY g +) WHERE NOT datasketch_bloom_filter_contains(b, g) OR NOT datasketch_bloom_filter_contains(b, g + 400000) + OR NOT datasketch_bloom_filter_contains(b, g + 800000) + OR datasketch_count_min_total_weight(c) != 3 OR datasketch_count_min_estimate(c, g) < 1 + OR datasketch_count_min_total_weight(cw) != total + OR datasketch_frequent_items_total_weight(f) != 3 OR datasketch_frequent_items_estimate(f, g) != 1 + OR datasketch_frequent_items_total_weight(fw) != total + OR datasketch_varopt_n(vo) != 3 OR datasketch_varopt_num_samples(vo) != 3 + OR datasketch_varopt_subset_sum(vow, [g, g + 400000, g + 800000]).estimate != total + OR datasketch_ebpps_n(e) != 3 OR datasketch_ebpps_cumulative_weight(ew) != total; +---- +0 + +# Large groups of the sampling sketches: n and total weights are exact +query I +SELECT count(*) FROM ( + SELECT g, datasketch_varopt(16, v) AS vo, datasketch_varopt_weighted(16, v, w::DOUBLE) AS vow, + datasketch_ebpps(64, v) AS e, datasketch_ebpps_weighted(64, v, w::DOUBLE) AS ew, sum(w) AS total + FROM big GROUP BY g +) WHERE datasketch_varopt_n(vo) != 40 OR datasketch_varopt_num_samples(vo) != 16 + OR abs(datasketch_varopt_subset_sum(vow, [g]).total_weight - total) > 1e-6 + OR datasketch_ebpps_n(e) != 40 OR datasketch_ebpps_cumulative_weight(ew) != total; +---- +0 + +# Merging stored sketches spills too +statement ok +SET memory_limit='4GB'; + +statement ok +CREATE TABLE parts AS +SELECT g, p, datasketch_bloom_filter(100, 0.05, v) AS b, datasketch_count_min(2, 16, v) AS c, + datasketch_frequent_items(10, v) AS f, datasketch_varopt(16, v) AS vo, datasketch_ebpps(16, v) AS e +FROM (SELECT i % 100000 AS g, i % 3 AS p, i AS v FROM range(1200000) t(i)) +GROUP BY g, p; + +statement ok +CREATE TABLE merged_ref AS +SELECT g, hash(datasketch_bloom_filter(b)) AS b, hash(datasketch_count_min(c)) AS c FROM parts GROUP BY g; + +statement ok +SET memory_limit='128MB'; + +query I +SELECT count(*) FROM ( + SELECT g, datasketch_bloom_filter(b) AS b, datasketch_count_min(c) AS c FROM parts GROUP BY g +) t JOIN merged_ref USING (g) +WHERE hash(t.b) != merged_ref.b OR hash(t.c) != merged_ref.c OR NOT datasketch_bloom_filter_contains(t.b, g + 1100000); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT g, datasketch_varopt(16, vo) AS vo, datasketch_ebpps(16, e) AS e FROM parts GROUP BY g +) WHERE datasketch_varopt_n(vo) != 12 OR datasketch_varopt_num_samples(vo) != 12 OR datasketch_ebpps_n(e) != 12; +---- +0 + +# The frequent items merge keeps its heap state (its items may be any strings), which DuckDB does not spill: on its own, +# so that the other aggregates of a query still can +query I +SELECT count(*) FROM (SELECT g, datasketch_frequent_items(f) AS f FROM parts GROUP BY g) +WHERE datasketch_frequent_items_total_weight(f) != 12; +---- +0 + +statement ok +SET memory_limit='4GB'; + +# Exported states finalize to the aggregate's result, buffered (10 rows) or not (100000 rows). Frequent items of +# several threads are combined in any order, which places items differently: their estimates are compared +foreach n 10 100000 + +query IIIIIIII +SELECT finalize(datasketch_bloom_filter(1000, 0.01, i) EXPORT_STATE) = datasketch_bloom_filter(1000, 0.01, i), + finalize(datasketch_bloom_filter(1000, 0.01, i::VARCHAR) EXPORT_STATE) = datasketch_bloom_filter(1000, 0.01, i::VARCHAR), + finalize(datasketch_count_min(3, 128, i) EXPORT_STATE) = datasketch_count_min(3, 128, i), + finalize(datasketch_count_min_weighted(3, 128, i, i % 3) EXPORT_STATE) = datasketch_count_min_weighted(3, 128, i, i % 3), + datasketch_frequent_items_estimate(finalize(datasketch_frequent_items(10, i % 50) EXPORT_STATE), 7) = count(*) FILTER (i % 50 = 7), + datasketch_varopt_n(finalize(datasketch_varopt_weighted(16, i, 1.0 + i % 3) EXPORT_STATE)) = ${n}, + datasketch_ebpps_cumulative_weight(finalize(datasketch_ebpps_weighted(16, i, 1.0 + i % 3) EXPORT_STATE)) = sum(1.0 + i % 3), + datasketch_frequent_items_total_weight(finalize(datasketch_frequent_items_weighted(i % 50, i % 4) EXPORT_STATE)) = sum(i % 4) +FROM range(${n}) t(i); +---- +true true true true true true true true + +endloop + +# Exported merge states +query III +SELECT finalize(datasketch_bloom_filter(b) EXPORT_STATE) = datasketch_bloom_filter(b), + finalize(datasketch_count_min(c) EXPORT_STATE) = datasketch_count_min(c), + datasketch_varopt_n(finalize(datasketch_varopt(16, vo) EXPORT_STATE)) = 1200000 +FROM parts; +---- +true true true + +# VARCHAR frequent items and samples keep their heap states and still work under the limit +statement ok +SET memory_limit='128MB'; + +query I +SELECT count(*) FROM ( + SELECT g, datasketch_frequent_items(v::VARCHAR) AS f, datasketch_varopt(16, v::VARCHAR) AS vo, + datasketch_ebpps(16, v::VARCHAR) AS e + FROM (SELECT i % 1000 AS g, i AS v FROM range(100000) t(i)) GROUP BY g +) WHERE datasketch_frequent_items_total_weight(f) != 100 OR datasketch_varopt_n(vo) != 100 OR datasketch_ebpps_n(e) != 100; +---- +0 + +# Weights are checked when a row arrives, also while a state buffers it +statement error +SELECT datasketch_varopt_weighted(16, i, 0.0) FROM range(3) t(i); +---- +Item weights must be positive and finite + +statement error +SELECT datasketch_ebpps_weighted(16, i, -1.0) FROM range(3) t(i); +---- +Item weights must be nonnegative and finite + +statement error +SELECT datasketch_frequent_items_weighted(i, -1) FROM range(3) t(i); +---- +Frequent Items weight must be non-negative diff --git a/vcpkg-overlay/apache-datasketches/count-min-hash-seeds-allocator.patch b/vcpkg-overlay/apache-datasketches/count-min-hash-seeds-allocator.patch new file mode 100644 index 0000000..80c0551 --- /dev/null +++ b/vcpkg-overlay/apache-datasketches/count-min-hash-seeds-allocator.patch @@ -0,0 +1,23 @@ +--- a/count/include/count_min.hpp ++++ b/count/include/count_min.hpp +@@ -382,7 +382,7 @@ + std::vector _sketch_array; // the array stored by the sketch + uint64_t _seed; + W _total_weight; +- std::vector hash_seeds; ++ std::vector::template rebind_alloc> hash_seeds; + + enum flags {IS_EMPTY}; + static const uint8_t PREAMBLE_LONGS_SHORT = 2; // Empty -> need second byte for sketch parameters +--- a/count/include/count_min_impl.hpp ++++ b/count/include/count_min_impl.hpp +@@ -72,7 +72,8 @@ + _num_buckets(num_buckets), + _sketch_array((static_cast(num_hashes) * num_buckets < 1<<30) ? static_cast(num_hashes) * num_buckets : 0, 0, _allocator), + _seed(seed), +-_total_weight(0) { ++_total_weight(0), ++hash_seeds(_allocator) { + if (num_buckets < 3) { + throw std::invalid_argument("Using fewer than 3 buckets incurs relative error greater than 1."); + } diff --git a/vcpkg-overlay/apache-datasketches/frequent-items-purge-samples.patch b/vcpkg-overlay/apache-datasketches/frequent-items-purge-samples.patch new file mode 100644 index 0000000..f2985c5 --- /dev/null +++ b/vcpkg-overlay/apache-datasketches/frequent-items-purge-samples.patch @@ -0,0 +1,22 @@ +--- a/fi/include/reverse_purge_hash_map_impl.hpp ++++ b/fi/include/reverse_purge_hash_map_impl.hpp +@@ -353,8 +353,9 @@ + const uint32_t limit = std::min(MAX_SAMPLE_SIZE, num_active_); + uint32_t num_samples = 0; + uint32_t i = 0; +- AllocV av(allocator_); +- V* samples = av.allocate(limit); ++ // A temporary of at most MAX_SAMPLE_SIZE values: on the stack rather than from the allocator, which need not be ++ // able to give memory back (e.g. an arena) ++ V samples[MAX_SAMPLE_SIZE]; + while (num_samples < limit) { + if (is_active(i)) { + samples[num_samples++] = values_[i]; +@@ -363,7 +364,6 @@ + } + std::nth_element(samples, samples+ (num_samples / 2), samples + num_samples); + const V median = samples[num_samples / 2]; +- av.deallocate(samples, limit); + subtract_and_keep_positive_only(median); + return median; + } diff --git a/vcpkg-overlay/apache-datasketches/portfile.cmake b/vcpkg-overlay/apache-datasketches/portfile.cmake index 8db8859..b9f6142 100644 --- a/vcpkg-overlay/apache-datasketches/portfile.cmake +++ b/vcpkg-overlay/apache-datasketches/portfile.cmake @@ -20,6 +20,11 @@ vcpkg_from_github( # kll_sketch allocated K items in its constructor; it now allocates them on first use # (https://github.com/apache/datasketches-cpp/pull/558). kll-lazy-items.patch + # count_min_sketch kept its hash seeds in a std::vector with std::allocator, ignoring the sketch's allocator. + count-min-hash-seeds-allocator.patch + # reverse_purge_hash_map::purge (frequent items) took its temporary median sample from the sketch's + # allocator, which an arena never gets back; it now lives on the stack. + frequent-items-purge-samples.patch ) set(VCPKG_BUILD_TYPE release) # header-only port diff --git a/vcpkg-overlay/apache-datasketches/vcpkg.json b/vcpkg-overlay/apache-datasketches/vcpkg.json index 9b144ba..1d6dc26 100644 --- a/vcpkg-overlay/apache-datasketches/vcpkg.json +++ b/vcpkg-overlay/apache-datasketches/vcpkg.json @@ -1,7 +1,7 @@ { "name": "apache-datasketches", "version": "5.3.0", - "port-version": 3, + "port-version": 4, "description": "Apache DataSketches Core C++ Library Component.", "homepage": "https://datasketches.apache.org/", "license": "Apache-2.0",