From b2d7a7d0de850412d12e13656baa1ab8f1367d04 Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Wed, 28 Oct 2026 13:17:28 -0400 Subject: [PATCH 1/5] fix(vcpkg): allocator fixes for Count-Min and frequent items Two overlay patches, so these sketches can keep all their memory in an arena allocator (DSAllocator's arena mode), which never frees: - count-min-hash-seeds-allocator.patch: count_min_sketch kept its hash seeds in a std::vector with std::allocator, ignoring the sketch's allocator. The vector now uses the sketch's allocator (rebound to uint64_t). - frequent-items-purge-samples.patch: reverse_purge_hash_map::purge took its temporary median sample (up to 1024 values) from the sketch's allocator on every purge; it now lives on the stack. Co-Authored-By: Claude Opus 5.5 --- .../count-min-hash-seeds-allocator.patch | 23 +++++++++++++++++++ .../frequent-items-purge-samples.patch | 22 ++++++++++++++++++ .../apache-datasketches/portfile.cmake | 5 ++++ vcpkg-overlay/apache-datasketches/vcpkg.json | 2 +- 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 vcpkg-overlay/apache-datasketches/count-min-hash-seeds-allocator.patch create mode 100644 vcpkg-overlay/apache-datasketches/frequent-items-purge-samples.patch diff --git a/vcpkg-overlay/apache-datasketches/count-min-hash-seeds-allocator.patch b/vcpkg-overlay/apache-datasketches/count-min-hash-seeds-allocator.patch new file mode 100644 index 0000000..80c0551 --- /dev/null +++ b/vcpkg-overlay/apache-datasketches/count-min-hash-seeds-allocator.patch @@ -0,0 +1,23 @@ +--- a/count/include/count_min.hpp ++++ b/count/include/count_min.hpp +@@ -382,7 +382,7 @@ + std::vector _sketch_array; // the array stored by the sketch + uint64_t _seed; + W _total_weight; +- std::vector hash_seeds; ++ std::vector::template rebind_alloc> hash_seeds; + + enum flags {IS_EMPTY}; + static const uint8_t PREAMBLE_LONGS_SHORT = 2; // Empty -> need second byte for sketch parameters +--- a/count/include/count_min_impl.hpp ++++ b/count/include/count_min_impl.hpp +@@ -72,7 +72,8 @@ + _num_buckets(num_buckets), + _sketch_array((static_cast(num_hashes) * num_buckets < 1<<30) ? static_cast(num_hashes) * num_buckets : 0, 0, _allocator), + _seed(seed), +-_total_weight(0) { ++_total_weight(0), ++hash_seeds(_allocator) { + if (num_buckets < 3) { + throw std::invalid_argument("Using fewer than 3 buckets incurs relative error greater than 1."); + } diff --git a/vcpkg-overlay/apache-datasketches/frequent-items-purge-samples.patch b/vcpkg-overlay/apache-datasketches/frequent-items-purge-samples.patch new file mode 100644 index 0000000..f2985c5 --- /dev/null +++ b/vcpkg-overlay/apache-datasketches/frequent-items-purge-samples.patch @@ -0,0 +1,22 @@ +--- a/fi/include/reverse_purge_hash_map_impl.hpp ++++ b/fi/include/reverse_purge_hash_map_impl.hpp +@@ -353,8 +353,9 @@ + const uint32_t limit = std::min(MAX_SAMPLE_SIZE, num_active_); + uint32_t num_samples = 0; + uint32_t i = 0; +- AllocV av(allocator_); +- V* samples = av.allocate(limit); ++ // A temporary of at most MAX_SAMPLE_SIZE values: on the stack rather than from the allocator, which need not be ++ // able to give memory back (e.g. an arena) ++ V samples[MAX_SAMPLE_SIZE]; + while (num_samples < limit) { + if (is_active(i)) { + samples[num_samples++] = values_[i]; +@@ -363,7 +364,6 @@ + } + std::nth_element(samples, samples+ (num_samples / 2), samples + num_samples); + const V median = samples[num_samples / 2]; +- av.deallocate(samples, limit); + subtract_and_keep_positive_only(median); + return median; + } diff --git a/vcpkg-overlay/apache-datasketches/portfile.cmake b/vcpkg-overlay/apache-datasketches/portfile.cmake index 8db8859..b9f6142 100644 --- a/vcpkg-overlay/apache-datasketches/portfile.cmake +++ b/vcpkg-overlay/apache-datasketches/portfile.cmake @@ -20,6 +20,11 @@ vcpkg_from_github( # kll_sketch allocated K items in its constructor; it now allocates them on first use # (https://github.com/apache/datasketches-cpp/pull/558). kll-lazy-items.patch + # count_min_sketch kept its hash seeds in a std::vector with std::allocator, ignoring the sketch's allocator. + count-min-hash-seeds-allocator.patch + # reverse_purge_hash_map::purge (frequent items) took its temporary median sample from the sketch's + # allocator, which an arena never gets back; it now lives on the stack. + frequent-items-purge-samples.patch ) set(VCPKG_BUILD_TYPE release) # header-only port diff --git a/vcpkg-overlay/apache-datasketches/vcpkg.json b/vcpkg-overlay/apache-datasketches/vcpkg.json index 9b144ba..1d6dc26 100644 --- a/vcpkg-overlay/apache-datasketches/vcpkg.json +++ b/vcpkg-overlay/apache-datasketches/vcpkg.json @@ -1,7 +1,7 @@ { "name": "apache-datasketches", "version": "5.3.0", - "port-version": 3, + "port-version": 4, "description": "Apache DataSketches Core C++ Library Component.", "homepage": "https://datasketches.apache.org/", "license": "Apache-2.0", From 5b54013828f0f216a195cc68060a117fdd4f7997 Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Wed, 28 Oct 2026 13:17:28 -0400 Subject: [PATCH 2/5] perf: keep Bloom filter and Count-Min states in DuckDB's arena so they can spill The Bloom filter and Count-Min aggregates (build, weighted build and merge, every input type including VARCHAR and BLOB) now keep their states in the aggregate arena, have no destructor, and export and import them as BLOBs, so they count toward memory_limit and DuckDB can spill them. A filter allocates its whole bit array and a Count-Min sketch all its counters up front, and DuckDB creates a state per input row when grouping reduces little, so a state first buffers up to 32 raw values (with their weights; strings are copied into the arena), bounded by the sketch's size, and builds the sketch only when more arrive. Both sketches are order-independent, so results are byte for byte the same. Buffered states export as their values, so importing them does not inflate them. The shared machinery (DSBufferedArenaState and its policies' interface, a weighted arena operation, scratch arenas for temporaries) is in src/include/datasketches_arena_buffer.hpp. Co-Authored-By: Claude Opus 5.5 --- src/bloom_filter.cpp | 164 +++++----- src/count_min_sketch.cpp | 184 ++++------- src/include/datasketches_arena_buffer.hpp | 382 ++++++++++++++++++++++ 3 files changed, 528 insertions(+), 202 deletions(-) create mode 100644 src/include/datasketches_arena_buffer.hpp diff --git a/src/bloom_filter.cpp b/src/bloom_filter.cpp index e89ff7c..771a7f5 100644 --- a/src/bloom_filter.cpp +++ b/src/bloom_filter.cpp @@ -1,5 +1,4 @@ -#include "datasketches_extension.hpp" -#include "datasketches_items.hpp" +#include "datasketches_arena_buffer.hpp" #include "duckdb/common/vector/flat_vector.hpp" #include "duckdb/common/vector/string_vector.hpp" @@ -79,25 +78,32 @@ namespace duckdb // ============================================================ struct DSBloomBindData : public FunctionData { + // The merge aggregate's: its filters carry their own sizes. + DSBloomBindData() : max_distinct_items(0), false_positive_rate(0), filter_bytes(0) + { + } DSBloomBindData(uint64_t max_distinct_items, double false_positive_rate) - : max_distinct_items(max_distinct_items), false_positive_rate(false_positive_rate) + : max_distinct_items(max_distinct_items), false_positive_rate(false_positive_rate), + filter_bytes(((BloomFilter::builder::suggest_num_filter_bits(max_distinct_items, false_positive_rate) + 63) / 64) * 8) { } unique_ptr Copy() const override { - return make_uniq(max_distinct_items, false_positive_rate); + return make_uniq(*this); } bool Equals(const FunctionData &other_p) const override { auto &other = other_p.Cast(); return max_distinct_items == other.max_distinct_items && false_positive_rate == other.false_positive_rate; } - BloomFilter Create() const + BloomFilter Create(const DSAllocator &allocator = DSAllocator()) const { - return BloomFilter::builder::create_by_accuracy(max_distinct_items, false_positive_rate, DS_BLOOM_SEED); + return BloomFilter::builder::create_by_accuracy(max_distinct_items, false_positive_rate, DS_BLOOM_SEED, allocator); } uint64_t max_distinct_items; double false_positive_rate; + // The size of the filter's bit array. + idx_t filter_bytes; }; // datasketch_bloom_filter(max_distinct_items, false_positive_rate, data): both constants size the filter. @@ -131,109 +137,84 @@ namespace duckdb unique_ptr DSBloomUnionBind(BindAggregateFunctionInput &) { - return nullptr; + return make_uniq(); + } + + // The constants an exported state is re-bound with. + void DSBloomConstants(const DSBloomBindData &bind, unordered_map &constants) + { + constants.emplace(0, Value::BIGINT(static_cast(bind.max_distinct_items))); + constants.emplace(1, Value::DOUBLE(bind.false_positive_rate)); } // ============================================================ // 2. State & Operations // ============================================================ - struct DSBloomState + // The aggregates' state, in the aggregate arena (see datasketches_arena_buffer.hpp): the first values, or the + // filter. Values of every type are buffered as they are (strings copied into the arena): a filter never keeps + // them, only their hashes. + template + struct DSBloomPolicy { - BloomFilter *filter = nullptr; - ~DSBloomState() + using BIND = DSBloomBindData; + using VALUE = T; + using WEIGHT = void; + using SKETCH = BloomFilter; + using ENTRY = DSBufferedEntry; + static constexpr const char *NAME = "Bloom filter"; + static constexpr uint32_t BUFFERED = 32; + + static DSAllocator ArenaMode() { - delete filter; + return DSAllocator(DSArenaMode()); } - }; - - // Shared by the build and merge aggregates. With no input rows, the result is NULL. - struct DSBloomOperationBase - { - template - static void Initialize(STATE &state) { state.filter = nullptr; } - - template - static void Destroy(STATE &state, AggregateInputData &) + static idx_t Budget(const BIND &bind) { return bind.filter_bytes; } + static ENTRY Entry(const T &value) { return ENTRY {value}; } + static BloomFilter *Create(const BIND &bind, ArenaAllocator &arena) { - delete state.filter; - state.filter = nullptr; + return arena.Make(bind.Create(ArenaMode())); } - - static bool IgnoreNull() { return true; } - - // Adding a value (or a filter) again does not change the filter. - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static BloomFilter Temporary(const BIND &bind) { return bind.Create(); } + static void Update(BloomFilter &filter, const ENTRY &entry) { DSBloomUpdate(filter, entry.value); } + // See DSBloomSerialize. + static auto Serialize(BloomFilter &filter) { - OP::template Operation(state, input, unary_input); + filter.get_bits_used(); + return filter.serialize(); } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static BloomFilter *Deserialize(const string_t &blob, ArenaAllocator &arena) { - if (!source.filter) - { - return; - } - if (!target.filter) - { - if (aggr_input_data.combine_type == AggregateCombineType::ALLOW_DESTRUCTIVE) - { - // The source is discarded after this call, so take its filter rather than copy it. - target.filter = source.filter; - const_cast(source).filter = nullptr; - } - else - { - target.filter = new BloomFilter(*source.filter); - } - return; - } - DSBloomCombine(*target.filter, *source.filter, "datasketch_bloom_filter"); + return arena.Make(DSDeserialize(blob, "Bloom filter", ArenaMode())); } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + // The copy allocates through the source's allocator, which in arena mode is the current (target's) arena. + static BloomFilter *Copy(const BloomFilter &source, ArenaAllocator &arena) { - if (state.filter) - { - target = DSBloomSerialize(finalize_data.result, *state.filter); - } - else - { - finalize_data.ReturnNull(); - } + return arena.Make(source); } - }; - - template - struct DSBloomBuildOperation : DSBloomOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &input, AggregateUnaryInput &idata) + static void Combine(BloomFilter &target, const BloomFilter &source, const BIND &, ArenaAllocator &) + { + DSBloomCombine(target, source, "datasketch_bloom_filter"); + } + // An input filter is read on the heap and freed once combined, except the first, which becomes the state's. + static void Merge(BloomFilter *&filter, const string_t &blob, const BIND &, ArenaAllocator &arena) { - if (!state.filter) + if (!filter) { - state.filter = new BloomFilter(idata.input.bind_data->template Cast().Create()); + filter = Deserialize(blob, arena); + return; } - DSBloomUpdate(*state.filter, input); + DSBloomCombine(*filter, DSBloomDeserialize(blob), "datasketch_bloom_filter"); } }; - struct DSBloomUnionOperation : DSBloomOperationBase + // Adding a value (or a filter) again does not change the filter. + template + struct DSBloomIdempotent : BASE { - template - static void Operation(STATE &state, const A_TYPE &input, AggregateUnaryInput &) + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t) { - auto filter = DSBloomDeserialize(input); - if (!state.filter) - { - state.filter = new BloomFilter(std::move(filter)); - } - else - { - DSBloomCombine(*state.filter, filter, "datasketch_bloom_filter"); - } + BASE::template Operation(state, input, unary_input); } }; @@ -342,10 +323,12 @@ namespace duckdb DSForEachHashedType([&](auto marker, const LogicalType &item_type) { using INPUT = DSCountingInput; - using OP = DSBloomBuildOperation; using T = typename INPUT::physical_type; - auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); - DSReadDataArgumentAt<2, DSBloomState, T, OP>(fun); + using STATE = DSBufferedArenaState>; + using OP = DSBloomIdempotent>; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + DSReadDataArgumentAt<2, STATE, T, OP>(fun); + DSSetBufferedStateExport(fun); DSAddAggregate(set, descs, fun, DSBloomBind, {LogicalType::BIGINT, LogicalType::DOUBLE, item_type}, {"max_distinct_items", "false_positive_rate", "data"}, "Creates a Bloom filter sized to hold max_distinct_items distinct values with the given false positive rate. " @@ -353,7 +336,10 @@ namespace duckdb {"SELECT datasketch_bloom_filter(1000, 0.01, i) FROM range(100) t(i)"}); }); // datasketch_bloom_filter(filter): combines filters. - auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + using MERGE_STATE = DSBufferedArenaState>; + using MERGE = DSBloomIdempotent>; + auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSSetBufferedStateExport>(merge); DSAddAggregate(set, descs, merge, DSBloomUnionBind, {sketch_type}, {"filter"}, "Combines Bloom filters built with the same max_distinct_items and false_positive_rate into one that " "contains the values of each", diff --git a/src/count_min_sketch.cpp b/src/count_min_sketch.cpp index 88bc25e..27a9031 100644 --- a/src/count_min_sketch.cpp +++ b/src/count_min_sketch.cpp @@ -1,5 +1,4 @@ -#include "datasketches_extension.hpp" -#include "datasketches_items.hpp" +#include "datasketches_arena_buffer.hpp" #include "duckdb/common/vector/flat_vector.hpp" #include "duckdb/common/vector/string_vector.hpp" @@ -132,139 +131,92 @@ namespace duckdb return make_uniq(static_cast(num_hashes), static_cast(num_buckets)); } + // The merge aggregate's bind data: its sketches carry their own sizes. unique_ptr DSCountMinMergeBind(BindAggregateFunctionInput &) { - return nullptr; + return make_uniq(0, 0); + } + + // The constants an exported state is re-bound with. + void DSCountMinConstants(const DSCountMinBindData &bind, unordered_map &constants) + { + constants.emplace(0, Value::INTEGER(bind.num_hashes)); + constants.emplace(1, Value::INTEGER(static_cast(bind.num_buckets))); } // ============================================================ // 2. State & Operations // ============================================================ - struct DSCountMinState - { - CountMinSketch *sketch = nullptr; - ~DSCountMinState() + // The aggregates' state, in the aggregate arena (see datasketches_arena_buffer.hpp): the first values with + // their weights, or the sketch. Values of every type are buffered as they are (strings copied into the arena): + // a sketch never keeps them, only their hashes. Counters are sums, so feeding the buffered values later gives + // the same sketch. + template + struct DSCountMinPolicy + { + using BIND = DSCountMinBindData; + using VALUE = T; + using WEIGHT = int64_t; + using SKETCH = CountMinSketch; + using ENTRY = DSBufferedEntry; + static constexpr const char *NAME = "Count-Min"; + static constexpr uint32_t BUFFERED = 32; + + static DSAllocator ArenaMode() { - delete sketch; + return DSAllocator(DSArenaMode()); } - void Create(AggregateInputData &input) + static idx_t Budget(const BIND &bind) { - if (!sketch) - { - auto &bind = input.bind_data->Cast(); - sketch = new CountMinSketch(bind.num_hashes, bind.num_buckets); - } + return static_cast(bind.num_hashes) * bind.num_buckets * sizeof(int64_t); } - }; - - // Shared by the build, weighted build and merge aggregates. With no input rows, the result is NULL. - struct DSCountMinOperationBase - { - template - static void Initialize(STATE &state) { state.sketch = nullptr; } - - template - static void Destroy(STATE &state, AggregateInputData &) + static ENTRY Entry(const T &value, int64_t weight = 1) { return ENTRY {value, weight}; } + static CountMinSketch *Create(const BIND &bind, ArenaAllocator &arena) { - delete state.sketch; - state.sketch = nullptr; + return arena.Make(bind.num_hashes, bind.num_buckets, datasketches::DEFAULT_SEED, ArenaMode()); } - - static bool IgnoreNull() { return true; } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static CountMinSketch Temporary(const BIND &bind) { return CountMinSketch(bind.num_hashes, bind.num_buckets); } + static void Update(CountMinSketch &sketch, const ENTRY &entry) { - if (!source.sketch) - { - return; - } - if (!target.sketch) - { - if (aggr_input_data.combine_type == AggregateCombineType::ALLOW_DESTRUCTIVE) - { - // The source is discarded after this call, so take its sketch rather than copy it. - target.sketch = source.sketch; - const_cast(source).sketch = nullptr; - } - else - { - target.sketch = new CountMinSketch(*source.sketch); - } - return; - } - DSCountMinMerge(*target.sketch, *source.sketch, "datasketch_count_min"); + DSCountMinUpdate(sketch, entry.value, entry.weight); } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + static auto Serialize(const CountMinSketch &sketch) { return sketch.serialize(); } + static CountMinSketch *Deserialize(const string_t &blob, ArenaAllocator &arena) { - if (state.sketch) - { - target = DSCountMinSerialize(finalize_data.result, *state.sketch); - } - else - { - finalize_data.ReturnNull(); - } + return arena.Make(DSDeserialize(blob, "Count-Min", datasketches::DEFAULT_SEED, ArenaMode())); } - }; - - // Each row counts once. - template - struct DSCountMinOperation : DSCountMinOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &input, AggregateUnaryInput &idata) - { - state.Create(idata.input); - DSCountMinUpdate(*state.sketch, input, 1); - } - - // A repeated value counts as many times as it repeats. - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + // The copy allocates through the source's allocator, which in arena mode is the current (target's) arena + // (with vcpkg-overlay/apache-datasketches/count-min-hash-seeds-allocator.patch). + static CountMinSketch *Copy(const CountMinSketch &source, ArenaAllocator &arena) { - state.Create(unary_input.input); - DSCountMinUpdate(*state.sketch, input, static_cast(count)); + return arena.Make(source); } - }; - - // Each row adds its weight. - template - struct DSCountMinWeightedOperation : DSCountMinOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &input, const W_TYPE &weight, AggregateBinaryInput &idata) + static void Combine(CountMinSketch &target, const CountMinSketch &source, const BIND &, ArenaAllocator &) { - state.Create(idata.input); - DSCountMinUpdate(*state.sketch, input, weight); + DSCountMinMerge(target, source, "datasketch_count_min"); } - }; - - struct DSCountMinMergeOperation : DSCountMinOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &input, AggregateUnaryInput &) + // An input sketch is read on the heap and freed once merged, except the first, which becomes the state's. + static void Merge(CountMinSketch *&sketch, const string_t &blob, const BIND &, ArenaAllocator &arena) { - auto sketch = DSCountMinDeserialize(input); - if (!state.sketch) - { - state.sketch = new CountMinSketch(std::move(sketch)); - } - else + if (!sketch) { - DSCountMinMerge(*state.sketch, sketch, "datasketch_count_min"); + sketch = Deserialize(blob, arena); + return; } + DSCountMinMerge(*sketch, DSCountMinDeserialize(blob), "datasketch_count_min"); } + }; + // Each row counts once; a repeated value counts as many times as it repeats. + template + struct DSCountMinOperation : DSArenaCreateOperation + { template static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) { - for (idx_t i = 0; i < count; i++) - { - Operation(state, input, unary_input); - } + DSArenaScope scope(unary_input.input.allocator); + state.UpdateWeighted(input, static_cast(count), unary_input.input.bind_data->template Cast(), + unary_input.input.allocator); } }; @@ -396,15 +348,20 @@ namespace duckdb using INPUT = DSCountingInput; using OP = DSCountMinOperation; using T = typename INPUT::physical_type; - auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); - DSReadDataArgumentAt<2, DSCountMinState, T, OP>(fun); + using STATE = DSBufferedArenaState>; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + DSReadDataArgumentAt<2, STATE, T, OP>(fun); + DSSetBufferedStateExport(fun); DSAddAggregate(set, descs, fun, DSCountMinBind, {LogicalType::INTEGER, LogicalType::INTEGER, item_type}, {"num_hashes", "num_buckets", "data"}, "Creates a Count-Min sketch that estimates how often each value occurs. An estimate exceeds the true " "count by at most e / num_buckets of the total count, with probability 1 - e^-num_hashes " "(see datasketch_count_min_suggest_num_buckets and _suggest_num_hashes)", {"SELECT datasketch_count_min(3, 128, item)" + rows}); }); - auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + using MERGE_STATE = DSBufferedArenaState>; + using MERGE = DSArenaMergeOperation; + auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSSetBufferedStateExport>(merge); DSAddAggregate(set, descs, merge, DSCountMinMergeBind, {sketch_type}, {"sketch"}, "Merges Count-Min sketches of the same size", {ex.QueryPartials("datasketch_count_min(sketch)")}); @@ -418,11 +375,12 @@ namespace duckdb DSForEachHashedType([&](auto marker, const LogicalType &item_type) { using INPUT = DSCountingInput; - using OP = DSCountMinWeightedOperation; + using OP = DSArenaWeightedOperation; using T = typename INPUT::physical_type; - auto fun = AggregateFunction::BinaryAggregate(item_type, LogicalType::BIGINT, sketch_type); - fun.SetStateDestructorCallback(AggregateFunction::StateDestroy); - DSReadDataArgumentsAt<2, 3, DSCountMinState, T, int64_t, OP>(fun); + using STATE = DSBufferedArenaState>; + auto fun = AggregateFunction::BinaryAggregate(item_type, LogicalType::BIGINT, sketch_type); + DSReadDataArgumentsAt<2, 3, STATE, T, int64_t, OP>(fun); + DSSetBufferedStateExport(fun); DSAddAggregate(set, descs, fun, DSCountMinBind, {LogicalType::INTEGER, LogicalType::INTEGER, item_type, LogicalType::BIGINT}, {"num_hashes", "num_buckets", "data", "weight"}, "Creates a Count-Min sketch of the total weight of each value: each row adds its weight (which may be " diff --git a/src/include/datasketches_arena_buffer.hpp b/src/include/datasketches_arena_buffer.hpp new file mode 100644 index 0000000..1c0ca26 --- /dev/null +++ b/src/include/datasketches_arena_buffer.hpp @@ -0,0 +1,382 @@ +#pragma once + +// Arena states that buffer their first raw inputs, for the hand-written sketches (frequent items, VarOpt, EBPPS, +// Bloom filter, Count-Min). They build on the arena states of datasketches_generated_common.hpp (the operations, the +// export and import callbacks, DSArenaItems) and follow the same rules: everything a state keeps lives in the +// aggregate arena of the callback that allocated it, a state has no destructor, and Combine copies from the source +// state without taking over or growing its memory. +// +// DuckDB creates one state per input row when grouping reduces little (it skips the hash table lookups), and these +// sketches allocate a lot up front: a Bloom filter its whole bit array, a Count-Min sketch num_hashes * num_buckets +// counters, an EBPPS sketch room for K items. So a state keeps its first inputs (with their weights) as they are, and +// builds its sketch, feeding them in order, only when more arrive. + +#include "datasketches_generated_common.hpp" + +#include +#include +#include + +namespace duckdb +{ + +// A scratch arena for temporaries of arena-mode sketches (e.g. a union's result, serialized bytes): while it lives, +// arena-mode allocations go to it, and it is freed as a whole when it goes out of scope. A temporary must not +// outlive it, and nothing a state keeps may be allocated while it is the current arena (open a DSArenaScope on the +// state's arena for that). +struct DSArenaScratch +{ + DSArenaScratch() : arena(Allocator::DefaultAllocator()), scope(arena) + { + } + + ArenaAllocator arena; + DSArenaScope scope; +}; + +// The bytes fun() returns (a library's serialize(), whose vector uses the sketch's allocator), produced in a scratch +// arena and returned on the heap: serializing an arena-mode sketch leaves nothing behind in the callback's arena. +template +static std::vector DSScratchBytes(FUNC &&fun) +{ + DSArenaScratch scratch; + auto bytes = fun(); + return std::vector(bytes.begin(), bytes.end()); +} + +// One buffered input: a value and, unless WEIGHT is void, its weight. +template +struct DSBufferedEntry +{ + T value; + WEIGHT weight; +}; + +template +struct DSBufferedEntry +{ + T value; +}; + +// Strings are copied into the arena (unless inlined), so a buffered string_t never points at input or at another +// state's memory. +template +static inline idx_t DSArenaValueBytes(const T &value) +{ + if constexpr (std::is_same_v) + { + return value.IsInlined() ? 0 : value.GetSize(); + } + else + { + return 0; + } +} + +template +static inline T DSArenaCopyValue(const T &value, ArenaAllocator &arena) +{ + if constexpr (std::is_same_v) + { + if (value.IsInlined()) + { + return value; + } + auto size = value.GetSize(); + auto data = arena.Allocate(size); + memcpy(data, value.GetData(), size); + return string_t(const_char_ptr_cast(data), UnsafeNumericCast(size)); + } + else + { + return value; + } +} + +// An entry's bytes in an exported state: the weight, then the value as it is (a string as its uint32 length and +// bytes). +template +static inline void DSWriteEntry(std::vector &out, const DSBufferedEntry &entry) +{ + auto append = [&](const void *data, idx_t size) + { + auto bytes = static_cast(data); + out.insert(out.end(), bytes, bytes + size); + }; + if constexpr (!std::is_void_v) + { + append(&entry.weight, sizeof(WEIGHT)); + } + if constexpr (std::is_same_v) + { + const uint32_t size = entry.value.GetSize(); + append(&size, sizeof(size)); + append(entry.value.GetData(), size); + } + else + { + append(&entry.value, sizeof(T)); + } +} + +// Reads an entry written by DSWriteEntry, copying a string into the arena. False if the bytes end too soon. +template +static inline bool DSReadEntry(const_data_ptr_t &ptr, const_data_ptr_t end, DSBufferedEntry &entry, + ArenaAllocator &arena) +{ + auto read = [&](void *data, idx_t size) + { + if (idx_t(end - ptr) < size) + { + return false; + } + memcpy(data, ptr, size); + ptr += size; + return true; + }; + if constexpr (!std::is_void_v) + { + if (!read(&entry.weight, sizeof(WEIGHT))) + { + return false; + } + } + if constexpr (std::is_same_v) + { + uint32_t size; + if (!read(&size, sizeof(size)) || idx_t(end - ptr) < size) + { + return false; + } + entry.value = DSArenaCopyValue(string_t(const_char_ptr_cast(ptr), size), arena); + ptr += size; + return true; + } + else + { + return read(&entry.value, sizeof(T)); + } +} + +// The aggregates' state: up to POLICY::BUFFERED raw inputs, or the sketch they were fed to. POLICY (with BIND its +// bind data, SKETCH what the state keeps in the arena, ENTRY = DSBufferedEntry) provides: +// static constexpr uint32_t BUFFERED; the most inputs a state buffers +// static idx_t Budget(const BIND &); the most bytes a buffer may take (about the sketch's size) +// static SKETCH *Create(const BIND &, ArenaAllocator &); an empty sketch in the arena +// static auto Temporary(const BIND &); an empty sketch on the heap +// static void Update(S &sketch, const ENTRY &); feeds an input to a sketch (SKETCH or the temporary) +// static auto Serialize(S &sketch); the sketch's bytes (run in a scratch arena) +// static SKETCH *Deserialize(const string_t &, ArenaAllocator &); a sketch read into the arena +// static SKETCH *Copy(const SKETCH &, ArenaAllocator &); a copy in the arena +// static void Combine(SKETCH &target, const SKETCH &source, const BIND &, ArenaAllocator &); +// static void Merge(SKETCH *&sketch, const string_t &blob, const BIND &, ArenaAllocator &); merges a serialized +// sketch, creating the state's sketch if it has none +template +struct DSBufferedArenaState +{ + using BIND = typename POLICY::BIND; + using VALUE = typename POLICY::VALUE; + using WEIGHT = typename POLICY::WEIGHT; + using SKETCH = typename POLICY::SKETCH; + using ENTRY = DSBufferedEntry; + + SKETCH *sketch; + DSArenaItems buffer; + // bytes of the buffered strings that are not inlined + uint32_t string_bytes; + + void Initialize() + { + sketch = nullptr; + buffer.Initialize(); + string_bytes = 0; + } + bool IsEmpty() const + { + return !sketch && buffer.count == 0; + } + + template + void Replay(TARGET &target) const + { + for (uint32_t i = 0; i < buffer.count; i++) + { + POLICY::Update(target, buffer.items[i]); + } + } + void ClearBuffer() + { + buffer.Initialize(); + string_bytes = 0; + } + // Moves the buffered inputs into a new sketch. + SKETCH &Promote(const BIND &bind, ArenaAllocator &arena) + { + if (!sketch) + { + sketch = POLICY::Create(bind, arena); + Replay(*sketch); + ClearBuffer(); + } + return *sketch; + } + void Add(const ENTRY &entry, const BIND &bind, ArenaAllocator &arena) + { + if (!sketch && buffer.count < POLICY::BUFFERED) + { + const idx_t extra = DSArenaValueBytes(entry.value); + if ((buffer.count + 1) * sizeof(ENTRY) + string_bytes + extra <= POLICY::Budget(bind)) + { + ENTRY copy = entry; + copy.value = DSArenaCopyValue(entry.value, arena); + buffer.Append(copy, arena); + string_bytes += UnsafeNumericCast(extra); + return; + } + } + POLICY::Update(Promote(bind, arena), entry); + } + + // An input of the default weight (DSArenaCreateOperation). + template + void Update(const V &value, const BIND &bind, ArenaAllocator &arena) + { + Add(POLICY::Entry(value), bind, arena); + } + // An input with its weight (DSArenaWeightedOperation). + template + void UpdateWeighted(const V &value, const W &weight, const BIND &bind, ArenaAllocator &arena) + { + Add(POLICY::Entry(value, weight), bind, arena); + } + void Merge(const string_t &blob, const BIND &bind, ArenaAllocator &arena) + { + if (!sketch && buffer.count) + { + Promote(bind, arena); + } + POLICY::Merge(sketch, blob, bind, arena); + } + void Combine(const DSBufferedArenaState &source, const BIND &bind, ArenaAllocator &arena) + { + if (!source.sketch) + { + for (uint32_t i = 0; i < source.buffer.count; i++) + { + Add(source.buffer.items[i], bind, arena); + } + } + else if (!sketch) + { + // Copy the source's sketch, then feed it what this state buffered. + sketch = POLICY::Copy(*source.sketch, arena); + Replay(*sketch); + ClearBuffer(); + } + else + { + POLICY::Combine(*sketch, *source.sketch, bind, arena); + } + } + + // A state that still buffers its inputs is serialized through a temporary sketch on the heap, freed right away. + std::vector Serialize(const BIND &bind, ArenaAllocator &) const + { + if (sketch) + { + return DSScratchBytes([&]() { return POLICY::Serialize(*sketch); }); + } + auto temporary = POLICY::Temporary(bind); + Replay(temporary); + auto bytes = POLICY::Serialize(temporary); + return std::vector(bytes.begin(), bytes.end()); + } + + // The exported state is a tag byte, then the buffered inputs (EXPORT_BUFFER, see DSWriteEntry) or the serialized + // sketch (EXPORT_SKETCH). Buffered states stay buffered when they are imported. + static constexpr uint8_t EXPORT_BUFFER = 0; + static constexpr uint8_t EXPORT_SKETCH = 1; + std::vector Export(const BIND &, ArenaAllocator &) const + { + std::vector bytes; + if (sketch) + { + auto serialized = DSScratchBytes([&]() { return POLICY::Serialize(*sketch); }); + bytes.reserve(1 + serialized.size()); + bytes.push_back(EXPORT_SKETCH); + bytes.insert(bytes.end(), serialized.begin(), serialized.end()); + return bytes; + } + bytes.push_back(EXPORT_BUFFER); + for (uint32_t i = 0; i < buffer.count; i++) + { + DSWriteEntry(bytes, buffer.items[i]); + } + return bytes; + } + void Import(const string_t &data, ArenaAllocator &arena) + { + const auto size = data.GetSize(); + const auto bytes = const_data_ptr_cast(data.GetData()); + if (size >= 1 && bytes[0] == EXPORT_SKETCH) + { + sketch = POLICY::Deserialize(string_t(data.GetData() + 1, UnsafeNumericCast(size - 1)), arena); + return; + } + if (size < 1 || bytes[0] != EXPORT_BUFFER) + { + throw InvalidInputException("Invalid %s aggregate state", POLICY::NAME); + } + auto ptr = bytes + 1; + const auto end = bytes + size; + while (ptr < end) + { + ENTRY entry; + if (buffer.count == POLICY::BUFFERED || !DSReadEntry(ptr, end, entry, arena)) + { + throw InvalidInputException("Invalid %s aggregate state", POLICY::NAME); + } + buffer.Append(entry, arena); + string_bytes += UnsafeNumericCast(DSArenaValueBytes(entry.value)); + } + } +}; + +// Builds the sketch from values with weights (a binary aggregate). +template +struct DSArenaWeightedOperation : DSArenaOperationBase +{ + template + static void Operation(STATE &state, const A_TYPE &a_data, const B_TYPE &b_data, AggregateBinaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + state.UpdateWeighted(a_data, b_data, idata.input.bind_data->template Cast(), idata.input.allocator); + } +}; + +// The exported state is a BLOB. CONSTANTS(bind, constant_parameters) records the aggregate's constant arguments +// (e.g. K), so that re-binding the exported state supplies them. +template &)> +static AggregateStateLayout DSBufferedGetStateType(AggregateLayoutInput &input) +{ + AggregateStateLayout layout(LogicalType::BLOB, AlignValue(sizeof(STATE))); + if (input.bind_data) + { + CONSTANTS(input.bind_data->template Cast(), layout.constant_parameters); + } + return layout; +} + +template +static void DSNoConstants(const BIND_DATA_TYPE &, unordered_map &) +{ +} + +template &)> +static void DSSetBufferedStateExport(AggregateFunction &fun) +{ + fun.SetStateExportCallbacks(DSBufferedGetStateType, + DSArenaExportState, DSArenaImportState); +} + +} // namespace duckdb From 97eed466be383b87fde55584bf6d9687e444c6df Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Wed, 28 Oct 2026 13:17:28 -0400 Subject: [PATCH 3/5] perf: keep VarOpt and EBPPS states in DuckDB's arena so they can spill The VarOpt and EBPPS aggregates over every item type but VARCHAR (whose std::string items own heap memory and keep the heap state) now keep their states in the aggregate arena and export and import them, so they can spill. States buffer their first 32 items with their weights (weights are checked as they arrive, with the library's messages); an EBPPS sketch reserves K items up front. A VarOpt state switches to a var_opt_union (in the arena) when it is first combined or merged, so later combines and merges update it in place instead of leaving a replaced sketch behind in the arena. The EBPPS merge aggregate reproduces the heap state's merge (the lighter sample is merged into the heavier, K the smaller of the two) without swapping a heap sketch into an arena state. Co-Authored-By: Claude Opus 5.5 --- src/sampling_sketches.cpp | 307 +++++++++++++++++++++++++++++++++++--- 1 file changed, 290 insertions(+), 17 deletions(-) diff --git a/src/sampling_sketches.cpp b/src/sampling_sketches.cpp index d852c70..c704c9b 100644 --- a/src/sampling_sketches.cpp +++ b/src/sampling_sketches.cpp @@ -1,7 +1,7 @@ // The weighted sampling sketches: VarOpt (a variance-optimal sample of K items, with an estimate of the total // weight of any subset) and EBPPS (a sample whose inclusion probabilities are exactly proportional to the weights). -#include "datasketches_generated_common.hpp" +#include "datasketches_arena_buffer.hpp" #include #include @@ -50,6 +50,15 @@ namespace duckdb static constexpr const char *NAME = "VarOpt"; static constexpr const char *SQL = "varopt"; + // The library's check of an item's weight (see var_opt_sketch::update), for weights that are buffered. + static void CheckWeight(double weight) + { + if (weight <= 0.0 || std::isnan(weight) || std::isinf(weight)) + { + throw std::invalid_argument("Item weights must be positive and finite. Found: " + std::to_string(weight)); + } + } + // Merging goes through a union of at most `k` samples. static void Merge(Sketch &target, Sketch &&source, uint32_t k) { @@ -74,6 +83,15 @@ namespace duckdb static constexpr const char *NAME = "EBPPS"; static constexpr const char *SQL = "ebpps"; + // The library's check of an item's weight (see ebpps_sketch::update), for weights that are buffered. + static void CheckWeight(double weight) + { + if (weight < 0.0 || std::isnan(weight) || std::isinf(weight)) + { + throw std::invalid_argument("Item weights must be nonnegative and finite. Found: " + std::to_string(weight)); + } + } + // The merged sample has the smaller K of the two. static void Merge(Sketch &target, Sketch &&source, uint32_t) { @@ -257,6 +275,273 @@ namespace duckdb } }; + // The aggregates' state over items that own no heap memory (every item type but VARCHAR, which keeps + // DSSampleState), in the aggregate arena (see datasketches_arena_buffer.hpp): the first items with their + // weights, or the sample they were fed to, in order. An EBPPS sketch reserves room for K items up front. + template + struct DSSamplePolicyBase + { + using BIND = DSSampleBindData; + using VALUE = T; + using WEIGHT = double; + using ENTRY = DSBufferedEntry; + static constexpr const char *NAME = TRAITS::NAME; + static constexpr uint32_t BUFFERED = 32; + + static DSAllocator ArenaMode() + { + return DSAllocator(DSArenaMode()); + } + static idx_t Budget(const BIND &) { return NumericLimits::Maximum(); } + static ENTRY Entry(const T &value) { return ENTRY {value, 1.0}; } + // The weight is checked now, as the library would: a buffered item reaches a sketch only later. + static ENTRY Entry(const T &value, double weight) + { + DSSampleGuard([&]() { TRAITS::CheckWeight(weight); }); + return ENTRY {value, weight}; + } + }; + + // A VarOpt state's sample: a sketch, which takes items, until it is combined with another sample. Combining + // goes through a union of at most K samples, which the state then keeps, so that combining more samples + // updates it in place (in the arena, replacing a sketch leaves the old one behind). + template + struct DSVarOptArenaSample + { + DSVarOptSketch *sketch; + DSVarOptUnion *u; + uint32_t k; + }; + + template + struct DSVarOptArenaPolicy : DSSamplePolicyBase, T> + { + using BASE = DSSamplePolicyBase, T>; + using typename BASE::BIND; + using typename BASE::ENTRY; + using Sketch = DSVarOptSketch; + using Union = DSVarOptUnion; + using SKETCH = DSVarOptArenaSample; + + static SKETCH *Make(ArenaAllocator &arena, uint32_t k) + { + auto sample = arena.Make(); + sample->sketch = nullptr; + sample->u = nullptr; + sample->k = k; + return sample; + } + static SKETCH *Create(const BIND &bind, ArenaAllocator &arena) + { + auto sample = Make(arena, bind.k); + sample->sketch = arena.Make(bind.k, Sketch::DEFAULT_RESIZE_FACTOR, BASE::ArenaMode()); + return sample; + } + static Sketch Temporary(const BIND &bind) { return Sketch(bind.k); } + static void Update(Sketch &sketch, const ENTRY &entry) { sketch.update(entry.value, entry.weight); } + static void Update(SKETCH &sample, const ENTRY &entry) + { + if (sample.sketch) + { + sample.sketch->update(entry.value, entry.weight); + return; + } + // A union takes samples only: the item's, on the heap. + Sketch one(sample.k); + one.update(entry.value, entry.weight); + sample.u->update(std::move(one)); + } + static auto Serialize(const Sketch &sketch) { return sketch.serialize(); } + static auto Serialize(const SKETCH &sample) + { + return sample.sketch ? sample.sketch->serialize() : sample.u->get_result().serialize(); + } + static SKETCH *Deserialize(const string_t &blob, ArenaAllocator &arena) + { + auto sketch = arena.Make(DSDeserialize(blob, BASE::NAME, datasketches::serde(), BASE::ArenaMode())); + auto sample = Make(arena, sketch->get_k()); + sample->sketch = sketch; + return sample; + } + // Copies allocate through the source's allocator, which in arena mode is the current (target's) arena. + static SKETCH *Copy(const SKETCH &source, ArenaAllocator &arena) + { + auto sample = Make(arena, source.k); + if (source.sketch) + { + sample->sketch = arena.Make(*source.sketch); + } + else + { + sample->u = arena.Make(*source.u); + } + return sample; + } + static void ToUnion(SKETCH &sample, const BIND &bind, ArenaAllocator &arena) + { + if (sample.u) + { + return; + } + sample.u = arena.Make(bind.k, BASE::ArenaMode()); + if (sample.sketch) + { + sample.u->update(*sample.sketch); + sample.sketch = nullptr; + } + sample.k = bind.k; + } + static void Combine(SKETCH &target, const SKETCH &source, const BIND &bind, ArenaAllocator &arena) + { + ToUnion(target, bind, arena); + if (source.sketch) + { + target.u->update(*source.sketch); + return; + } + // The source union's result is a temporary, built in a scratch arena; the target grows in its own. + DSArenaScratch scratch; + auto result = source.u->get_result(); + DSArenaScope scope(arena); + target.u->update(std::move(result)); + } + // An input sample is read on the heap and freed once merged into the state's union. + static void Merge(SKETCH *&sample, const string_t &blob, const BIND &bind, ArenaAllocator &arena) + { + auto incoming = DSDeserialize(blob, BASE::NAME); + if (!sample) + { + sample = Make(arena, bind.k); + } + ToUnion(*sample, bind, arena); + sample->u->update(std::move(incoming)); + } + }; + + template + struct DSEbppsArenaPolicy : DSSamplePolicyBase, T> + { + using BASE = DSSamplePolicyBase, T>; + using typename BASE::BIND; + using typename BASE::ENTRY; + using SKETCH = DSEbppsSketch; + + static SKETCH *Create(const BIND &bind, ArenaAllocator &arena) + { + return arena.Make(bind.k, BASE::ArenaMode()); + } + static SKETCH Temporary(const BIND &bind) { return SKETCH(bind.k); } + static void Update(SKETCH &sketch, const ENTRY &entry) { sketch.update(entry.value, entry.weight); } + static auto Serialize(const SKETCH &sketch) { return sketch.serialize(); } + static SKETCH *Deserialize(const string_t &blob, ArenaAllocator &arena) + { + return arena.Make(DSDeserialize(blob, BASE::NAME, datasketches::serde(), BASE::ArenaMode())); + } + // Copies allocate through the source's allocator, which in arena mode is the current (target's) arena: + // also the copy merge() makes of a heavier source, which the target then keeps. + static SKETCH *Copy(const SKETCH &source, ArenaAllocator &arena) + { + return arena.Make(source); + } + static void Combine(SKETCH &target, const SKETCH &source, const BIND &, ArenaAllocator &) + { + target.merge(source); + } + // As the heap state's merge into a sketch of K: the lighter sample is merged into the heavier one, and the + // result has the smaller K. A lighter input is read on the heap and freed once merged. A heavier one + // becomes the state's sample, so it is read into the arena: merging into the state's sketch would swap + // the heap sketch in. + static void Merge(SKETCH *&sketch, const string_t &blob, const BIND &bind, ArenaAllocator &arena) + { + auto incoming = DSDeserialize(blob, BASE::NAME); + if (incoming.get_cumulative_weight() == 0.0) + { + if (!sketch) + { + sketch = Create(bind, arena); + } + return; + } + if (sketch && incoming.get_cumulative_weight() <= sketch->get_cumulative_weight()) + { + sketch->merge(std::move(incoming)); + return; + } + auto base = Deserialize(blob, arena); + if (sketch && sketch->get_cumulative_weight() > 0.0) + { + base->merge(*sketch); + } + else + { + // Merging a sample of no weight only lowers K to its K. + const uint32_t k = sketch ? sketch->get_k() : bind.k; + if (base->get_k() > k) + { + auto shrunk = arena.Make(k, BASE::ArenaMode()); + shrunk->merge(*base); + base = shrunk; + } + } + sketch = base; + } + }; + + template + struct DSSampleArena; + template + struct DSSampleArena> + { + using type = DSVarOptArenaPolicy; + }; + template + struct DSSampleArena> + { + using type = DSEbppsArenaPolicy; + }; + + // The create, merge and weighted create aggregates. + template + vector DSSampleAggregates(const LogicalType &item_type, const LogicalType &sketch_type) + { + if constexpr (std::is_same_v) + { + using STATE = DSSampleState; + using OP = DSSampleOperation; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + DSReadDataArgumentAt<1, STATE, T, OP>(fun); + + using MERGE = DSSampleMergeOperation; + auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSReadDataArgumentAt<1, STATE, string_t, MERGE>(merge); + + using WEIGHTED = DSSampleWeightedOperation; + auto weighted = AggregateFunction::BinaryAggregate(item_type, LogicalType::DOUBLE, sketch_type); + weighted.SetStateDestructorCallback(AggregateFunction::StateDestroy); + DSReadDataArgumentsAt<1, 2, STATE, T, double, WEIGHTED>(weighted); + return {fun, merge, weighted}; + } + else + { + using STATE = DSBufferedArenaState::type>; + using OP = DSArenaCreateOperation; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + DSReadDataArgumentAt<1, STATE, T, OP>(fun); + DSSetArenaStateExport(fun); + + using MERGE = DSArenaMergeOperation; + auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSReadDataArgumentAt<1, STATE, string_t, MERGE>(merge); + DSSetArenaStateExport(merge); + + using WEIGHTED = DSArenaWeightedOperation; + auto weighted = AggregateFunction::BinaryAggregate(item_type, LogicalType::DOUBLE, sketch_type); + DSReadDataArgumentsAt<1, 2, STATE, T, double, WEIGHTED>(weighted); + DSSetArenaStateExport(weighted); + return {fun, merge, weighted}; + } + } + // ============================================================ // 3. Scalar Functions // ============================================================ @@ -448,29 +733,17 @@ namespace duckdb { using T = decltype(item); using TR = TRAITS; - using STATE = DSSampleState; auto sketch_type = DSCreateSketchType(loader, "sketch_" + sql + "_" + StringUtil::Lower(item_type.ToString()), type_description + " with embedded type " + item_type.ToString()); + auto functions = DSSampleAggregates(item_type, sketch_type); - using OP = DSSampleOperation; - auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); - DSReadDataArgumentAt<1, STATE, T, OP>(fun); - DSAddAggregate(aggregates, aggregate_descs, fun, DSSampleBind, {LogicalType::INTEGER, item_type}, {"k", "data"}, + DSAddAggregate(aggregates, aggregate_descs, functions[0], DSSampleBind, {LogicalType::INTEGER, item_type}, {"k", "data"}, "Creates a sketch_" + sql + " sample of at most K of the values, each of weight 1", {"SELECT " + build + " FROM range(1000) t(i)"}); - - using MERGE = DSSampleMergeOperation; - auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); - DSReadDataArgumentAt<1, STATE, string_t, MERGE>(merge); - DSAddAggregate(aggregates, aggregate_descs, merge, DSSampleBind, {LogicalType::INTEGER, sketch_type}, {"k", "sketch"}, + DSAddAggregate(aggregates, aggregate_descs, functions[1], DSSampleBind, {LogicalType::INTEGER, sketch_type}, {"k", "sketch"}, "Merges sketch_" + sql + " samples into one sample of at most K items", {"SELECT " + name + "(16, sketch) FROM " + partials}); - - using WEIGHTED = DSSampleWeightedOperation; - auto weighted = AggregateFunction::BinaryAggregate(item_type, LogicalType::DOUBLE, sketch_type); - weighted.SetStateDestructorCallback(AggregateFunction::StateDestroy); - DSReadDataArgumentsAt<1, 2, STATE, T, double, WEIGHTED>(weighted); - DSAddAggregate(weighted_aggregates, weighted_descs, weighted, DSSampleBind, {LogicalType::INTEGER, item_type, LogicalType::DOUBLE}, + DSAddAggregate(weighted_aggregates, weighted_descs, functions[2], DSSampleBind, {LogicalType::INTEGER, item_type, LogicalType::DOUBLE}, {"k", "data", "weight"}, "Creates a sketch_" + sql + " sample of at most K of the values, sampled in proportion to their weights", {"SELECT " + name + "_weighted(16, i::DOUBLE, (i % 10 + 1)::DOUBLE) FROM range(1000) t(i)"}); From ff93c12ea2240ae95d8bc9d7e6863ed65add301d Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Wed, 28 Oct 2026 13:17:28 -0400 Subject: [PATCH 4/5] perf: keep frequent items states of numbers in DuckDB's arena so they can spill A frequent items sketch keeps its items as std::string, which owns heap memory. The build and weighted build aggregates over numbers now keep their items as an inline key (the number's string, at most 24 bytes) that compares, hashes and serializes exactly as the std::string, in a sketch in the aggregate arena, so results are the same bytes and the states can spill. States buffer their first 32 items with their weights. VARCHAR items and the merge aggregates (whose input sketches may hold any strings) keep the heap state. Co-Authored-By: Claude Opus 5.5 --- src/frequent_items_sketch.cpp | 209 ++++++++++++++++++++++++++++++---- 1 file changed, 190 insertions(+), 19 deletions(-) diff --git a/src/frequent_items_sketch.cpp b/src/frequent_items_sketch.cpp index 65afde3..1bf838d 100644 --- a/src/frequent_items_sketch.cpp +++ b/src/frequent_items_sketch.cpp @@ -1,4 +1,4 @@ -#include "datasketches_extension.hpp" +#include "datasketches_arena_buffer.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/types/vector.hpp" @@ -10,7 +10,9 @@ #include "duckdb/common/types/cast_helpers.hpp" // duckdb_fmt #include +#include #include +#include #include namespace duckdb @@ -254,6 +256,137 @@ namespace duckdb } }; + // ------------------------------------------------------------ + // Arena states, for numbers + // ------------------------------------------------------------ + // A sketch's items are strings, which own heap memory, so VARCHAR items and the merge aggregates keep + // DSFreqItemsState. A number's string (see DSFreqItemsToString) is at most 24 bytes, so sketches of numbers + // keep it inline in a key that owns no heap memory, in the aggregate arena (see datasketches_arena_buffer.hpp). + // A key compares, hashes and serializes as the std::string of the same bytes, so a sketch of keys holds its + // items in the same places, purges the same ones and serializes to the same bytes as one of strings. + struct DSFreqItemsKey + { + static constexpr idx_t CAPACITY = 31; + uint8_t size; + char data[CAPACITY]; + + std::string_view View() const { return std::string_view(data, size); } + bool operator==(const DSFreqItemsKey &other) const { return View() == other.View(); } + + static DSFreqItemsKey From(const std::string &text) + { + if (text.size() > CAPACITY) + { + throw InternalException("Frequent Items: the item '%s' is too long for an inline key", text); + } + DSFreqItemsKey key; + key.size = static_cast(text.size()); + memcpy(key.data, text.data(), text.size()); + return key; + } + }; + + // std::hash of a std::string equals that of its string_view. + struct DSFreqItemsKeyHash + { + size_t operator()(const DSFreqItemsKey &key) const { return std::hash()(key.View()); } + }; + + // As datasketches::serde: each item is its uint32 length, then its bytes. + struct DSFreqItemsKeySerde + { + size_t size_of_item(const DSFreqItemsKey &key) const { return sizeof(uint32_t) + key.size; } + + size_t serialize(void *ptr, size_t capacity, const DSFreqItemsKey *items, unsigned num) const + { + auto out = static_cast(ptr); + size_t written = 0; + for (unsigned i = 0; i < num; i++) + { + const uint32_t length = items[i].size; + datasketches::check_memory_size(written + sizeof(length) + length, capacity); + memcpy(out + written, &length, sizeof(length)); + memcpy(out + written + sizeof(length), items[i].data, length); + written += sizeof(length) + length; + } + return written; + } + + size_t deserialize(const void *ptr, size_t capacity, DSFreqItemsKey *items, unsigned num) const + { + auto in = static_cast(ptr); + size_t read = 0; + for (unsigned i = 0; i < num; i++) + { + uint32_t length; + datasketches::check_memory_size(read + sizeof(length), capacity); + memcpy(&length, in + read, sizeof(length)); + read += sizeof(length); + datasketches::check_memory_size(read + length, capacity); + if (length > DSFreqItemsKey::CAPACITY) + { + throw std::invalid_argument("item of " + std::to_string(length) + " bytes in a sketch of numbers"); + } + auto key = new (&items[i]) DSFreqItemsKey(); + key->size = static_cast(length); + memcpy(key->data, in + read, length); + read += length; + } + return read; + } + }; + + using DSFreqItemsArenaSketch = datasketches::frequent_items_sketch, DSAllocator>; + + // The first items with their weights, or the sketch they were fed to, in order: a sketch reserves room for + // 8 items, and DuckDB creates a state per input row when grouping reduces little. + template + struct DSFreqItemsArenaPolicy + { + using BIND = DSFreqItemsBindData; + using VALUE = T; + using WEIGHT = uint64_t; + using SKETCH = DSFreqItemsArenaSketch; + using ENTRY = DSBufferedEntry; + static constexpr const char *NAME = "Frequent Items"; + static constexpr uint32_t BUFFERED = 32; + + static DSAllocator ArenaMode() + { + return DSAllocator(DSArenaMode()); + } + static idx_t Budget(const BIND &) { return NumericLimits::Maximum(); } + static ENTRY Entry(const T &value) { return ENTRY {value, 1}; } + // The weight is checked now, as the heap state checks it. + template + static ENTRY Entry(const T &value, const W &weight) { return ENTRY {value, DSFreqItemsToWeight(weight)}; } + + static SKETCH *Create(const BIND &bind, ArenaAllocator &arena) + { + return arena.Make(bind.lg_max_k, SKETCH::LG_MIN_MAP_SIZE, std::equal_to(), ArenaMode()); + } + static SKETCH Temporary(const BIND &bind) { return SKETCH(bind.lg_max_k); } + static void Update(SKETCH &sketch, const ENTRY &entry) + { + sketch.update(DSFreqItemsKey::From(DSFreqItemsToString(entry.value)), entry.weight); + } + static auto Serialize(const SKETCH &sketch) { return sketch.serialize(0, DSFreqItemsKeySerde()); } + static SKETCH *Deserialize(const string_t &blob, ArenaAllocator &arena) + { + return arena.Make(DSDeserialize(blob, NAME, DSFreqItemsKeySerde(), std::equal_to(), ArenaMode())); + } + // The copy allocates through the source's allocator, which in arena mode is the current (target's) arena. + static SKETCH *Copy(const SKETCH &source, ArenaAllocator &arena) { return arena.Make(source); } + static void Combine(SKETCH &target, const SKETCH &source, const BIND &, ArenaAllocator &) { target.merge(source); } + }; + + // The constants an exported state is re-bound with: lg_max_k, for the overloads that take it. + void DSFreqItemsConstants(const DSFreqItemsBindData &bind, unordered_map &constants) + { + constants.emplace(0, Value::INTEGER(bind.lg_max_k)); + } + // ============================================================ // 3. Scalar Functions // ============================================================ @@ -364,14 +497,33 @@ namespace duckdb DSFreqItemsForEachItemType([&](auto item, const LogicalType &item_type) { using T = decltype(item); - auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); - DSAddAggregate(set, descs, fun, DSFreqItemsBind<2>, {item_type}, {"data"}, - "Creates a Frequent Items sketch to find heavy hitters in a data stream", - {"SELECT datasketch_frequent_items(item)" + rows}); - DSReadDataArgumentAt<1, DSFreqItemsState, T, DSFreqItemsOperation>(fun); - DSAddAggregate(set, descs, fun, DSFreqItemsBind<2>, {LogicalType::INTEGER, item_type}, {"lg_max_k", "data"}, - "Creates a Frequent Items sketch with the given log2 of max map size (lg_max_k)", - {"SELECT datasketch_frequent_items(10, item)" + rows}); }); + auto add = [&](AggregateFunction fun, AggregateFunction with_lg_max_k) + { + DSAddAggregate(set, descs, fun, DSFreqItemsBind<2>, {item_type}, {"data"}, + "Creates a Frequent Items sketch to find heavy hitters in a data stream", + {"SELECT datasketch_frequent_items(item)" + rows}); + DSAddAggregate(set, descs, with_lg_max_k, DSFreqItemsBind<2>, {LogicalType::INTEGER, item_type}, {"lg_max_k", "data"}, + "Creates a Frequent Items sketch with the given log2 of max map size (lg_max_k)", + {"SELECT datasketch_frequent_items(10, item)" + rows}); + }; + if constexpr (std::is_same_v) + { + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + auto with_lg_max_k = fun; + DSReadDataArgumentAt<1, DSFreqItemsState, T, DSFreqItemsOperation>(with_lg_max_k); + add(fun, with_lg_max_k); + } + else + { + using STATE = DSBufferedArenaState>; + using OP = DSArenaCreateOperation; + auto fun = AggregateFunction::UnaryAggregate(item_type, sketch_type); + auto with_lg_max_k = fun; + DSSetBufferedStateExport>(fun); + DSReadDataArgumentAt<1, STATE, T, OP>(with_lg_max_k); + DSSetBufferedStateExport(with_lg_max_k); + add(fun, with_lg_max_k); + } }); auto merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); DSAddAggregate(set, descs, merge, DSFreqItemsBind<2>, {sketch_type}, {"sketch"}, @@ -394,16 +546,35 @@ namespace duckdb auto add = [&](auto weight, const LogicalType &weight_type) { using W = decltype(weight); - using OP = DSFreqItemsWeightedOperation; - auto fun = AggregateFunction::BinaryAggregate(item_type, weight_type, sketch_type); - fun.SetStateDestructorCallback(AggregateFunction::StateDestroy); - DSAddAggregate(set, descs, fun, DSFreqItemsBind<3>, {item_type, weight_type}, {"data", "weight"}, - "Creates a weighted Frequent Items sketch to find heavy hitters by total item weight; each row adds its weight to the item's count", - {"SELECT datasketch_frequent_items_weighted(item, weight)" + rows}); - DSReadDataArgumentsAt<1, 2, DSFreqItemsState, T, W, OP>(fun); - DSAddAggregate(set, descs, fun, DSFreqItemsBind<3>, {LogicalType::INTEGER, item_type, weight_type}, {"lg_max_k", "data", "weight"}, - "Creates a weighted Frequent Items sketch with the given log2 of max map size (lg_max_k); each row adds its weight to the item's count", - {"SELECT datasketch_frequent_items_weighted(10, item, weight)" + rows}); + auto add_functions = [&](AggregateFunction fun, AggregateFunction with_lg_max_k) + { + DSAddAggregate(set, descs, fun, DSFreqItemsBind<3>, {item_type, weight_type}, {"data", "weight"}, + "Creates a weighted Frequent Items sketch to find heavy hitters by total item weight; each row adds its weight to the item's count", + {"SELECT datasketch_frequent_items_weighted(item, weight)" + rows}); + DSAddAggregate(set, descs, with_lg_max_k, DSFreqItemsBind<3>, {LogicalType::INTEGER, item_type, weight_type}, {"lg_max_k", "data", "weight"}, + "Creates a weighted Frequent Items sketch with the given log2 of max map size (lg_max_k); each row adds its weight to the item's count", + {"SELECT datasketch_frequent_items_weighted(10, item, weight)" + rows}); + }; + if constexpr (std::is_same_v) + { + using OP = DSFreqItemsWeightedOperation; + auto fun = AggregateFunction::BinaryAggregate(item_type, weight_type, sketch_type); + fun.SetStateDestructorCallback(AggregateFunction::StateDestroy); + auto with_lg_max_k = fun; + DSReadDataArgumentsAt<1, 2, DSFreqItemsState, T, W, OP>(with_lg_max_k); + add_functions(fun, with_lg_max_k); + } + else + { + using STATE = DSBufferedArenaState>; + using OP = DSArenaWeightedOperation; + auto fun = AggregateFunction::BinaryAggregate(item_type, weight_type, sketch_type); + auto with_lg_max_k = fun; + DSSetBufferedStateExport>(fun); + DSReadDataArgumentsAt<1, 2, STATE, T, W, OP>(with_lg_max_k); + DSSetBufferedStateExport(with_lg_max_k); + add_functions(fun, with_lg_max_k); + } }; add(int64_t(), LogicalType::BIGINT); add(uint64_t(), LogicalType::UBIGINT); }); From 90efdc3119a935821b7b7d0fbeb122a9fb75a118 Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Wed, 28 Oct 2026 13:17:28 -0400 Subject: [PATCH 5/5] test: spill frequent items, VarOpt, EBPPS, Bloom and Count-Min states Runs the aggregates under a 128MB limit with 4 threads: many small (buffered) and larger groups, weighted variants, merges of stored sketches, EXPORT_STATE round trips and the VARCHAR heap paths. Bloom and Count-Min results are compared to the same query with plenty of memory, the others by exact facts. Co-Authored-By: Claude Opus 5.5 --- test/sql/datasketch_spill_items.test | 200 +++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 test/sql/datasketch_spill_items.test diff --git a/test/sql/datasketch_spill_items.test b/test/sql/datasketch_spill_items.test new file mode 100644 index 0000000..7f1fb92 --- /dev/null +++ b/test/sql/datasketch_spill_items.test @@ -0,0 +1,200 @@ +# name: test/sql/datasketch_spill_items.test +# description: frequent items, VarOpt, EBPPS, Bloom filter and Count-Min states live in the arena and spill under memory_limit +# group: [datasketches] + +require datasketches + +require skip_reload + +statement ok +SET threads=4; + +statement ok +SET temp_directory='{TEST_DIR}/datasketch_spill_items'; + +statement ok +SET preserve_insertion_order=false; + +# Inputs (views, so they take no memory) and reference results, computed with plenty of memory: 100k groups of 40 rows (more than a state buffers) and 400k groups of +# 3 rows (fewer) +statement ok +SET memory_limit='4GB'; + +statement ok +CREATE VIEW big AS SELECT i % 100000 AS g, i AS v, (i % 5 + 1) AS w FROM range(4000000) t(i); + +statement ok +CREATE VIEW small AS SELECT i % 400000 AS g, i AS v, (i % 5 + 1) AS w FROM range(1200000) t(i); + +# (hashes of the deterministic sketches, so the reference itself stays small) +statement ok +CREATE TABLE ref AS +SELECT g, hash(datasketch_bloom_filter(1000, 0.01, v)) AS b, hash(datasketch_count_min(3, 128, v)) AS c, + hash(datasketch_count_min_weighted(3, 128, v, w)) AS cw, datasketch_frequent_items(10, v % 7) AS f, + hash(datasketch_bloom_filter(1000, 0.01, v::VARCHAR)) AS bs, hash(datasketch_count_min(3, 128, v::VARCHAR)) AS cs +FROM big GROUP BY g; + +statement ok +SET memory_limit='128MB'; + +# Bloom filters and Count-Min sketches are deterministic: the same bytes as with plenty of memory +query I +SELECT count(*) FROM ( + SELECT g, hash(datasketch_bloom_filter(1000, 0.01, v)) AS b, hash(datasketch_count_min(3, 128, v)) AS c, + hash(datasketch_count_min_weighted(3, 128, v, w)) AS cw, hash(datasketch_bloom_filter(1000, 0.01, v::VARCHAR)) AS bs, + hash(datasketch_count_min(3, 128, v::VARCHAR)) AS cs + FROM big GROUP BY g +) t JOIN ref USING (g) +WHERE t.b != ref.b OR t.c != ref.c OR t.cw != ref.cw OR t.bs != ref.bs OR t.cs != ref.cs; +---- +0 + +# Frequent items of 7 distinct values are exact +query I +SELECT count(*) FROM ( + SELECT g, datasketch_frequent_items(10, v % 7) AS f FROM big GROUP BY g +) t JOIN ref USING (g) +WHERE datasketch_frequent_items_total_weight(t.f) != 40 + OR datasketch_frequent_items_estimate(t.f, (g % 7)::BIGINT) != datasketch_frequent_items_estimate(ref.f, (g % 7)::BIGINT); +---- +0 + +# Many small groups: every sketch's exact facts +query I +SELECT count(*) FROM ( + SELECT g, + datasketch_bloom_filter(1000, 0.01, v) AS b, + datasketch_count_min(3, 128, v) AS c, + datasketch_count_min_weighted(3, 128, v, w) AS cw, + datasketch_frequent_items(10, v) AS f, + datasketch_frequent_items_weighted(v, w) AS fw, + datasketch_varopt(16, v) AS vo, + datasketch_varopt_weighted(16, v, w::DOUBLE) AS vow, + datasketch_ebpps(16, v) AS e, + datasketch_ebpps_weighted(16, v, w::DOUBLE) AS ew, + sum(w) AS total + FROM small GROUP BY g +) WHERE NOT datasketch_bloom_filter_contains(b, g) OR NOT datasketch_bloom_filter_contains(b, g + 400000) + OR NOT datasketch_bloom_filter_contains(b, g + 800000) + OR datasketch_count_min_total_weight(c) != 3 OR datasketch_count_min_estimate(c, g) < 1 + OR datasketch_count_min_total_weight(cw) != total + OR datasketch_frequent_items_total_weight(f) != 3 OR datasketch_frequent_items_estimate(f, g) != 1 + OR datasketch_frequent_items_total_weight(fw) != total + OR datasketch_varopt_n(vo) != 3 OR datasketch_varopt_num_samples(vo) != 3 + OR datasketch_varopt_subset_sum(vow, [g, g + 400000, g + 800000]).estimate != total + OR datasketch_ebpps_n(e) != 3 OR datasketch_ebpps_cumulative_weight(ew) != total; +---- +0 + +# Large groups of the sampling sketches: n and total weights are exact +query I +SELECT count(*) FROM ( + SELECT g, datasketch_varopt(16, v) AS vo, datasketch_varopt_weighted(16, v, w::DOUBLE) AS vow, + datasketch_ebpps(64, v) AS e, datasketch_ebpps_weighted(64, v, w::DOUBLE) AS ew, sum(w) AS total + FROM big GROUP BY g +) WHERE datasketch_varopt_n(vo) != 40 OR datasketch_varopt_num_samples(vo) != 16 + OR abs(datasketch_varopt_subset_sum(vow, [g]).total_weight - total) > 1e-6 + OR datasketch_ebpps_n(e) != 40 OR datasketch_ebpps_cumulative_weight(ew) != total; +---- +0 + +# Merging stored sketches spills too +statement ok +SET memory_limit='4GB'; + +statement ok +CREATE TABLE parts AS +SELECT g, p, datasketch_bloom_filter(100, 0.05, v) AS b, datasketch_count_min(2, 16, v) AS c, + datasketch_frequent_items(10, v) AS f, datasketch_varopt(16, v) AS vo, datasketch_ebpps(16, v) AS e +FROM (SELECT i % 100000 AS g, i % 3 AS p, i AS v FROM range(1200000) t(i)) +GROUP BY g, p; + +statement ok +CREATE TABLE merged_ref AS +SELECT g, hash(datasketch_bloom_filter(b)) AS b, hash(datasketch_count_min(c)) AS c FROM parts GROUP BY g; + +statement ok +SET memory_limit='128MB'; + +query I +SELECT count(*) FROM ( + SELECT g, datasketch_bloom_filter(b) AS b, datasketch_count_min(c) AS c FROM parts GROUP BY g +) t JOIN merged_ref USING (g) +WHERE hash(t.b) != merged_ref.b OR hash(t.c) != merged_ref.c OR NOT datasketch_bloom_filter_contains(t.b, g + 1100000); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT g, datasketch_varopt(16, vo) AS vo, datasketch_ebpps(16, e) AS e FROM parts GROUP BY g +) WHERE datasketch_varopt_n(vo) != 12 OR datasketch_varopt_num_samples(vo) != 12 OR datasketch_ebpps_n(e) != 12; +---- +0 + +# The frequent items merge keeps its heap state (its items may be any strings), which DuckDB does not spill: on its own, +# so that the other aggregates of a query still can +query I +SELECT count(*) FROM (SELECT g, datasketch_frequent_items(f) AS f FROM parts GROUP BY g) +WHERE datasketch_frequent_items_total_weight(f) != 12; +---- +0 + +statement ok +SET memory_limit='4GB'; + +# Exported states finalize to the aggregate's result, buffered (10 rows) or not (100000 rows). Frequent items of +# several threads are combined in any order, which places items differently: their estimates are compared +foreach n 10 100000 + +query IIIIIIII +SELECT finalize(datasketch_bloom_filter(1000, 0.01, i) EXPORT_STATE) = datasketch_bloom_filter(1000, 0.01, i), + finalize(datasketch_bloom_filter(1000, 0.01, i::VARCHAR) EXPORT_STATE) = datasketch_bloom_filter(1000, 0.01, i::VARCHAR), + finalize(datasketch_count_min(3, 128, i) EXPORT_STATE) = datasketch_count_min(3, 128, i), + finalize(datasketch_count_min_weighted(3, 128, i, i % 3) EXPORT_STATE) = datasketch_count_min_weighted(3, 128, i, i % 3), + datasketch_frequent_items_estimate(finalize(datasketch_frequent_items(10, i % 50) EXPORT_STATE), 7) = count(*) FILTER (i % 50 = 7), + datasketch_varopt_n(finalize(datasketch_varopt_weighted(16, i, 1.0 + i % 3) EXPORT_STATE)) = ${n}, + datasketch_ebpps_cumulative_weight(finalize(datasketch_ebpps_weighted(16, i, 1.0 + i % 3) EXPORT_STATE)) = sum(1.0 + i % 3), + datasketch_frequent_items_total_weight(finalize(datasketch_frequent_items_weighted(i % 50, i % 4) EXPORT_STATE)) = sum(i % 4) +FROM range(${n}) t(i); +---- +true true true true true true true true + +endloop + +# Exported merge states +query III +SELECT finalize(datasketch_bloom_filter(b) EXPORT_STATE) = datasketch_bloom_filter(b), + finalize(datasketch_count_min(c) EXPORT_STATE) = datasketch_count_min(c), + datasketch_varopt_n(finalize(datasketch_varopt(16, vo) EXPORT_STATE)) = 1200000 +FROM parts; +---- +true true true + +# VARCHAR frequent items and samples keep their heap states and still work under the limit +statement ok +SET memory_limit='128MB'; + +query I +SELECT count(*) FROM ( + SELECT g, datasketch_frequent_items(v::VARCHAR) AS f, datasketch_varopt(16, v::VARCHAR) AS vo, + datasketch_ebpps(16, v::VARCHAR) AS e + FROM (SELECT i % 1000 AS g, i AS v FROM range(100000) t(i)) GROUP BY g +) WHERE datasketch_frequent_items_total_weight(f) != 100 OR datasketch_varopt_n(vo) != 100 OR datasketch_ebpps_n(e) != 100; +---- +0 + +# Weights are checked when a row arrives, also while a state buffers it +statement error +SELECT datasketch_varopt_weighted(16, i, 0.0) FROM range(3) t(i); +---- +Item weights must be positive and finite + +statement error +SELECT datasketch_ebpps_weighted(16, i, -1.0) FROM range(3) t(i); +---- +Item weights must be nonnegative and finite + +statement error +SELECT datasketch_frequent_items_weighted(i, -1) FROM range(3) t(i); +---- +Frequent Items weight must be non-negative