From 5683a03d3d810a8ecc014652b5246ad0cba7a22b Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Wed, 28 Oct 2026 12:54:24 -0400 Subject: [PATCH] perf: keep Theta, Tuple and AoD aggregate states in DuckDB's arena so they can spill The set-operation sketches' aggregates (datasketch_theta, datasketch_theta_blob, datasketch_theta_intersect_agg, datasketch_tuple, datasketch_tuple_intersect_agg, datasketch_aod, datasketch_aod_intersect_agg: builds and merges) kept heap states with a destructor, so DuckDB neither counted nor spilled them. They now live in the aggregate arena, have no destructor, and export and import themselves as BLOBs. DuckDB creates a state per row when grouping reduces little, and an update sketch or union allocates a hash table up front, so the states buffer small inputs: - builds keep their first 32 raw rows (up to 1 KB) and replay them, in order, into an update sketch when more arrive, or into a temporary one for a result; - unions keep up to 32 distinct keys with their combined summaries while that is no more than k, which is exactly what the library's union would hold; - intersections keep their current result and overwrite it in place (the library's intersection reallocates its table on every update). Results are identical to unbuffered building; buffered states export as raw rows or keys, so importing them keeps them small. Temporaries (input sketches, results, serialized bytes) go to a per-thread scratch arena (DSScratchScope). Every sketch object, AoD arrays included, allocates in arena mode, so the library's copies, which take the source's allocator, land in whichever arena is current, never on the heap. That avoids the allocator bug class of hll_union without a library patch. Co-Authored-By: Claude Opus 5.5 --- src/aod_sketch.cpp | 420 ++----- src/include/datasketches_theta_arena.hpp | 1460 ++++++++++++++++++++++ src/theta_sketch.cpp | 413 ++---- src/tuple_sketch.cpp | 412 ++---- test/sql/datasketch_spill_theta.test | 129 ++ 5 files changed, 1888 insertions(+), 946 deletions(-) create mode 100644 src/include/datasketches_theta_arena.hpp create mode 100644 test/sql/datasketch_spill_theta.test diff --git a/src/aod_sketch.cpp b/src/aod_sketch.cpp index df353ce..cefeb33 100644 --- a/src/aod_sketch.cpp +++ b/src/aod_sketch.cpp @@ -4,6 +4,7 @@ // same key are summed element-wise when building, merging, unioning and intersecting. #include "datasketches_tuple_common.hpp" +#include "datasketches_theta_arena.hpp" #include @@ -101,14 +102,6 @@ namespace duckdb using DSAodScalars = DSTupleFunctions; - DSAodCompact DSAodEmptySketch(uint8_t lg_k, uint64_t seed, uint8_t num_values) - { - DSAodUpdate::builder b{datasketches::default_array_tuple_update_policy(num_values)}; - b.set_lg_k(lg_k); - b.set_seed(seed); - return b.build().compact(); - } - // Tracks the number of values of the sketches being combined (empty sketches are ignored). struct DSAodNumValuesTracker { @@ -148,24 +141,7 @@ namespace duckdb // Bind Data // ============================================================ - struct DSAodBindData : public FunctionData - { - unique_ptr Copy() const override - { - return make_uniq(*this); - } - - bool Equals(const FunctionData &other_p) const override - { - auto &other = other_p.Cast(); - return lg_k == other.lg_k && infer_lg_k == other.infer_lg_k && seed == other.seed && p == other.p; - } - - uint8_t lg_k = DS_THETA_DEFAULT_LG_K; - bool infer_lg_k = false; - uint64_t seed = DS_THETA_DEFAULT_SEED; - float p = 1.0f; - }; + using DSAodBindData = DSThetaFamilyBindData; template unique_ptr DSAodBindBuild(BindAggregateFunctionInput &input) @@ -176,6 +152,7 @@ namespace duckdb result->lg_k = DSTupleBindLgK(input, 0); } DSTupleBindSeedAndP(input, result->seed, result->p); + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 3 : 2); return std::move(result); } @@ -191,213 +168,144 @@ namespace duckdb { result->infer_lg_k = true; } + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 2 : 1); return std::move(result); } // ============================================================ - // Build / Union Aggregate + // Aggregate states (see datasketches_theta_arena.hpp) // ============================================================ - struct DSAodState + // Every array (and so every entry) allocates in arena mode, so copies the library makes of entries land in + // the current arena. + struct DSAodStateTraits { - DSAodUpdate *update_sketch = nullptr; - DSAodUnion *union_sketch = nullptr; - uint8_t union_lg_k = 0; - uint64_t union_seed = DS_THETA_DEFAULT_SEED; - //! The number of values per key (0 while unknown) - uint8_t num_values = 0; - //! A merged input sketch was empty: with no other input, the result is an empty sketch rather than NULL - bool saw_empty_sketch = false; + using Alloc = DSAllocator; + using Entry = std::pair; + using Update = DSAodUpdate; + using Compact = DSAodCompact; + using Union = DSAodUnion; + using Intersection = DSAodIntersection; + static constexpr bool SET_ONLY = false; + static constexpr bool HAS_NUM_VALUES = true; + static constexpr const char *KIND = "Array of doubles"; + static constexpr const char *BUILD_NAME = "datasketch_aod"; + static constexpr const char *INTERSECT_NAME = "datasketch_aod_intersect_agg"; - ~DSAodState() + static uint64_t Key(const Entry &entry) { return entry.first; } + static idx_t SummarySize(const DSThetaParams ¶ms) { return sizeof(double) * params.num_values; } + static void WriteSummary(const Entry &entry, data_ptr_t data, const DSThetaParams ¶ms) { - delete update_sketch; - delete union_sketch; + memcpy(data, entry.second.data(), SummarySize(params)); } - - void CheckNumValues(idx_t count, const char *function_name) + static Entry MakeEntry(uint64_t key, const_data_ptr_t data, const DSThetaParams ¶ms) { - if (num_values == 0) - { - num_values = static_cast(count); - } - else if (num_values != count) - { - DSAodThrowNumValues(function_name, num_values, count); - } + DSAodArray values(params.num_values, 0.0, Alloc(DSArenaMode())); + memcpy(values.data(), data, SummarySize(params)); + return Entry(key, std::move(values)); } - - void CreateUpdateSketch(const DSAodBindData &bind_data, idx_t count) + // default_array_tuple_union_policy: element-wise sum + static void Fold(data_ptr_t data, const Entry &incoming, const DSThetaParams ¶ms) { - if (num_values != 0 && num_values != count) + for (uint8_t i = 0; i < params.num_values; i++) { - throw InvalidInputException("datasketch_aod: every values list must have the same length (got %llu and %llu)", idx_t(num_values), count); - } - CheckNumValues(count, "datasketch_aod"); - if (!update_sketch) - { - DSAodUpdate::builder b{datasketches::default_array_tuple_update_policy(num_values)}; - b.set_lg_k(bind_data.lg_k); - b.set_seed(bind_data.seed); - b.set_p(bind_data.p); - update_sketch = new DSAodUpdate(b.build()); + double value; + memcpy(&value, data + i * sizeof(double), sizeof(double)); + value += incoming.second[i]; + memcpy(data + i * sizeof(double), &value, sizeof(double)); } } + static idx_t PayloadSize(const DSThetaParams ¶ms) { return SummarySize(params); } + static uint8_t NumValues(const Compact &sketch) { return sketch.get_num_values(); } - // Creates the union, or grows it to a larger lg_k (re-adding what it holds so far). - void EnsureUnion(uint8_t lg_k, uint64_t seed) + template + static void Apply(Update &sketch, const_data_ptr_t key, uint32_t length, const_data_ptr_t payload, const DSThetaParams ¶ms) { - if (union_sketch && lg_k <= union_lg_k) + double values[DS_AOD_MAX_VALUES]; + memcpy(values, payload, SummarySize(params)); + const double *update = values; + if constexpr (std::is_same::value) { - return; + sketch.update(key, length, update); } - if (union_sketch) - { - seed = union_seed; - } - DSAodUnion::builder b{datasketches::default_array_tuple_union_policy(num_values)}; - b.set_lg_k(lg_k); - b.set_seed(seed); - auto grown = new DSAodUnion(b.build()); - if (union_sketch) + else { - grown->update(union_sketch->get_result()); - delete union_sketch; + KEY typed; + memcpy(&typed, key, sizeof(KEY)); + sketch.update(typed, update); } - union_sketch = grown; - union_lg_k = lg_k; - union_seed = seed; } - bool HasData() const + static Update NewUpdate(const DSAodBindData &bind, const DSThetaParams ¶ms) { - return update_sketch || union_sketch; + DSAodUpdate::builder b{datasketches::default_array_tuple_update_policy(params.num_values, Alloc(DSArenaMode())), + Alloc(DSArenaMode())}; + b.set_lg_k(bind.lg_k); + b.set_seed(bind.seed); + b.set_p(bind.p); + return b.build(); } - uint64_t Seed(const DSAodBindData &bind_data) const + static Union NewUnion(uint8_t lg_k, uint64_t seed, const DSThetaParams ¶ms) { - return union_sketch ? union_seed : bind_data.seed; - } - - uint8_t LgK(const DSAodBindData &bind_data) const - { - return union_sketch ? union_lg_k : bind_data.lg_k; + DSAodUnion::builder b{datasketches::default_array_tuple_union_policy(params.num_values), Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build(); } - }; - struct DSAodOperation - { - template - static void Initialize(STATE &state) + static Intersection NewIntersection(uint64_t seed, const DSThetaParams ¶ms) { - state.update_sketch = nullptr; - state.union_sketch = nullptr; - state.union_lg_k = 0; - state.union_seed = DS_THETA_DEFAULT_SEED; - state.num_values = 0; - state.saw_empty_sketch = false; + return Intersection(seed, DSAodSumPolicy(params.num_values), Alloc(DSArenaMode())); } - template - static void Destroy(STATE &state, AggregateInputData &) + static Compact Deserialize(const string_t &blob) { - delete state.update_sketch; - delete state.union_sketch; - state.update_sketch = nullptr; - state.union_sketch = nullptr; + auto seed = DSTupleSeedFor(blob); + return ThetaGuard([&]() + { return Compact::deserialize(blob.GetData(), blob.GetSize(), seed, Alloc(DSArenaMode())); }, + KIND); } - static bool IgnoreNull() { return true; } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static Compact Empty(uint8_t lg_k, uint64_t seed, const DSThetaParams ¶ms) { - target.saw_empty_sketch = target.saw_empty_sketch || source.saw_empty_sketch; - if (!source.HasData()) - return; - auto &bind_data = aggr_input_data.bind_data->template Cast(); - auto source_seed = source.Seed(bind_data); - if (target.HasData() && ThetaSeedHash(source_seed) != ThetaSeedHash(target.Seed(bind_data))) - { - DSAodThrowSeedMismatch("datasketch_aod", ThetaSeedHash(target.Seed(bind_data)), ThetaSeedHash(source_seed)); - } - target.CheckNumValues(source.num_values, "datasketch_aod"); - auto lg_k = MaxValue(source.LgK(bind_data), target.LgK(bind_data)); - target.EnsureUnion(lg_k, target.HasData() ? target.Seed(bind_data) : source_seed); - if (target.update_sketch) - { - target.union_sketch->update(target.update_sketch->compact()); - delete target.update_sketch; - target.update_sketch = nullptr; - } - if (source.update_sketch) - target.union_sketch->update(source.update_sketch->compact()); - if (source.union_sketch) - target.union_sketch->update(source.union_sketch->get_result()); + DSAodUpdate::builder b{datasketches::default_array_tuple_update_policy(params.num_values, Alloc(DSArenaMode())), + Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build().compact(); } - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + static auto Serialize(const Compact &sketch) -> decltype(sketch.serialize()) { - if (state.union_sketch) - { - target = DSAodTraits::Serialize(finalize_data.result, state.union_sketch->get_result()); - } - else if (state.update_sketch) - { - target = DSAodTraits::Serialize(finalize_data.result, state.update_sketch->compact()); - } - else if (state.saw_empty_sketch) - { - auto &bind_data = finalize_data.input.bind_data->template Cast(); - target = DSAodTraits::Serialize(finalize_data.result, DSAodEmptySketch(bind_data.lg_k, bind_data.seed, MaxValue(state.num_values, 1))); - } - else - { - // no input rows: NULL, as for the other sketches' aggregates - finalize_data.ReturnNull(); - } + return sketch.serialize(); } - // Merging serialized sketches. - template - static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + static void ThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b) { - auto &bind_data = idata.input.bind_data->template Cast(); - auto sketch = DSAodTraits::Deserialize(blob); - if (sketch.is_empty()) - { - state.saw_empty_sketch = true; - return; - } - if (state.union_sketch && sketch.get_seed_hash() != ThetaSeedHash(state.union_seed)) - { - DSAodThrowSeedMismatch("datasketch_aod", ThetaSeedHash(state.union_seed), sketch.get_seed_hash()); - } - state.CheckNumValues(sketch.get_num_values(), "datasketch_aod"); - auto lg_k = bind_data.infer_lg_k ? MaxValue(bind_data.lg_k, ThetaInferLgK(sketch)) : bind_data.lg_k; - state.EnsureUnion(lg_k, ThetaSeedForHash(sketch.get_seed_hash())); - ThetaGuard([&]() - { state.union_sketch->update(sketch); }, - DSAodTraits::KIND); + DSAodThrowSeedMismatch(function_name, a, b); } - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static void ThrowNumValues(const char *function_name, idx_t a, idx_t b) { - // the values of the same key add up, so merging a sketch twice is not the same as once - for (idx_t i = 0; i < count; i++) - { - Operation(state, input, unary_input); - } + DSAodThrowNumValues(function_name, a, b); } }; + template + using DSAodState = DSThetaFamilyState; + using DSAodMergeState = DSAodState; + using DSAodIntersectState = DSThetaFamilyIntersectState; + // the values of the same key add up, so merging a sketch twice is not the same as once + using DSAodOperation = DSThetaFamilyOperation; + using DSAodIntersectOperation = DSThetaFamilyIntersectOperation; + // The update of the build aggregate: (key, values) at KEY_INDEX and KEY_INDEX + 1. template void DSAodBuildUpdate(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, Vector &states, idx_t count) { D_ASSERT(input_count > KEY_INDEX + 1); + using STATE = DSAodState; auto &bind_data = aggr_input_data.bind_data->Cast(); auto &list = inputs[KEY_INDEX + 1]; UnifiedVectorFormat key_format, list_format, child_format, state_format; @@ -408,8 +316,9 @@ namespace duckdb auto keys = UnifiedVectorFormat::GetData(key_format); auto lists = UnifiedVectorFormat::GetData(list_format); auto values = UnifiedVectorFormat::GetData(child_format); - auto state_pointers = UnifiedVectorFormat::GetData(state_format); - std::vector buffer; + auto state_pointers = UnifiedVectorFormat::GetData(state_format); + DSArenaScope scope(aggr_input_data.allocator); + double buffer[DS_AOD_MAX_VALUES]; for (idx_t i = 0; i < count; i++) { auto key_index = key_format.sel->get_index(i); @@ -423,7 +332,6 @@ namespace duckdb { throw InvalidInputException("datasketch_aod: values must have between 1 and %llu elements, got %llu", DS_AOD_MAX_VALUES, entry.length); } - buffer.resize(entry.length); for (idx_t j = 0; j < entry.length; j++) { auto child_index = child_format.sel->get_index(entry.offset + j); @@ -434,138 +342,16 @@ namespace duckdb buffer[j] = values[child_index]; } auto &state = *state_pointers[state_format.sel->get_index(i)]; - state.CreateUpdateSketch(bind_data, entry.length); - if constexpr (std::is_same_v) + if (state.num_values != 0 && state.num_values != entry.length) { - state.update_sketch->update(keys[key_index].GetData(), keys[key_index].GetSize(), buffer.data()); - } - else - { - state.update_sketch->update(keys[key_index], buffer.data()); + throw InvalidInputException("datasketch_aod: every values list must have the same length (got %llu and %llu)", idx_t(state.num_values), entry.length); } + state.num_values = static_cast(entry.length); + auto bytes = DSThetaKeyBytes(keys[key_index]); + state.AddRow(bytes.first, bytes.second, const_data_ptr_cast(buffer), bind_data, aggr_input_data.allocator); } } - // ============================================================ - // Intersection Aggregate - // ============================================================ - - struct DSAodIntersectState - { - DSAodIntersection *intersection = nullptr; - uint64_t seed = DS_THETA_DEFAULT_SEED; - uint8_t num_values = 0; - bool saw_empty = false; - - ~DSAodIntersectState() - { - delete intersection; - } - - void Add(const DSAodCompact &sketch) - { - if (sketch.is_empty()) - { - saw_empty = true; - if (num_values == 0) - { - num_values = sketch.get_num_values(); - } - return; - } - if (!intersection) - { - seed = ThetaSeedForHash(sketch.get_seed_hash()); - num_values = sketch.get_num_values(); - intersection = new DSAodIntersection(seed, DSAodSumPolicy(num_values)); - return Update(sketch); - } - if (sketch.get_seed_hash() != ThetaSeedHash(seed)) - { - DSAodThrowSeedMismatch("datasketch_aod_intersect_agg", ThetaSeedHash(seed), sketch.get_seed_hash()); - } - if (sketch.get_num_values() != num_values) - { - DSAodThrowNumValues("datasketch_aod_intersect_agg", num_values, sketch.get_num_values()); - } - Update(sketch); - } - - void Update(const DSAodCompact &sketch) - { - ThetaGuard([&]() - { intersection->update(sketch); }, - DSAodTraits::KIND); - } - }; - - struct DSAodIntersectOperation - { - template - static void Initialize(STATE &state) - { - state.intersection = nullptr; - state.seed = DS_THETA_DEFAULT_SEED; - state.num_values = 0; - state.saw_empty = false; - } - - template - static void Destroy(STATE &state, AggregateInputData &) - { - delete state.intersection; - state.intersection = nullptr; - } - - static bool IgnoreNull() { return true; } - - template - static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &) - { - state.Add(DSAodTraits::Deserialize(blob)); - } - - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) - { - for (idx_t i = 0; i < count; i++) - { - Operation(state, input, unary_input); - } - } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &) - { - target.saw_empty = target.saw_empty || source.saw_empty; - if (target.num_values == 0) - { - target.num_values = source.num_values; - } - if (source.intersection) - { - target.Add(source.intersection->get_result()); - } - } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) - { - if (state.saw_empty) - { - target = DSAodTraits::Serialize(finalize_data.result, DSAodEmptySketch(DS_THETA_DEFAULT_LG_K, state.seed, MaxValue(state.num_values, 1))); - } - else if (state.intersection) - { - target = DSAodTraits::Serialize(finalize_data.result, state.intersection->get_result()); - } - else - { - finalize_data.ReturnNull(); - } - } - }; - // ============================================================ // Scalar Functions // ============================================================ @@ -743,8 +529,8 @@ namespace duckdb const auto values_type = LogicalType::LIST(LogicalType::DOUBLE); for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::UnaryAggregate( - key_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate, string_t, string_t, DSAodOperation>(key_type, sketch_type); + DSThetaSetStateExport>(fun); auto signature = DSTupleSignature(with_lg_k, {key_type, values_type}, sketch_type); DSTupleAddSeedAndP(signature); fun.GetSignature() = std::move(signature); @@ -808,13 +594,13 @@ namespace duckdb for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); fun.GetSignature() = DSTupleSignature(with_lg_k, {sketch_type}, sketch_type); if (with_lg_k) { fun.SetBindCallback(DSAodBindMerge); - DSReadDataArgumentAt<1, DSAodState, string_t, DSAodOperation>(fun); + DSReadDataArgumentAt<1, DSAodMergeState, string_t, DSAodOperation>(fun); descs.push_back(DSAodDescription("Merges (unions) array-of-doubles sketches into one with the given log2 of nominal entries (lg_k); " "the values of a key are summed element-wise", {ex.QueryPartials("datasketch_aod(16, sketch)")}, {"lg_k", "sketch"}, {LogicalType::INTEGER, sketch_type})); @@ -822,7 +608,7 @@ namespace duckdb else { fun.SetBindCallback(DSAodBindMerge); - DSReadDataArgumentAt<0, DSAodState, string_t, DSAodOperation>(fun); + DSReadDataArgumentAt<0, DSAodMergeState, string_t, DSAodOperation>(fun); descs.push_back(DSAodDescription("Merges (unions) array-of-doubles sketches into one, keeping as many entries as the largest input " "(at least lg_k 12); the values of a key are summed element-wise", {ex.QueryPartials("datasketch_aod(sketch)")}, {"sketch"}, {sketch_type})); @@ -836,8 +622,8 @@ namespace duckdb // --- datasketch_aod_intersect_agg { AggregateFunctionSet set("datasketch_aod_intersect_agg"); - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); DSSetArguments(fun, {sketch_type}); fun.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); set.AddFunction(fun); diff --git a/src/include/datasketches_theta_arena.hpp b/src/include/datasketches_theta_arena.hpp new file mode 100644 index 0000000..4922ac3 --- /dev/null +++ b/src/include/datasketches_theta_arena.hpp @@ -0,0 +1,1460 @@ +#pragma once + +// Arena states of the Theta-family aggregates (theta_sketch.cpp, tuple_sketch.cpp, aod_sketch.cpp). +// +// The states keep everything in the aggregate arena (DSAllocator's arena mode), have no destructor and export +// themselves as BLOBs, so DuckDB counts them toward memory_limit and can spill them. Every sketch object here +// allocates in arena mode; where its memory lands is decided by the arena of the scope it allocates in: +// - the callback's arena (DSArenaScope), for what a state keeps; +// - a per-thread scratch arena (DSScratchScope), for temporaries: input sketches, results, serialized bytes. +// Copies made by the library use the source object's allocator, which in arena mode means "the current arena", so a +// temporary in the scratch arena copied while the callback's arena is current lands in the state, and a state copied +// while the scratch arena is current lands in the scratch arena. Nothing of a temporary may outlive its scratch +// scope, and nothing a state keeps may be created under one. +// +// DuckDB creates a state per input row when grouping reduces little, and a Theta update sketch or union allocates a +// hash table up front (about 1 KB at lg_k 12). So the states buffer small inputs: +// - the build aggregates keep their first raw input rows and replay them, in order, into an update sketch when more +// arrive (or into a temporary one when a result is needed), so the result is the one an update sketch of all rows +// gives; +// - unions keep up to BUFFERED distinct keys with their combined summaries while that is no more than the union's +// nominal size k. A union's result is the k smallest keys below its theta (the minimum theta of its inputs), each +// with the summaries of its inputs combined in the order they arrived; with at most k keys nothing is trimmed, so +// the buffer holds exactly what the union would hold, and replaying it into a union gives the same union. +// Intersections keep their current result (an intersection only shrinks) and intersect it with each input in a +// temporary intersection, overwriting the result in place: the library's intersection reallocates its table on every +// update, which in an arena would never be freed. +// +// A TRAITS class describes a sketch family: +// using Entry, Alloc (DSAllocator), Update, Compact, Union, Intersection; +// using BindData; // DSThetaFamilyBindData +// static constexpr bool SET_ONLY; // no summaries: raw buffers can be concatenated +// static constexpr const char *KIND, *BUILD_NAME, *INTERSECT_NAME; +// static uint64_t Key(const Entry &); +// static idx_t SummarySize(const DSThetaParams &); +// static void WriteSummary(const Entry &, data_ptr_t, const DSThetaParams &); +// static Entry MakeEntry(uint64_t key, const_data_ptr_t summary, const DSThetaParams &); +// static void Fold(data_ptr_t summary, const Entry &incoming, const DSThetaParams &); // union policy +// static idx_t PayloadSize(const DSThetaParams &); // raw row value bytes +// template static void Apply(Update &, const_data_ptr_t key, uint32_t length, const_data_ptr_t payload, +// const DSThetaParams &); // update with a raw row +// static Update NewUpdate(const BindData &, const DSThetaParams &); +// static Union NewUnion(uint8_t lg_k, uint64_t seed, const DSThetaParams &); +// static Intersection NewIntersection(uint64_t seed, const DSThetaParams &); +// static Compact Deserialize(const string_t &blob); +// static Compact Empty(uint8_t lg_k, uint64_t seed, const DSThetaParams &); +// static auto Serialize(const Compact &); // the stored format +// static void ThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b); +// static void ThrowNumValues(const char *function_name, idx_t a, idx_t b); // AoD only + +#include "datasketches_theta_common.hpp" +#include "duckdb/common/vector/flat_vector.hpp" +#include "duckdb/common/vector/string_vector.hpp" +#include "duckdb/storage/arena_allocator.hpp" + +#include + +#include +#include + +namespace duckdb +{ + // ============================================================ + // Scratch arena and arena buffers + // ============================================================ + + // Makes the arena allocators allocate from this thread's scratch arena while the scope lasts. The arena is + // reset when the outermost scope ends, so temporaries cost a pointer bump instead of a malloc and free. + struct DSScratchScope + { + DSScratchScope() : scope(Arena()) + { + depth++; + } + ~DSScratchScope() + { + if (--depth == 0) + { + auto &arena = Arena(); + if (arena.SizeInBytes() > MAX_RETAINED) + { + arena.Destroy(); + } + else + { + arena.Reset(); + } + } + } + DSScratchScope(const DSScratchScope &) = delete; + DSScratchScope &operator=(const DSScratchScope &) = delete; + + static ArenaAllocator &Arena() + { + static thread_local ArenaAllocator arena(Allocator::DefaultAllocator()); + return arena; + } + + private: + static constexpr idx_t MAX_RETAINED = idx_t(1) << 20; + static inline thread_local idx_t depth = 0; + DSArenaScope scope; + }; + + // Bytes in the aggregate arena, in an array that doubles as it fills. Memory it gives up stays in the arena. + struct DSArenaBytes + { + data_ptr_t data; + uint32_t size; + uint32_t capacity; + + void Initialize() + { + data = nullptr; + size = 0; + capacity = 0; + } + + // Appends n bytes, returning where they go. + data_ptr_t Grow(idx_t n, ArenaAllocator &arena) + { + const idx_t needed = idx_t(size) + n; + if (needed > NumericLimits::Maximum()) + { + throw InvalidInputException("Theta sketch aggregate state is too large"); + } + if (needed > capacity) + { + const idx_t new_capacity = MinValue(MaxValue(idx_t(capacity) * 2, needed), NumericLimits::Maximum()); + auto new_data = arena.AllocateAligned(new_capacity); + if (size) + { + memcpy(new_data, data, size); + } + data = new_data; + capacity = static_cast(new_capacity); + } + auto result = data + size; + size = static_cast(needed); + return result; + } + + void Append(const void *source, idx_t n, ArenaAllocator &arena) + { + if (n) + { + memcpy(Grow(n, arena), source, n); + } + } + }; + + // Writes and reads the exported states. + struct DSThetaWriter + { + std::vector bytes; + + template + void Put(const T &value) + { + auto p = reinterpret_cast(&value); + bytes.insert(bytes.end(), p, p + sizeof(T)); + } + void PutBytes(const void *data, idx_t size) + { + auto p = reinterpret_cast(data); + bytes.insert(bytes.end(), p, p + size); + } + // A length-prefixed block. + template + void PutBlock(const BYTES &block) + { + Put(static_cast(block.size())); + PutBytes(block.data(), block.size()); + } + }; + + struct DSThetaReader + { + DSThetaReader(const string_t &blob, const char *kind) + : data(const_data_ptr_cast(blob.GetData())), size(blob.GetSize()), kind(kind) {} + + const_data_ptr_t data; + idx_t size; + idx_t pos = 0; + const char *kind; + + void Check(idx_t n) const + { + if (n > size - pos) + { + throw InvalidInputException("Invalid %s sketch aggregate state", kind); + } + } + template + T Get() + { + Check(sizeof(T)); + T value; + memcpy(&value, data + pos, sizeof(T)); + pos += sizeof(T); + return value; + } + const_data_ptr_t GetBytes(idx_t n) + { + Check(n); + auto result = data + pos; + pos += n; + return result; + } + string_t GetBlock() + { + auto n = Get(); + auto p = GetBytes(n); + return string_t(const_char_ptr_cast(p), n); + } + void Fail() const + { + throw InvalidInputException("Invalid %s sketch aggregate state", kind); + } + void End() const + { + if (pos != size) + { + Fail(); + } + } + }; + + // ============================================================ + // Bind data and parameters + // ============================================================ + + // The bind data of every Theta-family aggregate. `constants` are the constant arguments (lg_k, seed, p, mode) + // by argument index, which the exported state records so that re-binding it gets the same bind data. + struct DSThetaFamilyBindData : public FunctionData + { + unique_ptr Copy() const override + { + return make_uniq(*this); + } + + bool Equals(const FunctionData &other_p) const override + { + auto &other = other_p.Cast(); + return lg_k == other.lg_k && infer_lg_k == other.infer_lg_k && seed == other.seed && p == other.p && mode == other.mode; + } + + //! lg_k of the sketches built (or the minimum lg_k of the union when infer_lg_k is set) + uint8_t lg_k = DS_THETA_DEFAULT_LG_K; + //! Merging: grow the union to the lg_k of the sketches merged + bool infer_lg_k = false; + //! Building: the hash seed and the sampling probability + uint64_t seed = DS_THETA_DEFAULT_SEED; + float p = 1.0f; + //! Tuple sketches: how summaries combine (DSTupleMode) + uint8_t mode = 0; + vector> constants; + }; + + // Records the constant arguments of a call: every argument but the data arguments [data_begin, data_end). + inline void DSThetaRecordConstants(BindAggregateFunctionInput &input, DSThetaFamilyBindData &bind_data, idx_t data_begin, idx_t data_end) + { + auto &arguments = input.GetArguments(); + for (idx_t i = 0; i < arguments.size(); i++) + { + if (i >= data_begin && i < data_end) + { + continue; + } + bind_data.constants.emplace_back(i, input.GetNonNullConstant(i)); + } + } + + inline const DSThetaFamilyBindData &DSThetaBind(optional_ptr bind_data) + { + return bind_data->Cast(); + } + + inline uint8_t DSThetaMode(optional_ptr bind_data) + { + return bind_data ? bind_data->Cast().mode : 0; + } + + // What the sketch operations need besides the bind data: the summary mode and the number of values (AoD). + struct DSThetaParams + { + uint8_t mode = 0; + uint8_t num_values = 1; + }; + + // A sketch over entries held in a vector, in the shape the library's unions and intersections read. + template + struct DSThetaEntriesView + { + using Entry = typename TRAITS::Entry; + using EntryAlloc = typename std::allocator_traits::template rebind_alloc; + using Entries = std::vector; + + DSThetaEntriesView(bool empty, uint64_t seed, uint64_t theta) + : entries(EntryAlloc(DSArenaMode())), empty(empty), seed_hash(ThetaSeedHash(seed)), theta(theta) {} + + // Makes the entries from count buffered entries (key, then summary). + void Load(const_data_ptr_t data, idx_t count, const DSThetaParams ¶ms) + { + const idx_t stride = sizeof(uint64_t) + TRAITS::SummarySize(params); + entries.reserve(count); + for (idx_t i = 0; i < count; i++) + { + uint64_t key; + memcpy(&key, data + i * stride, sizeof(key)); + entries.push_back(TRAITS::MakeEntry(key, data + i * stride + sizeof(uint64_t), params)); + } + } + + bool is_empty() const { return empty; } + bool is_ordered() const { return false; } + uint16_t get_seed_hash() const { return seed_hash; } + uint64_t get_theta64() const { return theta; } + uint32_t get_num_retained() const { return static_cast(entries.size()); } + typename Entries::const_iterator begin() const { return entries.begin(); } + typename Entries::const_iterator end() const { return entries.end(); } + + Entries entries; + bool empty; + uint16_t seed_hash; + uint64_t theta; + }; + + // Writes the entries of a sketch (key, then summary). + template + void DSThetaWriteEntries(const SKETCH &sketch, DSThetaWriter &writer, const DSThetaParams ¶ms) + { + vector summary(TRAITS::SummarySize(params)); + for (const auto &entry : sketch) + { + writer.Put(TRAITS::Key(entry)); + TRAITS::WriteSummary(entry, summary.data(), params); + writer.PutBytes(summary.data(), summary.size()); + } + } + + // ============================================================ + // Unions + // ============================================================ + + // A union in the arena: buffered keys while it holds few, then the library's union. + template + struct DSThetaUnionPart + { + using Union = typename TRAITS::Union; + using Compact = typename TRAITS::Compact; + using View = DSThetaEntriesView; + static constexpr uint32_t BUFFERED = 32; + + Union *sketch; + //! The buffered keys (distinct, each followed by its summary) while there is no sketch + DSArenaBytes entries; + uint32_t count; + uint64_t theta; + uint64_t seed; + uint8_t lg_k; + bool active; + //! A non-empty sketch was added + bool nonempty; + + void Initialize() + { + sketch = nullptr; + entries.Initialize(); + count = 0; + theta = datasketches::theta_constants::MAX_THETA; + seed = DS_THETA_DEFAULT_SEED; + lg_k = 0; + active = false; + nonempty = false; + } + + uint32_t Limit() const + { + return MinValue(BUFFERED, uint32_t(1) << lg_k); + } + + static idx_t Stride(const DSThetaParams ¶ms) + { + return sizeof(uint64_t) + TRAITS::SummarySize(params); + } + + uint64_t KeyAt(idx_t i, const DSThetaParams ¶ms) const + { + uint64_t key; + memcpy(&key, entries.data + i * Stride(params), sizeof(key)); + return key; + } + + idx_t Find(uint64_t key, const DSThetaParams ¶ms) const + { + for (idx_t i = 0; i < count; i++) + { + if (KeyAt(i, params) == key) + { + return i; + } + } + return DConstants::INVALID_INDEX; + } + + // The buffered keys as a sketch (allocated in the current, scratch, arena). + View Buffered(const DSThetaParams ¶ms) const + { + View view(false, seed, theta); + view.Load(entries.data, count, params); + return view; + } + + // The library's EnsureUnion: creates the union, or grows it to a larger lg_k. + void Ensure(uint8_t new_lg_k, uint64_t new_seed, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + if (!active) + { + Initialize(); + active = true; + lg_k = new_lg_k; + seed = new_seed; + return; + } + if (new_lg_k <= lg_k) + { + return; + } + lg_k = new_lg_k; + if (sketch) + { + // re-add what the union holds to a larger one + DSScratchScope scratch; + auto held = sketch->get_result(); + DSArenaScope back(arena); + auto grown = arena.Make(TRAITS::NewUnion(lg_k, seed, params)); + grown->update(held); + sketch = grown; + } + } + + // Buffers a sketch's entries if the buffer stays within its limit. + template + bool TryBuffer(const SKETCH &incoming, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + const uint64_t new_theta = MinValue(theta, incoming.get_theta64()); + const uint32_t limit = Limit(); + idx_t kept = 0; + for (idx_t i = 0; i < count; i++) + { + kept += KeyAt(i, params) < new_theta; + } + idx_t added = 0; + for (const auto &entry : incoming) + { + const auto key = TRAITS::Key(entry); + if (key < new_theta && Find(key, params) == DConstants::INVALID_INDEX && kept + ++added > limit) + { + return false; + } + } + // drop the keys at or above the new theta: the union's result leaves them out + const idx_t stride = Stride(params); + if (new_theta < theta) + { + uint32_t write = 0; + for (idx_t i = 0; i < count; i++) + { + if (KeyAt(i, params) < new_theta) + { + if (write != i) + { + memmove(entries.data + write * stride, entries.data + i * stride, stride); + } + write++; + } + } + count = write; + entries.size = static_cast(count * stride); + theta = new_theta; + } + for (const auto &entry : incoming) + { + const auto key = TRAITS::Key(entry); + if (key >= theta) + { + continue; + } + auto index = Find(key, params); + if (index != DConstants::INVALID_INDEX) + { + TRAITS::Fold(entries.data + index * stride + sizeof(uint64_t), entry, params); + continue; + } + auto slot = entries.Grow(stride, arena); + memcpy(slot, &key, sizeof(key)); + TRAITS::WriteSummary(entry, slot + sizeof(uint64_t), params); + count++; + } + return true; + } + + // Replaces the buffer with the library's union. + void Promote(const DSThetaParams ¶ms, ArenaAllocator &arena) + { + D_ASSERT(!sketch); + auto promoted = arena.Make(TRAITS::NewUnion(lg_k, seed, params)); + if (nonempty) + { + DSScratchScope scratch; + auto buffered = Buffered(params); + DSArenaScope back(arena); + promoted->update(buffered); + } + sketch = promoted; + entries.Initialize(); + count = 0; + } + + // EnsureUnion(lg_k, seed), then the union's update with `incoming`, which is read only. It may live in the + // scratch arena or another state's arena: what the union keeps of it is copied into `arena`. + template + void Add(const SKETCH &incoming, uint8_t new_lg_k, uint64_t new_seed, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + DSArenaScope scope(arena); + Ensure(new_lg_k, new_seed, params, arena); + if (incoming.is_empty()) + { + return; + } + if (!sketch) + { + if (TryBuffer(incoming, params, arena)) + { + nonempty = true; + return; + } + Promote(params, arena); + } + nonempty = true; + ThetaGuard([&]() + { sketch->update(incoming); }, + TRAITS::KIND); + } + + // Adds another union (EnsureUnion(lg_k, seed), then the update with its result). + void AddUnion(const DSThetaUnionPart &source, uint8_t new_lg_k, uint64_t new_seed, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + D_ASSERT(source.active); + if (!active && source.sketch && source.lg_k == new_lg_k) + { + // a copy allocates through the source's allocator, which in arena mode is the current (target's) arena + DSArenaScope scope(arena); + Initialize(); + active = true; + lg_k = source.lg_k; + seed = source.seed; + nonempty = source.nonempty; + sketch = arena.Make(*source.sketch); + return; + } + DSScratchScope scratch; + if (source.sketch) + { + auto result = source.sketch->get_result(); + Add(result, new_lg_k, new_seed, params, arena); + } + else if (source.nonempty) + { + Add(source.Buffered(params), new_lg_k, new_seed, params, arena); + } + else + { + DSArenaScope back(arena); + Ensure(new_lg_k, new_seed, params, arena); + } + } + + // The union's result, in the current (scratch) arena. + Compact Result(const DSThetaParams ¶ms) const + { + if (sketch) + { + return sketch->get_result(); + } + auto temporary = TRAITS::NewUnion(lg_k, seed, params); + if (nonempty) + { + temporary.update(Buffered(params)); + } + return temporary.get_result(); + } + + // lg_k, seed and flags, then the buffered entries or the union's result. + void Export(DSThetaWriter &writer, const DSThetaParams ¶ms) const + { + writer.Put(sketch ? 2 : 1); + writer.Put(lg_k); + writer.Put(seed); + writer.Put(nonempty); + if (sketch) + { + DSScratchScope scratch; + writer.PutBlock(TRAITS::Serialize(sketch->get_result())); + return; + } + writer.Put(theta); + writer.Put(count); + writer.PutBytes(entries.data, count * Stride(params)); + } + + void Import(DSThetaReader &reader, uint8_t kind, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + const auto new_lg_k = reader.Get(); + const auto new_seed = reader.Get(); + const auto new_nonempty = reader.Get(); + if (new_lg_k < datasketches::theta_constants::MIN_LG_K || new_lg_k > datasketches::theta_constants::MAX_LG_K || new_nonempty > 1) + { + reader.Fail(); + } + if (kind == 2) + { + // the union's result: a union of it is the union (buffered again if it is small) + auto block = reader.GetBlock(); + DSScratchScope scratch; + auto result = TRAITS::Deserialize(block); + { + DSArenaScope back(arena); + Ensure(new_lg_k, new_seed, params, arena); + } + if (new_nonempty) + { + Add(result, new_lg_k, new_seed, params, arena); + } + return; + } + const auto new_theta = reader.Get(); + const auto new_count = reader.Get(); + if (new_count > BUFFERED || new_count > (uint32_t(1) << new_lg_k) || (new_count && !new_nonempty)) + { + reader.Fail(); + } + auto data = reader.GetBytes(new_count * Stride(params)); + Initialize(); + active = true; + lg_k = new_lg_k; + seed = new_seed; + nonempty = new_nonempty; + theta = new_theta; + count = new_count; + entries.Append(data, new_count * Stride(params), arena); + } + }; + + // ============================================================ + // Build and union aggregates + // ============================================================ + + // The state of the aggregates that build a sketch from rows (an update sketch) and of those that merge + // sketches (a union); combining states makes a union, as before. KEY is the type of the rows' keys. + template + struct DSThetaFamilyState + { + using Update = typename TRAITS::Update; + using Compact = typename TRAITS::Compact; + using BindData = DSThetaFamilyBindData; + //! Raw rows kept before an update sketch is built + static constexpr uint32_t RAW_ROWS = 32; + static constexpr uint32_t RAW_BYTES = 1024; + + DSThetaUnionPart part; + //! The update sketch of the rows + Update *update; + //! An update sketch's compact form, from an imported state + Compact *compact; + //! The first rows (key length, key, value) while there is no update sketch + DSArenaBytes raw; + uint32_t raw_rows; + //! The number of values per key of AoD sketches (0 while unknown) + uint8_t num_values; + //! A merged input sketch was empty: with no other input, the result is an empty sketch rather than NULL + bool saw_empty_sketch; + + void Initialize() + { + part.Initialize(); + update = nullptr; + compact = nullptr; + raw.Initialize(); + raw_rows = 0; + num_values = 0; + saw_empty_sketch = false; + } + + DSThetaParams Params(const BindData &bind) const + { + DSThetaParams params; + params.mode = bind.mode; + params.num_values = MaxValue(num_values, 1); + return params; + } + + bool HasData() const + { + return update || compact || raw_rows || part.active; + } + bool IsEmpty() const + { + return !HasData() && !saw_empty_sketch; + } + bool OnlyRaw() const + { + return raw_rows && !update && !compact && !part.active; + } + uint64_t Seed(const BindData &bind) const + { + return part.active ? part.seed : bind.seed; + } + uint8_t LgK(const BindData &bind) const + { + return part.active ? part.lg_k : bind.lg_k; + } + + void CheckNumValues(idx_t count, const char *function_name) + { + if (num_values == 0) + { + num_values = static_cast(count); + } + else if (num_values != count) + { + TRAITS::ThrowNumValues(function_name, num_values, count); + } + } + + // Replays raw rows (as written by AddRow) into an update sketch. + static void Replay(Update &sketch, const_data_ptr_t data, idx_t size, const DSThetaParams ¶ms) + { + const idx_t payload = TRAITS::PayloadSize(params); + idx_t pos = 0; + while (pos < size) + { + uint32_t length; + memcpy(&length, data + pos, sizeof(length)); + pos += sizeof(length); + TRAITS::template Apply(sketch, data + pos, length, data + pos + length, params); + pos += length + payload; + } + } + + // The update sketch of the rows: the one built so far, or one replayed from the raw rows in the current + // (scratch) arena. + template + void WithRowsSketch(const BindData &bind, FUNC &&fun) const + { + if (update) + { + fun(update->compact()); + return; + } + auto params = Params(bind); + auto temporary = TRAITS::NewUpdate(bind, params); + Replay(temporary, raw.data, raw.size, params); + fun(temporary.compact()); + } + + // Adds a row: key bytes (the value of a numeric key, or a string's bytes) and value bytes. + void AddRow(const_data_ptr_t key, uint32_t length, const_data_ptr_t payload, const BindData &bind, ArenaAllocator &arena) + { + auto params = Params(bind); + if (update) + { + TRAITS::template Apply(*update, key, length, payload, params); + return; + } + const idx_t payload_size = TRAITS::PayloadSize(params); + const idx_t row_size = sizeof(uint32_t) + length + payload_size; + if (raw_rows < RAW_ROWS && raw.size + row_size <= RAW_BYTES) + { + auto slot = raw.Grow(row_size, arena); + memcpy(slot, &length, sizeof(length)); + memcpy(slot + sizeof(length), key, length); + memcpy(slot + sizeof(length) + length, payload, payload_size); + raw_rows++; + return; + } + update = arena.Make(TRAITS::NewUpdate(bind, params)); + Replay(*update, raw.data, raw.size, params); + raw.Initialize(); + raw_rows = 0; + TRAITS::template Apply(*update, key, length, payload, params); + } + + // Merges a serialized sketch. + void Merge(const string_t &blob, const BindData &bind, ArenaAllocator &arena) + { + DSScratchScope scratch; + auto sketch = TRAITS::Deserialize(blob); + if (sketch.is_empty()) + { + // an empty sketch adds nothing to a union + saw_empty_sketch = true; + return; + } + if (part.active && sketch.get_seed_hash() != ThetaSeedHash(part.seed)) + { + TRAITS::ThrowSeedMismatch(TRAITS::BUILD_NAME, ThetaSeedHash(part.seed), sketch.get_seed_hash()); + } + if (TRAITS::HAS_NUM_VALUES) + { + CheckNumValues(TRAITS::NumValues(sketch), TRAITS::BUILD_NAME); + } + auto lg_k = bind.infer_lg_k ? MaxValue(bind.lg_k, ThetaInferLgK(sketch)) : bind.lg_k; + part.Add(sketch, lg_k, ThetaSeedForHash(sketch.get_seed_hash()), Params(bind), arena); + } + + // Moves what the rows built into the union. + void FlushRows(uint8_t lg_k, uint64_t seed, const BindData &bind, ArenaAllocator &arena) + { + auto params = Params(bind); + if (update || raw_rows) + { + DSScratchScope scratch; + WithRowsSketch(bind, [&](const Compact &rows) + { part.Add(rows, lg_k, seed, params, arena); }); + } + else if (compact) + { + part.Add(*compact, lg_k, seed, params, arena); + } + else + { + DSArenaScope scope(arena); + part.Ensure(lg_k, seed, params, arena); + } + update = nullptr; + compact = nullptr; + raw.Initialize(); + raw_rows = 0; + } + + // Combines a partial state, which may live in another thread's arena: nothing of it is taken over. + void Combine(const DSThetaFamilyState &source, const BindData &bind, ArenaAllocator &arena) + { + saw_empty_sketch = saw_empty_sketch || source.saw_empty_sketch; + if (!source.HasData()) + { + return; + } + auto source_seed = source.Seed(bind); + if (HasData() && ThetaSeedHash(source_seed) != ThetaSeedHash(Seed(bind))) + { + TRAITS::ThrowSeedMismatch(TRAITS::BUILD_NAME, ThetaSeedHash(Seed(bind)), ThetaSeedHash(source_seed)); + } + if (TRAITS::HAS_NUM_VALUES) + { + CheckNumValues(source.num_values, TRAITS::BUILD_NAME); + } + const uint32_t k = uint32_t(1) << bind.lg_k; + if (source.OnlyRaw() && source.raw_rows <= k) + { + // The rows' update sketch is exact (at most k keys), so the union of it holds just its keys: keeping + // the rows gives the same result. Without summaries, the rows of two such states give the union of + // both (at most k keys in all). + const bool fresh = !HasData(); + const bool concatenate = TRAITS::SET_ONLY && OnlyRaw() && raw_rows + source.raw_rows <= MinValue(RAW_ROWS, k) && + raw.size + source.raw.size <= RAW_BYTES; + if (fresh || concatenate) + { + raw.Append(source.raw.data, source.raw.size, arena); + raw_rows += source.raw_rows; + return; + } + } + auto params = Params(bind); + auto lg_k = MaxValue(source.LgK(bind), LgK(bind)); + auto seed = HasData() ? Seed(bind) : source_seed; + if (!HasData() && !source.update && !source.raw_rows && !source.compact) + { + // only a union: copied if it is the library's + part.AddUnion(source.part, lg_k, seed, params, arena); + return; + } + FlushRows(lg_k, seed, bind, arena); + if (source.update || source.raw_rows) + { + DSScratchScope scratch; + source.WithRowsSketch(bind, [&](const Compact &rows) + { part.Add(rows, lg_k, seed, params, arena); }); + } + if (source.compact) + { + part.Add(*source.compact, lg_k, seed, params, arena); + } + if (source.part.active) + { + part.AddUnion(source.part, lg_k, seed, params, arena); + } + } + + // The aggregate's result. + string_t Finalize(Vector &result, const BindData &bind) const + { + auto params = Params(bind); + DSScratchScope scratch; + if (part.active) + { + return Store(result, part.Result(params)); + } + if (update || raw_rows) + { + string_t stored; + WithRowsSketch(bind, [&](const Compact &rows) + { stored = Store(result, rows); }); + return stored; + } + if (compact) + { + return Store(result, *compact); + } + return Store(result, TRAITS::Empty(bind.lg_k, bind.seed, params)); + } + + static string_t Store(Vector &result, const Compact &sketch) + { + auto bytes = TRAITS::Serialize(sketch); + return StringVector::AddStringOrBlob(result, const_char_ptr_cast(bytes.data()), bytes.size()); + } + + // The exported state: a version, flags and num_values, then the rows (raw, or the update sketch's compact + // form) and the union. Raw rows are exported as they are, so importing a small state keeps it small. + static constexpr uint8_t EXPORT_VERSION = 1; + static constexpr uint8_t ROWS_NONE = 0; + static constexpr uint8_t ROWS_RAW = 1; + static constexpr uint8_t ROWS_COMPACT = 2; + + std::vector Export(optional_ptr bind_data) const + { + const auto &bind = bind_data->Cast(); + auto params = Params(bind); + DSThetaWriter writer; + writer.Put(EXPORT_VERSION); + writer.Put(saw_empty_sketch); + writer.Put(num_values); + if (update) + { + DSScratchScope scratch; + writer.Put(ROWS_COMPACT); + writer.PutBlock(TRAITS::Serialize(update->compact())); + } + else if (compact) + { + DSScratchScope scratch; + writer.Put(ROWS_COMPACT); + writer.PutBlock(TRAITS::Serialize(*compact)); + } + else if (raw_rows) + { + writer.Put(ROWS_RAW); + writer.Put(raw_rows); + writer.Put(raw.size); + writer.PutBytes(raw.data, raw.size); + } + else + { + writer.Put(ROWS_NONE); + } + if (part.active) + { + part.Export(writer, params); + } + else + { + writer.Put(0); + } + return std::move(writer.bytes); + } + + void Import(const string_t &blob, ArenaAllocator &arena) + { + DSThetaReader reader(blob, TRAITS::KIND); + if (reader.Get() != EXPORT_VERSION) + { + reader.Fail(); + } + const auto flags = reader.Get(); + if (flags > 1) + { + reader.Fail(); + } + saw_empty_sketch = flags; + num_values = reader.Get(); + DSThetaParams params; + params.num_values = MaxValue(num_values, 1); + // mode is not needed: imported states only copy summaries + const auto rows = reader.Get(); + if (rows == ROWS_COMPACT) + { + auto block = reader.GetBlock(); + DSScratchScope scratch; + auto sketch = TRAITS::Deserialize(block); + DSArenaScope back(arena); + compact = arena.Make(sketch); + } + else if (rows == ROWS_RAW) + { + const auto count = reader.Get(); + const auto size = reader.Get(); + auto data = reader.GetBytes(size); + // check the rows' framing, which Replay trusts + const idx_t payload = TRAITS::PayloadSize(params); + idx_t pos = 0; + uint32_t found = 0; + while (pos < size) + { + if (size - pos < sizeof(uint32_t)) + { + reader.Fail(); + } + uint32_t length; + memcpy(&length, data + pos, sizeof(length)); + pos += sizeof(length); + if ((!std::is_same::value && length != sizeof(KEY)) || size - pos < idx_t(length) + payload) + { + reader.Fail(); + } + pos += length + payload; + found++; + } + if (found != count || count == 0 || count > RAW_ROWS || size > RAW_BYTES) + { + reader.Fail(); + } + raw.Append(data, size, arena); + raw_rows = count; + } + else if (rows != ROWS_NONE) + { + reader.Fail(); + } + const auto union_kind = reader.Get(); + if (union_kind == 1 || union_kind == 2) + { + part.Import(reader, union_kind, params, arena); + } + else if (union_kind != 0) + { + reader.Fail(); + } + reader.End(); + } + }; + + // ============================================================ + // Intersection aggregates + // ============================================================ + + // Intersects sketches across rows. The state is the current result: each input is intersected with it in a + // temporary intersection, which gives what the library's intersection would hold. + template + struct DSThetaFamilyIntersectState + { + using Compact = typename TRAITS::Compact; + using View = DSThetaEntriesView; + + //! The result's entries (key, then summary) + DSArenaBytes entries; + uint32_t count; + uint64_t theta; + uint64_t seed; + bool has_result; + bool result_empty; + //! An empty sketch was intersected: the result is empty + bool saw_empty; + uint8_t num_values; + + void Initialize() + { + entries.Initialize(); + count = 0; + theta = datasketches::theta_constants::MAX_THETA; + seed = DS_THETA_DEFAULT_SEED; + has_result = false; + result_empty = false; + saw_empty = false; + num_values = 0; + } + + bool IsEmpty() const + { + return !has_result && !saw_empty; + } + + DSThetaParams Params(uint8_t mode) const + { + DSThetaParams params; + params.mode = mode; + params.num_values = MaxValue(num_values, 1); + return params; + } + + // The current result as a sketch (allocated in the current, scratch, arena). + View Current(const DSThetaParams ¶ms) const + { + View view(result_empty, seed, theta); + view.Load(entries.data, count, params); + return view; + } + + // Intersects a sketch (read only; it may be another state's) with `sketch_num_values` values per key. + template + void Add(const SKETCH &sketch, uint8_t sketch_num_values, uint8_t mode, ArenaAllocator &arena) + { + if (sketch.is_empty()) + { + saw_empty = true; + if (num_values == 0) + { + num_values = sketch_num_values; + } + return; + } + if (!has_result) + { + seed = ThetaSeedForHash(sketch.get_seed_hash()); + num_values = sketch_num_values; + } + else + { + if (sketch.get_seed_hash() != ThetaSeedHash(seed)) + { + TRAITS::ThrowSeedMismatch(TRAITS::INTERSECT_NAME, ThetaSeedHash(seed), sketch.get_seed_hash()); + } + if (TRAITS::HAS_NUM_VALUES && sketch_num_values != num_values) + { + TRAITS::ThrowNumValues(TRAITS::INTERSECT_NAME, num_values, sketch_num_values); + } + } + auto params = Params(mode); + DSScratchScope scratch; + auto intersection = TRAITS::NewIntersection(seed, params); + ThetaGuard([&]() + { + if (has_result) + { + intersection.update(Current(params)); + } + intersection.update(sketch); }, + TRAITS::KIND); + auto result = intersection.get_result(false); + // an intersection only shrinks, so the result usually fits where the previous one was + DSThetaWriter writer; + DSThetaWriteEntries(result, writer, params); + entries.size = 0; + entries.Append(writer.bytes.data(), writer.bytes.size(), arena); + count = result.get_num_retained(); + theta = result.get_theta64(); + result_empty = result.is_empty(); + has_result = true; + } + + void AddBlob(const string_t &blob, uint8_t mode, ArenaAllocator &arena) + { + DSScratchScope scratch; + auto sketch = TRAITS::Deserialize(blob); + Add(sketch, TRAITS::NumValues(sketch), mode, arena); + } + + void Combine(const DSThetaFamilyIntersectState &source, uint8_t mode, ArenaAllocator &arena) + { + saw_empty = saw_empty || source.saw_empty; + if (num_values == 0) + { + num_values = source.num_values; + } + if (source.has_result) + { + DSScratchScope scratch; + Add(source.Current(source.Params(mode)), MaxValue(source.num_values, 1), mode, arena); + } + } + + string_t Finalize(Vector &result, uint8_t mode) const + { + auto params = Params(mode); + DSScratchScope scratch; + if (saw_empty) + { + return Store(result, TRAITS::Empty(DS_THETA_DEFAULT_LG_K, seed, params)); + } + auto intersection = TRAITS::NewIntersection(seed, params); + intersection.update(Current(params)); + return Store(result, intersection.get_result()); + } + + static string_t Store(Vector &result, const Compact &sketch) + { + auto bytes = TRAITS::Serialize(sketch); + return StringVector::AddStringOrBlob(result, const_char_ptr_cast(bytes.data()), bytes.size()); + } + + static constexpr uint8_t EXPORT_VERSION = 1; + + std::vector Export(optional_ptr bind_data) const + { + const auto mode = DSThetaMode(bind_data); + auto params = Params(mode); + DSThetaWriter writer; + writer.Put(EXPORT_VERSION); + writer.Put(uint8_t(has_result) | uint8_t(result_empty) << 1 | uint8_t(saw_empty) << 2); + writer.Put(num_values); + writer.Put(theta); + writer.Put(seed); + writer.Put(count); + writer.PutBytes(entries.data, count * (sizeof(uint64_t) + TRAITS::SummarySize(params))); + return std::move(writer.bytes); + } + + void Import(const string_t &blob, ArenaAllocator &arena) + { + DSThetaReader reader(blob, TRAITS::KIND); + if (reader.Get() != EXPORT_VERSION) + { + reader.Fail(); + } + const auto flags = reader.Get(); + if (flags > 7) + { + reader.Fail(); + } + has_result = flags & 1; + result_empty = flags & 2; + saw_empty = flags & 4; + num_values = reader.Get(); + theta = reader.Get(); + seed = reader.Get(); + count = reader.Get(); + const idx_t size = idx_t(count) * (sizeof(uint64_t) + TRAITS::SummarySize(Params(0))); + entries.Append(reader.GetBytes(size), size, arena); + reader.End(); + } + + }; + + // ============================================================ + // Aggregate operations and state export + // ============================================================ + + // Operations of the build and merge aggregates. IDEMPOTENT: merging the same sketch twice is merging it once. + template + struct DSThetaFamilyOperation + { + template + static void Initialize(STATE &state) + { + state.Initialize(); + } + + static bool IgnoreNull() { return true; } + + template + static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + { + if (source.IsEmpty()) + { + return; + } + DSArenaScope scope(aggr_input_data.allocator); + target.Combine(source, DSThetaBind(aggr_input_data.bind_data), aggr_input_data.allocator); + } + + template + static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + { + if (state.IsEmpty()) + { + // no input rows: NULL, as for the other sketches' aggregates + finalize_data.ReturnNull(); + return; + } + DSArenaScope scope(finalize_data.input.allocator); + target = state.Finalize(finalize_data.result, DSThetaBind(finalize_data.input.bind_data)); + } + + // Merging serialized sketches. + template + static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + state.Merge(blob, DSThetaBind(idata.input.bind_data), idata.input.allocator); + } + + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + { + for (idx_t i = 0; i < (IDEMPOTENT ? 1 : count); i++) + { + Operation(state, input, unary_input); + } + } + }; + + // The key of a row as bytes: the value of a numeric key, the bytes of a string. + template + inline std::pair DSThetaKeyBytes(const KEY &key) + { + if constexpr (std::is_same::value) + { + return {const_data_ptr_cast(key.GetData()), static_cast(key.GetSize())}; + } + else + { + return {const_data_ptr_cast(&key), static_cast(sizeof(KEY))}; + } + } + + // Builds from keys (Theta): adding the same key again does not change the sketch. + struct DSThetaFamilyCreateOperation : DSThetaFamilyOperation + { + template + static void Operation(STATE &state, const A_TYPE &key, AggregateUnaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + auto bytes = DSThetaKeyBytes(key); + state.AddRow(bytes.first, bytes.second, nullptr, DSThetaBind(idata.input.bind_data), idata.input.allocator); + } + + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + { + Operation(state, input, unary_input); + } + }; + + // Builds from (key, DOUBLE value) rows (Tuple). + struct DSThetaFamilyCreatePairOperation : DSThetaFamilyOperation + { + template + static void Operation(STATE &state, const KEY_TYPE &key, const VALUE_TYPE &value, AggregateBinaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + auto bytes = DSThetaKeyBytes(key); + state.AddRow(bytes.first, bytes.second, const_data_ptr_cast(&value), DSThetaBind(idata.input.bind_data), idata.input.allocator); + } + }; + + template + struct DSThetaFamilyIntersectOperation + { + template + static void Initialize(STATE &state) + { + state.Initialize(); + } + + static bool IgnoreNull() { return true; } + + template + static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + state.AddBlob(blob, DSThetaMode(idata.input.bind_data), idata.input.allocator); + } + + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + { + for (idx_t i = 0; i < (IDEMPOTENT ? 1 : count); i++) + { + Operation(state, input, unary_input); + } + } + + template + static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + { + if (source.IsEmpty()) + { + return; + } + DSArenaScope scope(aggr_input_data.allocator); + target.Combine(source, DSThetaMode(aggr_input_data.bind_data), aggr_input_data.allocator); + } + + template + static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + { + if (state.IsEmpty()) + { + // no sketches: the intersection of nothing is undefined + finalize_data.ReturnNull(); + return; + } + DSArenaScope scope(finalize_data.input.allocator); + target = state.Finalize(finalize_data.result, DSThetaMode(finalize_data.input.bind_data)); + } + }; + + // Exports each state (NULL for a state that saw no rows) for spilling. + template + static void DSThetaExportState(Vector &state_vector, AggregateFinalizeInputData &aggr_input_data, Vector &result, + idx_t count, idx_t offset) + { + auto states = state_vector.Values(); + auto results = FlatVector::GetDataMutable(result); + DSArenaScope scope(aggr_input_data.allocator); + for (idx_t i = 0; i < count; i++) + { + auto &state = *states[i].GetValue(); + if (state.IsEmpty()) + { + FlatVector::SetNull(result, offset + i, true); + continue; + } + auto bytes = state.Export(aggr_input_data.bind_data); + results[offset + i] = StringVector::AddStringOrBlob(result, const_char_ptr_cast(bytes.data()), bytes.size()); + } + } + + // Rebuilds exported states in the arena. + template + static void DSThetaImportState(AggregateImportInputData &input) + { + const auto count = input.input_vec.size(); + auto entries = input.input_vec.Values(); + DSArenaScope scope(input.allocator); + for (idx_t i = 0; i < count; i++) + { + auto &state = *reinterpret_cast(input.dest_buffer + i * input.layout.total_state_size); + state.Initialize(); + const auto entry = entries[i]; + if (entry.IsValid()) + { + state.Import(entry.GetValue(), input.allocator); + } + } + } + + // The exported state is a BLOB; the constant arguments are recorded so that re-binding it gets them. + template + static AggregateStateLayout DSThetaGetStateType(AggregateLayoutInput &input) + { + AggregateStateLayout layout(LogicalType::BLOB, AlignValue(sizeof(STATE))); + if (input.bind_data) + { + for (auto &constant : input.bind_data->template Cast().constants) + { + layout.constant_parameters.emplace(constant.first, constant.second); + } + } + return layout; + } + + template + static void DSThetaSetStateExport(AggregateFunction &fun) + { + fun.SetStateExportCallbacks(DSThetaGetStateType, DSThetaExportState, DSThetaImportState); + } + +} // namespace duckdb diff --git a/src/theta_sketch.cpp b/src/theta_sketch.cpp index d42bb0e..a46a7b0 100644 --- a/src/theta_sketch.cpp +++ b/src/theta_sketch.cpp @@ -7,6 +7,7 @@ #include "duckdb/common/vector/struct_vector.hpp" #include "duckdb/common/vector/unified_vector_format.hpp" #include "datasketches_theta_common.hpp" +#include "datasketches_theta_arena.hpp" // Apache DataSketches Headers #include @@ -77,44 +78,11 @@ namespace duckdb return StringVector::AddStringOrBlob(result, reinterpret_cast(bytes.data()), bytes.size()); } - compact_theta_sketch ThetaEmptySketch(uint8_t lg_k, uint64_t seed) - { - update_theta_sketch::builder b; - b.set_lg_k(lg_k); - b.set_seed(seed); - return b.build().compact(); - } - - // ============================================================ // 2. Bind Data // ============================================================ - struct DSThetaBindData : public FunctionData - { - DSThetaBindData() = default; - DSThetaBindData(uint8_t lg_k, bool infer_lg_k, uint64_t seed, float p) - : lg_k(lg_k), infer_lg_k(infer_lg_k), seed(seed), p(p) {} - - unique_ptr Copy() const override - { - return make_uniq(lg_k, infer_lg_k, seed, p); - } - - bool Equals(const FunctionData &other_p) const override - { - auto &other = other_p.Cast(); - return lg_k == other.lg_k && infer_lg_k == other.infer_lg_k && seed == other.seed && p == other.p; - } - - //! lg_k of the sketches built (or the minimum lg_k of the union when infer_lg_k is set) - uint8_t lg_k = DS_THETA_DEFAULT_LG_K; - //! Merging: grow the union to the lg_k of the sketches merged - bool infer_lg_k = false; - //! Building: the hash seed and the sampling probability - uint64_t seed = DS_THETA_DEFAULT_SEED; - float p = 1.0f; - }; + using DSThetaBindData = DSThetaFamilyBindData; uint8_t DSThetaBindLgK(BindAggregateFunctionInput &input, idx_t index) { @@ -132,14 +100,17 @@ namespace duckdb template unique_ptr DSThetaBindBuild(BindAggregateFunctionInput &input) { - uint8_t lg_k = HAS_LG_K ? DSThetaBindLgK(input, 0) : DS_THETA_DEFAULT_LG_K; - auto seed = static_cast(input.GetNonNullConstant(Identifier("seed")).GetValue()); + auto result = make_uniq(); + result->lg_k = HAS_LG_K ? DSThetaBindLgK(input, 0) : DS_THETA_DEFAULT_LG_K; + result->seed = static_cast(input.GetNonNullConstant(Identifier("seed")).GetValue()); auto p = input.GetNonNullConstant(Identifier("p")).GetValue(); if (!(p > 0.0 && p <= 1.0)) { throw BinderException("Theta sketch sampling probability p must be in (0, 1], got %f", p); } - return make_uniq(lg_k, false, seed, static_cast(p)); + result->p = static_cast(p); + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 2 : 1); + return std::move(result); } // Merging sketches with an explicit lg_k. The lg_k stays in the argument list (the update reads the sketch @@ -147,322 +118,122 @@ namespace duckdb // aggregates that differ only in lg_k as identical. unique_ptr DSThetaBindMergeWithK(BindAggregateFunctionInput &input) { - auto lg_k = DSThetaBindLgK(input, 0); - return make_uniq(lg_k, false, DS_THETA_DEFAULT_SEED, 1.0f); + auto result = make_uniq(); + result->lg_k = DSThetaBindLgK(input, 0); + DSThetaRecordConstants(input, *result, 1, 2); + return std::move(result); } // Merging sketches: the union grows to the lg_k of the sketches merged. unique_ptr DSThetaBindMergeDefault(BindAggregateFunctionInput &input) { - return make_uniq(DS_THETA_DEFAULT_LG_K, true, DS_THETA_DEFAULT_SEED, 1.0f); + auto result = make_uniq(); + result->infer_lg_k = true; + return std::move(result); } // ============================================================ - // 3. Build / Union Aggregate State & Operations + // 3. Aggregate states (see datasketches_theta_arena.hpp) // ============================================================ - struct DSThetaState - { - update_theta_sketch *update_sketch = nullptr; - theta_union *union_sketch = nullptr; - uint8_t union_lg_k = 0; - uint64_t union_seed = DS_THETA_DEFAULT_SEED; - //! A merged input sketch was empty: with no other input, the result is an empty sketch rather than NULL - bool saw_empty_sketch = false; - - ~DSThetaState() - { - if (update_sketch) - delete update_sketch; - if (union_sketch) - delete union_sketch; - } - - void CreateUpdateSketch(const DSThetaBindData &bind_data) - { - if (!update_sketch) - { - update_theta_sketch::builder b; - b.set_lg_k(bind_data.lg_k); - b.set_seed(bind_data.seed); - b.set_p(bind_data.p); - update_sketch = new update_theta_sketch(b.build()); - } - } - - // Creates the union, or grows it to a larger lg_k (re-adding what it holds so far). - void EnsureUnion(uint8_t lg_k, uint64_t seed) - { - if (union_sketch && lg_k <= union_lg_k) - { - return; - } - if (union_sketch) - { - seed = union_seed; - } - theta_union::builder b; - b.set_lg_k(lg_k); - b.set_seed(seed); - auto grown = new theta_union(b.build()); - if (union_sketch) - { - grown->update(union_sketch->get_result()); - delete union_sketch; - } - union_sketch = grown; - union_lg_k = lg_k; - union_seed = seed; - } - - bool HasData() const - { - return update_sketch || union_sketch; - } - - uint64_t Seed(const DSThetaBindData &bind_data) const - { - return union_sketch ? union_seed : bind_data.seed; - } - - uint8_t LgK(const DSThetaBindData &bind_data) const - { - return union_sketch ? union_lg_k : bind_data.lg_k; - } - }; - - struct DSThetaOperationBase - { - template - static void Initialize(STATE &state) - { - state.update_sketch = nullptr; - state.union_sketch = nullptr; - state.union_lg_k = 0; - state.union_seed = DS_THETA_DEFAULT_SEED; - state.saw_empty_sketch = false; - } - template - static void Destroy(STATE &state, AggregateInputData &) - { - if (state.update_sketch) - delete state.update_sketch; - if (state.union_sketch) - delete state.union_sketch; - state.update_sketch = nullptr; - state.union_sketch = nullptr; - } - static bool IgnoreNull() { return true; } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) - { - target.saw_empty_sketch = target.saw_empty_sketch || source.saw_empty_sketch; - if (!source.HasData()) - return; - auto &bind_data = aggr_input_data.bind_data->template Cast(); - auto source_seed = source.Seed(bind_data); - if (target.HasData() && ThetaSeedHash(source_seed) != ThetaSeedHash(target.Seed(bind_data))) - { - throw InvalidInputException("datasketch_theta: cannot merge Theta sketches built with different seeds (seed hashes %d and %d)", - ThetaSeedHash(target.Seed(bind_data)), ThetaSeedHash(source_seed)); - } - auto lg_k = MaxValue(source.LgK(bind_data), target.LgK(bind_data)); - target.EnsureUnion(lg_k, target.HasData() ? target.Seed(bind_data) : source_seed); - if (target.update_sketch) - { - target.union_sketch->update(*target.update_sketch); - delete target.update_sketch; - target.update_sketch = nullptr; - } - if (source.update_sketch) - target.union_sketch->update(*source.update_sketch); - if (source.union_sketch) - target.union_sketch->update(source.union_sketch->get_result()); - } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + struct DSThetaTraits + { + using Alloc = ThetaAllocator; + using Entry = uint64_t; + using Update = update_theta_sketch; + using Compact = compact_theta_sketch; + using Union = theta_union; + using Intersection = theta_intersection; + static constexpr bool SET_ONLY = true; + static constexpr bool HAS_NUM_VALUES = false; + static constexpr const char *KIND = "Theta"; + static constexpr const char *BUILD_NAME = "datasketch_theta"; + static constexpr const char *INTERSECT_NAME = "datasketch_theta_intersect_agg"; + + static uint64_t Key(const Entry &entry) { return entry; } + static idx_t SummarySize(const DSThetaParams &) { return 0; } + static void WriteSummary(const Entry &, data_ptr_t, const DSThetaParams &) {} + static Entry MakeEntry(uint64_t key, const_data_ptr_t, const DSThetaParams &) { return key; } + static void Fold(data_ptr_t, const Entry &, const DSThetaParams &) {} + static idx_t PayloadSize(const DSThetaParams &) { return 0; } + static uint8_t NumValues(const Compact &) { return 1; } + + template + static void Apply(Update &sketch, const_data_ptr_t key, uint32_t length, const_data_ptr_t, const DSThetaParams &) { - if (state.union_sketch) + if constexpr (std::is_same::value) { - target = ThetaSerialize(finalize_data.result, state.union_sketch->get_result()); - } - else if (state.update_sketch) - { - target = ThetaSerialize(finalize_data.result, state.update_sketch->compact()); - } - else if (state.saw_empty_sketch) - { - auto &bind_data = finalize_data.input.bind_data->template Cast(); - target = ThetaSerialize(finalize_data.result, ThetaEmptySketch(bind_data.lg_k, bind_data.seed)); + sketch.update(key, length); } else { - // no input rows: NULL, as for the other sketches' aggregates - finalize_data.ReturnNull(); + KEY value; + memcpy(&value, key, sizeof(KEY)); + sketch.update(value); } } - }; - struct DSThetaCreateOperation : DSThetaOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &a_data, AggregateUnaryInput &idata) + static Update NewUpdate(const DSThetaBindData &bind, const DSThetaParams &) { - auto &bind_data = idata.input.bind_data->template Cast(); - state.CreateUpdateSketch(bind_data); - if constexpr (std::is_same_v) - { - state.update_sketch->update(a_data.GetData(), a_data.GetSize()); - } - else - { - state.update_sketch->update(a_data); - } + update_theta_sketch::builder b{Alloc(DSArenaMode())}; + b.set_lg_k(bind.lg_k); + b.set_seed(bind.seed); + b.set_p(bind.p); + return b.build(); } - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static Union NewUnion(uint8_t lg_k, uint64_t seed, const DSThetaParams &) { - // Adding the same value again does not change the sketch. - Operation(state, input, unary_input); - } - }; - - struct DSThetaMergeOperation : DSThetaOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &a_data, AggregateUnaryInput &idata) - { - auto &bind_data = idata.input.bind_data->template Cast(); - auto sketch = ThetaDeserialize(a_data); - if (sketch.is_empty()) - { - // an empty sketch adds nothing to a union - state.saw_empty_sketch = true; - return; - } - if (state.union_sketch && sketch.get_seed_hash() != ThetaSeedHash(state.union_seed)) - { - throw InvalidInputException("datasketch_theta: cannot merge Theta sketches built with different seeds (seed hashes %d and %d)", - ThetaSeedHash(state.union_seed), sketch.get_seed_hash()); - } - auto lg_k = bind_data.infer_lg_k ? MaxValue(bind_data.lg_k, ThetaInferLgK(sketch)) : bind_data.lg_k; - state.EnsureUnion(lg_k, ThetaSeedForHash(sketch.get_seed_hash())); - ThetaGuard([&]() - { state.union_sketch->update(sketch); }); - } - - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) - { - // A union is idempotent: merging the same sketch again does not change it. - Operation(state, input, unary_input); - } - }; - - // ============================================================ - // 4. Intersection Aggregate - // ============================================================ - - struct DSThetaIntersectState - { - theta_intersection *intersection = nullptr; - uint64_t seed = DS_THETA_DEFAULT_SEED; - //! An empty sketch was intersected: the result is empty - bool saw_empty = false; - - ~DSThetaIntersectState() - { - if (intersection) - delete intersection; - } - - void Add(const compact_theta_sketch &sketch) - { - if (sketch.is_empty()) - { - saw_empty = true; - return; - } - if (!intersection) - { - seed = ThetaSeedForHash(sketch.get_seed_hash()); - intersection = new theta_intersection(seed); - } - else if (sketch.get_seed_hash() != ThetaSeedHash(seed)) - { - throw InvalidInputException("datasketch_theta_intersect_agg: cannot combine Theta sketches built with different seeds (seed hashes %d and %d)", - ThetaSeedHash(seed), sketch.get_seed_hash()); - } - ThetaGuard([&]() - { intersection->update(sketch); }); + theta_union::builder b{Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build(); } - }; - struct DSThetaIntersectOperation - { - template - static void Initialize(STATE &state) + static Intersection NewIntersection(uint64_t seed, const DSThetaParams &) { - state.intersection = nullptr; - state.seed = DS_THETA_DEFAULT_SEED; - state.saw_empty = false; + return Intersection(seed, Alloc(DSArenaMode())); } - template - static void Destroy(STATE &state, AggregateInputData &) + static Compact Deserialize(const string_t &blob) { - if (state.intersection) - delete state.intersection; - state.intersection = nullptr; + auto data = blob.GetData(); + auto size = blob.GetSize(); + auto seed = ThetaSeedForHash(ThetaPeekSeedHash(data, size)); + return ThetaGuard([&]() + { return Compact::deserialize(data, size, seed, Alloc(DSArenaMode())); }); } - static bool IgnoreNull() { return true; } - - template - static void Operation(STATE &state, const A_TYPE &a_data, AggregateUnaryInput &) + static Compact Empty(uint8_t lg_k, uint64_t seed, const DSThetaParams &) { - state.Add(ThetaDeserialize(a_data)); + update_theta_sketch::builder b{Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build().compact(); } - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static auto Serialize(const Compact &sketch) -> decltype(sketch.serialize_compressed()) { - // An intersection is idempotent: intersecting the same sketch again does not change it. - Operation(state, input, unary_input); + return sketch.serialize_compressed(); } - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &) + static void ThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b) { - target.saw_empty = target.saw_empty || source.saw_empty; - if (source.intersection) + if (string(function_name) == BUILD_NAME) { - target.Add(source.intersection->get_result()); + throw InvalidInputException("datasketch_theta: cannot merge Theta sketches built with different seeds (seed hashes %d and %d)", a, b); } + throw InvalidInputException("%s: cannot combine Theta sketches built with different seeds (seed hashes %d and %d)", function_name, a, b); } - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) - { - if (state.saw_empty) - { - target = ThetaSerialize(finalize_data.result, ThetaEmptySketch(DS_THETA_DEFAULT_LG_K, state.seed)); - } - else if (state.intersection) - { - target = ThetaSerialize(finalize_data.result, state.intersection->get_result()); - } - else - { - // no sketches: the intersection of nothing is undefined - finalize_data.ReturnNull(); - } - } + static void ThrowNumValues(const char *, idx_t, idx_t) {} }; + template + using DSThetaState = DSThetaFamilyState; + using DSThetaIntersectState = DSThetaFamilyIntersectState; + using DSThetaMergeOperation = DSThetaFamilyOperation; + using DSThetaIntersectOperation = DSThetaFamilyIntersectOperation; // ============================================================ // 5. Scalar Functions @@ -704,13 +475,14 @@ namespace duckdb { for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::UnaryAggregate( - input_type, sketch_type); + using STATE = DSThetaState; + auto fun = AggregateFunction::UnaryAggregate(input_type, sketch_type); + DSThetaSetStateExport(fun); FunctionSignature signature(vector(), sketch_type); if (with_lg_k) { signature.AddParameter(LogicalType::INTEGER); - DSReadDataArgumentAt<1, DSThetaState, T, DSThetaCreateOperation>(fun); + DSReadDataArgumentAt<1, STATE, T, DSThetaFamilyCreateOperation>(fun); fun.SetBindCallback(DSThetaBindBuild); } else @@ -773,8 +545,9 @@ namespace duckdb RegisterThetaBuild(sketch_agg, agg_descs, "datasketch_theta", LogicalType::DOUBLE, sketch_type); RegisterThetaBuild(sketch_agg, agg_descs, "datasketch_theta", LogicalType::VARCHAR, sketch_type); - auto fun_merge = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + using MERGE_STATE = DSThetaState; + auto fun_merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun_merge); fun_merge.SetBindCallback(DSThetaBindMergeDefault); fun_merge.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); DSSetArguments(fun_merge, {sketch_type}); @@ -783,12 +556,12 @@ namespace duckdb "largest input sketch (inferred from the sketches, at least lg_k 12)", {ex.QueryPartials("datasketch_theta(sketch)")}, {"sketch"}, {sketch_type})); - auto fun_merge_k = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun_merge_k = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun_merge_k); fun_merge_k.SetBindCallback(DSThetaBindMergeWithK); fun_merge_k.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); DSSetArguments(fun_merge_k, {LogicalType::INTEGER, sketch_type}); - DSReadDataArgumentAt<1, DSThetaState, string_t, DSThetaMergeOperation>(fun_merge_k); + DSReadDataArgumentAt<1, MERGE_STATE, string_t, DSThetaMergeOperation>(fun_merge_k); sketch_agg.AddFunction(fun_merge_k); agg_descs.push_back(ThetaDescription("Merges (unions) Theta sketches into a single sketch with the given log2 of nominal entries (lg_k)", {ex.QueryPartials("datasketch_theta(16, sketch)")}, {"lg_k", "sketch"}, {LogicalType::INTEGER, sketch_type})); @@ -813,8 +586,8 @@ namespace duckdb // --- datasketch_theta_intersect_agg: intersect sketches across rows { AggregateFunctionSet intersect_agg("datasketch_theta_intersect_agg"); - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); fun.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); DSSetArguments(fun, {sketch_type}); intersect_agg.AddFunction(fun); diff --git a/src/tuple_sketch.cpp b/src/tuple_sketch.cpp index 976a732..06faf0a 100644 --- a/src/tuple_sketch.cpp +++ b/src/tuple_sketch.cpp @@ -5,6 +5,7 @@ // DataSketches libraries), so functions that combine summaries (merge, union, intersection) take it as an option. #include "datasketches_tuple_common.hpp" +#include "datasketches_theta_arena.hpp" #include #include @@ -123,43 +124,15 @@ namespace duckdb using DSTupleScalars = DSTupleFunctions; - DSTupleCompact DSTupleEmptySketch(uint8_t lg_k, uint64_t seed) - { - DSTupleUpdate::builder b; - b.set_lg_k(lg_k); - b.set_seed(seed); - return b.build().compact(); - } - // ============================================================ // Bind Data // ============================================================ - struct DSTupleBindData : public FunctionData - { - DSTupleBindData() = default; + using DSTupleBindData = DSThetaFamilyBindData; - unique_ptr Copy() const override - { - return make_uniq(*this); - } - - bool Equals(const FunctionData &other_p) const override - { - auto &other = other_p.Cast(); - return lg_k == other.lg_k && infer_lg_k == other.infer_lg_k && seed == other.seed && p == other.p && mode == other.mode; - } - - uint8_t lg_k = DS_THETA_DEFAULT_LG_K; - bool infer_lg_k = false; - uint64_t seed = DS_THETA_DEFAULT_SEED; - float p = 1.0f; - DSTupleMode mode = DSTupleMode::SUM; - }; - - DSTupleMode DSTupleBindMode(BindAggregateFunctionInput &input) + uint8_t DSTupleBindMode(BindAggregateFunctionInput &input) { - return DSTupleParseMode(input.GetNonNullConstant(Identifier("mode")).GetValue()); + return static_cast(DSTupleParseMode(input.GetNonNullConstant(Identifier("mode")).GetValue())); } template @@ -172,6 +145,7 @@ namespace duckdb } DSTupleBindSeedAndP(input, result->seed, result->p); result->mode = DSTupleBindMode(input); + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 3 : 2); return std::move(result); } @@ -188,325 +162,144 @@ namespace duckdb result->infer_lg_k = true; } result->mode = DSTupleBindMode(input); + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 2 : 1); return std::move(result); } - // ============================================================ - // Build / Union Aggregate - // ============================================================ - - struct DSTupleState + unique_ptr DSTupleBindIntersect(BindAggregateFunctionInput &input) { - DSTupleUpdate *update_sketch = nullptr; - DSTupleUnion *union_sketch = nullptr; - uint8_t union_lg_k = 0; - uint64_t union_seed = DS_THETA_DEFAULT_SEED; - //! A merged input sketch was empty: with no other input, the result is an empty sketch rather than NULL - bool saw_empty_sketch = false; - - ~DSTupleState() - { - delete update_sketch; - delete union_sketch; - } - - void CreateUpdateSketch(const DSTupleBindData &bind_data) - { - if (!update_sketch) - { - DSTupleUpdate::builder b{DSTupleDoublePolicy(bind_data.mode)}; - b.set_lg_k(bind_data.lg_k); - b.set_seed(bind_data.seed); - b.set_p(bind_data.p); - update_sketch = new DSTupleUpdate(b.build()); - } - } - - // Creates the union, or grows it to a larger lg_k (re-adding what it holds so far). - void EnsureUnion(uint8_t lg_k, uint64_t seed, DSTupleMode mode) - { - if (union_sketch && lg_k <= union_lg_k) - { - return; - } - if (union_sketch) - { - seed = union_seed; - } - DSTupleUnion::builder b{DSTupleDoublePolicy(mode)}; - b.set_lg_k(lg_k); - b.set_seed(seed); - auto grown = new DSTupleUnion(b.build()); - if (union_sketch) - { - grown->update(union_sketch->get_result()); - delete union_sketch; - } - union_sketch = grown; - union_lg_k = lg_k; - union_seed = seed; - } - - bool HasData() const - { - return update_sketch || union_sketch; - } - - uint64_t Seed(const DSTupleBindData &bind_data) const - { - return union_sketch ? union_seed : bind_data.seed; - } - - uint8_t LgK(const DSTupleBindData &bind_data) const - { - return union_sketch ? union_lg_k : bind_data.lg_k; - } - }; + auto result = make_uniq(); + result->mode = DSTupleBindMode(input); + DSThetaRecordConstants(input, *result, 0, 1); + return std::move(result); + } void DSTupleThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b) { throw InvalidInputException("%s: cannot combine Tuple sketches built with different seeds (seed hashes %d and %d)", function_name, a, b); } - struct DSTupleOperationBase + // ============================================================ + // Aggregate states (see datasketches_theta_arena.hpp) + // ============================================================ + + struct DSTupleTraits { - template - static void Initialize(STATE &state) - { - state.update_sketch = nullptr; - state.union_sketch = nullptr; - state.union_lg_k = 0; - state.union_seed = DS_THETA_DEFAULT_SEED; - state.saw_empty_sketch = false; - } + using Alloc = DSTupleAllocator; + using Entry = std::pair; + using Update = DSTupleUpdate; + using Compact = DSTupleCompact; + using Union = DSTupleUnion; + using Intersection = DSTupleIntersection; + static constexpr bool SET_ONLY = false; + static constexpr bool HAS_NUM_VALUES = false; + static constexpr const char *KIND = "Tuple"; + static constexpr const char *BUILD_NAME = "datasketch_tuple"; + static constexpr const char *INTERSECT_NAME = "datasketch_tuple_intersect_agg"; - template - static void Destroy(STATE &state, AggregateInputData &) + static DSTupleDoublePolicy Policy(const DSThetaParams ¶ms) { - delete state.update_sketch; - delete state.union_sketch; - state.update_sketch = nullptr; - state.union_sketch = nullptr; + return DSTupleDoublePolicy(static_cast(params.mode)); } - static bool IgnoreNull() { return true; } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static uint64_t Key(const Entry &entry) { return entry.first; } + static idx_t SummarySize(const DSThetaParams &) { return sizeof(double); } + static void WriteSummary(const Entry &entry, data_ptr_t data, const DSThetaParams &) { - target.saw_empty_sketch = target.saw_empty_sketch || source.saw_empty_sketch; - if (!source.HasData()) - return; - auto &bind_data = aggr_input_data.bind_data->template Cast(); - auto source_seed = source.Seed(bind_data); - if (target.HasData() && ThetaSeedHash(source_seed) != ThetaSeedHash(target.Seed(bind_data))) - { - DSTupleThrowSeedMismatch("datasketch_tuple", ThetaSeedHash(target.Seed(bind_data)), ThetaSeedHash(source_seed)); - } - auto lg_k = MaxValue(source.LgK(bind_data), target.LgK(bind_data)); - target.EnsureUnion(lg_k, target.HasData() ? target.Seed(bind_data) : source_seed, bind_data.mode); - if (target.update_sketch) - { - target.union_sketch->update(target.update_sketch->compact()); - delete target.update_sketch; - target.update_sketch = nullptr; - } - if (source.update_sketch) - target.union_sketch->update(source.update_sketch->compact()); - if (source.union_sketch) - target.union_sketch->update(source.union_sketch->get_result()); + memcpy(data, &entry.second, sizeof(double)); } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + static Entry MakeEntry(uint64_t key, const_data_ptr_t data, const DSThetaParams &) { - if (state.union_sketch) - { - target = DSTupleDoubleTraits::Serialize(finalize_data.result, state.union_sketch->get_result()); - } - else if (state.update_sketch) - { - target = DSTupleDoubleTraits::Serialize(finalize_data.result, state.update_sketch->compact()); - } - else if (state.saw_empty_sketch) - { - auto &bind_data = finalize_data.input.bind_data->template Cast(); - target = DSTupleDoubleTraits::Serialize(finalize_data.result, DSTupleEmptySketch(bind_data.lg_k, bind_data.seed)); - } - else - { - // no input rows: NULL, as for the other sketches' aggregates - finalize_data.ReturnNull(); - } + double value; + memcpy(&value, data, sizeof(double)); + return Entry(key, value); } - }; - - struct DSTupleCreateOperation : DSTupleOperationBase - { - template - static void Operation(STATE &state, const KEY_TYPE &key, const VALUE_TYPE &value, AggregateBinaryInput &idata) + static void Fold(data_ptr_t data, const Entry &incoming, const DSThetaParams ¶ms) { - auto &bind_data = idata.input.bind_data->template Cast(); - state.CreateUpdateSketch(bind_data); - if constexpr (std::is_same_v) - { - state.update_sketch->update(key.GetData(), key.GetSize(), value); - } - else - { - state.update_sketch->update(key, value); - } + double value; + memcpy(&value, data, sizeof(double)); + Policy(params)(value, incoming.second); + memcpy(data, &value, sizeof(double)); } - }; + static idx_t PayloadSize(const DSThetaParams &) { return sizeof(double); } + static uint8_t NumValues(const Compact &) { return 1; } - struct DSTupleMergeOperation : DSTupleOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + template + static void Apply(Update &sketch, const_data_ptr_t key, uint32_t length, const_data_ptr_t payload, const DSThetaParams &) { - auto &bind_data = idata.input.bind_data->template Cast(); - auto sketch = DSTupleDoubleTraits::Deserialize(blob); - if (sketch.is_empty()) + double value; + memcpy(&value, payload, sizeof(double)); + if constexpr (std::is_same::value) { - state.saw_empty_sketch = true; - return; + sketch.update(key, length, value); } - if (state.union_sketch && sketch.get_seed_hash() != ThetaSeedHash(state.union_seed)) - { - DSTupleThrowSeedMismatch("datasketch_tuple", ThetaSeedHash(state.union_seed), sketch.get_seed_hash()); - } - auto lg_k = bind_data.infer_lg_k ? MaxValue(bind_data.lg_k, ThetaInferLgK(sketch)) : bind_data.lg_k; - state.EnsureUnion(lg_k, ThetaSeedForHash(sketch.get_seed_hash()), bind_data.mode); - ThetaGuard([&]() - { state.union_sketch->update(sketch); }, - "Tuple"); - } - - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) - { - // Merging the same sketch repeatedly is not idempotent for mode sum: merge it count times. - for (idx_t i = 0; i < count; i++) + else { - Operation(state, input, unary_input); + KEY typed; + memcpy(&typed, key, sizeof(KEY)); + sketch.update(typed, value); } } - }; - - // ============================================================ - // Intersection Aggregate - // ============================================================ - - struct DSTupleIntersectBindData : public FunctionData - { - explicit DSTupleIntersectBindData(DSTupleMode mode) : mode(mode) {} - unique_ptr Copy() const override { return make_uniq(mode); } - bool Equals(const FunctionData &other) const override { return mode == other.Cast().mode; } - DSTupleMode mode; - }; - - unique_ptr DSTupleBindIntersect(BindAggregateFunctionInput &input) - { - return make_uniq(DSTupleBindMode(input)); - } - struct DSTupleIntersectState - { - DSTupleIntersection *intersection = nullptr; - uint64_t seed = DS_THETA_DEFAULT_SEED; - bool saw_empty = false; - - ~DSTupleIntersectState() + static Update NewUpdate(const DSTupleBindData &bind, const DSThetaParams ¶ms) { - delete intersection; + DSTupleUpdate::builder b{Policy(params), Alloc(DSArenaMode())}; + b.set_lg_k(bind.lg_k); + b.set_seed(bind.seed); + b.set_p(bind.p); + return b.build(); } - void Add(const DSTupleCompact &sketch, DSTupleMode mode) + static Union NewUnion(uint8_t lg_k, uint64_t seed, const DSThetaParams ¶ms) { - if (sketch.is_empty()) - { - saw_empty = true; - return; - } - if (!intersection) - { - seed = ThetaSeedForHash(sketch.get_seed_hash()); - intersection = new DSTupleIntersection(seed, DSTupleDoublePolicy(mode)); - } - else if (sketch.get_seed_hash() != ThetaSeedHash(seed)) - { - DSTupleThrowSeedMismatch("datasketch_tuple_intersect_agg", ThetaSeedHash(seed), sketch.get_seed_hash()); - } - ThetaGuard([&]() - { intersection->update(sketch); }, - "Tuple"); + DSTupleUnion::builder b{Policy(params), Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build(); } - }; - struct DSTupleIntersectOperation - { - template - static void Initialize(STATE &state) + static Intersection NewIntersection(uint64_t seed, const DSThetaParams ¶ms) { - state.intersection = nullptr; - state.seed = DS_THETA_DEFAULT_SEED; - state.saw_empty = false; + return Intersection(seed, Policy(params), Alloc(DSArenaMode())); } - template - static void Destroy(STATE &state, AggregateInputData &) + static Compact Deserialize(const string_t &blob) { - delete state.intersection; - state.intersection = nullptr; + auto seed = DSTupleSeedFor(blob); + return ThetaGuard([&]() + { return Compact::deserialize(blob.GetData(), blob.GetSize(), seed, datasketches::serde(), Alloc(DSArenaMode())); }, + KIND); } - static bool IgnoreNull() { return true; } - - template - static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + static Compact Empty(uint8_t lg_k, uint64_t seed, const DSThetaParams &) { - auto &bind_data = idata.input.bind_data->template Cast(); - state.Add(DSTupleDoubleTraits::Deserialize(blob), bind_data.mode); + DSTupleUpdate::builder b{DSTupleDoublePolicy(), Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build().compact(); } - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static auto Serialize(const Compact &sketch) -> decltype(sketch.serialize()) { - for (idx_t i = 0; i < count; i++) - { - Operation(state, input, unary_input); - } + return sketch.serialize(); } - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static void ThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b) { - target.saw_empty = target.saw_empty || source.saw_empty; - if (source.intersection) - { - auto &bind_data = aggr_input_data.bind_data->template Cast(); - target.Add(source.intersection->get_result(), bind_data.mode); - } + DSTupleThrowSeedMismatch(function_name, a, b); } - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) - { - if (state.saw_empty) - { - target = DSTupleDoubleTraits::Serialize(finalize_data.result, DSTupleEmptySketch(DS_THETA_DEFAULT_LG_K, state.seed)); - } - else if (state.intersection) - { - target = DSTupleDoubleTraits::Serialize(finalize_data.result, state.intersection->get_result()); - } - else - { - finalize_data.ReturnNull(); - } - } + static void ThrowNumValues(const char *, idx_t, idx_t) {} }; + template + using DSTupleState = DSThetaFamilyState; + using DSTupleMergeState = DSTupleState; + using DSTupleIntersectState = DSThetaFamilyIntersectState; + // merging the same sketch repeatedly is not idempotent for mode sum + using DSTupleMergeOperation = DSThetaFamilyOperation; + using DSTupleIntersectOperation = DSThetaFamilyIntersectOperation; + // ============================================================ // Scalar Functions // ============================================================ @@ -671,9 +464,10 @@ namespace duckdb { for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::BinaryAggregate( - key_type, LogicalType::DOUBLE, sketch_type); - fun.SetStateDestructorCallback(AggregateFunction::StateDestroy); + using STATE = DSTupleState; + using OP = DSThetaFamilyCreatePairOperation; + auto fun = AggregateFunction::BinaryAggregate(key_type, LogicalType::DOUBLE, sketch_type); + DSThetaSetStateExport(fun); auto signature = DSTupleSignature(with_lg_k, {key_type, LogicalType::DOUBLE}, sketch_type); DSTupleAddMode(signature); DSTupleAddSeedAndP(signature); @@ -681,12 +475,12 @@ namespace duckdb if (with_lg_k) { fun.SetBindCallback(DSTupleBindBuild); - DSReadDataArgumentsAt<1, 2, DSTupleState, T, double, DSTupleCreateOperation>(fun); + DSReadDataArgumentsAt<1, 2, STATE, T, double, OP>(fun); } else { fun.SetBindCallback(DSTupleBindBuild); - DSReadDataArgumentsAt<0, 1, DSTupleState, T, double, DSTupleCreateOperation>(fun); + DSReadDataArgumentsAt<0, 1, STATE, T, double, OP>(fun); } fun.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); set.AddFunction(fun); @@ -748,15 +542,15 @@ namespace duckdb for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); auto signature = DSTupleSignature(with_lg_k, {sketch_type}, sketch_type); DSTupleAddMode(signature); fun.GetSignature() = std::move(signature); if (with_lg_k) { fun.SetBindCallback(DSTupleBindMerge); - DSReadDataArgumentAt<1, DSTupleState, string_t, DSTupleMergeOperation>(fun); + DSReadDataArgumentAt<1, DSTupleMergeState, string_t, DSTupleMergeOperation>(fun); descs.push_back(DSTupleDescription("Merges (unions) Tuple sketches into one with the given log2 of nominal entries (lg_k); the " "summaries of a key are combined by mode := 'sum' (default), 'min', 'max' or 'one'", {ex.QueryPartials("datasketch_tuple(16, sketch)"), ex.QueryPartials("datasketch_tuple(16, sketch, mode := 'max')")}, @@ -765,7 +559,7 @@ namespace duckdb else { fun.SetBindCallback(DSTupleBindMerge); - DSReadDataArgumentAt<0, DSTupleState, string_t, DSTupleMergeOperation>(fun); + DSReadDataArgumentAt<0, DSTupleMergeState, string_t, DSTupleMergeOperation>(fun); descs.push_back(DSTupleDescription("Merges (unions) Tuple sketches into one, keeping as many entries as the largest input " "(at least lg_k 12); the summaries of a key are combined by mode := 'sum' (default), 'min', 'max' or 'one'", {ex.QueryPartials("datasketch_tuple(sketch)"), ex.QueryPartials("datasketch_tuple(sketch, mode := 'max')")}, @@ -780,8 +574,8 @@ namespace duckdb // --- datasketch_tuple_intersect_agg { AggregateFunctionSet set("datasketch_tuple_intersect_agg"); - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); auto signature = DSTupleSignature(false, {sketch_type}, sketch_type); DSTupleAddMode(signature); fun.GetSignature() = std::move(signature); diff --git a/test/sql/datasketch_spill_theta.test b/test/sql/datasketch_spill_theta.test new file mode 100644 index 0000000..3feed4b --- /dev/null +++ b/test/sql/datasketch_spill_theta.test @@ -0,0 +1,129 @@ +# name: test/sql/datasketch_spill_theta.test +# description: Theta, Tuple and array-of-doubles aggregate states live in the arena, spill, and give the results they give in memory +# group: [datasketches] + +require datasketches + +require skip_reload + +statement ok +SET threads=4; + +statement ok +SET temp_directory='{TEST_DIR}/datasketch_spill_theta'; + +statement ok +SET preserve_insertion_order=false; + +statement ok +SET memory_limit='4GB'; + +# Few rows per group (buffered states) and many rows per group (sketches), several input types. Summaries are +# small integers, so sums do not depend on the order they are added in. The expected results are computed in +# memory and kept as hashes of the sketches. +statement ok +CREATE VIEW small_rows AS SELECT i % 300000 AS g, i AS v, (i % 1000)::VARCHAR AS s, (i % 7)::DOUBLE AS d FROM range(900000) t(i); + +statement ok +CREATE VIEW large_rows AS SELECT i % 40000 AS g, i AS v, (i % 100000)::VARCHAR AS s, (i % 7)::DOUBLE AS d FROM range(4000000) t(i); + +statement ok +CREATE TABLE small_expected AS SELECT g, hash(datasketch_theta(v)) AS th, hash(datasketch_theta(s)) AS ths, hash(datasketch_theta(10, d::INTEGER, seed := 42, p := 0.5)) AS thp, hash(datasketch_tuple(v, d)) AS tu, hash(datasketch_tuple(s, d, mode := 'max')) AS tus, hash(datasketch_aod(v::INTEGER, [d, 1.0])) AS ao FROM small_rows GROUP BY g; + +statement ok +CREATE TABLE large_expected AS SELECT g, hash(datasketch_theta(v)) AS th, hash(datasketch_theta(s)) AS ths, hash(datasketch_theta(5, v::DOUBLE)) AS thd, hash(datasketch_tuple(v, d)) AS tu, hash(datasketch_tuple(s, d, mode := 'min')) AS tus, hash(datasketch_aod(v::INTEGER, [d, 1.0])) AS ao FROM large_rows GROUP BY g; + +# Partial sketches of each group, which overlap, for the merge and intersection aggregates +statement ok +CREATE TABLE parts AS +SELECT g, p, datasketch_theta(v) AS th, datasketch_tuple(v, d) AS tu, datasketch_aod(v, [d]) AS ao +FROM (SELECT i % 100000 AS g, (i // 100000) % 3 AS p, (i // 100000) % 4 + (i % 100000) * 10 AS v, (i % 5)::DOUBLE AS d FROM range(1200000) t(i)) +GROUP BY g, p; + +statement ok +CREATE TABLE merged_expected AS SELECT g, hash(datasketch_theta(th)) AS th, hash(datasketch_theta(14, th)) AS thk, hash(datasketch_theta_intersect_agg(th)) AS thi, hash(datasketch_tuple(tu)) AS tu, hash(datasketch_tuple(16, tu, mode := 'max')) AS tuk, hash(datasketch_tuple_intersect_agg(tu, mode := 'min')) AS tui, hash(datasketch_aod(ao)) AS ao, hash(datasketch_aod_intersect_agg(ao)) AS aoi FROM parts GROUP BY g; + +# The merged sketches are what they should be: each group has 4 distinct values, in every part +query IIII +SELECT min(datasketch_theta_estimate(th)), max(datasketch_theta_estimate(th)), min(datasketch_theta_estimate(thi)), max(datasketch_tuple_sum(tu)) +FROM (SELECT datasketch_theta(th) AS th, datasketch_theta_intersect_agg(th) AS thi, datasketch_tuple(tu) AS tu FROM parts GROUP BY g); +---- +4.0 4.0 4.0 48.0 + +statement ok +SET memory_limit='128MB'; + +query I +SELECT count(*) FROM small_expected e JOIN (SELECT g, hash(datasketch_theta(v)) AS th, hash(datasketch_theta(s)) AS ths, hash(datasketch_theta(10, d::INTEGER, seed := 42, p := 0.5)) AS thp, hash(datasketch_tuple(v, d)) AS tu, hash(datasketch_tuple(s, d, mode := 'max')) AS tus, hash(datasketch_aod(v::INTEGER, [d, 1.0])) AS ao FROM small_rows GROUP BY g) a USING (g) WHERE e.th = a.th AND e.ths = a.ths AND e.thp = a.thp AND e.tu = a.tu AND e.tus = a.tus AND e.ao = a.ao; +---- +300000 + +query I +SELECT count(*) FROM merged_expected e JOIN (SELECT g, hash(datasketch_theta(th)) AS th, hash(datasketch_theta(14, th)) AS thk, hash(datasketch_theta_intersect_agg(th)) AS thi, hash(datasketch_tuple(tu)) AS tu, hash(datasketch_tuple(16, tu, mode := 'max')) AS tuk, hash(datasketch_tuple_intersect_agg(tu, mode := 'min')) AS tui, hash(datasketch_aod(ao)) AS ao, hash(datasketch_aod_intersect_agg(ao)) AS aoi FROM parts GROUP BY g) a USING (g) WHERE e.th = a.th AND e.thk = a.thk AND e.thi = a.thi AND e.tu = a.tu AND e.tuk = a.tuk AND e.tui = a.tui AND e.ao = a.ao AND e.aoi = a.aoi; +---- +100000 + +# Large states: one aggregate per query, so that a partition of states fits in the limit +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_theta(v)) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.th = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_theta(s)) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.ths = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_theta(5, v::DOUBLE)) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.thd = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_tuple(v, d)) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.tu = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_tuple(s, d, mode := 'min')) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.tus = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_aod(v::INTEGER, [d, 1.0])) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.ao = a.x; +---- +40000 + +statement ok +SET memory_limit='4GB'; + +# Exported states finalize to the aggregate's result, buffered (small inputs) or not. DuckDB re-binds an exported +# state with its arguments as positional ones, so this works only for aggregates without keyword-only parameters +# (the builds take seed := and p :=, the Tuple merges mode :=); spilling does not re-bind and covers those above. +statement ok +CREATE TABLE stored AS +SELECT n, i % n AS k, datasketch_theta(i) AS th, datasketch_aod(i, [1.0, i::DOUBLE]) AS ao +FROM (VALUES (1), (3), (1000)) n(n), range(20000) t(i) GROUP BY n, k; + +query IIIIII +SELECT bool_and(a), bool_and(b), bool_and(c), bool_and(d), bool_and(e), bool_and(f) FROM ( + SELECT finalize(datasketch_theta(th) EXPORT_STATE) = datasketch_theta(th) AS a, + finalize(datasketch_theta(5, th) EXPORT_STATE) = datasketch_theta(5, th) AS b, + finalize(datasketch_theta_intersect_agg(th) EXPORT_STATE) = datasketch_theta_intersect_agg(th) AS c, + finalize(datasketch_aod(ao) EXPORT_STATE) = datasketch_aod(ao) AS d, + finalize(datasketch_aod(16, ao) EXPORT_STATE) = datasketch_aod(16, ao) AS e, + finalize(datasketch_aod_intersect_agg(ao) EXPORT_STATE) = datasketch_aod_intersect_agg(ao) AS f + FROM stored GROUP BY n, k % 5 +); +---- +true true true true true true + +query III +SELECT bool_and(a), bool_and(b), bool_and(c) FROM ( + SELECT finalize(datasketch_theta(th) EXPORT_STATE) = datasketch_theta(th) AS a, + finalize(datasketch_theta_intersect_agg(th) EXPORT_STATE) = datasketch_theta_intersect_agg(th) AS b, + finalize(datasketch_aod_intersect_agg(ao) EXPORT_STATE) = datasketch_aod_intersect_agg(ao) AS c + FROM parts GROUP BY g % 1000 +); +---- +true true true