diff --git a/src/aod_sketch.cpp b/src/aod_sketch.cpp index df353ce..cefeb33 100644 --- a/src/aod_sketch.cpp +++ b/src/aod_sketch.cpp @@ -4,6 +4,7 @@ // same key are summed element-wise when building, merging, unioning and intersecting. #include "datasketches_tuple_common.hpp" +#include "datasketches_theta_arena.hpp" #include @@ -101,14 +102,6 @@ namespace duckdb using DSAodScalars = DSTupleFunctions; - DSAodCompact DSAodEmptySketch(uint8_t lg_k, uint64_t seed, uint8_t num_values) - { - DSAodUpdate::builder b{datasketches::default_array_tuple_update_policy(num_values)}; - b.set_lg_k(lg_k); - b.set_seed(seed); - return b.build().compact(); - } - // Tracks the number of values of the sketches being combined (empty sketches are ignored). struct DSAodNumValuesTracker { @@ -148,24 +141,7 @@ namespace duckdb // Bind Data // ============================================================ - struct DSAodBindData : public FunctionData - { - unique_ptr Copy() const override - { - return make_uniq(*this); - } - - bool Equals(const FunctionData &other_p) const override - { - auto &other = other_p.Cast(); - return lg_k == other.lg_k && infer_lg_k == other.infer_lg_k && seed == other.seed && p == other.p; - } - - uint8_t lg_k = DS_THETA_DEFAULT_LG_K; - bool infer_lg_k = false; - uint64_t seed = DS_THETA_DEFAULT_SEED; - float p = 1.0f; - }; + using DSAodBindData = DSThetaFamilyBindData; template unique_ptr DSAodBindBuild(BindAggregateFunctionInput &input) @@ -176,6 +152,7 @@ namespace duckdb result->lg_k = DSTupleBindLgK(input, 0); } DSTupleBindSeedAndP(input, result->seed, result->p); + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 3 : 2); return std::move(result); } @@ -191,213 +168,144 @@ namespace duckdb { result->infer_lg_k = true; } + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 2 : 1); return std::move(result); } // ============================================================ - // Build / Union Aggregate + // Aggregate states (see datasketches_theta_arena.hpp) // ============================================================ - struct DSAodState + // Every array (and so every entry) allocates in arena mode, so copies the library makes of entries land in + // the current arena. + struct DSAodStateTraits { - DSAodUpdate *update_sketch = nullptr; - DSAodUnion *union_sketch = nullptr; - uint8_t union_lg_k = 0; - uint64_t union_seed = DS_THETA_DEFAULT_SEED; - //! The number of values per key (0 while unknown) - uint8_t num_values = 0; - //! A merged input sketch was empty: with no other input, the result is an empty sketch rather than NULL - bool saw_empty_sketch = false; + using Alloc = DSAllocator; + using Entry = std::pair; + using Update = DSAodUpdate; + using Compact = DSAodCompact; + using Union = DSAodUnion; + using Intersection = DSAodIntersection; + static constexpr bool SET_ONLY = false; + static constexpr bool HAS_NUM_VALUES = true; + static constexpr const char *KIND = "Array of doubles"; + static constexpr const char *BUILD_NAME = "datasketch_aod"; + static constexpr const char *INTERSECT_NAME = "datasketch_aod_intersect_agg"; - ~DSAodState() + static uint64_t Key(const Entry &entry) { return entry.first; } + static idx_t SummarySize(const DSThetaParams ¶ms) { return sizeof(double) * params.num_values; } + static void WriteSummary(const Entry &entry, data_ptr_t data, const DSThetaParams ¶ms) { - delete update_sketch; - delete union_sketch; + memcpy(data, entry.second.data(), SummarySize(params)); } - - void CheckNumValues(idx_t count, const char *function_name) + static Entry MakeEntry(uint64_t key, const_data_ptr_t data, const DSThetaParams ¶ms) { - if (num_values == 0) - { - num_values = static_cast(count); - } - else if (num_values != count) - { - DSAodThrowNumValues(function_name, num_values, count); - } + DSAodArray values(params.num_values, 0.0, Alloc(DSArenaMode())); + memcpy(values.data(), data, SummarySize(params)); + return Entry(key, std::move(values)); } - - void CreateUpdateSketch(const DSAodBindData &bind_data, idx_t count) + // default_array_tuple_union_policy: element-wise sum + static void Fold(data_ptr_t data, const Entry &incoming, const DSThetaParams ¶ms) { - if (num_values != 0 && num_values != count) + for (uint8_t i = 0; i < params.num_values; i++) { - throw InvalidInputException("datasketch_aod: every values list must have the same length (got %llu and %llu)", idx_t(num_values), count); - } - CheckNumValues(count, "datasketch_aod"); - if (!update_sketch) - { - DSAodUpdate::builder b{datasketches::default_array_tuple_update_policy(num_values)}; - b.set_lg_k(bind_data.lg_k); - b.set_seed(bind_data.seed); - b.set_p(bind_data.p); - update_sketch = new DSAodUpdate(b.build()); + double value; + memcpy(&value, data + i * sizeof(double), sizeof(double)); + value += incoming.second[i]; + memcpy(data + i * sizeof(double), &value, sizeof(double)); } } + static idx_t PayloadSize(const DSThetaParams ¶ms) { return SummarySize(params); } + static uint8_t NumValues(const Compact &sketch) { return sketch.get_num_values(); } - // Creates the union, or grows it to a larger lg_k (re-adding what it holds so far). - void EnsureUnion(uint8_t lg_k, uint64_t seed) + template + static void Apply(Update &sketch, const_data_ptr_t key, uint32_t length, const_data_ptr_t payload, const DSThetaParams ¶ms) { - if (union_sketch && lg_k <= union_lg_k) + double values[DS_AOD_MAX_VALUES]; + memcpy(values, payload, SummarySize(params)); + const double *update = values; + if constexpr (std::is_same::value) { - return; + sketch.update(key, length, update); } - if (union_sketch) - { - seed = union_seed; - } - DSAodUnion::builder b{datasketches::default_array_tuple_union_policy(num_values)}; - b.set_lg_k(lg_k); - b.set_seed(seed); - auto grown = new DSAodUnion(b.build()); - if (union_sketch) + else { - grown->update(union_sketch->get_result()); - delete union_sketch; + KEY typed; + memcpy(&typed, key, sizeof(KEY)); + sketch.update(typed, update); } - union_sketch = grown; - union_lg_k = lg_k; - union_seed = seed; } - bool HasData() const + static Update NewUpdate(const DSAodBindData &bind, const DSThetaParams ¶ms) { - return update_sketch || union_sketch; + DSAodUpdate::builder b{datasketches::default_array_tuple_update_policy(params.num_values, Alloc(DSArenaMode())), + Alloc(DSArenaMode())}; + b.set_lg_k(bind.lg_k); + b.set_seed(bind.seed); + b.set_p(bind.p); + return b.build(); } - uint64_t Seed(const DSAodBindData &bind_data) const + static Union NewUnion(uint8_t lg_k, uint64_t seed, const DSThetaParams ¶ms) { - return union_sketch ? union_seed : bind_data.seed; - } - - uint8_t LgK(const DSAodBindData &bind_data) const - { - return union_sketch ? union_lg_k : bind_data.lg_k; + DSAodUnion::builder b{datasketches::default_array_tuple_union_policy(params.num_values), Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build(); } - }; - struct DSAodOperation - { - template - static void Initialize(STATE &state) + static Intersection NewIntersection(uint64_t seed, const DSThetaParams ¶ms) { - state.update_sketch = nullptr; - state.union_sketch = nullptr; - state.union_lg_k = 0; - state.union_seed = DS_THETA_DEFAULT_SEED; - state.num_values = 0; - state.saw_empty_sketch = false; + return Intersection(seed, DSAodSumPolicy(params.num_values), Alloc(DSArenaMode())); } - template - static void Destroy(STATE &state, AggregateInputData &) + static Compact Deserialize(const string_t &blob) { - delete state.update_sketch; - delete state.union_sketch; - state.update_sketch = nullptr; - state.union_sketch = nullptr; + auto seed = DSTupleSeedFor(blob); + return ThetaGuard([&]() + { return Compact::deserialize(blob.GetData(), blob.GetSize(), seed, Alloc(DSArenaMode())); }, + KIND); } - static bool IgnoreNull() { return true; } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static Compact Empty(uint8_t lg_k, uint64_t seed, const DSThetaParams ¶ms) { - target.saw_empty_sketch = target.saw_empty_sketch || source.saw_empty_sketch; - if (!source.HasData()) - return; - auto &bind_data = aggr_input_data.bind_data->template Cast(); - auto source_seed = source.Seed(bind_data); - if (target.HasData() && ThetaSeedHash(source_seed) != ThetaSeedHash(target.Seed(bind_data))) - { - DSAodThrowSeedMismatch("datasketch_aod", ThetaSeedHash(target.Seed(bind_data)), ThetaSeedHash(source_seed)); - } - target.CheckNumValues(source.num_values, "datasketch_aod"); - auto lg_k = MaxValue(source.LgK(bind_data), target.LgK(bind_data)); - target.EnsureUnion(lg_k, target.HasData() ? target.Seed(bind_data) : source_seed); - if (target.update_sketch) - { - target.union_sketch->update(target.update_sketch->compact()); - delete target.update_sketch; - target.update_sketch = nullptr; - } - if (source.update_sketch) - target.union_sketch->update(source.update_sketch->compact()); - if (source.union_sketch) - target.union_sketch->update(source.union_sketch->get_result()); + DSAodUpdate::builder b{datasketches::default_array_tuple_update_policy(params.num_values, Alloc(DSArenaMode())), + Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build().compact(); } - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + static auto Serialize(const Compact &sketch) -> decltype(sketch.serialize()) { - if (state.union_sketch) - { - target = DSAodTraits::Serialize(finalize_data.result, state.union_sketch->get_result()); - } - else if (state.update_sketch) - { - target = DSAodTraits::Serialize(finalize_data.result, state.update_sketch->compact()); - } - else if (state.saw_empty_sketch) - { - auto &bind_data = finalize_data.input.bind_data->template Cast(); - target = DSAodTraits::Serialize(finalize_data.result, DSAodEmptySketch(bind_data.lg_k, bind_data.seed, MaxValue(state.num_values, 1))); - } - else - { - // no input rows: NULL, as for the other sketches' aggregates - finalize_data.ReturnNull(); - } + return sketch.serialize(); } - // Merging serialized sketches. - template - static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + static void ThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b) { - auto &bind_data = idata.input.bind_data->template Cast(); - auto sketch = DSAodTraits::Deserialize(blob); - if (sketch.is_empty()) - { - state.saw_empty_sketch = true; - return; - } - if (state.union_sketch && sketch.get_seed_hash() != ThetaSeedHash(state.union_seed)) - { - DSAodThrowSeedMismatch("datasketch_aod", ThetaSeedHash(state.union_seed), sketch.get_seed_hash()); - } - state.CheckNumValues(sketch.get_num_values(), "datasketch_aod"); - auto lg_k = bind_data.infer_lg_k ? MaxValue(bind_data.lg_k, ThetaInferLgK(sketch)) : bind_data.lg_k; - state.EnsureUnion(lg_k, ThetaSeedForHash(sketch.get_seed_hash())); - ThetaGuard([&]() - { state.union_sketch->update(sketch); }, - DSAodTraits::KIND); + DSAodThrowSeedMismatch(function_name, a, b); } - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static void ThrowNumValues(const char *function_name, idx_t a, idx_t b) { - // the values of the same key add up, so merging a sketch twice is not the same as once - for (idx_t i = 0; i < count; i++) - { - Operation(state, input, unary_input); - } + DSAodThrowNumValues(function_name, a, b); } }; + template + using DSAodState = DSThetaFamilyState; + using DSAodMergeState = DSAodState; + using DSAodIntersectState = DSThetaFamilyIntersectState; + // the values of the same key add up, so merging a sketch twice is not the same as once + using DSAodOperation = DSThetaFamilyOperation; + using DSAodIntersectOperation = DSThetaFamilyIntersectOperation; + // The update of the build aggregate: (key, values) at KEY_INDEX and KEY_INDEX + 1. template void DSAodBuildUpdate(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, Vector &states, idx_t count) { D_ASSERT(input_count > KEY_INDEX + 1); + using STATE = DSAodState; auto &bind_data = aggr_input_data.bind_data->Cast(); auto &list = inputs[KEY_INDEX + 1]; UnifiedVectorFormat key_format, list_format, child_format, state_format; @@ -408,8 +316,9 @@ namespace duckdb auto keys = UnifiedVectorFormat::GetData(key_format); auto lists = UnifiedVectorFormat::GetData(list_format); auto values = UnifiedVectorFormat::GetData(child_format); - auto state_pointers = UnifiedVectorFormat::GetData(state_format); - std::vector buffer; + auto state_pointers = UnifiedVectorFormat::GetData(state_format); + DSArenaScope scope(aggr_input_data.allocator); + double buffer[DS_AOD_MAX_VALUES]; for (idx_t i = 0; i < count; i++) { auto key_index = key_format.sel->get_index(i); @@ -423,7 +332,6 @@ namespace duckdb { throw InvalidInputException("datasketch_aod: values must have between 1 and %llu elements, got %llu", DS_AOD_MAX_VALUES, entry.length); } - buffer.resize(entry.length); for (idx_t j = 0; j < entry.length; j++) { auto child_index = child_format.sel->get_index(entry.offset + j); @@ -434,138 +342,16 @@ namespace duckdb buffer[j] = values[child_index]; } auto &state = *state_pointers[state_format.sel->get_index(i)]; - state.CreateUpdateSketch(bind_data, entry.length); - if constexpr (std::is_same_v) + if (state.num_values != 0 && state.num_values != entry.length) { - state.update_sketch->update(keys[key_index].GetData(), keys[key_index].GetSize(), buffer.data()); - } - else - { - state.update_sketch->update(keys[key_index], buffer.data()); + throw InvalidInputException("datasketch_aod: every values list must have the same length (got %llu and %llu)", idx_t(state.num_values), entry.length); } + state.num_values = static_cast(entry.length); + auto bytes = DSThetaKeyBytes(keys[key_index]); + state.AddRow(bytes.first, bytes.second, const_data_ptr_cast(buffer), bind_data, aggr_input_data.allocator); } } - // ============================================================ - // Intersection Aggregate - // ============================================================ - - struct DSAodIntersectState - { - DSAodIntersection *intersection = nullptr; - uint64_t seed = DS_THETA_DEFAULT_SEED; - uint8_t num_values = 0; - bool saw_empty = false; - - ~DSAodIntersectState() - { - delete intersection; - } - - void Add(const DSAodCompact &sketch) - { - if (sketch.is_empty()) - { - saw_empty = true; - if (num_values == 0) - { - num_values = sketch.get_num_values(); - } - return; - } - if (!intersection) - { - seed = ThetaSeedForHash(sketch.get_seed_hash()); - num_values = sketch.get_num_values(); - intersection = new DSAodIntersection(seed, DSAodSumPolicy(num_values)); - return Update(sketch); - } - if (sketch.get_seed_hash() != ThetaSeedHash(seed)) - { - DSAodThrowSeedMismatch("datasketch_aod_intersect_agg", ThetaSeedHash(seed), sketch.get_seed_hash()); - } - if (sketch.get_num_values() != num_values) - { - DSAodThrowNumValues("datasketch_aod_intersect_agg", num_values, sketch.get_num_values()); - } - Update(sketch); - } - - void Update(const DSAodCompact &sketch) - { - ThetaGuard([&]() - { intersection->update(sketch); }, - DSAodTraits::KIND); - } - }; - - struct DSAodIntersectOperation - { - template - static void Initialize(STATE &state) - { - state.intersection = nullptr; - state.seed = DS_THETA_DEFAULT_SEED; - state.num_values = 0; - state.saw_empty = false; - } - - template - static void Destroy(STATE &state, AggregateInputData &) - { - delete state.intersection; - state.intersection = nullptr; - } - - static bool IgnoreNull() { return true; } - - template - static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &) - { - state.Add(DSAodTraits::Deserialize(blob)); - } - - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) - { - for (idx_t i = 0; i < count; i++) - { - Operation(state, input, unary_input); - } - } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &) - { - target.saw_empty = target.saw_empty || source.saw_empty; - if (target.num_values == 0) - { - target.num_values = source.num_values; - } - if (source.intersection) - { - target.Add(source.intersection->get_result()); - } - } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) - { - if (state.saw_empty) - { - target = DSAodTraits::Serialize(finalize_data.result, DSAodEmptySketch(DS_THETA_DEFAULT_LG_K, state.seed, MaxValue(state.num_values, 1))); - } - else if (state.intersection) - { - target = DSAodTraits::Serialize(finalize_data.result, state.intersection->get_result()); - } - else - { - finalize_data.ReturnNull(); - } - } - }; - // ============================================================ // Scalar Functions // ============================================================ @@ -743,8 +529,8 @@ namespace duckdb const auto values_type = LogicalType::LIST(LogicalType::DOUBLE); for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::UnaryAggregate( - key_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate, string_t, string_t, DSAodOperation>(key_type, sketch_type); + DSThetaSetStateExport>(fun); auto signature = DSTupleSignature(with_lg_k, {key_type, values_type}, sketch_type); DSTupleAddSeedAndP(signature); fun.GetSignature() = std::move(signature); @@ -808,13 +594,13 @@ namespace duckdb for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); fun.GetSignature() = DSTupleSignature(with_lg_k, {sketch_type}, sketch_type); if (with_lg_k) { fun.SetBindCallback(DSAodBindMerge); - DSReadDataArgumentAt<1, DSAodState, string_t, DSAodOperation>(fun); + DSReadDataArgumentAt<1, DSAodMergeState, string_t, DSAodOperation>(fun); descs.push_back(DSAodDescription("Merges (unions) array-of-doubles sketches into one with the given log2 of nominal entries (lg_k); " "the values of a key are summed element-wise", {ex.QueryPartials("datasketch_aod(16, sketch)")}, {"lg_k", "sketch"}, {LogicalType::INTEGER, sketch_type})); @@ -822,7 +608,7 @@ namespace duckdb else { fun.SetBindCallback(DSAodBindMerge); - DSReadDataArgumentAt<0, DSAodState, string_t, DSAodOperation>(fun); + DSReadDataArgumentAt<0, DSAodMergeState, string_t, DSAodOperation>(fun); descs.push_back(DSAodDescription("Merges (unions) array-of-doubles sketches into one, keeping as many entries as the largest input " "(at least lg_k 12); the values of a key are summed element-wise", {ex.QueryPartials("datasketch_aod(sketch)")}, {"sketch"}, {sketch_type})); @@ -836,8 +622,8 @@ namespace duckdb // --- datasketch_aod_intersect_agg { AggregateFunctionSet set("datasketch_aod_intersect_agg"); - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); DSSetArguments(fun, {sketch_type}); fun.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); set.AddFunction(fun); diff --git a/src/include/datasketches_theta_arena.hpp b/src/include/datasketches_theta_arena.hpp new file mode 100644 index 0000000..4922ac3 --- /dev/null +++ b/src/include/datasketches_theta_arena.hpp @@ -0,0 +1,1460 @@ +#pragma once + +// Arena states of the Theta-family aggregates (theta_sketch.cpp, tuple_sketch.cpp, aod_sketch.cpp). +// +// The states keep everything in the aggregate arena (DSAllocator's arena mode), have no destructor and export +// themselves as BLOBs, so DuckDB counts them toward memory_limit and can spill them. Every sketch object here +// allocates in arena mode; where its memory lands is decided by the arena of the scope it allocates in: +// - the callback's arena (DSArenaScope), for what a state keeps; +// - a per-thread scratch arena (DSScratchScope), for temporaries: input sketches, results, serialized bytes. +// Copies made by the library use the source object's allocator, which in arena mode means "the current arena", so a +// temporary in the scratch arena copied while the callback's arena is current lands in the state, and a state copied +// while the scratch arena is current lands in the scratch arena. Nothing of a temporary may outlive its scratch +// scope, and nothing a state keeps may be created under one. +// +// DuckDB creates a state per input row when grouping reduces little, and a Theta update sketch or union allocates a +// hash table up front (about 1 KB at lg_k 12). So the states buffer small inputs: +// - the build aggregates keep their first raw input rows and replay them, in order, into an update sketch when more +// arrive (or into a temporary one when a result is needed), so the result is the one an update sketch of all rows +// gives; +// - unions keep up to BUFFERED distinct keys with their combined summaries while that is no more than the union's +// nominal size k. A union's result is the k smallest keys below its theta (the minimum theta of its inputs), each +// with the summaries of its inputs combined in the order they arrived; with at most k keys nothing is trimmed, so +// the buffer holds exactly what the union would hold, and replaying it into a union gives the same union. +// Intersections keep their current result (an intersection only shrinks) and intersect it with each input in a +// temporary intersection, overwriting the result in place: the library's intersection reallocates its table on every +// update, which in an arena would never be freed. +// +// A TRAITS class describes a sketch family: +// using Entry, Alloc (DSAllocator), Update, Compact, Union, Intersection; +// using BindData; // DSThetaFamilyBindData +// static constexpr bool SET_ONLY; // no summaries: raw buffers can be concatenated +// static constexpr const char *KIND, *BUILD_NAME, *INTERSECT_NAME; +// static uint64_t Key(const Entry &); +// static idx_t SummarySize(const DSThetaParams &); +// static void WriteSummary(const Entry &, data_ptr_t, const DSThetaParams &); +// static Entry MakeEntry(uint64_t key, const_data_ptr_t summary, const DSThetaParams &); +// static void Fold(data_ptr_t summary, const Entry &incoming, const DSThetaParams &); // union policy +// static idx_t PayloadSize(const DSThetaParams &); // raw row value bytes +// template static void Apply(Update &, const_data_ptr_t key, uint32_t length, const_data_ptr_t payload, +// const DSThetaParams &); // update with a raw row +// static Update NewUpdate(const BindData &, const DSThetaParams &); +// static Union NewUnion(uint8_t lg_k, uint64_t seed, const DSThetaParams &); +// static Intersection NewIntersection(uint64_t seed, const DSThetaParams &); +// static Compact Deserialize(const string_t &blob); +// static Compact Empty(uint8_t lg_k, uint64_t seed, const DSThetaParams &); +// static auto Serialize(const Compact &); // the stored format +// static void ThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b); +// static void ThrowNumValues(const char *function_name, idx_t a, idx_t b); // AoD only + +#include "datasketches_theta_common.hpp" +#include "duckdb/common/vector/flat_vector.hpp" +#include "duckdb/common/vector/string_vector.hpp" +#include "duckdb/storage/arena_allocator.hpp" + +#include + +#include +#include + +namespace duckdb +{ + // ============================================================ + // Scratch arena and arena buffers + // ============================================================ + + // Makes the arena allocators allocate from this thread's scratch arena while the scope lasts. The arena is + // reset when the outermost scope ends, so temporaries cost a pointer bump instead of a malloc and free. + struct DSScratchScope + { + DSScratchScope() : scope(Arena()) + { + depth++; + } + ~DSScratchScope() + { + if (--depth == 0) + { + auto &arena = Arena(); + if (arena.SizeInBytes() > MAX_RETAINED) + { + arena.Destroy(); + } + else + { + arena.Reset(); + } + } + } + DSScratchScope(const DSScratchScope &) = delete; + DSScratchScope &operator=(const DSScratchScope &) = delete; + + static ArenaAllocator &Arena() + { + static thread_local ArenaAllocator arena(Allocator::DefaultAllocator()); + return arena; + } + + private: + static constexpr idx_t MAX_RETAINED = idx_t(1) << 20; + static inline thread_local idx_t depth = 0; + DSArenaScope scope; + }; + + // Bytes in the aggregate arena, in an array that doubles as it fills. Memory it gives up stays in the arena. + struct DSArenaBytes + { + data_ptr_t data; + uint32_t size; + uint32_t capacity; + + void Initialize() + { + data = nullptr; + size = 0; + capacity = 0; + } + + // Appends n bytes, returning where they go. + data_ptr_t Grow(idx_t n, ArenaAllocator &arena) + { + const idx_t needed = idx_t(size) + n; + if (needed > NumericLimits::Maximum()) + { + throw InvalidInputException("Theta sketch aggregate state is too large"); + } + if (needed > capacity) + { + const idx_t new_capacity = MinValue(MaxValue(idx_t(capacity) * 2, needed), NumericLimits::Maximum()); + auto new_data = arena.AllocateAligned(new_capacity); + if (size) + { + memcpy(new_data, data, size); + } + data = new_data; + capacity = static_cast(new_capacity); + } + auto result = data + size; + size = static_cast(needed); + return result; + } + + void Append(const void *source, idx_t n, ArenaAllocator &arena) + { + if (n) + { + memcpy(Grow(n, arena), source, n); + } + } + }; + + // Writes and reads the exported states. + struct DSThetaWriter + { + std::vector bytes; + + template + void Put(const T &value) + { + auto p = reinterpret_cast(&value); + bytes.insert(bytes.end(), p, p + sizeof(T)); + } + void PutBytes(const void *data, idx_t size) + { + auto p = reinterpret_cast(data); + bytes.insert(bytes.end(), p, p + size); + } + // A length-prefixed block. + template + void PutBlock(const BYTES &block) + { + Put(static_cast(block.size())); + PutBytes(block.data(), block.size()); + } + }; + + struct DSThetaReader + { + DSThetaReader(const string_t &blob, const char *kind) + : data(const_data_ptr_cast(blob.GetData())), size(blob.GetSize()), kind(kind) {} + + const_data_ptr_t data; + idx_t size; + idx_t pos = 0; + const char *kind; + + void Check(idx_t n) const + { + if (n > size - pos) + { + throw InvalidInputException("Invalid %s sketch aggregate state", kind); + } + } + template + T Get() + { + Check(sizeof(T)); + T value; + memcpy(&value, data + pos, sizeof(T)); + pos += sizeof(T); + return value; + } + const_data_ptr_t GetBytes(idx_t n) + { + Check(n); + auto result = data + pos; + pos += n; + return result; + } + string_t GetBlock() + { + auto n = Get(); + auto p = GetBytes(n); + return string_t(const_char_ptr_cast(p), n); + } + void Fail() const + { + throw InvalidInputException("Invalid %s sketch aggregate state", kind); + } + void End() const + { + if (pos != size) + { + Fail(); + } + } + }; + + // ============================================================ + // Bind data and parameters + // ============================================================ + + // The bind data of every Theta-family aggregate. `constants` are the constant arguments (lg_k, seed, p, mode) + // by argument index, which the exported state records so that re-binding it gets the same bind data. + struct DSThetaFamilyBindData : public FunctionData + { + unique_ptr Copy() const override + { + return make_uniq(*this); + } + + bool Equals(const FunctionData &other_p) const override + { + auto &other = other_p.Cast(); + return lg_k == other.lg_k && infer_lg_k == other.infer_lg_k && seed == other.seed && p == other.p && mode == other.mode; + } + + //! lg_k of the sketches built (or the minimum lg_k of the union when infer_lg_k is set) + uint8_t lg_k = DS_THETA_DEFAULT_LG_K; + //! Merging: grow the union to the lg_k of the sketches merged + bool infer_lg_k = false; + //! Building: the hash seed and the sampling probability + uint64_t seed = DS_THETA_DEFAULT_SEED; + float p = 1.0f; + //! Tuple sketches: how summaries combine (DSTupleMode) + uint8_t mode = 0; + vector> constants; + }; + + // Records the constant arguments of a call: every argument but the data arguments [data_begin, data_end). + inline void DSThetaRecordConstants(BindAggregateFunctionInput &input, DSThetaFamilyBindData &bind_data, idx_t data_begin, idx_t data_end) + { + auto &arguments = input.GetArguments(); + for (idx_t i = 0; i < arguments.size(); i++) + { + if (i >= data_begin && i < data_end) + { + continue; + } + bind_data.constants.emplace_back(i, input.GetNonNullConstant(i)); + } + } + + inline const DSThetaFamilyBindData &DSThetaBind(optional_ptr bind_data) + { + return bind_data->Cast(); + } + + inline uint8_t DSThetaMode(optional_ptr bind_data) + { + return bind_data ? bind_data->Cast().mode : 0; + } + + // What the sketch operations need besides the bind data: the summary mode and the number of values (AoD). + struct DSThetaParams + { + uint8_t mode = 0; + uint8_t num_values = 1; + }; + + // A sketch over entries held in a vector, in the shape the library's unions and intersections read. + template + struct DSThetaEntriesView + { + using Entry = typename TRAITS::Entry; + using EntryAlloc = typename std::allocator_traits::template rebind_alloc; + using Entries = std::vector; + + DSThetaEntriesView(bool empty, uint64_t seed, uint64_t theta) + : entries(EntryAlloc(DSArenaMode())), empty(empty), seed_hash(ThetaSeedHash(seed)), theta(theta) {} + + // Makes the entries from count buffered entries (key, then summary). + void Load(const_data_ptr_t data, idx_t count, const DSThetaParams ¶ms) + { + const idx_t stride = sizeof(uint64_t) + TRAITS::SummarySize(params); + entries.reserve(count); + for (idx_t i = 0; i < count; i++) + { + uint64_t key; + memcpy(&key, data + i * stride, sizeof(key)); + entries.push_back(TRAITS::MakeEntry(key, data + i * stride + sizeof(uint64_t), params)); + } + } + + bool is_empty() const { return empty; } + bool is_ordered() const { return false; } + uint16_t get_seed_hash() const { return seed_hash; } + uint64_t get_theta64() const { return theta; } + uint32_t get_num_retained() const { return static_cast(entries.size()); } + typename Entries::const_iterator begin() const { return entries.begin(); } + typename Entries::const_iterator end() const { return entries.end(); } + + Entries entries; + bool empty; + uint16_t seed_hash; + uint64_t theta; + }; + + // Writes the entries of a sketch (key, then summary). + template + void DSThetaWriteEntries(const SKETCH &sketch, DSThetaWriter &writer, const DSThetaParams ¶ms) + { + vector summary(TRAITS::SummarySize(params)); + for (const auto &entry : sketch) + { + writer.Put(TRAITS::Key(entry)); + TRAITS::WriteSummary(entry, summary.data(), params); + writer.PutBytes(summary.data(), summary.size()); + } + } + + // ============================================================ + // Unions + // ============================================================ + + // A union in the arena: buffered keys while it holds few, then the library's union. + template + struct DSThetaUnionPart + { + using Union = typename TRAITS::Union; + using Compact = typename TRAITS::Compact; + using View = DSThetaEntriesView; + static constexpr uint32_t BUFFERED = 32; + + Union *sketch; + //! The buffered keys (distinct, each followed by its summary) while there is no sketch + DSArenaBytes entries; + uint32_t count; + uint64_t theta; + uint64_t seed; + uint8_t lg_k; + bool active; + //! A non-empty sketch was added + bool nonempty; + + void Initialize() + { + sketch = nullptr; + entries.Initialize(); + count = 0; + theta = datasketches::theta_constants::MAX_THETA; + seed = DS_THETA_DEFAULT_SEED; + lg_k = 0; + active = false; + nonempty = false; + } + + uint32_t Limit() const + { + return MinValue(BUFFERED, uint32_t(1) << lg_k); + } + + static idx_t Stride(const DSThetaParams ¶ms) + { + return sizeof(uint64_t) + TRAITS::SummarySize(params); + } + + uint64_t KeyAt(idx_t i, const DSThetaParams ¶ms) const + { + uint64_t key; + memcpy(&key, entries.data + i * Stride(params), sizeof(key)); + return key; + } + + idx_t Find(uint64_t key, const DSThetaParams ¶ms) const + { + for (idx_t i = 0; i < count; i++) + { + if (KeyAt(i, params) == key) + { + return i; + } + } + return DConstants::INVALID_INDEX; + } + + // The buffered keys as a sketch (allocated in the current, scratch, arena). + View Buffered(const DSThetaParams ¶ms) const + { + View view(false, seed, theta); + view.Load(entries.data, count, params); + return view; + } + + // The library's EnsureUnion: creates the union, or grows it to a larger lg_k. + void Ensure(uint8_t new_lg_k, uint64_t new_seed, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + if (!active) + { + Initialize(); + active = true; + lg_k = new_lg_k; + seed = new_seed; + return; + } + if (new_lg_k <= lg_k) + { + return; + } + lg_k = new_lg_k; + if (sketch) + { + // re-add what the union holds to a larger one + DSScratchScope scratch; + auto held = sketch->get_result(); + DSArenaScope back(arena); + auto grown = arena.Make(TRAITS::NewUnion(lg_k, seed, params)); + grown->update(held); + sketch = grown; + } + } + + // Buffers a sketch's entries if the buffer stays within its limit. + template + bool TryBuffer(const SKETCH &incoming, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + const uint64_t new_theta = MinValue(theta, incoming.get_theta64()); + const uint32_t limit = Limit(); + idx_t kept = 0; + for (idx_t i = 0; i < count; i++) + { + kept += KeyAt(i, params) < new_theta; + } + idx_t added = 0; + for (const auto &entry : incoming) + { + const auto key = TRAITS::Key(entry); + if (key < new_theta && Find(key, params) == DConstants::INVALID_INDEX && kept + ++added > limit) + { + return false; + } + } + // drop the keys at or above the new theta: the union's result leaves them out + const idx_t stride = Stride(params); + if (new_theta < theta) + { + uint32_t write = 0; + for (idx_t i = 0; i < count; i++) + { + if (KeyAt(i, params) < new_theta) + { + if (write != i) + { + memmove(entries.data + write * stride, entries.data + i * stride, stride); + } + write++; + } + } + count = write; + entries.size = static_cast(count * stride); + theta = new_theta; + } + for (const auto &entry : incoming) + { + const auto key = TRAITS::Key(entry); + if (key >= theta) + { + continue; + } + auto index = Find(key, params); + if (index != DConstants::INVALID_INDEX) + { + TRAITS::Fold(entries.data + index * stride + sizeof(uint64_t), entry, params); + continue; + } + auto slot = entries.Grow(stride, arena); + memcpy(slot, &key, sizeof(key)); + TRAITS::WriteSummary(entry, slot + sizeof(uint64_t), params); + count++; + } + return true; + } + + // Replaces the buffer with the library's union. + void Promote(const DSThetaParams ¶ms, ArenaAllocator &arena) + { + D_ASSERT(!sketch); + auto promoted = arena.Make(TRAITS::NewUnion(lg_k, seed, params)); + if (nonempty) + { + DSScratchScope scratch; + auto buffered = Buffered(params); + DSArenaScope back(arena); + promoted->update(buffered); + } + sketch = promoted; + entries.Initialize(); + count = 0; + } + + // EnsureUnion(lg_k, seed), then the union's update with `incoming`, which is read only. It may live in the + // scratch arena or another state's arena: what the union keeps of it is copied into `arena`. + template + void Add(const SKETCH &incoming, uint8_t new_lg_k, uint64_t new_seed, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + DSArenaScope scope(arena); + Ensure(new_lg_k, new_seed, params, arena); + if (incoming.is_empty()) + { + return; + } + if (!sketch) + { + if (TryBuffer(incoming, params, arena)) + { + nonempty = true; + return; + } + Promote(params, arena); + } + nonempty = true; + ThetaGuard([&]() + { sketch->update(incoming); }, + TRAITS::KIND); + } + + // Adds another union (EnsureUnion(lg_k, seed), then the update with its result). + void AddUnion(const DSThetaUnionPart &source, uint8_t new_lg_k, uint64_t new_seed, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + D_ASSERT(source.active); + if (!active && source.sketch && source.lg_k == new_lg_k) + { + // a copy allocates through the source's allocator, which in arena mode is the current (target's) arena + DSArenaScope scope(arena); + Initialize(); + active = true; + lg_k = source.lg_k; + seed = source.seed; + nonempty = source.nonempty; + sketch = arena.Make(*source.sketch); + return; + } + DSScratchScope scratch; + if (source.sketch) + { + auto result = source.sketch->get_result(); + Add(result, new_lg_k, new_seed, params, arena); + } + else if (source.nonempty) + { + Add(source.Buffered(params), new_lg_k, new_seed, params, arena); + } + else + { + DSArenaScope back(arena); + Ensure(new_lg_k, new_seed, params, arena); + } + } + + // The union's result, in the current (scratch) arena. + Compact Result(const DSThetaParams ¶ms) const + { + if (sketch) + { + return sketch->get_result(); + } + auto temporary = TRAITS::NewUnion(lg_k, seed, params); + if (nonempty) + { + temporary.update(Buffered(params)); + } + return temporary.get_result(); + } + + // lg_k, seed and flags, then the buffered entries or the union's result. + void Export(DSThetaWriter &writer, const DSThetaParams ¶ms) const + { + writer.Put(sketch ? 2 : 1); + writer.Put(lg_k); + writer.Put(seed); + writer.Put(nonempty); + if (sketch) + { + DSScratchScope scratch; + writer.PutBlock(TRAITS::Serialize(sketch->get_result())); + return; + } + writer.Put(theta); + writer.Put(count); + writer.PutBytes(entries.data, count * Stride(params)); + } + + void Import(DSThetaReader &reader, uint8_t kind, const DSThetaParams ¶ms, ArenaAllocator &arena) + { + const auto new_lg_k = reader.Get(); + const auto new_seed = reader.Get(); + const auto new_nonempty = reader.Get(); + if (new_lg_k < datasketches::theta_constants::MIN_LG_K || new_lg_k > datasketches::theta_constants::MAX_LG_K || new_nonempty > 1) + { + reader.Fail(); + } + if (kind == 2) + { + // the union's result: a union of it is the union (buffered again if it is small) + auto block = reader.GetBlock(); + DSScratchScope scratch; + auto result = TRAITS::Deserialize(block); + { + DSArenaScope back(arena); + Ensure(new_lg_k, new_seed, params, arena); + } + if (new_nonempty) + { + Add(result, new_lg_k, new_seed, params, arena); + } + return; + } + const auto new_theta = reader.Get(); + const auto new_count = reader.Get(); + if (new_count > BUFFERED || new_count > (uint32_t(1) << new_lg_k) || (new_count && !new_nonempty)) + { + reader.Fail(); + } + auto data = reader.GetBytes(new_count * Stride(params)); + Initialize(); + active = true; + lg_k = new_lg_k; + seed = new_seed; + nonempty = new_nonempty; + theta = new_theta; + count = new_count; + entries.Append(data, new_count * Stride(params), arena); + } + }; + + // ============================================================ + // Build and union aggregates + // ============================================================ + + // The state of the aggregates that build a sketch from rows (an update sketch) and of those that merge + // sketches (a union); combining states makes a union, as before. KEY is the type of the rows' keys. + template + struct DSThetaFamilyState + { + using Update = typename TRAITS::Update; + using Compact = typename TRAITS::Compact; + using BindData = DSThetaFamilyBindData; + //! Raw rows kept before an update sketch is built + static constexpr uint32_t RAW_ROWS = 32; + static constexpr uint32_t RAW_BYTES = 1024; + + DSThetaUnionPart part; + //! The update sketch of the rows + Update *update; + //! An update sketch's compact form, from an imported state + Compact *compact; + //! The first rows (key length, key, value) while there is no update sketch + DSArenaBytes raw; + uint32_t raw_rows; + //! The number of values per key of AoD sketches (0 while unknown) + uint8_t num_values; + //! A merged input sketch was empty: with no other input, the result is an empty sketch rather than NULL + bool saw_empty_sketch; + + void Initialize() + { + part.Initialize(); + update = nullptr; + compact = nullptr; + raw.Initialize(); + raw_rows = 0; + num_values = 0; + saw_empty_sketch = false; + } + + DSThetaParams Params(const BindData &bind) const + { + DSThetaParams params; + params.mode = bind.mode; + params.num_values = MaxValue(num_values, 1); + return params; + } + + bool HasData() const + { + return update || compact || raw_rows || part.active; + } + bool IsEmpty() const + { + return !HasData() && !saw_empty_sketch; + } + bool OnlyRaw() const + { + return raw_rows && !update && !compact && !part.active; + } + uint64_t Seed(const BindData &bind) const + { + return part.active ? part.seed : bind.seed; + } + uint8_t LgK(const BindData &bind) const + { + return part.active ? part.lg_k : bind.lg_k; + } + + void CheckNumValues(idx_t count, const char *function_name) + { + if (num_values == 0) + { + num_values = static_cast(count); + } + else if (num_values != count) + { + TRAITS::ThrowNumValues(function_name, num_values, count); + } + } + + // Replays raw rows (as written by AddRow) into an update sketch. + static void Replay(Update &sketch, const_data_ptr_t data, idx_t size, const DSThetaParams ¶ms) + { + const idx_t payload = TRAITS::PayloadSize(params); + idx_t pos = 0; + while (pos < size) + { + uint32_t length; + memcpy(&length, data + pos, sizeof(length)); + pos += sizeof(length); + TRAITS::template Apply(sketch, data + pos, length, data + pos + length, params); + pos += length + payload; + } + } + + // The update sketch of the rows: the one built so far, or one replayed from the raw rows in the current + // (scratch) arena. + template + void WithRowsSketch(const BindData &bind, FUNC &&fun) const + { + if (update) + { + fun(update->compact()); + return; + } + auto params = Params(bind); + auto temporary = TRAITS::NewUpdate(bind, params); + Replay(temporary, raw.data, raw.size, params); + fun(temporary.compact()); + } + + // Adds a row: key bytes (the value of a numeric key, or a string's bytes) and value bytes. + void AddRow(const_data_ptr_t key, uint32_t length, const_data_ptr_t payload, const BindData &bind, ArenaAllocator &arena) + { + auto params = Params(bind); + if (update) + { + TRAITS::template Apply(*update, key, length, payload, params); + return; + } + const idx_t payload_size = TRAITS::PayloadSize(params); + const idx_t row_size = sizeof(uint32_t) + length + payload_size; + if (raw_rows < RAW_ROWS && raw.size + row_size <= RAW_BYTES) + { + auto slot = raw.Grow(row_size, arena); + memcpy(slot, &length, sizeof(length)); + memcpy(slot + sizeof(length), key, length); + memcpy(slot + sizeof(length) + length, payload, payload_size); + raw_rows++; + return; + } + update = arena.Make(TRAITS::NewUpdate(bind, params)); + Replay(*update, raw.data, raw.size, params); + raw.Initialize(); + raw_rows = 0; + TRAITS::template Apply(*update, key, length, payload, params); + } + + // Merges a serialized sketch. + void Merge(const string_t &blob, const BindData &bind, ArenaAllocator &arena) + { + DSScratchScope scratch; + auto sketch = TRAITS::Deserialize(blob); + if (sketch.is_empty()) + { + // an empty sketch adds nothing to a union + saw_empty_sketch = true; + return; + } + if (part.active && sketch.get_seed_hash() != ThetaSeedHash(part.seed)) + { + TRAITS::ThrowSeedMismatch(TRAITS::BUILD_NAME, ThetaSeedHash(part.seed), sketch.get_seed_hash()); + } + if (TRAITS::HAS_NUM_VALUES) + { + CheckNumValues(TRAITS::NumValues(sketch), TRAITS::BUILD_NAME); + } + auto lg_k = bind.infer_lg_k ? MaxValue(bind.lg_k, ThetaInferLgK(sketch)) : bind.lg_k; + part.Add(sketch, lg_k, ThetaSeedForHash(sketch.get_seed_hash()), Params(bind), arena); + } + + // Moves what the rows built into the union. + void FlushRows(uint8_t lg_k, uint64_t seed, const BindData &bind, ArenaAllocator &arena) + { + auto params = Params(bind); + if (update || raw_rows) + { + DSScratchScope scratch; + WithRowsSketch(bind, [&](const Compact &rows) + { part.Add(rows, lg_k, seed, params, arena); }); + } + else if (compact) + { + part.Add(*compact, lg_k, seed, params, arena); + } + else + { + DSArenaScope scope(arena); + part.Ensure(lg_k, seed, params, arena); + } + update = nullptr; + compact = nullptr; + raw.Initialize(); + raw_rows = 0; + } + + // Combines a partial state, which may live in another thread's arena: nothing of it is taken over. + void Combine(const DSThetaFamilyState &source, const BindData &bind, ArenaAllocator &arena) + { + saw_empty_sketch = saw_empty_sketch || source.saw_empty_sketch; + if (!source.HasData()) + { + return; + } + auto source_seed = source.Seed(bind); + if (HasData() && ThetaSeedHash(source_seed) != ThetaSeedHash(Seed(bind))) + { + TRAITS::ThrowSeedMismatch(TRAITS::BUILD_NAME, ThetaSeedHash(Seed(bind)), ThetaSeedHash(source_seed)); + } + if (TRAITS::HAS_NUM_VALUES) + { + CheckNumValues(source.num_values, TRAITS::BUILD_NAME); + } + const uint32_t k = uint32_t(1) << bind.lg_k; + if (source.OnlyRaw() && source.raw_rows <= k) + { + // The rows' update sketch is exact (at most k keys), so the union of it holds just its keys: keeping + // the rows gives the same result. Without summaries, the rows of two such states give the union of + // both (at most k keys in all). + const bool fresh = !HasData(); + const bool concatenate = TRAITS::SET_ONLY && OnlyRaw() && raw_rows + source.raw_rows <= MinValue(RAW_ROWS, k) && + raw.size + source.raw.size <= RAW_BYTES; + if (fresh || concatenate) + { + raw.Append(source.raw.data, source.raw.size, arena); + raw_rows += source.raw_rows; + return; + } + } + auto params = Params(bind); + auto lg_k = MaxValue(source.LgK(bind), LgK(bind)); + auto seed = HasData() ? Seed(bind) : source_seed; + if (!HasData() && !source.update && !source.raw_rows && !source.compact) + { + // only a union: copied if it is the library's + part.AddUnion(source.part, lg_k, seed, params, arena); + return; + } + FlushRows(lg_k, seed, bind, arena); + if (source.update || source.raw_rows) + { + DSScratchScope scratch; + source.WithRowsSketch(bind, [&](const Compact &rows) + { part.Add(rows, lg_k, seed, params, arena); }); + } + if (source.compact) + { + part.Add(*source.compact, lg_k, seed, params, arena); + } + if (source.part.active) + { + part.AddUnion(source.part, lg_k, seed, params, arena); + } + } + + // The aggregate's result. + string_t Finalize(Vector &result, const BindData &bind) const + { + auto params = Params(bind); + DSScratchScope scratch; + if (part.active) + { + return Store(result, part.Result(params)); + } + if (update || raw_rows) + { + string_t stored; + WithRowsSketch(bind, [&](const Compact &rows) + { stored = Store(result, rows); }); + return stored; + } + if (compact) + { + return Store(result, *compact); + } + return Store(result, TRAITS::Empty(bind.lg_k, bind.seed, params)); + } + + static string_t Store(Vector &result, const Compact &sketch) + { + auto bytes = TRAITS::Serialize(sketch); + return StringVector::AddStringOrBlob(result, const_char_ptr_cast(bytes.data()), bytes.size()); + } + + // The exported state: a version, flags and num_values, then the rows (raw, or the update sketch's compact + // form) and the union. Raw rows are exported as they are, so importing a small state keeps it small. + static constexpr uint8_t EXPORT_VERSION = 1; + static constexpr uint8_t ROWS_NONE = 0; + static constexpr uint8_t ROWS_RAW = 1; + static constexpr uint8_t ROWS_COMPACT = 2; + + std::vector Export(optional_ptr bind_data) const + { + const auto &bind = bind_data->Cast(); + auto params = Params(bind); + DSThetaWriter writer; + writer.Put(EXPORT_VERSION); + writer.Put(saw_empty_sketch); + writer.Put(num_values); + if (update) + { + DSScratchScope scratch; + writer.Put(ROWS_COMPACT); + writer.PutBlock(TRAITS::Serialize(update->compact())); + } + else if (compact) + { + DSScratchScope scratch; + writer.Put(ROWS_COMPACT); + writer.PutBlock(TRAITS::Serialize(*compact)); + } + else if (raw_rows) + { + writer.Put(ROWS_RAW); + writer.Put(raw_rows); + writer.Put(raw.size); + writer.PutBytes(raw.data, raw.size); + } + else + { + writer.Put(ROWS_NONE); + } + if (part.active) + { + part.Export(writer, params); + } + else + { + writer.Put(0); + } + return std::move(writer.bytes); + } + + void Import(const string_t &blob, ArenaAllocator &arena) + { + DSThetaReader reader(blob, TRAITS::KIND); + if (reader.Get() != EXPORT_VERSION) + { + reader.Fail(); + } + const auto flags = reader.Get(); + if (flags > 1) + { + reader.Fail(); + } + saw_empty_sketch = flags; + num_values = reader.Get(); + DSThetaParams params; + params.num_values = MaxValue(num_values, 1); + // mode is not needed: imported states only copy summaries + const auto rows = reader.Get(); + if (rows == ROWS_COMPACT) + { + auto block = reader.GetBlock(); + DSScratchScope scratch; + auto sketch = TRAITS::Deserialize(block); + DSArenaScope back(arena); + compact = arena.Make(sketch); + } + else if (rows == ROWS_RAW) + { + const auto count = reader.Get(); + const auto size = reader.Get(); + auto data = reader.GetBytes(size); + // check the rows' framing, which Replay trusts + const idx_t payload = TRAITS::PayloadSize(params); + idx_t pos = 0; + uint32_t found = 0; + while (pos < size) + { + if (size - pos < sizeof(uint32_t)) + { + reader.Fail(); + } + uint32_t length; + memcpy(&length, data + pos, sizeof(length)); + pos += sizeof(length); + if ((!std::is_same::value && length != sizeof(KEY)) || size - pos < idx_t(length) + payload) + { + reader.Fail(); + } + pos += length + payload; + found++; + } + if (found != count || count == 0 || count > RAW_ROWS || size > RAW_BYTES) + { + reader.Fail(); + } + raw.Append(data, size, arena); + raw_rows = count; + } + else if (rows != ROWS_NONE) + { + reader.Fail(); + } + const auto union_kind = reader.Get(); + if (union_kind == 1 || union_kind == 2) + { + part.Import(reader, union_kind, params, arena); + } + else if (union_kind != 0) + { + reader.Fail(); + } + reader.End(); + } + }; + + // ============================================================ + // Intersection aggregates + // ============================================================ + + // Intersects sketches across rows. The state is the current result: each input is intersected with it in a + // temporary intersection, which gives what the library's intersection would hold. + template + struct DSThetaFamilyIntersectState + { + using Compact = typename TRAITS::Compact; + using View = DSThetaEntriesView; + + //! The result's entries (key, then summary) + DSArenaBytes entries; + uint32_t count; + uint64_t theta; + uint64_t seed; + bool has_result; + bool result_empty; + //! An empty sketch was intersected: the result is empty + bool saw_empty; + uint8_t num_values; + + void Initialize() + { + entries.Initialize(); + count = 0; + theta = datasketches::theta_constants::MAX_THETA; + seed = DS_THETA_DEFAULT_SEED; + has_result = false; + result_empty = false; + saw_empty = false; + num_values = 0; + } + + bool IsEmpty() const + { + return !has_result && !saw_empty; + } + + DSThetaParams Params(uint8_t mode) const + { + DSThetaParams params; + params.mode = mode; + params.num_values = MaxValue(num_values, 1); + return params; + } + + // The current result as a sketch (allocated in the current, scratch, arena). + View Current(const DSThetaParams ¶ms) const + { + View view(result_empty, seed, theta); + view.Load(entries.data, count, params); + return view; + } + + // Intersects a sketch (read only; it may be another state's) with `sketch_num_values` values per key. + template + void Add(const SKETCH &sketch, uint8_t sketch_num_values, uint8_t mode, ArenaAllocator &arena) + { + if (sketch.is_empty()) + { + saw_empty = true; + if (num_values == 0) + { + num_values = sketch_num_values; + } + return; + } + if (!has_result) + { + seed = ThetaSeedForHash(sketch.get_seed_hash()); + num_values = sketch_num_values; + } + else + { + if (sketch.get_seed_hash() != ThetaSeedHash(seed)) + { + TRAITS::ThrowSeedMismatch(TRAITS::INTERSECT_NAME, ThetaSeedHash(seed), sketch.get_seed_hash()); + } + if (TRAITS::HAS_NUM_VALUES && sketch_num_values != num_values) + { + TRAITS::ThrowNumValues(TRAITS::INTERSECT_NAME, num_values, sketch_num_values); + } + } + auto params = Params(mode); + DSScratchScope scratch; + auto intersection = TRAITS::NewIntersection(seed, params); + ThetaGuard([&]() + { + if (has_result) + { + intersection.update(Current(params)); + } + intersection.update(sketch); }, + TRAITS::KIND); + auto result = intersection.get_result(false); + // an intersection only shrinks, so the result usually fits where the previous one was + DSThetaWriter writer; + DSThetaWriteEntries(result, writer, params); + entries.size = 0; + entries.Append(writer.bytes.data(), writer.bytes.size(), arena); + count = result.get_num_retained(); + theta = result.get_theta64(); + result_empty = result.is_empty(); + has_result = true; + } + + void AddBlob(const string_t &blob, uint8_t mode, ArenaAllocator &arena) + { + DSScratchScope scratch; + auto sketch = TRAITS::Deserialize(blob); + Add(sketch, TRAITS::NumValues(sketch), mode, arena); + } + + void Combine(const DSThetaFamilyIntersectState &source, uint8_t mode, ArenaAllocator &arena) + { + saw_empty = saw_empty || source.saw_empty; + if (num_values == 0) + { + num_values = source.num_values; + } + if (source.has_result) + { + DSScratchScope scratch; + Add(source.Current(source.Params(mode)), MaxValue(source.num_values, 1), mode, arena); + } + } + + string_t Finalize(Vector &result, uint8_t mode) const + { + auto params = Params(mode); + DSScratchScope scratch; + if (saw_empty) + { + return Store(result, TRAITS::Empty(DS_THETA_DEFAULT_LG_K, seed, params)); + } + auto intersection = TRAITS::NewIntersection(seed, params); + intersection.update(Current(params)); + return Store(result, intersection.get_result()); + } + + static string_t Store(Vector &result, const Compact &sketch) + { + auto bytes = TRAITS::Serialize(sketch); + return StringVector::AddStringOrBlob(result, const_char_ptr_cast(bytes.data()), bytes.size()); + } + + static constexpr uint8_t EXPORT_VERSION = 1; + + std::vector Export(optional_ptr bind_data) const + { + const auto mode = DSThetaMode(bind_data); + auto params = Params(mode); + DSThetaWriter writer; + writer.Put(EXPORT_VERSION); + writer.Put(uint8_t(has_result) | uint8_t(result_empty) << 1 | uint8_t(saw_empty) << 2); + writer.Put(num_values); + writer.Put(theta); + writer.Put(seed); + writer.Put(count); + writer.PutBytes(entries.data, count * (sizeof(uint64_t) + TRAITS::SummarySize(params))); + return std::move(writer.bytes); + } + + void Import(const string_t &blob, ArenaAllocator &arena) + { + DSThetaReader reader(blob, TRAITS::KIND); + if (reader.Get() != EXPORT_VERSION) + { + reader.Fail(); + } + const auto flags = reader.Get(); + if (flags > 7) + { + reader.Fail(); + } + has_result = flags & 1; + result_empty = flags & 2; + saw_empty = flags & 4; + num_values = reader.Get(); + theta = reader.Get(); + seed = reader.Get(); + count = reader.Get(); + const idx_t size = idx_t(count) * (sizeof(uint64_t) + TRAITS::SummarySize(Params(0))); + entries.Append(reader.GetBytes(size), size, arena); + reader.End(); + } + + }; + + // ============================================================ + // Aggregate operations and state export + // ============================================================ + + // Operations of the build and merge aggregates. IDEMPOTENT: merging the same sketch twice is merging it once. + template + struct DSThetaFamilyOperation + { + template + static void Initialize(STATE &state) + { + state.Initialize(); + } + + static bool IgnoreNull() { return true; } + + template + static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + { + if (source.IsEmpty()) + { + return; + } + DSArenaScope scope(aggr_input_data.allocator); + target.Combine(source, DSThetaBind(aggr_input_data.bind_data), aggr_input_data.allocator); + } + + template + static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + { + if (state.IsEmpty()) + { + // no input rows: NULL, as for the other sketches' aggregates + finalize_data.ReturnNull(); + return; + } + DSArenaScope scope(finalize_data.input.allocator); + target = state.Finalize(finalize_data.result, DSThetaBind(finalize_data.input.bind_data)); + } + + // Merging serialized sketches. + template + static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + state.Merge(blob, DSThetaBind(idata.input.bind_data), idata.input.allocator); + } + + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + { + for (idx_t i = 0; i < (IDEMPOTENT ? 1 : count); i++) + { + Operation(state, input, unary_input); + } + } + }; + + // The key of a row as bytes: the value of a numeric key, the bytes of a string. + template + inline std::pair DSThetaKeyBytes(const KEY &key) + { + if constexpr (std::is_same::value) + { + return {const_data_ptr_cast(key.GetData()), static_cast(key.GetSize())}; + } + else + { + return {const_data_ptr_cast(&key), static_cast(sizeof(KEY))}; + } + } + + // Builds from keys (Theta): adding the same key again does not change the sketch. + struct DSThetaFamilyCreateOperation : DSThetaFamilyOperation + { + template + static void Operation(STATE &state, const A_TYPE &key, AggregateUnaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + auto bytes = DSThetaKeyBytes(key); + state.AddRow(bytes.first, bytes.second, nullptr, DSThetaBind(idata.input.bind_data), idata.input.allocator); + } + + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + { + Operation(state, input, unary_input); + } + }; + + // Builds from (key, DOUBLE value) rows (Tuple). + struct DSThetaFamilyCreatePairOperation : DSThetaFamilyOperation + { + template + static void Operation(STATE &state, const KEY_TYPE &key, const VALUE_TYPE &value, AggregateBinaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + auto bytes = DSThetaKeyBytes(key); + state.AddRow(bytes.first, bytes.second, const_data_ptr_cast(&value), DSThetaBind(idata.input.bind_data), idata.input.allocator); + } + }; + + template + struct DSThetaFamilyIntersectOperation + { + template + static void Initialize(STATE &state) + { + state.Initialize(); + } + + static bool IgnoreNull() { return true; } + + template + static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + { + DSArenaScope scope(idata.input.allocator); + state.AddBlob(blob, DSThetaMode(idata.input.bind_data), idata.input.allocator); + } + + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + { + for (idx_t i = 0; i < (IDEMPOTENT ? 1 : count); i++) + { + Operation(state, input, unary_input); + } + } + + template + static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + { + if (source.IsEmpty()) + { + return; + } + DSArenaScope scope(aggr_input_data.allocator); + target.Combine(source, DSThetaMode(aggr_input_data.bind_data), aggr_input_data.allocator); + } + + template + static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + { + if (state.IsEmpty()) + { + // no sketches: the intersection of nothing is undefined + finalize_data.ReturnNull(); + return; + } + DSArenaScope scope(finalize_data.input.allocator); + target = state.Finalize(finalize_data.result, DSThetaMode(finalize_data.input.bind_data)); + } + }; + + // Exports each state (NULL for a state that saw no rows) for spilling. + template + static void DSThetaExportState(Vector &state_vector, AggregateFinalizeInputData &aggr_input_data, Vector &result, + idx_t count, idx_t offset) + { + auto states = state_vector.Values(); + auto results = FlatVector::GetDataMutable(result); + DSArenaScope scope(aggr_input_data.allocator); + for (idx_t i = 0; i < count; i++) + { + auto &state = *states[i].GetValue(); + if (state.IsEmpty()) + { + FlatVector::SetNull(result, offset + i, true); + continue; + } + auto bytes = state.Export(aggr_input_data.bind_data); + results[offset + i] = StringVector::AddStringOrBlob(result, const_char_ptr_cast(bytes.data()), bytes.size()); + } + } + + // Rebuilds exported states in the arena. + template + static void DSThetaImportState(AggregateImportInputData &input) + { + const auto count = input.input_vec.size(); + auto entries = input.input_vec.Values(); + DSArenaScope scope(input.allocator); + for (idx_t i = 0; i < count; i++) + { + auto &state = *reinterpret_cast(input.dest_buffer + i * input.layout.total_state_size); + state.Initialize(); + const auto entry = entries[i]; + if (entry.IsValid()) + { + state.Import(entry.GetValue(), input.allocator); + } + } + } + + // The exported state is a BLOB; the constant arguments are recorded so that re-binding it gets them. + template + static AggregateStateLayout DSThetaGetStateType(AggregateLayoutInput &input) + { + AggregateStateLayout layout(LogicalType::BLOB, AlignValue(sizeof(STATE))); + if (input.bind_data) + { + for (auto &constant : input.bind_data->template Cast().constants) + { + layout.constant_parameters.emplace(constant.first, constant.second); + } + } + return layout; + } + + template + static void DSThetaSetStateExport(AggregateFunction &fun) + { + fun.SetStateExportCallbacks(DSThetaGetStateType, DSThetaExportState, DSThetaImportState); + } + +} // namespace duckdb diff --git a/src/theta_sketch.cpp b/src/theta_sketch.cpp index d42bb0e..a46a7b0 100644 --- a/src/theta_sketch.cpp +++ b/src/theta_sketch.cpp @@ -7,6 +7,7 @@ #include "duckdb/common/vector/struct_vector.hpp" #include "duckdb/common/vector/unified_vector_format.hpp" #include "datasketches_theta_common.hpp" +#include "datasketches_theta_arena.hpp" // Apache DataSketches Headers #include @@ -77,44 +78,11 @@ namespace duckdb return StringVector::AddStringOrBlob(result, reinterpret_cast(bytes.data()), bytes.size()); } - compact_theta_sketch ThetaEmptySketch(uint8_t lg_k, uint64_t seed) - { - update_theta_sketch::builder b; - b.set_lg_k(lg_k); - b.set_seed(seed); - return b.build().compact(); - } - - // ============================================================ // 2. Bind Data // ============================================================ - struct DSThetaBindData : public FunctionData - { - DSThetaBindData() = default; - DSThetaBindData(uint8_t lg_k, bool infer_lg_k, uint64_t seed, float p) - : lg_k(lg_k), infer_lg_k(infer_lg_k), seed(seed), p(p) {} - - unique_ptr Copy() const override - { - return make_uniq(lg_k, infer_lg_k, seed, p); - } - - bool Equals(const FunctionData &other_p) const override - { - auto &other = other_p.Cast(); - return lg_k == other.lg_k && infer_lg_k == other.infer_lg_k && seed == other.seed && p == other.p; - } - - //! lg_k of the sketches built (or the minimum lg_k of the union when infer_lg_k is set) - uint8_t lg_k = DS_THETA_DEFAULT_LG_K; - //! Merging: grow the union to the lg_k of the sketches merged - bool infer_lg_k = false; - //! Building: the hash seed and the sampling probability - uint64_t seed = DS_THETA_DEFAULT_SEED; - float p = 1.0f; - }; + using DSThetaBindData = DSThetaFamilyBindData; uint8_t DSThetaBindLgK(BindAggregateFunctionInput &input, idx_t index) { @@ -132,14 +100,17 @@ namespace duckdb template unique_ptr DSThetaBindBuild(BindAggregateFunctionInput &input) { - uint8_t lg_k = HAS_LG_K ? DSThetaBindLgK(input, 0) : DS_THETA_DEFAULT_LG_K; - auto seed = static_cast(input.GetNonNullConstant(Identifier("seed")).GetValue()); + auto result = make_uniq(); + result->lg_k = HAS_LG_K ? DSThetaBindLgK(input, 0) : DS_THETA_DEFAULT_LG_K; + result->seed = static_cast(input.GetNonNullConstant(Identifier("seed")).GetValue()); auto p = input.GetNonNullConstant(Identifier("p")).GetValue(); if (!(p > 0.0 && p <= 1.0)) { throw BinderException("Theta sketch sampling probability p must be in (0, 1], got %f", p); } - return make_uniq(lg_k, false, seed, static_cast(p)); + result->p = static_cast(p); + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 2 : 1); + return std::move(result); } // Merging sketches with an explicit lg_k. The lg_k stays in the argument list (the update reads the sketch @@ -147,322 +118,122 @@ namespace duckdb // aggregates that differ only in lg_k as identical. unique_ptr DSThetaBindMergeWithK(BindAggregateFunctionInput &input) { - auto lg_k = DSThetaBindLgK(input, 0); - return make_uniq(lg_k, false, DS_THETA_DEFAULT_SEED, 1.0f); + auto result = make_uniq(); + result->lg_k = DSThetaBindLgK(input, 0); + DSThetaRecordConstants(input, *result, 1, 2); + return std::move(result); } // Merging sketches: the union grows to the lg_k of the sketches merged. unique_ptr DSThetaBindMergeDefault(BindAggregateFunctionInput &input) { - return make_uniq(DS_THETA_DEFAULT_LG_K, true, DS_THETA_DEFAULT_SEED, 1.0f); + auto result = make_uniq(); + result->infer_lg_k = true; + return std::move(result); } // ============================================================ - // 3. Build / Union Aggregate State & Operations + // 3. Aggregate states (see datasketches_theta_arena.hpp) // ============================================================ - struct DSThetaState - { - update_theta_sketch *update_sketch = nullptr; - theta_union *union_sketch = nullptr; - uint8_t union_lg_k = 0; - uint64_t union_seed = DS_THETA_DEFAULT_SEED; - //! A merged input sketch was empty: with no other input, the result is an empty sketch rather than NULL - bool saw_empty_sketch = false; - - ~DSThetaState() - { - if (update_sketch) - delete update_sketch; - if (union_sketch) - delete union_sketch; - } - - void CreateUpdateSketch(const DSThetaBindData &bind_data) - { - if (!update_sketch) - { - update_theta_sketch::builder b; - b.set_lg_k(bind_data.lg_k); - b.set_seed(bind_data.seed); - b.set_p(bind_data.p); - update_sketch = new update_theta_sketch(b.build()); - } - } - - // Creates the union, or grows it to a larger lg_k (re-adding what it holds so far). - void EnsureUnion(uint8_t lg_k, uint64_t seed) - { - if (union_sketch && lg_k <= union_lg_k) - { - return; - } - if (union_sketch) - { - seed = union_seed; - } - theta_union::builder b; - b.set_lg_k(lg_k); - b.set_seed(seed); - auto grown = new theta_union(b.build()); - if (union_sketch) - { - grown->update(union_sketch->get_result()); - delete union_sketch; - } - union_sketch = grown; - union_lg_k = lg_k; - union_seed = seed; - } - - bool HasData() const - { - return update_sketch || union_sketch; - } - - uint64_t Seed(const DSThetaBindData &bind_data) const - { - return union_sketch ? union_seed : bind_data.seed; - } - - uint8_t LgK(const DSThetaBindData &bind_data) const - { - return union_sketch ? union_lg_k : bind_data.lg_k; - } - }; - - struct DSThetaOperationBase - { - template - static void Initialize(STATE &state) - { - state.update_sketch = nullptr; - state.union_sketch = nullptr; - state.union_lg_k = 0; - state.union_seed = DS_THETA_DEFAULT_SEED; - state.saw_empty_sketch = false; - } - template - static void Destroy(STATE &state, AggregateInputData &) - { - if (state.update_sketch) - delete state.update_sketch; - if (state.union_sketch) - delete state.union_sketch; - state.update_sketch = nullptr; - state.union_sketch = nullptr; - } - static bool IgnoreNull() { return true; } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) - { - target.saw_empty_sketch = target.saw_empty_sketch || source.saw_empty_sketch; - if (!source.HasData()) - return; - auto &bind_data = aggr_input_data.bind_data->template Cast(); - auto source_seed = source.Seed(bind_data); - if (target.HasData() && ThetaSeedHash(source_seed) != ThetaSeedHash(target.Seed(bind_data))) - { - throw InvalidInputException("datasketch_theta: cannot merge Theta sketches built with different seeds (seed hashes %d and %d)", - ThetaSeedHash(target.Seed(bind_data)), ThetaSeedHash(source_seed)); - } - auto lg_k = MaxValue(source.LgK(bind_data), target.LgK(bind_data)); - target.EnsureUnion(lg_k, target.HasData() ? target.Seed(bind_data) : source_seed); - if (target.update_sketch) - { - target.union_sketch->update(*target.update_sketch); - delete target.update_sketch; - target.update_sketch = nullptr; - } - if (source.update_sketch) - target.union_sketch->update(*source.update_sketch); - if (source.union_sketch) - target.union_sketch->update(source.union_sketch->get_result()); - } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + struct DSThetaTraits + { + using Alloc = ThetaAllocator; + using Entry = uint64_t; + using Update = update_theta_sketch; + using Compact = compact_theta_sketch; + using Union = theta_union; + using Intersection = theta_intersection; + static constexpr bool SET_ONLY = true; + static constexpr bool HAS_NUM_VALUES = false; + static constexpr const char *KIND = "Theta"; + static constexpr const char *BUILD_NAME = "datasketch_theta"; + static constexpr const char *INTERSECT_NAME = "datasketch_theta_intersect_agg"; + + static uint64_t Key(const Entry &entry) { return entry; } + static idx_t SummarySize(const DSThetaParams &) { return 0; } + static void WriteSummary(const Entry &, data_ptr_t, const DSThetaParams &) {} + static Entry MakeEntry(uint64_t key, const_data_ptr_t, const DSThetaParams &) { return key; } + static void Fold(data_ptr_t, const Entry &, const DSThetaParams &) {} + static idx_t PayloadSize(const DSThetaParams &) { return 0; } + static uint8_t NumValues(const Compact &) { return 1; } + + template + static void Apply(Update &sketch, const_data_ptr_t key, uint32_t length, const_data_ptr_t, const DSThetaParams &) { - if (state.union_sketch) + if constexpr (std::is_same::value) { - target = ThetaSerialize(finalize_data.result, state.union_sketch->get_result()); - } - else if (state.update_sketch) - { - target = ThetaSerialize(finalize_data.result, state.update_sketch->compact()); - } - else if (state.saw_empty_sketch) - { - auto &bind_data = finalize_data.input.bind_data->template Cast(); - target = ThetaSerialize(finalize_data.result, ThetaEmptySketch(bind_data.lg_k, bind_data.seed)); + sketch.update(key, length); } else { - // no input rows: NULL, as for the other sketches' aggregates - finalize_data.ReturnNull(); + KEY value; + memcpy(&value, key, sizeof(KEY)); + sketch.update(value); } } - }; - struct DSThetaCreateOperation : DSThetaOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &a_data, AggregateUnaryInput &idata) + static Update NewUpdate(const DSThetaBindData &bind, const DSThetaParams &) { - auto &bind_data = idata.input.bind_data->template Cast(); - state.CreateUpdateSketch(bind_data); - if constexpr (std::is_same_v) - { - state.update_sketch->update(a_data.GetData(), a_data.GetSize()); - } - else - { - state.update_sketch->update(a_data); - } + update_theta_sketch::builder b{Alloc(DSArenaMode())}; + b.set_lg_k(bind.lg_k); + b.set_seed(bind.seed); + b.set_p(bind.p); + return b.build(); } - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static Union NewUnion(uint8_t lg_k, uint64_t seed, const DSThetaParams &) { - // Adding the same value again does not change the sketch. - Operation(state, input, unary_input); - } - }; - - struct DSThetaMergeOperation : DSThetaOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &a_data, AggregateUnaryInput &idata) - { - auto &bind_data = idata.input.bind_data->template Cast(); - auto sketch = ThetaDeserialize(a_data); - if (sketch.is_empty()) - { - // an empty sketch adds nothing to a union - state.saw_empty_sketch = true; - return; - } - if (state.union_sketch && sketch.get_seed_hash() != ThetaSeedHash(state.union_seed)) - { - throw InvalidInputException("datasketch_theta: cannot merge Theta sketches built with different seeds (seed hashes %d and %d)", - ThetaSeedHash(state.union_seed), sketch.get_seed_hash()); - } - auto lg_k = bind_data.infer_lg_k ? MaxValue(bind_data.lg_k, ThetaInferLgK(sketch)) : bind_data.lg_k; - state.EnsureUnion(lg_k, ThetaSeedForHash(sketch.get_seed_hash())); - ThetaGuard([&]() - { state.union_sketch->update(sketch); }); - } - - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) - { - // A union is idempotent: merging the same sketch again does not change it. - Operation(state, input, unary_input); - } - }; - - // ============================================================ - // 4. Intersection Aggregate - // ============================================================ - - struct DSThetaIntersectState - { - theta_intersection *intersection = nullptr; - uint64_t seed = DS_THETA_DEFAULT_SEED; - //! An empty sketch was intersected: the result is empty - bool saw_empty = false; - - ~DSThetaIntersectState() - { - if (intersection) - delete intersection; - } - - void Add(const compact_theta_sketch &sketch) - { - if (sketch.is_empty()) - { - saw_empty = true; - return; - } - if (!intersection) - { - seed = ThetaSeedForHash(sketch.get_seed_hash()); - intersection = new theta_intersection(seed); - } - else if (sketch.get_seed_hash() != ThetaSeedHash(seed)) - { - throw InvalidInputException("datasketch_theta_intersect_agg: cannot combine Theta sketches built with different seeds (seed hashes %d and %d)", - ThetaSeedHash(seed), sketch.get_seed_hash()); - } - ThetaGuard([&]() - { intersection->update(sketch); }); + theta_union::builder b{Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build(); } - }; - struct DSThetaIntersectOperation - { - template - static void Initialize(STATE &state) + static Intersection NewIntersection(uint64_t seed, const DSThetaParams &) { - state.intersection = nullptr; - state.seed = DS_THETA_DEFAULT_SEED; - state.saw_empty = false; + return Intersection(seed, Alloc(DSArenaMode())); } - template - static void Destroy(STATE &state, AggregateInputData &) + static Compact Deserialize(const string_t &blob) { - if (state.intersection) - delete state.intersection; - state.intersection = nullptr; + auto data = blob.GetData(); + auto size = blob.GetSize(); + auto seed = ThetaSeedForHash(ThetaPeekSeedHash(data, size)); + return ThetaGuard([&]() + { return Compact::deserialize(data, size, seed, Alloc(DSArenaMode())); }); } - static bool IgnoreNull() { return true; } - - template - static void Operation(STATE &state, const A_TYPE &a_data, AggregateUnaryInput &) + static Compact Empty(uint8_t lg_k, uint64_t seed, const DSThetaParams &) { - state.Add(ThetaDeserialize(a_data)); + update_theta_sketch::builder b{Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build().compact(); } - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static auto Serialize(const Compact &sketch) -> decltype(sketch.serialize_compressed()) { - // An intersection is idempotent: intersecting the same sketch again does not change it. - Operation(state, input, unary_input); + return sketch.serialize_compressed(); } - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &) + static void ThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b) { - target.saw_empty = target.saw_empty || source.saw_empty; - if (source.intersection) + if (string(function_name) == BUILD_NAME) { - target.Add(source.intersection->get_result()); + throw InvalidInputException("datasketch_theta: cannot merge Theta sketches built with different seeds (seed hashes %d and %d)", a, b); } + throw InvalidInputException("%s: cannot combine Theta sketches built with different seeds (seed hashes %d and %d)", function_name, a, b); } - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) - { - if (state.saw_empty) - { - target = ThetaSerialize(finalize_data.result, ThetaEmptySketch(DS_THETA_DEFAULT_LG_K, state.seed)); - } - else if (state.intersection) - { - target = ThetaSerialize(finalize_data.result, state.intersection->get_result()); - } - else - { - // no sketches: the intersection of nothing is undefined - finalize_data.ReturnNull(); - } - } + static void ThrowNumValues(const char *, idx_t, idx_t) {} }; + template + using DSThetaState = DSThetaFamilyState; + using DSThetaIntersectState = DSThetaFamilyIntersectState; + using DSThetaMergeOperation = DSThetaFamilyOperation; + using DSThetaIntersectOperation = DSThetaFamilyIntersectOperation; // ============================================================ // 5. Scalar Functions @@ -704,13 +475,14 @@ namespace duckdb { for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::UnaryAggregate( - input_type, sketch_type); + using STATE = DSThetaState; + auto fun = AggregateFunction::UnaryAggregate(input_type, sketch_type); + DSThetaSetStateExport(fun); FunctionSignature signature(vector(), sketch_type); if (with_lg_k) { signature.AddParameter(LogicalType::INTEGER); - DSReadDataArgumentAt<1, DSThetaState, T, DSThetaCreateOperation>(fun); + DSReadDataArgumentAt<1, STATE, T, DSThetaFamilyCreateOperation>(fun); fun.SetBindCallback(DSThetaBindBuild); } else @@ -773,8 +545,9 @@ namespace duckdb RegisterThetaBuild(sketch_agg, agg_descs, "datasketch_theta", LogicalType::DOUBLE, sketch_type); RegisterThetaBuild(sketch_agg, agg_descs, "datasketch_theta", LogicalType::VARCHAR, sketch_type); - auto fun_merge = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + using MERGE_STATE = DSThetaState; + auto fun_merge = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun_merge); fun_merge.SetBindCallback(DSThetaBindMergeDefault); fun_merge.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); DSSetArguments(fun_merge, {sketch_type}); @@ -783,12 +556,12 @@ namespace duckdb "largest input sketch (inferred from the sketches, at least lg_k 12)", {ex.QueryPartials("datasketch_theta(sketch)")}, {"sketch"}, {sketch_type})); - auto fun_merge_k = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun_merge_k = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun_merge_k); fun_merge_k.SetBindCallback(DSThetaBindMergeWithK); fun_merge_k.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); DSSetArguments(fun_merge_k, {LogicalType::INTEGER, sketch_type}); - DSReadDataArgumentAt<1, DSThetaState, string_t, DSThetaMergeOperation>(fun_merge_k); + DSReadDataArgumentAt<1, MERGE_STATE, string_t, DSThetaMergeOperation>(fun_merge_k); sketch_agg.AddFunction(fun_merge_k); agg_descs.push_back(ThetaDescription("Merges (unions) Theta sketches into a single sketch with the given log2 of nominal entries (lg_k)", {ex.QueryPartials("datasketch_theta(16, sketch)")}, {"lg_k", "sketch"}, {LogicalType::INTEGER, sketch_type})); @@ -813,8 +586,8 @@ namespace duckdb // --- datasketch_theta_intersect_agg: intersect sketches across rows { AggregateFunctionSet intersect_agg("datasketch_theta_intersect_agg"); - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); fun.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); DSSetArguments(fun, {sketch_type}); intersect_agg.AddFunction(fun); diff --git a/src/tuple_sketch.cpp b/src/tuple_sketch.cpp index 976a732..06faf0a 100644 --- a/src/tuple_sketch.cpp +++ b/src/tuple_sketch.cpp @@ -5,6 +5,7 @@ // DataSketches libraries), so functions that combine summaries (merge, union, intersection) take it as an option. #include "datasketches_tuple_common.hpp" +#include "datasketches_theta_arena.hpp" #include #include @@ -123,43 +124,15 @@ namespace duckdb using DSTupleScalars = DSTupleFunctions; - DSTupleCompact DSTupleEmptySketch(uint8_t lg_k, uint64_t seed) - { - DSTupleUpdate::builder b; - b.set_lg_k(lg_k); - b.set_seed(seed); - return b.build().compact(); - } - // ============================================================ // Bind Data // ============================================================ - struct DSTupleBindData : public FunctionData - { - DSTupleBindData() = default; + using DSTupleBindData = DSThetaFamilyBindData; - unique_ptr Copy() const override - { - return make_uniq(*this); - } - - bool Equals(const FunctionData &other_p) const override - { - auto &other = other_p.Cast(); - return lg_k == other.lg_k && infer_lg_k == other.infer_lg_k && seed == other.seed && p == other.p && mode == other.mode; - } - - uint8_t lg_k = DS_THETA_DEFAULT_LG_K; - bool infer_lg_k = false; - uint64_t seed = DS_THETA_DEFAULT_SEED; - float p = 1.0f; - DSTupleMode mode = DSTupleMode::SUM; - }; - - DSTupleMode DSTupleBindMode(BindAggregateFunctionInput &input) + uint8_t DSTupleBindMode(BindAggregateFunctionInput &input) { - return DSTupleParseMode(input.GetNonNullConstant(Identifier("mode")).GetValue()); + return static_cast(DSTupleParseMode(input.GetNonNullConstant(Identifier("mode")).GetValue())); } template @@ -172,6 +145,7 @@ namespace duckdb } DSTupleBindSeedAndP(input, result->seed, result->p); result->mode = DSTupleBindMode(input); + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 3 : 2); return std::move(result); } @@ -188,325 +162,144 @@ namespace duckdb result->infer_lg_k = true; } result->mode = DSTupleBindMode(input); + DSThetaRecordConstants(input, *result, HAS_LG_K ? 1 : 0, HAS_LG_K ? 2 : 1); return std::move(result); } - // ============================================================ - // Build / Union Aggregate - // ============================================================ - - struct DSTupleState + unique_ptr DSTupleBindIntersect(BindAggregateFunctionInput &input) { - DSTupleUpdate *update_sketch = nullptr; - DSTupleUnion *union_sketch = nullptr; - uint8_t union_lg_k = 0; - uint64_t union_seed = DS_THETA_DEFAULT_SEED; - //! A merged input sketch was empty: with no other input, the result is an empty sketch rather than NULL - bool saw_empty_sketch = false; - - ~DSTupleState() - { - delete update_sketch; - delete union_sketch; - } - - void CreateUpdateSketch(const DSTupleBindData &bind_data) - { - if (!update_sketch) - { - DSTupleUpdate::builder b{DSTupleDoublePolicy(bind_data.mode)}; - b.set_lg_k(bind_data.lg_k); - b.set_seed(bind_data.seed); - b.set_p(bind_data.p); - update_sketch = new DSTupleUpdate(b.build()); - } - } - - // Creates the union, or grows it to a larger lg_k (re-adding what it holds so far). - void EnsureUnion(uint8_t lg_k, uint64_t seed, DSTupleMode mode) - { - if (union_sketch && lg_k <= union_lg_k) - { - return; - } - if (union_sketch) - { - seed = union_seed; - } - DSTupleUnion::builder b{DSTupleDoublePolicy(mode)}; - b.set_lg_k(lg_k); - b.set_seed(seed); - auto grown = new DSTupleUnion(b.build()); - if (union_sketch) - { - grown->update(union_sketch->get_result()); - delete union_sketch; - } - union_sketch = grown; - union_lg_k = lg_k; - union_seed = seed; - } - - bool HasData() const - { - return update_sketch || union_sketch; - } - - uint64_t Seed(const DSTupleBindData &bind_data) const - { - return union_sketch ? union_seed : bind_data.seed; - } - - uint8_t LgK(const DSTupleBindData &bind_data) const - { - return union_sketch ? union_lg_k : bind_data.lg_k; - } - }; + auto result = make_uniq(); + result->mode = DSTupleBindMode(input); + DSThetaRecordConstants(input, *result, 0, 1); + return std::move(result); + } void DSTupleThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b) { throw InvalidInputException("%s: cannot combine Tuple sketches built with different seeds (seed hashes %d and %d)", function_name, a, b); } - struct DSTupleOperationBase + // ============================================================ + // Aggregate states (see datasketches_theta_arena.hpp) + // ============================================================ + + struct DSTupleTraits { - template - static void Initialize(STATE &state) - { - state.update_sketch = nullptr; - state.union_sketch = nullptr; - state.union_lg_k = 0; - state.union_seed = DS_THETA_DEFAULT_SEED; - state.saw_empty_sketch = false; - } + using Alloc = DSTupleAllocator; + using Entry = std::pair; + using Update = DSTupleUpdate; + using Compact = DSTupleCompact; + using Union = DSTupleUnion; + using Intersection = DSTupleIntersection; + static constexpr bool SET_ONLY = false; + static constexpr bool HAS_NUM_VALUES = false; + static constexpr const char *KIND = "Tuple"; + static constexpr const char *BUILD_NAME = "datasketch_tuple"; + static constexpr const char *INTERSECT_NAME = "datasketch_tuple_intersect_agg"; - template - static void Destroy(STATE &state, AggregateInputData &) + static DSTupleDoublePolicy Policy(const DSThetaParams ¶ms) { - delete state.update_sketch; - delete state.union_sketch; - state.update_sketch = nullptr; - state.union_sketch = nullptr; + return DSTupleDoublePolicy(static_cast(params.mode)); } - static bool IgnoreNull() { return true; } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static uint64_t Key(const Entry &entry) { return entry.first; } + static idx_t SummarySize(const DSThetaParams &) { return sizeof(double); } + static void WriteSummary(const Entry &entry, data_ptr_t data, const DSThetaParams &) { - target.saw_empty_sketch = target.saw_empty_sketch || source.saw_empty_sketch; - if (!source.HasData()) - return; - auto &bind_data = aggr_input_data.bind_data->template Cast(); - auto source_seed = source.Seed(bind_data); - if (target.HasData() && ThetaSeedHash(source_seed) != ThetaSeedHash(target.Seed(bind_data))) - { - DSTupleThrowSeedMismatch("datasketch_tuple", ThetaSeedHash(target.Seed(bind_data)), ThetaSeedHash(source_seed)); - } - auto lg_k = MaxValue(source.LgK(bind_data), target.LgK(bind_data)); - target.EnsureUnion(lg_k, target.HasData() ? target.Seed(bind_data) : source_seed, bind_data.mode); - if (target.update_sketch) - { - target.union_sketch->update(target.update_sketch->compact()); - delete target.update_sketch; - target.update_sketch = nullptr; - } - if (source.update_sketch) - target.union_sketch->update(source.update_sketch->compact()); - if (source.union_sketch) - target.union_sketch->update(source.union_sketch->get_result()); + memcpy(data, &entry.second, sizeof(double)); } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) + static Entry MakeEntry(uint64_t key, const_data_ptr_t data, const DSThetaParams &) { - if (state.union_sketch) - { - target = DSTupleDoubleTraits::Serialize(finalize_data.result, state.union_sketch->get_result()); - } - else if (state.update_sketch) - { - target = DSTupleDoubleTraits::Serialize(finalize_data.result, state.update_sketch->compact()); - } - else if (state.saw_empty_sketch) - { - auto &bind_data = finalize_data.input.bind_data->template Cast(); - target = DSTupleDoubleTraits::Serialize(finalize_data.result, DSTupleEmptySketch(bind_data.lg_k, bind_data.seed)); - } - else - { - // no input rows: NULL, as for the other sketches' aggregates - finalize_data.ReturnNull(); - } + double value; + memcpy(&value, data, sizeof(double)); + return Entry(key, value); } - }; - - struct DSTupleCreateOperation : DSTupleOperationBase - { - template - static void Operation(STATE &state, const KEY_TYPE &key, const VALUE_TYPE &value, AggregateBinaryInput &idata) + static void Fold(data_ptr_t data, const Entry &incoming, const DSThetaParams ¶ms) { - auto &bind_data = idata.input.bind_data->template Cast(); - state.CreateUpdateSketch(bind_data); - if constexpr (std::is_same_v) - { - state.update_sketch->update(key.GetData(), key.GetSize(), value); - } - else - { - state.update_sketch->update(key, value); - } + double value; + memcpy(&value, data, sizeof(double)); + Policy(params)(value, incoming.second); + memcpy(data, &value, sizeof(double)); } - }; + static idx_t PayloadSize(const DSThetaParams &) { return sizeof(double); } + static uint8_t NumValues(const Compact &) { return 1; } - struct DSTupleMergeOperation : DSTupleOperationBase - { - template - static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + template + static void Apply(Update &sketch, const_data_ptr_t key, uint32_t length, const_data_ptr_t payload, const DSThetaParams &) { - auto &bind_data = idata.input.bind_data->template Cast(); - auto sketch = DSTupleDoubleTraits::Deserialize(blob); - if (sketch.is_empty()) + double value; + memcpy(&value, payload, sizeof(double)); + if constexpr (std::is_same::value) { - state.saw_empty_sketch = true; - return; + sketch.update(key, length, value); } - if (state.union_sketch && sketch.get_seed_hash() != ThetaSeedHash(state.union_seed)) - { - DSTupleThrowSeedMismatch("datasketch_tuple", ThetaSeedHash(state.union_seed), sketch.get_seed_hash()); - } - auto lg_k = bind_data.infer_lg_k ? MaxValue(bind_data.lg_k, ThetaInferLgK(sketch)) : bind_data.lg_k; - state.EnsureUnion(lg_k, ThetaSeedForHash(sketch.get_seed_hash()), bind_data.mode); - ThetaGuard([&]() - { state.union_sketch->update(sketch); }, - "Tuple"); - } - - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) - { - // Merging the same sketch repeatedly is not idempotent for mode sum: merge it count times. - for (idx_t i = 0; i < count; i++) + else { - Operation(state, input, unary_input); + KEY typed; + memcpy(&typed, key, sizeof(KEY)); + sketch.update(typed, value); } } - }; - - // ============================================================ - // Intersection Aggregate - // ============================================================ - - struct DSTupleIntersectBindData : public FunctionData - { - explicit DSTupleIntersectBindData(DSTupleMode mode) : mode(mode) {} - unique_ptr Copy() const override { return make_uniq(mode); } - bool Equals(const FunctionData &other) const override { return mode == other.Cast().mode; } - DSTupleMode mode; - }; - - unique_ptr DSTupleBindIntersect(BindAggregateFunctionInput &input) - { - return make_uniq(DSTupleBindMode(input)); - } - struct DSTupleIntersectState - { - DSTupleIntersection *intersection = nullptr; - uint64_t seed = DS_THETA_DEFAULT_SEED; - bool saw_empty = false; - - ~DSTupleIntersectState() + static Update NewUpdate(const DSTupleBindData &bind, const DSThetaParams ¶ms) { - delete intersection; + DSTupleUpdate::builder b{Policy(params), Alloc(DSArenaMode())}; + b.set_lg_k(bind.lg_k); + b.set_seed(bind.seed); + b.set_p(bind.p); + return b.build(); } - void Add(const DSTupleCompact &sketch, DSTupleMode mode) + static Union NewUnion(uint8_t lg_k, uint64_t seed, const DSThetaParams ¶ms) { - if (sketch.is_empty()) - { - saw_empty = true; - return; - } - if (!intersection) - { - seed = ThetaSeedForHash(sketch.get_seed_hash()); - intersection = new DSTupleIntersection(seed, DSTupleDoublePolicy(mode)); - } - else if (sketch.get_seed_hash() != ThetaSeedHash(seed)) - { - DSTupleThrowSeedMismatch("datasketch_tuple_intersect_agg", ThetaSeedHash(seed), sketch.get_seed_hash()); - } - ThetaGuard([&]() - { intersection->update(sketch); }, - "Tuple"); + DSTupleUnion::builder b{Policy(params), Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build(); } - }; - struct DSTupleIntersectOperation - { - template - static void Initialize(STATE &state) + static Intersection NewIntersection(uint64_t seed, const DSThetaParams ¶ms) { - state.intersection = nullptr; - state.seed = DS_THETA_DEFAULT_SEED; - state.saw_empty = false; + return Intersection(seed, Policy(params), Alloc(DSArenaMode())); } - template - static void Destroy(STATE &state, AggregateInputData &) + static Compact Deserialize(const string_t &blob) { - delete state.intersection; - state.intersection = nullptr; + auto seed = DSTupleSeedFor(blob); + return ThetaGuard([&]() + { return Compact::deserialize(blob.GetData(), blob.GetSize(), seed, datasketches::serde(), Alloc(DSArenaMode())); }, + KIND); } - static bool IgnoreNull() { return true; } - - template - static void Operation(STATE &state, const A_TYPE &blob, AggregateUnaryInput &idata) + static Compact Empty(uint8_t lg_k, uint64_t seed, const DSThetaParams &) { - auto &bind_data = idata.input.bind_data->template Cast(); - state.Add(DSTupleDoubleTraits::Deserialize(blob), bind_data.mode); + DSTupleUpdate::builder b{DSTupleDoublePolicy(), Alloc(DSArenaMode())}; + b.set_lg_k(lg_k); + b.set_seed(seed); + return b.build().compact(); } - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, idx_t count) + static auto Serialize(const Compact &sketch) -> decltype(sketch.serialize()) { - for (idx_t i = 0; i < count; i++) - { - Operation(state, input, unary_input); - } + return sketch.serialize(); } - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) + static void ThrowSeedMismatch(const char *function_name, uint16_t a, uint16_t b) { - target.saw_empty = target.saw_empty || source.saw_empty; - if (source.intersection) - { - auto &bind_data = aggr_input_data.bind_data->template Cast(); - target.Add(source.intersection->get_result(), bind_data.mode); - } + DSTupleThrowSeedMismatch(function_name, a, b); } - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) - { - if (state.saw_empty) - { - target = DSTupleDoubleTraits::Serialize(finalize_data.result, DSTupleEmptySketch(DS_THETA_DEFAULT_LG_K, state.seed)); - } - else if (state.intersection) - { - target = DSTupleDoubleTraits::Serialize(finalize_data.result, state.intersection->get_result()); - } - else - { - finalize_data.ReturnNull(); - } - } + static void ThrowNumValues(const char *, idx_t, idx_t) {} }; + template + using DSTupleState = DSThetaFamilyState; + using DSTupleMergeState = DSTupleState; + using DSTupleIntersectState = DSThetaFamilyIntersectState; + // merging the same sketch repeatedly is not idempotent for mode sum + using DSTupleMergeOperation = DSThetaFamilyOperation; + using DSTupleIntersectOperation = DSThetaFamilyIntersectOperation; + // ============================================================ // Scalar Functions // ============================================================ @@ -671,9 +464,10 @@ namespace duckdb { for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::BinaryAggregate( - key_type, LogicalType::DOUBLE, sketch_type); - fun.SetStateDestructorCallback(AggregateFunction::StateDestroy); + using STATE = DSTupleState; + using OP = DSThetaFamilyCreatePairOperation; + auto fun = AggregateFunction::BinaryAggregate(key_type, LogicalType::DOUBLE, sketch_type); + DSThetaSetStateExport(fun); auto signature = DSTupleSignature(with_lg_k, {key_type, LogicalType::DOUBLE}, sketch_type); DSTupleAddMode(signature); DSTupleAddSeedAndP(signature); @@ -681,12 +475,12 @@ namespace duckdb if (with_lg_k) { fun.SetBindCallback(DSTupleBindBuild); - DSReadDataArgumentsAt<1, 2, DSTupleState, T, double, DSTupleCreateOperation>(fun); + DSReadDataArgumentsAt<1, 2, STATE, T, double, OP>(fun); } else { fun.SetBindCallback(DSTupleBindBuild); - DSReadDataArgumentsAt<0, 1, DSTupleState, T, double, DSTupleCreateOperation>(fun); + DSReadDataArgumentsAt<0, 1, STATE, T, double, OP>(fun); } fun.SetOrderDependent(AggregateOrderDependent::NOT_ORDER_DEPENDENT); set.AddFunction(fun); @@ -748,15 +542,15 @@ namespace duckdb for (bool with_lg_k : {false, true}) { - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); auto signature = DSTupleSignature(with_lg_k, {sketch_type}, sketch_type); DSTupleAddMode(signature); fun.GetSignature() = std::move(signature); if (with_lg_k) { fun.SetBindCallback(DSTupleBindMerge); - DSReadDataArgumentAt<1, DSTupleState, string_t, DSTupleMergeOperation>(fun); + DSReadDataArgumentAt<1, DSTupleMergeState, string_t, DSTupleMergeOperation>(fun); descs.push_back(DSTupleDescription("Merges (unions) Tuple sketches into one with the given log2 of nominal entries (lg_k); the " "summaries of a key are combined by mode := 'sum' (default), 'min', 'max' or 'one'", {ex.QueryPartials("datasketch_tuple(16, sketch)"), ex.QueryPartials("datasketch_tuple(16, sketch, mode := 'max')")}, @@ -765,7 +559,7 @@ namespace duckdb else { fun.SetBindCallback(DSTupleBindMerge); - DSReadDataArgumentAt<0, DSTupleState, string_t, DSTupleMergeOperation>(fun); + DSReadDataArgumentAt<0, DSTupleMergeState, string_t, DSTupleMergeOperation>(fun); descs.push_back(DSTupleDescription("Merges (unions) Tuple sketches into one, keeping as many entries as the largest input " "(at least lg_k 12); the summaries of a key are combined by mode := 'sum' (default), 'min', 'max' or 'one'", {ex.QueryPartials("datasketch_tuple(sketch)"), ex.QueryPartials("datasketch_tuple(sketch, mode := 'max')")}, @@ -780,8 +574,8 @@ namespace duckdb // --- datasketch_tuple_intersect_agg { AggregateFunctionSet set("datasketch_tuple_intersect_agg"); - auto fun = AggregateFunction::UnaryAggregate( - sketch_type, sketch_type); + auto fun = AggregateFunction::UnaryAggregate(sketch_type, sketch_type); + DSThetaSetStateExport(fun); auto signature = DSTupleSignature(false, {sketch_type}, sketch_type); DSTupleAddMode(signature); fun.GetSignature() = std::move(signature); diff --git a/test/sql/datasketch_spill_theta.test b/test/sql/datasketch_spill_theta.test new file mode 100644 index 0000000..3feed4b --- /dev/null +++ b/test/sql/datasketch_spill_theta.test @@ -0,0 +1,129 @@ +# name: test/sql/datasketch_spill_theta.test +# description: Theta, Tuple and array-of-doubles aggregate states live in the arena, spill, and give the results they give in memory +# group: [datasketches] + +require datasketches + +require skip_reload + +statement ok +SET threads=4; + +statement ok +SET temp_directory='{TEST_DIR}/datasketch_spill_theta'; + +statement ok +SET preserve_insertion_order=false; + +statement ok +SET memory_limit='4GB'; + +# Few rows per group (buffered states) and many rows per group (sketches), several input types. Summaries are +# small integers, so sums do not depend on the order they are added in. The expected results are computed in +# memory and kept as hashes of the sketches. +statement ok +CREATE VIEW small_rows AS SELECT i % 300000 AS g, i AS v, (i % 1000)::VARCHAR AS s, (i % 7)::DOUBLE AS d FROM range(900000) t(i); + +statement ok +CREATE VIEW large_rows AS SELECT i % 40000 AS g, i AS v, (i % 100000)::VARCHAR AS s, (i % 7)::DOUBLE AS d FROM range(4000000) t(i); + +statement ok +CREATE TABLE small_expected AS SELECT g, hash(datasketch_theta(v)) AS th, hash(datasketch_theta(s)) AS ths, hash(datasketch_theta(10, d::INTEGER, seed := 42, p := 0.5)) AS thp, hash(datasketch_tuple(v, d)) AS tu, hash(datasketch_tuple(s, d, mode := 'max')) AS tus, hash(datasketch_aod(v::INTEGER, [d, 1.0])) AS ao FROM small_rows GROUP BY g; + +statement ok +CREATE TABLE large_expected AS SELECT g, hash(datasketch_theta(v)) AS th, hash(datasketch_theta(s)) AS ths, hash(datasketch_theta(5, v::DOUBLE)) AS thd, hash(datasketch_tuple(v, d)) AS tu, hash(datasketch_tuple(s, d, mode := 'min')) AS tus, hash(datasketch_aod(v::INTEGER, [d, 1.0])) AS ao FROM large_rows GROUP BY g; + +# Partial sketches of each group, which overlap, for the merge and intersection aggregates +statement ok +CREATE TABLE parts AS +SELECT g, p, datasketch_theta(v) AS th, datasketch_tuple(v, d) AS tu, datasketch_aod(v, [d]) AS ao +FROM (SELECT i % 100000 AS g, (i // 100000) % 3 AS p, (i // 100000) % 4 + (i % 100000) * 10 AS v, (i % 5)::DOUBLE AS d FROM range(1200000) t(i)) +GROUP BY g, p; + +statement ok +CREATE TABLE merged_expected AS SELECT g, hash(datasketch_theta(th)) AS th, hash(datasketch_theta(14, th)) AS thk, hash(datasketch_theta_intersect_agg(th)) AS thi, hash(datasketch_tuple(tu)) AS tu, hash(datasketch_tuple(16, tu, mode := 'max')) AS tuk, hash(datasketch_tuple_intersect_agg(tu, mode := 'min')) AS tui, hash(datasketch_aod(ao)) AS ao, hash(datasketch_aod_intersect_agg(ao)) AS aoi FROM parts GROUP BY g; + +# The merged sketches are what they should be: each group has 4 distinct values, in every part +query IIII +SELECT min(datasketch_theta_estimate(th)), max(datasketch_theta_estimate(th)), min(datasketch_theta_estimate(thi)), max(datasketch_tuple_sum(tu)) +FROM (SELECT datasketch_theta(th) AS th, datasketch_theta_intersect_agg(th) AS thi, datasketch_tuple(tu) AS tu FROM parts GROUP BY g); +---- +4.0 4.0 4.0 48.0 + +statement ok +SET memory_limit='128MB'; + +query I +SELECT count(*) FROM small_expected e JOIN (SELECT g, hash(datasketch_theta(v)) AS th, hash(datasketch_theta(s)) AS ths, hash(datasketch_theta(10, d::INTEGER, seed := 42, p := 0.5)) AS thp, hash(datasketch_tuple(v, d)) AS tu, hash(datasketch_tuple(s, d, mode := 'max')) AS tus, hash(datasketch_aod(v::INTEGER, [d, 1.0])) AS ao FROM small_rows GROUP BY g) a USING (g) WHERE e.th = a.th AND e.ths = a.ths AND e.thp = a.thp AND e.tu = a.tu AND e.tus = a.tus AND e.ao = a.ao; +---- +300000 + +query I +SELECT count(*) FROM merged_expected e JOIN (SELECT g, hash(datasketch_theta(th)) AS th, hash(datasketch_theta(14, th)) AS thk, hash(datasketch_theta_intersect_agg(th)) AS thi, hash(datasketch_tuple(tu)) AS tu, hash(datasketch_tuple(16, tu, mode := 'max')) AS tuk, hash(datasketch_tuple_intersect_agg(tu, mode := 'min')) AS tui, hash(datasketch_aod(ao)) AS ao, hash(datasketch_aod_intersect_agg(ao)) AS aoi FROM parts GROUP BY g) a USING (g) WHERE e.th = a.th AND e.thk = a.thk AND e.thi = a.thi AND e.tu = a.tu AND e.tuk = a.tuk AND e.tui = a.tui AND e.ao = a.ao AND e.aoi = a.aoi; +---- +100000 + +# Large states: one aggregate per query, so that a partition of states fits in the limit +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_theta(v)) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.th = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_theta(s)) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.ths = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_theta(5, v::DOUBLE)) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.thd = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_tuple(v, d)) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.tu = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_tuple(s, d, mode := 'min')) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.tus = a.x; +---- +40000 + +query I +SELECT count(*) FROM large_expected e JOIN (SELECT g, hash(datasketch_aod(v::INTEGER, [d, 1.0])) AS x FROM large_rows GROUP BY g) a USING (g) WHERE e.ao = a.x; +---- +40000 + +statement ok +SET memory_limit='4GB'; + +# Exported states finalize to the aggregate's result, buffered (small inputs) or not. DuckDB re-binds an exported +# state with its arguments as positional ones, so this works only for aggregates without keyword-only parameters +# (the builds take seed := and p :=, the Tuple merges mode :=); spilling does not re-bind and covers those above. +statement ok +CREATE TABLE stored AS +SELECT n, i % n AS k, datasketch_theta(i) AS th, datasketch_aod(i, [1.0, i::DOUBLE]) AS ao +FROM (VALUES (1), (3), (1000)) n(n), range(20000) t(i) GROUP BY n, k; + +query IIIIII +SELECT bool_and(a), bool_and(b), bool_and(c), bool_and(d), bool_and(e), bool_and(f) FROM ( + SELECT finalize(datasketch_theta(th) EXPORT_STATE) = datasketch_theta(th) AS a, + finalize(datasketch_theta(5, th) EXPORT_STATE) = datasketch_theta(5, th) AS b, + finalize(datasketch_theta_intersect_agg(th) EXPORT_STATE) = datasketch_theta_intersect_agg(th) AS c, + finalize(datasketch_aod(ao) EXPORT_STATE) = datasketch_aod(ao) AS d, + finalize(datasketch_aod(16, ao) EXPORT_STATE) = datasketch_aod(16, ao) AS e, + finalize(datasketch_aod_intersect_agg(ao) EXPORT_STATE) = datasketch_aod_intersect_agg(ao) AS f + FROM stored GROUP BY n, k % 5 +); +---- +true true true true true true + +query III +SELECT bool_and(a), bool_and(b), bool_and(c) FROM ( + SELECT finalize(datasketch_theta(th) EXPORT_STATE) = datasketch_theta(th) AS a, + finalize(datasketch_theta_intersect_agg(th) EXPORT_STATE) = datasketch_theta_intersect_agg(th) AS b, + finalize(datasketch_aod_intersect_agg(ao) EXPORT_STATE) = datasketch_aod_intersect_agg(ao) AS c + FROM parts GROUP BY g % 1000 +); +---- +true true true