diff --git a/codegen/generated.cpp.j2 b/codegen/generated.cpp.j2 index 1e97f2a..4fe0705 100644 --- a/codegen/generated.cpp.j2 +++ b/codegen/generated.cpp.j2 @@ -65,7 +65,7 @@ unique_ptr DS{{ S }}Bind(BindAggregateFunctionInput &input) { return std::move(bind_data); } -{% if not sketch.union_accepts_values %} +{% if not sketch.counting %} {% if not sketch.counting %} template {% endif %} @@ -107,41 +107,25 @@ struct DS{{ S }}State { {% endif %} {% if sketch.counting %} -// Holds a union so partial states and input sketches merge into it in place. -struct DS{{ S }}UnionState { - {{ sketch.union_class }} *sketch = nullptr; - - ~DS{{ S }}UnionState() { - delete sketch; - } - - void CreateSketch(const DS{{ S }}BindData &bind) { - D_ASSERT(!sketch); - sketch = new {{ sketch.union_class }}(bind.k); - } - - void CreateSketch(const DS{{ S }}UnionState &existing) { - if (existing.sketch) { - sketch = new {{ sketch.union_class }}(*existing.sketch); - } - } - - {{ sketch.cpp_class }} deserialize_sketch(const string_t &data) { - return DSDeserialize<{{ sketch.cpp_class }}>(data, "{{ S }}"); - } -}; - -{% endif %} -{% if sketch.arena and sketch.counting %} -// The aggregates' state: a union in the aggregate arena (see the arena states in datasketches_generated_common.hpp). -struct DS{{ S }}ArenaState { +{% set cpc = S == "CPC" %} +{% set noun = "sketch" if cpc else "union" %} +{% set held_class = sketch.cpp_class if cpc else sketch.union_class %} +{% set held = "*sketch" if cpc else "*unioned.sketch" %} +// The union aggregate's state: a union in the aggregate arena (see the arena states in +// datasketches_generated_common.hpp).{{ "" if cpc else " The create aggregate keeps one once it outgrows its buffer." }} +struct DS{{ S }}ArenaUnionState { {{ sketch.union_class }} *sketch; static DSAllocator Allocator() { return DSAllocator(DSArenaMode()); } +{% if cpc %} + // Input sketches are read on the heap and freed once merged: the union assigns a sparse sketch to its + // accumulator, whose tables keep the accumulator's allocator, and ORs any other sketch into its bit matrix. +{% else %} // Input sketches are read on the heap and freed once merged: the union copies what it keeps into its own // memory (with vcpkg-overlay/apache-datasketches/hll-union-allocator.patch). +{% endif %} static {{ sketch.cpp_class }} Read(const string_t &data) { return DSDeserialize<{{ sketch.cpp_class }}>(data, "{{ S }}"); } @@ -152,35 +136,34 @@ struct DS{{ S }}ArenaState { bool IsEmpty() const { return !sketch; } - void Ensure(uint8_t lg_max_k, ArenaAllocator &arena) { + void Ensure(uint8_t lg_k, ArenaAllocator &arena) { if (!sketch) { - sketch = arena.Make<{{ sketch.union_class }}>(lg_max_k, Allocator()); + sketch = arena.Make<{{ sketch.union_class }}>(lg_k, {{ "datasketches::DEFAULT_SEED, " if cpc else "" }}Allocator()); } } - template - void Update(const V &value, const DS{{ S }}BindData &bind, ArenaAllocator &arena) { - Ensure(bind.k, arena); - DSCountingUpdate(*sketch, value); - } void Merge(const string_t &data, const DS{{ S }}BindData &bind, ArenaAllocator &arena) { Ensure(bind.k, arena); sketch->update(Read(data)); } - void Combine(const DS{{ S }}ArenaState &source, const DS{{ S }}BindData &bind, ArenaAllocator &arena) { + void Combine(const DS{{ S }}ArenaUnionState &source, const DS{{ S }}BindData &bind, ArenaAllocator &arena) { if (!sketch) { // A copy allocates through the source's allocator, which in arena mode is the current (target's) arena. sketch = arena.Make<{{ sketch.union_class }}>(*source.sketch); return; } + // The source's contents are copied into the current arena, then merged. sketch->update(UnionContents(*source.sketch)); } auto Serialize(const DS{{ S }}BindData &bind, ArenaAllocator &) const { return SerializeUnion(*sketch, bind); } - // The union's lg_max_k, then its contents. - auto Export(const DS{{ S }}BindData &bind, ArenaAllocator &) const { - auto bytes = UnionContents(*sketch).serialize_updatable(); - bytes.insert(bytes.begin(), static_cast(bind.k)); + // The union's lg_k, then its contents. + std::vector Export(const DS{{ S }}BindData &bind, ArenaAllocator &) const { + auto contents = UnionContents(*sketch).{{ "serialize" if cpc else "serialize_updatable" }}(); + std::vector bytes; + bytes.reserve(1 + contents.size()); + bytes.push_back(static_cast(bind.k)); + bytes.insert(bytes.end(), contents.begin(), contents.end()); return bytes; } void Import(const string_t &data, ArenaAllocator &arena) { @@ -193,6 +176,180 @@ struct DS{{ S }}ArenaState { } }; +// The create aggregate's state, for values read as INPUT (see DSCountingInput). DuckDB creates a state per input +// row when grouping reduces little (it skips the hash table lookups), and a {{ noun }} takes a few allocations, so the +// first BUFFERED distinct values are kept as they are (see DSArenaValues) and fed to a {{ noun }}, in order, only +// when more arrive, when the state is finalized or exported as a sketch, or when it merges a partial state +{% if cpc %} +// and both have values. Two partial states merge through a union, which marks the result merged (it drops the HIP +// estimate), so buffered values are fed to a sketch first: the result is the one unbuffered states give. +{% else %} +// that has a union. Up to 7 values make at most 7 coupons, which a union keeps in LIST mode in arrival order and +// merges into another union by feeding them in that order: so appending a partial state's buffered values to +// another state gives the same union as merging the two states' unions. +{% endif %} +template +struct DS{{ S }}ArenaState { + using T = typename INPUT::physical_type; + static constexpr uint32_t BUFFERED = {{ 16 if cpc else 7 }}; + +{% if cpc %} + {{ sketch.cpp_class }} *sketch; +{% else %} + // Empty while the values are buffered. + DS{{ S }}ArenaUnionState unioned; +{% endif %} + DSArenaValues buffer; + +{% if cpc %} + static DSAllocator Allocator() { + return DSAllocator(DSArenaMode()); + } +{% endif %} + template + static void Feed(TARGET &target, const T &value) { + DSCountingUpdate(target, value); + } + + void Initialize() { + {{ "sketch = nullptr" if cpc else "unioned.Initialize()" }}; + buffer.Initialize(); + } + bool HasSketch() const { + return {{ "sketch != nullptr" if cpc else "!unioned.IsEmpty()" }}; + } + bool IsEmpty() const { + return !HasSketch() && buffer.Count() == 0; + } + // Feeds the buffered values to a new {{ noun }}. + {{ held_class }} &Promote(const DS{{ S }}BindData &bind, ArenaAllocator &arena) { + if (!HasSketch()) { +{% if cpc %} + sketch = arena.Make<{{ sketch.cpp_class }}>(bind.k, datasketches::DEFAULT_SEED, Allocator()); +{% else %} + unioned.Ensure(bind.k, arena); +{% endif %} + auto &target = {{ held }}; + buffer.ForEach([&](const T &value) { Feed(target, value); }); + buffer.Initialize(); + } + return {{ held }}; + } + void Add(const T &value, const DS{{ S }}BindData &bind, ArenaAllocator &arena) { + if (!HasSketch()) { + if (buffer.Contains(value)) { + return; + } + if (buffer.Count() < BUFFERED) { + buffer.Append(value, arena); + return; + } + } + Feed(Promote(bind, arena), value); + } + template + void Update(const V &value, const DS{{ S }}BindData &bind, ArenaAllocator &arena) { + static_assert(IS_UUID == INPUT::is_uuid, "the state reads its values as INPUT"); + Add(value, bind, arena); + } +{% if cpc %} + void Combine(const DS{{ S }}ArenaState &source, const DS{{ S }}BindData &bind, ArenaAllocator &arena) { + if (IsEmpty()) { + if (source.sketch) { + // A copy allocates through the source's allocator, which in arena mode is the current (target's) + // arena. + sketch = arena.Make<{{ sketch.cpp_class }}>(*source.sketch); + } else { + source.buffer.ForEach([&](const T &value) { buffer.Append(value, arena); }); + } + return; + } + // The union and a sketch of the source's buffered values are temporaries on the heap. The union copies + // the sketches it reads; its result is assigned to the state's sketch, whose tables keep their (arena) + // allocator and so copy the result's. + auto &target = Promote(bind, arena); + {{ sketch.union_class }} u(target.get_lg_k()); + u.update(target); + if (source.sketch) { + u.update(*source.sketch); + } else { + {{ sketch.cpp_class }} values(bind.k); + source.buffer.ForEach([&](const T &value) { Feed(values, value); }); + u.update(std::move(values)); + } + target = u.get_result(); + } +{% else %} + void Combine(const DS{{ S }}ArenaState &source, const DS{{ S }}BindData &bind, ArenaAllocator &arena) { + if (!source.HasSketch()) { + source.buffer.ForEach([&](const T &value) { Add(value, bind, arena); }); + return; + } + if (!IsEmpty()) { + Promote(bind, arena); + } + unioned.Combine(source.unioned, bind, arena); + } +{% endif %} + // A state that still buffers its values is serialized through a temporary {{ noun }} on the heap, which is freed + // right away, instead of one left behind in the arena. + auto Serialize(const DS{{ S }}BindData &bind, ArenaAllocator &arena) const { + if (HasSketch()) { + return {{ "sketch->serialize()" if cpc else "unioned.Serialize(bind, arena)" }}; + } + {{ held_class }} temporary(bind.k); + buffer.ForEach([&](const T &value) { Feed(temporary, value); }); + return {{ "temporary.serialize()" if cpc else "SerializeUnion(temporary, bind)" }}; + } + // The exported state is a tag byte, then the buffered values (EXPORT_BUFFER, see DSArenaValues::Export) or the + // {{ "serialized sketch" if cpc else "exported union (see DS" ~ S ~ "ArenaUnionState::Export)" }} (EXPORT_SKETCH). Buffered states are exported as + // values, so spilled states that buffer stay small. + static constexpr uint8_t EXPORT_BUFFER = 0; + static constexpr uint8_t EXPORT_SKETCH = 1; + std::vector Export(const DS{{ S }}BindData &bind, ArenaAllocator &arena) const { + std::vector bytes; + if (!HasSketch()) { + bytes.push_back(EXPORT_BUFFER); + buffer.Export(bytes); + return bytes; + } + auto exported = {{ "sketch->serialize()" if cpc else "unioned.Export(bind, arena)" }}; + bytes.reserve(1 + exported.size()); + bytes.push_back(EXPORT_SKETCH); + bytes.insert(bytes.end(), exported.begin(), exported.end()); + return bytes; + } + void Import(const string_t &data, ArenaAllocator &arena) { + const auto size = data.GetSize(); + const auto bytes = const_data_ptr_cast(data.GetData()); + if (size >= 1 && bytes[0] == EXPORT_SKETCH) { + const string_t exported(data.GetData() + 1, UnsafeNumericCast(size - 1)); +{% if cpc %} + sketch = arena.Make<{{ sketch.cpp_class }}>(DSDeserialize<{{ sketch.cpp_class }}>( + exported, "{{ S }}", datasketches::DEFAULT_SEED, Allocator())); +{% else %} + unioned.Import(exported, arena); +{% endif %} + return; + } + if (size < 1 || bytes[0] != EXPORT_BUFFER || !buffer.Import(bytes + 1, size - 1, BUFFERED, arena)) { + throw InvalidInputException("Invalid {{ S }} aggregate state"); + } + } +}; + +// A sketch absorbs a repeated value once, so a constant input is added once. +template +struct DS{{ S }}ArenaCreateOperation : DSArenaCreateOperation { + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, + idx_t count) { + if (count > 0) { + OP::template Operation(state, input, unary_input); + } + } +}; + {% elif sketch.arena %} // The aggregates' state over items that own no heap memory (every item type but VARCHAR), in the aggregate arena // (see the arena states in datasketches_generated_common.hpp). The first BUFFERED items are kept as they are and @@ -330,9 +487,9 @@ static void DS{{ S }}{{ function.name }}(DataChunk &args, ExpressionState &state } {% endfor %} -{% if sketch.arena and sketch.counting %} +{% if sketch.counting %} static AggregateFunction DS{{ S }}MergeAggregate(const LogicalType &result_type) { - using STATE = DS{{ S }}ArenaState; + using STATE = DS{{ S }}ArenaUnionState; using OP = DSArenaMergeOperation; auto fun = AggregateFunction::UnaryAggregate(result_type, result_type); DSReadDataArgumentAt<1, STATE, string_t, OP>(fun); @@ -343,35 +500,13 @@ static AggregateFunction DS{{ S }}MergeAggregate(const LogicalType &result_type) template static AggregateFunction DS{{ S }}CreateAggregate(const LogicalType &type, const LogicalType &result_type) { using INPUT = DSCountingInput; - using STATE = DS{{ S }}ArenaState; - using OP = DSArenaCreateOperation; + using STATE = DS{{ S }}ArenaState; + using OP = DS{{ S }}ArenaCreateOperation; auto fun = AggregateFunction::UnaryAggregate(type, result_type); DSReadDataArgumentAt<1, STATE, typename INPUT::physical_type, OP>(fun); DSSetArenaStateExport(fun); return fun; } -{% elif sketch.counting %} -static AggregateFunction DS{{ S }}MergeAggregate(const LogicalType &result_type) { - using OP = DSUnionMergeOperation; - auto fun = AggregateFunction::UnaryAggregate(result_type, result_type); - DSReadDataArgumentAt<1, DS{{ S }}UnionState, string_t, OP>(fun); - return fun; -} - -template -static AggregateFunction DS{{ S }}CreateAggregate(const LogicalType &type, const LogicalType &result_type) { - using INPUT = DSCountingInput; -{% if sketch.union_accepts_values %} - using STATE = DS{{ S }}UnionState; - using OP = DSHLLCreateOperation; -{% else %} - using STATE = DS{{ S }}State; - using OP = DSCPCCreateOperation; -{% endif %} - auto fun = AggregateFunction::UnaryAggregate(type, result_type); - DSReadDataArgumentAt<1, STATE, typename INPUT::physical_type, OP>(fun); - return fun; -} {% else %} template static AggregateFunction DS{{ S }}MergeAggregate(const LogicalType &type, const LogicalType &result_type) { diff --git a/codegen/generator.py b/codegen/generator.py index 651cc5a..ca02374 100644 --- a/codegen/generator.py +++ b/codegen/generator.py @@ -110,12 +110,14 @@ class Sketch: option: Optional[Option] = None counting: bool = False """Counting sketches have one sketch type for every input type; the others have one per item type.""" - union_accepts_values: bool = False - """The library union also accepts raw values, so the create aggregate builds the union directly.""" arena: bool = False """The aggregates keep their sketches in the aggregate arena, so DuckDB can spill them (see the arena states in datasketches_generated_common.hpp). Item sketches over VARCHAR stay on the heap: their std::string items do not - allocate from the arena.""" + allocate from the arena. Counting sketches must use it: the template has no heap states for them.""" + + def __post_init__(self): + if self.counting and not self.arena: + raise ValueError(f"{self.name}: counting sketches keep their states in the arena (arena=True)") @property def sql(self) -> str: @@ -195,9 +197,9 @@ def k_error(self) -> str: example_k=100, k_min=10, k_max=65535, k_label="K (compression)"), Sketch("HLL", "DSHLLSketch", COUNTING_TYPES, example_k=12, k_min=4, k_max=21, k_label="K (lg_k)", option=HLL_TARGET_TYPE, - counting=True, union_accepts_values=True, arena=True), + counting=True, arena=True), Sketch("CPC", "DSCPCSketch", COUNTING_TYPES, - example_k=12, k_min=4, k_max=26, k_label="K (lg_k)", counting=True), + example_k=12, k_min=4, k_max=26, k_label="K (lg_k)", counting=True, arena=True), ] QUANTILE_SKETCHES = frozenset({"Quantiles", "KLL", "REQ", "TDigest"}) diff --git a/src/generated.cpp b/src/generated.cpp index 8df3ce9..eb0b422 100644 --- a/src/generated.cpp +++ b/src/generated.cpp @@ -5132,32 +5132,9 @@ unique_ptr DSHLLBind(BindAggregateFunctionInput &input) { return std::move(bind_data); } -// Holds a union so partial states and input sketches merge into it in place. -struct DSHLLUnionState { - DSHLLUnion *sketch = nullptr; - - ~DSHLLUnionState() { - delete sketch; - } - - void CreateSketch(const DSHLLBindData &bind) { - D_ASSERT(!sketch); - sketch = new DSHLLUnion(bind.k); - } - - void CreateSketch(const DSHLLUnionState &existing) { - if (existing.sketch) { - sketch = new DSHLLUnion(*existing.sketch); - } - } - - DSHLLSketch deserialize_sketch(const string_t &data) { - return DSDeserialize(data, "HLL"); - } -}; - -// The aggregates' state: a union in the aggregate arena (see the arena states in datasketches_generated_common.hpp). -struct DSHLLArenaState { +// The union aggregate's state: a union in the aggregate arena (see the arena states in +// datasketches_generated_common.hpp). The create aggregate keeps one once it outgrows its buffer. +struct DSHLLArenaUnionState { DSHLLUnion *sketch; static DSAllocator Allocator() { @@ -5175,35 +5152,34 @@ struct DSHLLArenaState { bool IsEmpty() const { return !sketch; } - void Ensure(uint8_t lg_max_k, ArenaAllocator &arena) { + void Ensure(uint8_t lg_k, ArenaAllocator &arena) { if (!sketch) { - sketch = arena.Make(lg_max_k, Allocator()); + sketch = arena.Make(lg_k, Allocator()); } } - template - void Update(const V &value, const DSHLLBindData &bind, ArenaAllocator &arena) { - Ensure(bind.k, arena); - DSCountingUpdate(*sketch, value); - } void Merge(const string_t &data, const DSHLLBindData &bind, ArenaAllocator &arena) { Ensure(bind.k, arena); sketch->update(Read(data)); } - void Combine(const DSHLLArenaState &source, const DSHLLBindData &bind, ArenaAllocator &arena) { + void Combine(const DSHLLArenaUnionState &source, const DSHLLBindData &bind, ArenaAllocator &arena) { if (!sketch) { // A copy allocates through the source's allocator, which in arena mode is the current (target's) arena. sketch = arena.Make(*source.sketch); return; } + // The source's contents are copied into the current arena, then merged. sketch->update(UnionContents(*source.sketch)); } auto Serialize(const DSHLLBindData &bind, ArenaAllocator &) const { return SerializeUnion(*sketch, bind); } - // The union's lg_max_k, then its contents. - auto Export(const DSHLLBindData &bind, ArenaAllocator &) const { - auto bytes = UnionContents(*sketch).serialize_updatable(); - bytes.insert(bytes.begin(), static_cast(bind.k)); + // The union's lg_k, then its contents. + std::vector Export(const DSHLLBindData &bind, ArenaAllocator &) const { + auto contents = UnionContents(*sketch).serialize_updatable(); + std::vector bytes; + bytes.reserve(1 + contents.size()); + bytes.push_back(static_cast(bind.k)); + bytes.insert(bytes.end(), contents.begin(), contents.end()); return bytes; } void Import(const string_t &data, ArenaAllocator &arena) { @@ -5216,6 +5192,128 @@ struct DSHLLArenaState { } }; +// The create aggregate's state, for values read as INPUT (see DSCountingInput). DuckDB creates a state per input +// row when grouping reduces little (it skips the hash table lookups), and a union takes a few allocations, so the +// first BUFFERED distinct values are kept as they are (see DSArenaValues) and fed to a union, in order, only +// when more arrive, when the state is finalized or exported as a sketch, or when it merges a partial state +// that has a union. Up to 7 values make at most 7 coupons, which a union keeps in LIST mode in arrival order and +// merges into another union by feeding them in that order: so appending a partial state's buffered values to +// another state gives the same union as merging the two states' unions. +template +struct DSHLLArenaState { + using T = typename INPUT::physical_type; + static constexpr uint32_t BUFFERED = 7; + + // Empty while the values are buffered. + DSHLLArenaUnionState unioned; + DSArenaValues buffer; + + template + static void Feed(TARGET &target, const T &value) { + DSCountingUpdate(target, value); + } + + void Initialize() { + unioned.Initialize(); + buffer.Initialize(); + } + bool HasSketch() const { + return !unioned.IsEmpty(); + } + bool IsEmpty() const { + return !HasSketch() && buffer.Count() == 0; + } + // Feeds the buffered values to a new union. + DSHLLUnion &Promote(const DSHLLBindData &bind, ArenaAllocator &arena) { + if (!HasSketch()) { + unioned.Ensure(bind.k, arena); + auto &target = *unioned.sketch; + buffer.ForEach([&](const T &value) { Feed(target, value); }); + buffer.Initialize(); + } + return *unioned.sketch; + } + void Add(const T &value, const DSHLLBindData &bind, ArenaAllocator &arena) { + if (!HasSketch()) { + if (buffer.Contains(value)) { + return; + } + if (buffer.Count() < BUFFERED) { + buffer.Append(value, arena); + return; + } + } + Feed(Promote(bind, arena), value); + } + template + void Update(const V &value, const DSHLLBindData &bind, ArenaAllocator &arena) { + static_assert(IS_UUID == INPUT::is_uuid, "the state reads its values as INPUT"); + Add(value, bind, arena); + } + void Combine(const DSHLLArenaState &source, const DSHLLBindData &bind, ArenaAllocator &arena) { + if (!source.HasSketch()) { + source.buffer.ForEach([&](const T &value) { Add(value, bind, arena); }); + return; + } + if (!IsEmpty()) { + Promote(bind, arena); + } + unioned.Combine(source.unioned, bind, arena); + } + // A state that still buffers its values is serialized through a temporary union on the heap, which is freed + // right away, instead of one left behind in the arena. + auto Serialize(const DSHLLBindData &bind, ArenaAllocator &arena) const { + if (HasSketch()) { + return unioned.Serialize(bind, arena); + } + DSHLLUnion temporary(bind.k); + buffer.ForEach([&](const T &value) { Feed(temporary, value); }); + return SerializeUnion(temporary, bind); + } + // The exported state is a tag byte, then the buffered values (EXPORT_BUFFER, see DSArenaValues::Export) or the + // exported union (see DSHLLArenaUnionState::Export) (EXPORT_SKETCH). Buffered states are exported as + // values, so spilled states that buffer stay small. + static constexpr uint8_t EXPORT_BUFFER = 0; + static constexpr uint8_t EXPORT_SKETCH = 1; + std::vector Export(const DSHLLBindData &bind, ArenaAllocator &arena) const { + std::vector bytes; + if (!HasSketch()) { + bytes.push_back(EXPORT_BUFFER); + buffer.Export(bytes); + return bytes; + } + auto exported = unioned.Export(bind, arena); + bytes.reserve(1 + exported.size()); + bytes.push_back(EXPORT_SKETCH); + bytes.insert(bytes.end(), exported.begin(), exported.end()); + return bytes; + } + void Import(const string_t &data, ArenaAllocator &arena) { + const auto size = data.GetSize(); + const auto bytes = const_data_ptr_cast(data.GetData()); + if (size >= 1 && bytes[0] == EXPORT_SKETCH) { + const string_t exported(data.GetData() + 1, UnsafeNumericCast(size - 1)); + unioned.Import(exported, arena); + return; + } + if (size < 1 || bytes[0] != EXPORT_BUFFER || !buffer.Import(bytes + 1, size - 1, BUFFERED, arena)) { + throw InvalidInputException("Invalid HLL aggregate state"); + } + } +}; + +// A sketch absorbs a repeated value once, so a constant input is added once. +template +struct DSHLLArenaCreateOperation : DSArenaCreateOperation { + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, + idx_t count) { + if (count > 0) { + OP::template Operation(state, input, unary_input); + } + } +}; + static void DSHLLis_empty(DataChunk &args, ExpressionState &state, Vector &result) { D_ASSERT(args.ColumnCount() == 1); auto &sketch_vector = args.data[0]; @@ -5344,7 +5442,7 @@ static void DSHLLupper_bound(DataChunk &args, ExpressionState &state, Vector &re } static AggregateFunction DSHLLMergeAggregate(const LogicalType &result_type) { - using STATE = DSHLLArenaState; + using STATE = DSHLLArenaUnionState; using OP = DSArenaMergeOperation; auto fun = AggregateFunction::UnaryAggregate(result_type, result_type); DSReadDataArgumentAt<1, STATE, string_t, OP>(fun); @@ -5355,8 +5453,8 @@ static AggregateFunction DSHLLMergeAggregate(const LogicalType &result_type) { template static AggregateFunction DSHLLCreateAggregate(const LogicalType &type, const LogicalType &result_type) { using INPUT = DSCountingInput; - using STATE = DSHLLArenaState; - using OP = DSArenaCreateOperation; + using STATE = DSHLLArenaState; + using OP = DSHLLArenaCreateOperation; auto fun = AggregateFunction::UnaryAggregate(type, result_type); DSReadDataArgumentAt<1, STATE, typename INPUT::physical_type, OP>(fun); DSSetArenaStateExport(fun); @@ -5645,50 +5743,203 @@ unique_ptr DSCPCBind(BindAggregateFunctionInput &input) { return std::move(bind_data); } -struct DSCPCState { - DSCPCSketch *sketch = nullptr; +// The union aggregate's state: a union in the aggregate arena (see the arena states in +// datasketches_generated_common.hpp). +struct DSCPCArenaUnionState { + DSCPCUnion *sketch; - ~DSCPCState() { - delete sketch; + static DSAllocator Allocator() { + return DSAllocator(DSArenaMode()); } - - void CreateSketch(const DSCPCBindData &bind) { - D_ASSERT(!sketch); - sketch = new DSCPCSketch(bind.k); + // Input sketches are read on the heap and freed once merged: the union assigns a sparse sketch to its + // accumulator, whose tables keep the accumulator's allocator, and ORs any other sketch into its bit matrix. + static DSCPCSketch Read(const string_t &data) { + return DSDeserialize(data, "CPC"); } - void CreateSketch(const DSCPCState &existing) { - if (existing.sketch) { - sketch = new DSCPCSketch(*existing.sketch); + void Initialize() { + sketch = nullptr; + } + bool IsEmpty() const { + return !sketch; + } + void Ensure(uint8_t lg_k, ArenaAllocator &arena) { + if (!sketch) { + sketch = arena.Make(lg_k, datasketches::DEFAULT_SEED, Allocator()); } } - - DSCPCSketch deserialize_sketch(const string_t &data) { - return DSDeserialize(data, "CPC"); + void Merge(const string_t &data, const DSCPCBindData &bind, ArenaAllocator &arena) { + Ensure(bind.k, arena); + sketch->update(Read(data)); + } + void Combine(const DSCPCArenaUnionState &source, const DSCPCBindData &bind, ArenaAllocator &arena) { + if (!sketch) { + // A copy allocates through the source's allocator, which in arena mode is the current (target's) arena. + sketch = arena.Make(*source.sketch); + return; + } + // The source's contents are copied into the current arena, then merged. + sketch->update(UnionContents(*source.sketch)); + } + auto Serialize(const DSCPCBindData &bind, ArenaAllocator &) const { + return SerializeUnion(*sketch, bind); + } + // The union's lg_k, then its contents. + std::vector Export(const DSCPCBindData &bind, ArenaAllocator &) const { + auto contents = UnionContents(*sketch).serialize(); + std::vector bytes; + bytes.reserve(1 + contents.size()); + bytes.push_back(static_cast(bind.k)); + bytes.insert(bytes.end(), contents.begin(), contents.end()); + return bytes; + } + void Import(const string_t &data, ArenaAllocator &arena) { + if (data.GetSize() < 1) { + throw InvalidInputException("Invalid CPC aggregate state"); + } + auto bytes = data.GetData(); + Ensure(static_cast(bytes[0]), arena); + sketch->update(Read(string_t(bytes + 1, UnsafeNumericCast(data.GetSize() - 1)))); } }; -// Holds a union so partial states and input sketches merge into it in place. -struct DSCPCUnionState { - DSCPCUnion *sketch = nullptr; +// The create aggregate's state, for values read as INPUT (see DSCountingInput). DuckDB creates a state per input +// row when grouping reduces little (it skips the hash table lookups), and a sketch takes a few allocations, so the +// first BUFFERED distinct values are kept as they are (see DSArenaValues) and fed to a sketch, in order, only +// when more arrive, when the state is finalized or exported as a sketch, or when it merges a partial state +// and both have values. Two partial states merge through a union, which marks the result merged (it drops the HIP +// estimate), so buffered values are fed to a sketch first: the result is the one unbuffered states give. +template +struct DSCPCArenaState { + using T = typename INPUT::physical_type; + static constexpr uint32_t BUFFERED = 16; - ~DSCPCUnionState() { - delete sketch; - } + DSCPCSketch *sketch; + DSArenaValues buffer; - void CreateSketch(const DSCPCBindData &bind) { - D_ASSERT(!sketch); - sketch = new DSCPCUnion(bind.k); + static DSAllocator Allocator() { + return DSAllocator(DSArenaMode()); + } + template + static void Feed(TARGET &target, const T &value) { + DSCountingUpdate(target, value); } - void CreateSketch(const DSCPCUnionState &existing) { - if (existing.sketch) { - sketch = new DSCPCUnion(*existing.sketch); + void Initialize() { + sketch = nullptr; + buffer.Initialize(); + } + bool HasSketch() const { + return sketch != nullptr; + } + bool IsEmpty() const { + return !HasSketch() && buffer.Count() == 0; + } + // Feeds the buffered values to a new sketch. + DSCPCSketch &Promote(const DSCPCBindData &bind, ArenaAllocator &arena) { + if (!HasSketch()) { + sketch = arena.Make(bind.k, datasketches::DEFAULT_SEED, Allocator()); + auto &target = *sketch; + buffer.ForEach([&](const T &value) { Feed(target, value); }); + buffer.Initialize(); + } + return *sketch; + } + void Add(const T &value, const DSCPCBindData &bind, ArenaAllocator &arena) { + if (!HasSketch()) { + if (buffer.Contains(value)) { + return; + } + if (buffer.Count() < BUFFERED) { + buffer.Append(value, arena); + return; + } + } + Feed(Promote(bind, arena), value); + } + template + void Update(const V &value, const DSCPCBindData &bind, ArenaAllocator &arena) { + static_assert(IS_UUID == INPUT::is_uuid, "the state reads its values as INPUT"); + Add(value, bind, arena); + } + void Combine(const DSCPCArenaState &source, const DSCPCBindData &bind, ArenaAllocator &arena) { + if (IsEmpty()) { + if (source.sketch) { + // A copy allocates through the source's allocator, which in arena mode is the current (target's) + // arena. + sketch = arena.Make(*source.sketch); + } else { + source.buffer.ForEach([&](const T &value) { buffer.Append(value, arena); }); + } + return; + } + // The union and a sketch of the source's buffered values are temporaries on the heap. The union copies + // the sketches it reads; its result is assigned to the state's sketch, whose tables keep their (arena) + // allocator and so copy the result's. + auto &target = Promote(bind, arena); + DSCPCUnion u(target.get_lg_k()); + u.update(target); + if (source.sketch) { + u.update(*source.sketch); + } else { + DSCPCSketch values(bind.k); + source.buffer.ForEach([&](const T &value) { Feed(values, value); }); + u.update(std::move(values)); + } + target = u.get_result(); + } + // A state that still buffers its values is serialized through a temporary sketch on the heap, which is freed + // right away, instead of one left behind in the arena. + auto Serialize(const DSCPCBindData &bind, ArenaAllocator &arena) const { + if (HasSketch()) { + return sketch->serialize(); } + DSCPCSketch temporary(bind.k); + buffer.ForEach([&](const T &value) { Feed(temporary, value); }); + return temporary.serialize(); } + // The exported state is a tag byte, then the buffered values (EXPORT_BUFFER, see DSArenaValues::Export) or the + // serialized sketch (EXPORT_SKETCH). Buffered states are exported as + // values, so spilled states that buffer stay small. + static constexpr uint8_t EXPORT_BUFFER = 0; + static constexpr uint8_t EXPORT_SKETCH = 1; + std::vector Export(const DSCPCBindData &bind, ArenaAllocator &arena) const { + std::vector bytes; + if (!HasSketch()) { + bytes.push_back(EXPORT_BUFFER); + buffer.Export(bytes); + return bytes; + } + auto exported = sketch->serialize(); + bytes.reserve(1 + exported.size()); + bytes.push_back(EXPORT_SKETCH); + bytes.insert(bytes.end(), exported.begin(), exported.end()); + return bytes; + } + void Import(const string_t &data, ArenaAllocator &arena) { + const auto size = data.GetSize(); + const auto bytes = const_data_ptr_cast(data.GetData()); + if (size >= 1 && bytes[0] == EXPORT_SKETCH) { + const string_t exported(data.GetData() + 1, UnsafeNumericCast(size - 1)); + sketch = arena.Make(DSDeserialize( + exported, "CPC", datasketches::DEFAULT_SEED, Allocator())); + return; + } + if (size < 1 || bytes[0] != EXPORT_BUFFER || !buffer.Import(bytes + 1, size - 1, BUFFERED, arena)) { + throw InvalidInputException("Invalid CPC aggregate state"); + } + } +}; - DSCPCSketch deserialize_sketch(const string_t &data) { - return DSDeserialize(data, "CPC"); +// A sketch absorbs a repeated value once, so a constant input is added once. +template +struct DSCPCArenaCreateOperation : DSArenaCreateOperation { + template + static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, + idx_t count) { + if (count > 0) { + OP::template Operation(state, input, unary_input); + } } }; @@ -5770,19 +6021,22 @@ static void DSCPCupper_bound(DataChunk &args, ExpressionState &state, Vector &re } static AggregateFunction DSCPCMergeAggregate(const LogicalType &result_type) { - using OP = DSUnionMergeOperation; - auto fun = AggregateFunction::UnaryAggregate(result_type, result_type); - DSReadDataArgumentAt<1, DSCPCUnionState, string_t, OP>(fun); + using STATE = DSCPCArenaUnionState; + using OP = DSArenaMergeOperation; + auto fun = AggregateFunction::UnaryAggregate(result_type, result_type); + DSReadDataArgumentAt<1, STATE, string_t, OP>(fun); + DSSetArenaStateExport(fun); return fun; } template static AggregateFunction DSCPCCreateAggregate(const LogicalType &type, const LogicalType &result_type) { using INPUT = DSCountingInput; - using STATE = DSCPCState; - using OP = DSCPCCreateOperation; - auto fun = AggregateFunction::UnaryAggregate(type, result_type); + using STATE = DSCPCArenaState; + using OP = DSCPCArenaCreateOperation; + auto fun = AggregateFunction::UnaryAggregate(type, result_type); DSReadDataArgumentAt<1, STATE, typename INPUT::physical_type, OP>(fun); + DSSetArenaStateExport(fun); return fun; } diff --git a/src/include/datasketches_generated_common.hpp b/src/include/datasketches_generated_common.hpp index 565de68..f0ccc79 100644 --- a/src/include/datasketches_generated_common.hpp +++ b/src/include/datasketches_generated_common.hpp @@ -421,135 +421,105 @@ static inline auto SerializeUnion(const DSCPCUnion &u, const BIND_DATA_TYPE &) { return u.get_result().serialize(); } -// For states holding a union. A union absorbs the same input any number of times with the same result, -// so repeats merge once, and partial states merge in place instead of being rebuilt. -template -struct DSUnionOperationBase : DSSketchOperationBase { - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, - idx_t count) { - if (count > 0) { - OP::template Operation(state, input, unary_input); - } - } +template +struct DSArenaItems; - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) { - if (!source.sketch) { - return; - } - if (!target.sketch) { - if (aggr_input_data.combine_type == AggregateCombineType::ALLOW_DESTRUCTIVE) { - // The source is discarded after this call, so take its union rather than copy it. - target.sketch = source.sketch; - const_cast(source).sketch = nullptr; - } else { - target.CreateSketch(source); - } - return; - } - target.sketch->update(UnionContents(*source.sketch)); - } +// Input values of a counting sketch (HLL, CPC) kept in the aggregate arena until they are fed to a sketch, in the +// order they arrived: fixed-width values as they are, VARCHAR and BLOB values with their bytes copied into the +// arena. The states keep a value once (see Contains): equal bytes hash to the same coupon, which a sketch absorbs +// only once. See the counting arena states in codegen/generated.cpp.j2. +template +struct DSArenaValues { + DSArenaItems values; - template - static void RepeatedCombine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data, - idx_t multiplicity) { - Combine(source, target, aggr_input_data); + void Initialize() { + values.Initialize(); } - - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) { - if (!state.sketch) { - finalize_data.ReturnNull(); - } else { - auto &bind_data = finalize_data.input.bind_data->template Cast(); - target = DSAddBytes(finalize_data.result, SerializeUnion(*state.sketch, bind_data)); - } + uint32_t Count() const { + return values.count; } -}; - -// Merges serialized sketches into a union state; shared by every counting sketch's union aggregate. -template -struct DSUnionMergeOperation : DSUnionOperationBase { - template - static void Operation(STATE &state, const A_TYPE &a_data, AggregateUnaryInput &idata) { - if (!state.sketch) { - auto &bind_data = idata.input.bind_data->template Cast(); - state.CreateSketch(bind_data); + bool Contains(const T &value) const { + for (uint32_t i = 0; i < values.count; i++) { + if (Same(values.items[i], value)) { + return true; + } } - - state.sketch->update(state.deserialize_sketch(a_data)); + return false; } -}; - -template -struct DSHLLCreateOperation : DSUnionOperationBase { - template - static void Operation(STATE &state, const A_TYPE &a_data, AggregateUnaryInput &idata) { - if (!state.sketch) { - auto &bind_data = idata.input.bind_data->template Cast(); - state.CreateSketch(bind_data); + void Append(const T &value, ArenaAllocator &arena) { + if constexpr (std::is_same_v) { + if (!value.IsInlined()) { + const auto size = value.GetSize(); + auto data = arena.Allocate(size); + memcpy(data, value.GetData(), size); + values.Append(string_t(const_char_ptr_cast(data), UnsafeNumericCast(size)), arena); + return; + } } - - DSCountingUpdate(*state.sketch, a_data); + values.Append(value, arena); } -}; - -template -struct DSCPCCreateOperation : DSSketchOperationBase { - template - static void Operation(STATE &state, const A_TYPE &a_data, AggregateUnaryInput &idata) { - if (!state.sketch) { - auto &bind_data = idata.input.bind_data->template Cast(); - state.CreateSketch(bind_data); + template + void ForEach(FUNC &&fun) const { + for (uint32_t i = 0; i < values.count; i++) { + fun(values.items[i]); } - - DSCountingUpdate(*state.sketch, a_data); } - template - static void ConstantOperation(STATE &state, const INPUT_TYPE &input, AggregateUnaryInput &unary_input, - idx_t count) { - for (idx_t i = 0; i < count; i++) { - Operation(state, input, unary_input); + // Appends the values to `bytes`: fixed-width values as they are, strings as their size (a uint32_t) and bytes. + void Export(std::vector &bytes) const { + if constexpr (std::is_same_v) { + for (uint32_t i = 0; i < values.count; i++) { + const auto &value = values.items[i]; + const auto size = static_cast(value.GetSize()); + auto size_bytes = reinterpret_cast(&size); + bytes.insert(bytes.end(), size_bytes, size_bytes + sizeof(size)); + auto data = const_data_ptr_cast(value.GetData()); + bytes.insert(bytes.end(), data, data + size); + } + } else { + auto data = reinterpret_cast(values.items); + bytes.insert(bytes.end(), data, data + values.count * sizeof(T)); } } - - template - static void Combine(const STATE &source, STATE &target, AggregateInputData &aggr_input_data) { - // A partial that saw no rows has nothing to add. Passing the target through a union anyway would mark it - // merged, which drops its HIP estimator, the more accurate estimate of a sketch that was never merged. - if (!source.sketch) { - return; - } - const bool destructive = aggr_input_data.combine_type == AggregateCombineType::ALLOW_DESTRUCTIVE; - if (!target.sketch) { - if (destructive) { - // The source is discarded after this call, so take its sketch rather than copy it. - target.sketch = source.sketch; - const_cast(source).sketch = nullptr; + // Rebuilds exported values (at most `max_count` of them); false if the bytes are not exported values. + bool Import(const_data_ptr_t data, idx_t size, uint32_t max_count, ArenaAllocator &arena) { + idx_t offset = 0; + while (offset < size) { + if (values.count == max_count) { + return false; + } + if constexpr (std::is_same_v) { + uint32_t length; + if (size - offset < sizeof(length)) { + return false; + } + memcpy(&length, data + offset, sizeof(length)); + offset += sizeof(length); + if (size - offset < length) { + return false; + } + Append(string_t(const_char_ptr_cast(data + offset), length), arena); + offset += length; } else { - target.CreateSketch(source); + if (size - offset < sizeof(T)) { + return false; + } + T value; + memcpy(&value, data + offset, sizeof(T)); + values.Append(value, arena); + offset += sizeof(T); } - return; - } - // Move the sketches into the union so it adopts their tables instead of copying them. - DSCPCUnion u(target.sketch->get_lg_k()); - u.update(std::move(*target.sketch)); - if (destructive) { - u.update(std::move(*const_cast(source).sketch)); - } else { - u.update(*source.sketch); } - *target.sketch = u.get_result(); + return true; } - template - static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) { - if (!state.sketch) { - finalize_data.ReturnNull(); +private: + static bool Same(const T &a, const T &b) { + if constexpr (std::is_same_v) { + return a == b; } else { - target = DSAddBytes(finalize_data.result, state.sketch->serialize()); + // bytewise, so e.g. 0.0 and -0.0 count as different values (the sketch decides whether they are) + return memcmp(&a, &b, sizeof(T)) == 0; } } }; diff --git a/test/sql/datasketch_spill_counting.test b/test/sql/datasketch_spill_counting.test new file mode 100644 index 0000000..76a6b2c --- /dev/null +++ b/test/sql/datasketch_spill_counting.test @@ -0,0 +1,357 @@ +# name: test/sql/datasketch_spill_counting.test +# description: HLL and CPC aggregate states live in the arena and spill under a memory limit, with the results of unspilled runs +# group: [datasketches] + +require datasketches + +require skip_reload + +statement ok +SET threads=4; + +statement ok +SET temp_directory='{TEST_DIR}/datasketch_spill_counting'; + +statement ok +SET preserve_insertion_order=false; + +# Every input type family. Each group gets 10 values: more than an HLL state buffers (7), fewer than a CPC state +# buffers (16). Strings are longer than 12 bytes so their bytes are copied into the arena. +statement ok +CREATE VIEW src AS +SELECT i % 100000 AS g, + i AS v_int, + (i::HUGEINT << 80) + i AS v_huge, + 'value number ' || i AS v_str, + ('blob value ' || i)::BLOB AS v_blob, + md5(i::VARCHAR)::UUID AS v_uuid +FROM range(1000000) t(i); + +# Distinct counts this small are estimated almost exactly, but two values can share a coupon, and whether a group's +# rows met in one partial state or were merged from several (which changes a CPC estimate slightly) depends on the +# threads: so each run is compared, rounded, against the same query without a limit, through a checksum of its rows +# (tables of an in-memory database count toward the limit). +statement ok +SET memory_limit='4GB'; + +statement ok +CREATE TABLE expected AS +SELECT 'hll_v_int' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_int))) AS e FROM src GROUP BY g) +UNION ALL +SELECT 'hll_v_huge' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_huge))) AS e FROM src GROUP BY g) +UNION ALL +SELECT 'hll_v_str' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_str))) AS e FROM src GROUP BY g) +UNION ALL +SELECT 'hll_v_blob' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_blob))) AS e FROM src GROUP BY g) +UNION ALL +SELECT 'hll_v_uuid' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_uuid))) AS e FROM src GROUP BY g) +UNION ALL +SELECT 'cpc_v_int' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_int))) AS e FROM src GROUP BY g) +UNION ALL +SELECT 'cpc_v_huge' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_huge))) AS e FROM src GROUP BY g) +UNION ALL +SELECT 'cpc_v_str' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_str))) AS e FROM src GROUP BY g) +UNION ALL +SELECT 'cpc_v_blob' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_blob))) AS e FROM src GROUP BY g) +UNION ALL +SELECT 'cpc_v_uuid' AS name, count(*) AS n, bit_xor(hash(g, e)) AS checksum, count(*) FILTER (WHERE abs(e - 10) > 3) AS off +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_uuid))) AS e FROM src GROUP BY g); + +query I +SELECT count(*) FROM expected WHERE n != 100000 OR off != 0; +---- +0 + +statement ok +SET memory_limit='128MB'; + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'hll_v_int') +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_int))) AS e FROM src GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'hll_v_huge') +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_huge))) AS e FROM src GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'hll_v_str') +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_str))) AS e FROM src GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'hll_v_blob') +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_blob))) AS e FROM src GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'hll_v_uuid') +FROM (SELECT g, round(datasketch_hll_estimate(datasketch_hll(12, v_uuid))) AS e FROM src GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'cpc_v_int') +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_int))) AS e FROM src GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'cpc_v_huge') +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_huge))) AS e FROM src GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'cpc_v_str') +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_str))) AS e FROM src GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'cpc_v_blob') +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_blob))) AS e FROM src GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, e)) = (SELECT checksum FROM expected WHERE name = 'cpc_v_uuid') +FROM (SELECT g, round(datasketch_cpc_estimate(datasketch_cpc(12, v_uuid))) AS e FROM src GROUP BY g); +---- +true + +# With one thread the runs are deterministic, so a spilled HLL run returns the very same sketches: a group's values +# make the same union whether its rows met in one state or in several. (A CPC sketch merged from several states is +# marked merged, which changes its estimate slightly, and spilling decides which rows meet: so CPC runs are compared, +# rounded, by estimate.) Groups of 2 values stay buffered; groups of 20 build a sketch. +statement ok +SET threads=1; + +statement ok +SET memory_limit='4GB'; + +statement ok +CREATE TABLE expected_sketches AS +SELECT 0 AS e, 2 AS rows_per_group, count(*) AS n, bit_xor(hash(g, s)) AS checksum +FROM (SELECT g, datasketch_hll(12, v) AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 0 AS e, 20 AS rows_per_group, count(*) AS n, bit_xor(hash(g, s)) AS checksum +FROM (SELECT g, datasketch_hll(12, v) AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 1 AS e, 2 AS rows_per_group, count(*) AS n, bit_xor(hash(g, s)) AS checksum +FROM (SELECT g, datasketch_hll(12, v::VARCHAR || ' padding the string') AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 1 AS e, 20 AS rows_per_group, count(*) AS n, bit_xor(hash(g, s)) AS checksum +FROM (SELECT g, datasketch_hll(12, v::VARCHAR || ' padding the string') AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 2 AS e, 2 AS rows_per_group, count(*) AS n, bit_xor(hash(g, s)) AS checksum +FROM (SELECT g, datasketch_hll(10, md5(v::VARCHAR)::UUID, 'HLL_8') AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 2 AS e, 20 AS rows_per_group, count(*) AS n, bit_xor(hash(g, s)) AS checksum +FROM (SELECT g, datasketch_hll(10, md5(v::VARCHAR)::UUID, 'HLL_8') AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 3 AS e, 2 AS rows_per_group, count(*) AS n, bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) AS checksum +FROM (SELECT g, datasketch_cpc(12, v) AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 3 AS e, 20 AS rows_per_group, count(*) AS n, bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) AS checksum +FROM (SELECT g, datasketch_cpc(12, v) AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 4 AS e, 2 AS rows_per_group, count(*) AS n, bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) AS checksum +FROM (SELECT g, datasketch_cpc(12, v::VARCHAR || ' padding the string') AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 4 AS e, 20 AS rows_per_group, count(*) AS n, bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) AS checksum +FROM (SELECT g, datasketch_cpc(12, v::VARCHAR || ' padding the string') AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 5 AS e, 2 AS rows_per_group, count(*) AS n, bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) AS checksum +FROM (SELECT g, datasketch_cpc(10, v::VARCHAR::BLOB) AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g) +UNION ALL +SELECT 5 AS e, 20 AS rows_per_group, count(*) AS n, bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) AS checksum +FROM (SELECT g, datasketch_cpc(10, v::VARCHAR::BLOB) AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); + +query I +SELECT count(*) FROM expected_sketches WHERE n != 1000000 // rows_per_group; +---- +0 + +statement ok +SET memory_limit='128MB'; + +query I +SELECT bit_xor(hash(g, s)) = (SELECT checksum FROM expected_sketches WHERE e = 0 AND rows_per_group = 2) +FROM (SELECT g, datasketch_hll(12, v) AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, s)) = (SELECT checksum FROM expected_sketches WHERE e = 0 AND rows_per_group = 20) +FROM (SELECT g, datasketch_hll(12, v) AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, s)) = (SELECT checksum FROM expected_sketches WHERE e = 1 AND rows_per_group = 2) +FROM (SELECT g, datasketch_hll(12, v::VARCHAR || ' padding the string') AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, s)) = (SELECT checksum FROM expected_sketches WHERE e = 1 AND rows_per_group = 20) +FROM (SELECT g, datasketch_hll(12, v::VARCHAR || ' padding the string') AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, s)) = (SELECT checksum FROM expected_sketches WHERE e = 2 AND rows_per_group = 2) +FROM (SELECT g, datasketch_hll(10, md5(v::VARCHAR)::UUID, 'HLL_8') AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, s)) = (SELECT checksum FROM expected_sketches WHERE e = 2 AND rows_per_group = 20) +FROM (SELECT g, datasketch_hll(10, md5(v::VARCHAR)::UUID, 'HLL_8') AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) = (SELECT checksum FROM expected_sketches WHERE e = 3 AND rows_per_group = 2) +FROM (SELECT g, datasketch_cpc(12, v) AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) = (SELECT checksum FROM expected_sketches WHERE e = 3 AND rows_per_group = 20) +FROM (SELECT g, datasketch_cpc(12, v) AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) = (SELECT checksum FROM expected_sketches WHERE e = 4 AND rows_per_group = 2) +FROM (SELECT g, datasketch_cpc(12, v::VARCHAR || ' padding the string') AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) = (SELECT checksum FROM expected_sketches WHERE e = 4 AND rows_per_group = 20) +FROM (SELECT g, datasketch_cpc(12, v::VARCHAR || ' padding the string') AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) = (SELECT checksum FROM expected_sketches WHERE e = 5 AND rows_per_group = 2) +FROM (SELECT g, datasketch_cpc(10, v::VARCHAR::BLOB) AS s FROM (SELECT i % (500000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +query I +SELECT bit_xor(hash(g, round(datasketch_cpc_estimate(s)))) = (SELECT checksum FROM expected_sketches WHERE e = 5 AND rows_per_group = 20) +FROM (SELECT g, datasketch_cpc(10, v::VARCHAR::BLOB) AS s FROM (SELECT i % (50000) AS g, i * 7919 AS v FROM range(1000000) t(i)) GROUP BY g); +---- +true + +statement ok +SET memory_limit='4GB'; + +statement ok +SET threads=4; + +# Merging stored sketches spills too +statement ok +CREATE TABLE parts AS +SELECT g, p, datasketch_hll(12, v) AS h, datasketch_cpc(12, v) AS c +FROM (SELECT i % 100000 AS g, i % 3 AS p, i AS v FROM range(900000) t(i)) +GROUP BY g, p; + +statement ok +CREATE MACRO merged() AS TABLE +SELECT g, round(datasketch_hll_estimate(datasketch_hll_union(12, h))) AS h, + round(datasketch_cpc_estimate(datasketch_cpc_union(12, c))) AS c +FROM parts GROUP BY g; + +statement ok +CREATE TABLE expected_merged AS +SELECT count(*) AS n, bit_xor(hash(t)) AS checksum, count(*) FILTER (WHERE abs(h - 9) > 3 OR abs(c - 9) > 3) AS off +FROM merged() t; + +query II +SELECT n, off FROM expected_merged; +---- +100000 0 + +statement ok +SET memory_limit='128MB'; + +query I +SELECT (SELECT bit_xor(hash(t)) FROM merged() t) = (SELECT checksum FROM expected_merged); +---- +true + +statement ok +SET memory_limit='4GB'; + +# Exported states finalize to the aggregate's result: buffered states (3 values) and states with a sketch, for every +# input type family +query IIIIII +SELECT finalize(datasketch_hll(12, i) EXPORT_STATE) = datasketch_hll(12, i), + finalize(datasketch_cpc(12, i) EXPORT_STATE) = datasketch_cpc(12, i), + finalize(datasketch_hll(12, i::HUGEINT << 70) EXPORT_STATE) = datasketch_hll(12, i::HUGEINT << 70), + finalize(datasketch_cpc(12, i::HUGEINT << 70) EXPORT_STATE) = datasketch_cpc(12, i::HUGEINT << 70), + finalize(datasketch_hll(12, 'a long string value ' || i) EXPORT_STATE) = datasketch_hll(12, 'a long string value ' || i), + finalize(datasketch_cpc(12, ('blob ' || i)::BLOB) EXPORT_STATE) = datasketch_cpc(12, ('blob ' || i)::BLOB) +FROM range(3) t(i); +---- +true true true true true true + +query IIIIII +SELECT finalize(datasketch_hll(12, i) EXPORT_STATE) = datasketch_hll(12, i), + finalize(datasketch_cpc(12, i) EXPORT_STATE) = datasketch_cpc(12, i), + finalize(datasketch_hll(12, md5(i::VARCHAR)::UUID) EXPORT_STATE) = datasketch_hll(12, md5(i::VARCHAR)::UUID), + finalize(datasketch_cpc(12, md5(i::VARCHAR)::UUID) EXPORT_STATE) = datasketch_cpc(12, md5(i::VARCHAR)::UUID), + finalize(datasketch_hll(12, 'a long string value ' || i) EXPORT_STATE) = datasketch_hll(12, 'a long string value ' || i), + finalize(datasketch_cpc(12, ('blob ' || i)::BLOB) EXPORT_STATE) = datasketch_cpc(12, ('blob ' || i)::BLOB) +FROM range(100000) t(i); +---- +true true true true true true + +# Exported states of the union aggregates +query II +SELECT round(datasketch_hll_estimate(finalize(datasketch_hll_union(12, h) EXPORT_STATE))) = round(datasketch_hll_estimate(datasketch_hll_union(12, h))), + round(datasketch_cpc_estimate(finalize(datasketch_cpc_union(12, c) EXPORT_STATE))) = round(datasketch_cpc_estimate(datasketch_cpc_union(12, c))) +FROM parts; +---- +true true + +# Combining exported states: buffered with buffered, buffered with a sketch, and two sketches +query IIIIII +SELECT round(datasketch_hll_estimate(finalize(combine(a.h, b.h)))), + round(datasketch_cpc_estimate(finalize(combine(a.c, b.c)))), + round(datasketch_hll_estimate(finalize(combine(l.h, a.h)))), + round(datasketch_cpc_estimate(finalize(combine(a.c, l.c)))), + round(datasketch_hll_estimate(finalize(combine(l.h, m.h)))), + round(datasketch_cpc_estimate(finalize(combine(l.c, m.c)))) +FROM (SELECT datasketch_hll(12, i) EXPORT_STATE AS h, datasketch_cpc(12, i) EXPORT_STATE AS c FROM range(3) t(i)) a, + (SELECT datasketch_hll(12, i) EXPORT_STATE AS h, datasketch_cpc(12, i) EXPORT_STATE AS c FROM range(2, 5) t(i)) b, + (SELECT datasketch_hll(12, i) EXPORT_STATE AS h, datasketch_cpc(12, i) EXPORT_STATE AS c FROM range(100, 200) t(i)) l, + (SELECT datasketch_hll(12, i) EXPORT_STATE AS h, datasketch_cpc(12, i) EXPORT_STATE AS c FROM range(150, 300) t(i)) m; +---- +5.0 5.0 103.0 103.0 200.0 200.0 + +# A constant input is added once +query II +SELECT (SELECT datasketch_hll(12, 42) FROM range(10000)) = (SELECT datasketch_hll(12, 42)), + (SELECT datasketch_cpc(12, 42) FROM range(10000)) = (SELECT datasketch_cpc(12, 42)); +---- +true true