diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 23f86dde9cd..c8f689f03f1 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -56,9 +56,9 @@ jobs: - { shard: 3, name: "Array dictionaries", packages: "vortex-array", array-group: "dictionary", features: "--features _test-harness" } - { shard: 4, name: "Array builders & views", packages: "vortex-array", array-group: "builders", features: "--features _test-harness" } - { shard: 5, name: "Core & encodings", packages: "vortex-buffer vortex-error vortex-mask vortex-compute vortex-file vortex-alp vortex-bytebool vortex-datetime-parts" } - - { shard: 6, name: "Main library & storage", packages: "vortex vortex-btrblocks vortex-compressor vortex-row" } + - { shard: 6, name: "Main library, functions & storage", packages: "vortex vortex-expr vortex-functions vortex-arrow vortex-btrblocks vortex-compressor vortex-row" } - { shard: 7, name: "FastLanes & decimals", packages: "vortex-decimal-byte-parts vortex-fastlanes", features: "--features _test-harness" } - - { shard: 8, name: "Encodings & layout", packages: "vortex-pco vortex-runend vortex-sequence vortex-sparse vortex-zigzag vortex-zstd vortex-layout" } + - { shard: 8, name: "Encodings & layout", packages: "vortex-onpair vortex-pco vortex-runend vortex-sequence vortex-sparse vortex-zigzag vortex-zstd vortex-layout" } - { shard: 9, name: "Strings, tensor & spatial", packages: "vortex-fsst vortex-tensor vortex-spatial", features: "--features _test-harness" } name: "Benchmark with Codspeed (Shard #${{ matrix.shard }} - ${{ matrix.name }})" timeout-minutes: 30 @@ -92,7 +92,7 @@ jobs: groups = { "dictionary": ("dict_", "chunked_dict_", "patches_"), "selection": ("take_", "filter_", "piecewise_sequence_"), - "compute": ("aggregate_", "cast_", "expr_", "binary_ops", "compare", "kleene_bool", "like", "list_sum"), + "compute": ("aggregate_", "cast_", "binary_ops", "compare", "kleene_bool"), } manifest = tomllib.loads(pathlib.Path("vortex-array/Cargo.toml").read_text()) benches = [] diff --git a/AGENTS.md b/AGENTS.md index 98e283f837d..0166a6a89e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,8 @@ documentation in `docs/`, and benchmark tooling in `vortex-bench/` and `benchmar - `vortex-array` contains the core `Array` trait and the base encodings, including most Apache Arrow-style encodings. - `encodings/*` contains more specialized compressed encodings. +- `vortex-expr` contains expression trees, their optimizer, and the analyses and rewrites built on + them. `vortex-functions` contains the scalar functions that `vortex-array` itself is not built on. - `vortex-file` implements file IO. It uses `LayoutReader` from `vortex-layout`. - `vortex-io` holds the core async and blocking IO traits, plus the generic `object_store` adapters that implement them. diff --git a/Cargo.lock b/Cargo.lock index 8de7450eea7..2c8dff209d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11351,6 +11351,7 @@ dependencies = [ "vortex-fastlanes", "vortex-file", "vortex-fsst", + "vortex-functions", "vortex-io", "vortex-ipc", "vortex-layout", @@ -11385,6 +11386,7 @@ dependencies = [ "vortex-buffer", "vortex-error", "vortex-fastlanes", + "vortex-functions", "vortex-mask", "vortex-session", ] @@ -11848,6 +11850,7 @@ dependencies = [ "vortex-buffer", "vortex-compute", "vortex-error", + "vortex-functions", "vortex-mask", "vortex-session", ] @@ -11867,6 +11870,7 @@ dependencies = [ "vortex-bench-support", "vortex-buffer", "vortex-error", + "vortex-functions", "vortex-mask", "vortex-session", ] @@ -11936,13 +11940,20 @@ dependencies = [ name = "vortex-expr" version = "0.1.0" dependencies = [ + "arbitrary", + "codspeed-divan-compat", + "insta", "itertools 0.14.0", + "mimalloc", "parking_lot", + "prost 0.14.4", "rstest", + "smallvec", "tracing", "vortex-array", "vortex-buffer", "vortex-error", + "vortex-mask", "vortex-session", "vortex-utils", ] @@ -11968,6 +11979,7 @@ dependencies = [ "vortex-compute", "vortex-error", "vortex-fastlanes", + "vortex-functions", "vortex-mask", "vortex-session", ] @@ -12030,6 +12042,7 @@ dependencies = [ "vortex-expr", "vortex-fastlanes", "vortex-fsst", + "vortex-functions", "vortex-io", "vortex-layout", "vortex-mask", @@ -12062,10 +12075,41 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", + "vortex-expr", + "vortex-functions", "vortex-mask", "vortex-session", ] +[[package]] +name = "vortex-functions" +version = "0.1.0" +dependencies = [ + "arrow-buffer 59.3.1", + "codspeed-divan-compat", + "insta", + "itertools 0.14.0", + "jiff", + "memchr", + "mimalloc", + "num-traits", + "parking_lot", + "prost 0.14.4", + "rand 0.10.3", + "regex", + "regex-syntax", + "rstest", + "tracing", + "vortex-array", + "vortex-buffer", + "vortex-compute", + "vortex-error", + "vortex-expr", + "vortex-mask", + "vortex-session", + "vortex-utils", +] + [[package]] name = "vortex-fuzz" version = "0.1.0" @@ -12085,8 +12129,10 @@ dependencies = [ "vortex-buffer", "vortex-cuda", "vortex-error", + "vortex-expr", "vortex-file", "vortex-fsst", + "vortex-functions", "vortex-io", "vortex-mask", "vortex-row", @@ -12182,6 +12228,8 @@ dependencies = [ "vortex-arrow", "vortex-edition", "vortex-error", + "vortex-expr", + "vortex-functions", "vortex-session", ] @@ -12232,6 +12280,7 @@ dependencies = [ "vortex-error", "vortex-expr", "vortex-file", + "vortex-functions", "vortex-io", "vortex-mask", "vortex-metrics", @@ -12290,6 +12339,7 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", + "vortex-functions", "vortex-mask", "vortex-session", ] @@ -12312,7 +12362,9 @@ dependencies = [ "vortex-buffer", "vortex-edition", "vortex-error", + "vortex-expr", "vortex-file", + "vortex-functions", "vortex-io", "vortex-json", "vortex-layout", @@ -12439,6 +12491,7 @@ dependencies = [ "vortex-array", "vortex-buffer", "vortex-error", + "vortex-expr", "vortex-io", "vortex-mask", "vortex-session", @@ -12457,6 +12510,8 @@ dependencies = [ "vortex-buffer", "vortex-compute", "vortex-error", + "vortex-expr", + "vortex-functions", "vortex-mask", "vortex-session", ] @@ -12486,6 +12541,7 @@ dependencies = [ "vortex-arrow", "vortex-buffer", "vortex-error", + "vortex-functions", "vortex-mask", "vortex-session", ] @@ -12511,6 +12567,7 @@ dependencies = [ "vortex-edition", "vortex-error", "vortex-expr", + "vortex-functions", "vortex-layout", "vortex-mask", "vortex-session", diff --git a/Cargo.toml b/Cargo.toml index f7d5a15e8a5..e6fc6a66b08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "vortex-io", "vortex-cloud", "vortex-array", + "vortex-functions", "vortex-expr", "vortex-arrow", "vortex-row", @@ -324,6 +325,7 @@ vortex-expr = { version = "0.1.0", path = "./vortex-expr", default-features = fa vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false } vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false } vortex-fsst = { version = "0.1.0", path = "./encodings/fsst", default-features = false } +vortex-functions = { version = "0.1.0", path = "./vortex-functions", default-features = false } vortex-io = { version = "0.1.0", path = "./vortex-io", default-features = false } vortex-ipc = { version = "0.1.0", path = "./vortex-ipc", default-features = false } vortex-json = { version = "0.1.0", path = "./vortex-json", default-features = false } diff --git a/docs/concepts/expressions.md b/docs/concepts/expressions.md index 81317a2b888..d6591ee80fb 100644 --- a/docs/concepts/expressions.md +++ b/docs/concepts/expressions.md @@ -9,8 +9,10 @@ to the returned data. Expressions are defined as abstract scalar functions. These vtables define the signature of the function, properties such as whether the function is strict, and the actual logic for executing the function over input arrays. -The built-in scalar functions can be found in the `vortex-array::expr` module, with additional use-case specific -functions provided by integration and plugin crates. +The scalar function framework and the handful of functions the array crate itself is built on live in +`vortex_array::scalar_fn`; expression trees, their optimizer and the analyses built on them live in the +`vortex-expr` crate; `vortex-functions` holds the rest of the built-in scalar functions, with additional +use-case specific functions provided by integration and plugin crates. In the future, we plan to add full support for spatial functions, date-time functions, and more. diff --git a/encodings/alp/Cargo.toml b/encodings/alp/Cargo.toml index 8907e85aec8..46daee1cbe0 100644 --- a/encodings/alp/Cargo.toml +++ b/encodings/alp/Cargo.toml @@ -24,6 +24,7 @@ vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-fastlanes = { workspace = true } +vortex-functions = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/encodings/alp/src/alp/compute/between.rs b/encodings/alp/src/alp/compute/between.rs index 48e2e7fb5ee..79c00d8cf05 100644 --- a/encodings/alp/src/alp/compute/between.rs +++ b/encodings/alp/src/alp/compute/between.rs @@ -8,16 +8,16 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::arrays::ConstantArray; -use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::NativeDType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::between::BetweenOptions; -use vortex_array::scalar_fn::fns::between::BetweenReduce; -use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_error::VortexResult; +use vortex_functions::between::BetweenOptions; +use vortex_functions::between::BetweenReduce; +use vortex_functions::between::StrictComparison; +use vortex_functions::builtins::ArrayFunctions; use crate::ALP; use crate::ALPFloat; @@ -145,8 +145,8 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison; + use vortex_functions::between::BetweenOptions; + use vortex_functions::between::StrictComparison; use vortex_session::VortexSession; use crate::ALPArray; diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index 6b05ed7d949..198fcdaaafc 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::ALPArrayExt; diff --git a/encodings/alp/src/alp/compute/compare.rs b/encodings/alp/src/alp/compute/compare.rs index 457aacee172..55b1d5843e1 100644 --- a/encodings/alp/src/alp/compute/compare.rs +++ b/encodings/alp/src/alp/compute/compare.rs @@ -11,9 +11,9 @@ use vortex_array::arrays::ConstantArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::NativePType; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::operators::CompareOperator; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; @@ -165,8 +165,8 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::operators::CompareOperator; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::operators::CompareOperator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_session::VortexSession; use super::*; diff --git a/encodings/alp/src/alp/compute/mask.rs b/encodings/alp/src/alp/compute/mask.rs index b67848ca6a4..890147f0ff1 100644 --- a/encodings/alp/src/alp/compute/mask.rs +++ b/encodings/alp/src/alp/compute/mask.rs @@ -6,8 +6,8 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; -use vortex_array::scalar_fn::fns::mask::MaskKernel; -use vortex_array::scalar_fn::fns::mask::MaskReduce; +use vortex_array::scalar_fn::builtin::mask::MaskKernel; +use vortex_array::scalar_fn::builtin::mask::MaskReduce; use vortex_array::validity::Validity; use vortex_error::VortexResult; @@ -58,7 +58,7 @@ mod test { use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::mask::test_mask_conformance; use vortex_array::dtype::Nullability; - use vortex_array::scalar_fn::fns::mask::MaskKernel; + use vortex_array::scalar_fn::builtin::mask::MaskKernel; use vortex_buffer::buffer; use vortex_session::VortexSession; diff --git a/encodings/alp/src/alp/rules.rs b/encodings/alp/src/alp/rules.rs index 3b888ae8269..e401e594369 100644 --- a/encodings/alp/src/alp/rules.rs +++ b/encodings/alp/src/alp/rules.rs @@ -11,13 +11,13 @@ use vortex_array::arrays::slice::SliceExecuteAdaptor; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::between::BetweenReduceAdaptor; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; -use vortex_array::scalar_fn::fns::mask::Mask; -use vortex_array::scalar_fn::fns::mask::MaskExecuteAdaptor; -use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::mask::Mask; +use vortex_array::scalar_fn::builtin::mask::MaskExecuteAdaptor; +use vortex_array::scalar_fn::builtin::mask::MaskReduceAdaptor; +use vortex_functions::between::BetweenReduceAdaptor; use vortex_session::VortexSession; use crate::ALP; diff --git a/encodings/alp/src/alp_rd/compute/cast.rs b/encodings/alp/src/alp_rd/compute/cast.rs index 70e1affeee9..c88817e8bc4 100644 --- a/encodings/alp/src/alp_rd/compute/cast.rs +++ b/encodings/alp/src/alp_rd/compute/cast.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::ALPRDArrayExt; diff --git a/encodings/alp/src/alp_rd/compute/mask.rs b/encodings/alp/src/alp_rd/compute/mask.rs index 9ffcfa06375..8c504f14d7b 100644 --- a/encodings/alp/src/alp_rd/compute/mask.rs +++ b/encodings/alp/src/alp_rd/compute/mask.rs @@ -4,8 +4,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; -use vortex_array::scalar_fn::fns::mask::MaskReduce; +use vortex_array::scalar_fn::builtin::mask::Mask as MaskExpr; +use vortex_array::scalar_fn::builtin::mask::MaskReduce; use vortex_error::VortexResult; use crate::ALPRD; diff --git a/encodings/alp/src/alp_rd/rules.rs b/encodings/alp/src/alp_rd/rules.rs index be5eda6e7f3..34b7a8272ce 100644 --- a/encodings/alp/src/alp_rd/rules.rs +++ b/encodings/alp/src/alp_rd/rules.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; -use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::mask::MaskReduceAdaptor; use crate::alp_rd::ALPRD; diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index 6a92a039ec8..e5d993282b9 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -12,13 +12,13 @@ use vortex_array::arrays::dict::TakeExecute; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::match_each_integer_ptype; -use vortex_array::scalar_fn::fns::binary::BooleanKernel; -use vortex_array::scalar_fn::fns::binary::kleene_boolean_buffer_scalar; -use vortex_array::scalar_fn::fns::binary::kleene_boolean_buffers; -use vortex_array::scalar_fn::fns::cast::CastKernel; -use vortex_array::scalar_fn::fns::cast::CastReduce; -use vortex_array::scalar_fn::fns::mask::MaskReduce; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::BooleanKernel; +use vortex_array::scalar_fn::builtin::binary::kleene_boolean_buffer_scalar; +use vortex_array::scalar_fn::builtin::binary::kleene_boolean_buffers; +use vortex_array::scalar_fn::builtin::cast::CastKernel; +use vortex_array::scalar_fn::builtin::cast::CastReduce; +use vortex_array::scalar_fn::builtin::mask::MaskReduce; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::ByteBuffer; @@ -182,7 +182,7 @@ mod tests { use vortex_array::compute::conformance::filter::test_filter_conformance; use vortex_array::compute::conformance::mask::test_mask_conformance; use vortex_array::compute::conformance::take::test_take_conformance; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::vortex_err; use vortex_session::VortexSession; diff --git a/encodings/bytebool/src/kernel.rs b/encodings/bytebool/src/kernel.rs index 678da8642a7..c480f68184d 100644 --- a/encodings/bytebool/src/kernel.rs +++ b/encodings/bytebool/src/kernel.rs @@ -6,10 +6,10 @@ use vortex_array::arrays::Dict; use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::BooleanExecuteAdaptor; -use vortex_array::scalar_fn::fns::cast::Cast; -use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::BooleanExecuteAdaptor; +use vortex_array::scalar_fn::builtin::cast::Cast; +use vortex_array::scalar_fn::builtin::cast::CastExecuteAdaptor; use vortex_session::VortexSession; use crate::ByteBool; diff --git a/encodings/bytebool/src/rules.rs b/encodings/bytebool/src/rules.rs index f67d3567326..a29efa0ea1a 100644 --- a/encodings/bytebool/src/rules.rs +++ b/encodings/bytebool/src/rules.rs @@ -3,8 +3,8 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; -use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::mask::MaskReduceAdaptor; use crate::ByteBool; diff --git a/encodings/datetime-parts/Cargo.toml b/encodings/datetime-parts/Cargo.toml index 5f5657c6e3d..c8eb48fee0c 100644 --- a/encodings/datetime-parts/Cargo.toml +++ b/encodings/datetime-parts/Cargo.toml @@ -22,6 +22,7 @@ vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-compute = { workspace = true } vortex-error = { workspace = true } +vortex-functions = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/encodings/datetime-parts/src/compute/cast.rs b/encodings/datetime-parts/src/compute/cast.rs index 8d5b0cd2e2c..7354871c6c7 100644 --- a/encodings/datetime-parts/src/compute/cast.rs +++ b/encodings/datetime-parts/src/compute/cast.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::DateTimeParts; diff --git a/encodings/datetime-parts/src/compute/compare.rs b/encodings/datetime-parts/src/compute/compare.rs index 96885800424..41b6cf3eefe 100644 --- a/encodings/datetime-parts/src/compute/compare.rs +++ b/encodings/datetime-parts/src/compute/compare.rs @@ -13,9 +13,9 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::extension::datetime::Timestamp; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::operators::CompareOperator; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_mask::AllOr; @@ -302,7 +302,7 @@ mod test { use vortex_array::extension::datetime::Timestamp; use vortex_array::extension::datetime::TimestampOptions; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::binary::CompareKernel; + use vortex_array::scalar_fn::builtin::binary::CompareKernel; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_session::VortexSession; diff --git a/encodings/datetime-parts/src/compute/kernel.rs b/encodings/datetime-parts/src/compute/kernel.rs index 4bd0cf2e1b3..46cc7b20b3a 100644 --- a/encodings/datetime-parts/src/compute/kernel.rs +++ b/encodings/datetime-parts/src/compute/kernel.rs @@ -6,8 +6,8 @@ use vortex_array::arrays::Dict; use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; use vortex_session::VortexSession; use crate::DateTimeParts; diff --git a/encodings/datetime-parts/src/compute/mask.rs b/encodings/datetime-parts/src/compute/mask.rs index bd81605c446..67132e7dbb4 100644 --- a/encodings/datetime-parts/src/compute/mask.rs +++ b/encodings/datetime-parts/src/compute/mask.rs @@ -5,7 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; -use vortex_array::scalar_fn::fns::mask::MaskReduce; +use vortex_array::scalar_fn::builtin::mask::MaskReduce; use vortex_error::VortexResult; use crate::DateTimeParts; diff --git a/encodings/datetime-parts/src/compute/rules.rs b/encodings/datetime-parts/src/compute/rules.rs index 5712b94961d..2614aa3c2c7 100644 --- a/encodings/datetime-parts/src/compute/rules.rs +++ b/encodings/datetime-parts/src/compute/rules.rs @@ -19,12 +19,12 @@ use vortex_array::extension::datetime::Timestamp; use vortex_array::optimizer::ArrayOptimizer; use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::between::Between; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; -use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::mask::MaskReduceAdaptor; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_functions::between::Between; use crate::DateTimeParts; use crate::array::DateTimePartsArraySlotsExt; @@ -192,11 +192,11 @@ mod tests { use vortex_array::extension::datetime::TimestampOptions; use vortex_array::optimizer::ArrayOptimizer; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::validity::Validity; use vortex_buffer::Buffer; + use vortex_functions::between::BetweenOptions; + use vortex_functions::between::StrictComparison; use vortex_session::VortexSession; use super::*; diff --git a/encodings/decimal-byte-parts/Cargo.toml b/encodings/decimal-byte-parts/Cargo.toml index dcccabf001a..be283bf96ba 100644 --- a/encodings/decimal-byte-parts/Cargo.toml +++ b/encodings/decimal-byte-parts/Cargo.toml @@ -21,6 +21,7 @@ prost = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-functions = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/between.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/between.rs index 8ef4d81daf3..2fe075465b1 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/between.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/between.rs @@ -7,17 +7,17 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; -use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; -use vortex_array::scalar_fn::fns::between::BetweenKernel; -use vortex_array::scalar_fn::fns::between::BetweenOptions; -use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_buffer::BitBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_functions::between::BetweenKernel; +use vortex_functions::between::BetweenOptions; +use vortex_functions::between::StrictComparison; +use vortex_functions::builtins::ArrayFunctions; use crate::DecimalByteParts; use crate::decimal_byte_parts::DecimalBytePartsArraySlotsExt; @@ -116,17 +116,17 @@ mod tests { use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; - use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DecimalDType; use vortex_array::dtype::Nullability; use vortex_array::scalar::DecimalValue; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_functions::between::BetweenOptions; + use vortex_functions::between::StrictComparison; + use vortex_functions::builtins::ArrayFunctions; use vortex_session::VortexSession; use crate::DecimalByteParts; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs index c700c6f2515..7b0080ed854 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::DecimalByteParts; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs index 4626b325215..6069b1c2b79 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs @@ -18,9 +18,9 @@ use vortex_array::match_each_integer_ptype; use vortex_array::scalar::DecimalValue; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::operators::CompareOperator; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -165,7 +165,7 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::scalar::DecimalValue; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexExpect; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/kernel.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/kernel.rs index 9f26305a94d..fc85a00931c 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/kernel.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/kernel.rs @@ -3,10 +3,10 @@ use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::between::Between; -use vortex_array::scalar_fn::fns::between::BetweenExecuteAdaptor; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; +use vortex_functions::between::Between; +use vortex_functions::between::BetweenExecuteAdaptor; use vortex_session::VortexSession; use crate::DecimalByteParts; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs index 9a022ef34ce..a898bbe35e7 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs @@ -4,8 +4,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; -use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; -use vortex_array::scalar_fn::fns::mask::MaskReduce; +use vortex_array::scalar_fn::builtin::mask::Mask as MaskExpr; +use vortex_array::scalar_fn::builtin::mask::MaskReduce; use vortex_error::VortexResult; use crate::DecimalByteParts; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs index 28503d5d8af..3c4c9ecf6d7 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs @@ -5,8 +5,8 @@ use vortex_array::arrays::dict::TakeReduceAdaptor; use vortex_array::arrays::filter::FilterReduceAdaptor; use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; -use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::mask::MaskReduceAdaptor; use crate::DecimalByteParts; diff --git a/encodings/fastlanes/Cargo.toml b/encodings/fastlanes/Cargo.toml index c1261092b61..4b69c99259a 100644 --- a/encodings/fastlanes/Cargo.toml +++ b/encodings/fastlanes/Cargo.toml @@ -29,6 +29,7 @@ vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-compute = { workspace = true } vortex-error = { workspace = true } +vortex-functions = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/encodings/fastlanes/benches/bitpack_compare.rs b/encodings/fastlanes/benches/bitpack_compare.rs index acc8c89f2e8..674ebbc342f 100644 --- a/encodings/fastlanes/benches/bitpack_compare.rs +++ b/encodings/fastlanes/benches/bitpack_compare.rs @@ -26,7 +26,7 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Alignment; diff --git a/encodings/fastlanes/benches/bitpack_compare_sweep.rs b/encodings/fastlanes/benches/bitpack_compare_sweep.rs index 3dcb3f50a11..f7ae88f341d 100644 --- a/encodings/fastlanes/benches/bitpack_compare_sweep.rs +++ b/encodings/fastlanes/benches/bitpack_compare_sweep.rs @@ -29,7 +29,7 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::NativePType; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Alignment; diff --git a/encodings/fastlanes/benches/compute_between.rs b/encodings/fastlanes/benches/compute_between.rs index d3e85f61e3d..080043a5f1f 100644 --- a/encodings/fastlanes/benches/compute_between.rs +++ b/encodings/fastlanes/benches/compute_between.rs @@ -99,11 +99,11 @@ mod primitive { use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; - use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::NativePType; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison::NonStrict; use vortex_error::VortexExpect; + use vortex_functions::between::BetweenOptions; + use vortex_functions::between::StrictComparison::NonStrict; + use vortex_functions::builtins::ArrayFunctions; use crate::BENCH_ARGS; use crate::SESSION; @@ -152,11 +152,11 @@ mod bitpack { use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; - use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::NativePType; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison::NonStrict; use vortex_error::VortexExpect; + use vortex_functions::between::BetweenOptions; + use vortex_functions::between::StrictComparison::NonStrict; + use vortex_functions::builtins::ArrayFunctions; use crate::BENCH_ARGS; use crate::SESSION; @@ -204,11 +204,11 @@ mod alp { use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; - use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::NativePType; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison::NonStrict; use vortex_error::VortexExpect; + use vortex_functions::between::BetweenOptions; + use vortex_functions::between::StrictComparison::NonStrict; + use vortex_functions::builtins::ArrayFunctions; use crate::BENCH_ARGS; use crate::SESSION; diff --git a/encodings/fastlanes/src/bitpacking/array/tests.rs b/encodings/fastlanes/src/bitpacking/array/tests.rs index 94a29adff73..0364c068499 100644 --- a/encodings/fastlanes/src/bitpacking/array/tests.rs +++ b/encodings/fastlanes/src/bitpacking/array/tests.rs @@ -21,8 +21,8 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; -use vortex_array::scalar_fn::fns::cast::CastKernel; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastKernel; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_array::validity::Validity; use vortex_buffer::ByteBuffer; use vortex_buffer::buffer; diff --git a/encodings/fastlanes/src/bitpacking/compute/between.rs b/encodings/fastlanes/src/bitpacking/compute/between.rs index a878fcb3c05..9734da3f119 100644 --- a/encodings/fastlanes/src/bitpacking/compute/between.rs +++ b/encodings/fastlanes/src/bitpacking/compute/between.rs @@ -21,11 +21,11 @@ use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PhysicalPType; use vortex_array::match_each_integer_ptype; -use vortex_array::scalar_fn::fns::between::BetweenKernel; -use vortex_array::scalar_fn::fns::between::BetweenOptions; -use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_functions::between::BetweenKernel; +use vortex_functions::between::BetweenOptions; +use vortex_functions::between::StrictComparison; use crate::BitPacked; use crate::BitPackedArrayExt; @@ -224,13 +224,13 @@ mod tests { use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; - use vortex_array::builtins::ArrayBuiltins; - use vortex_array::scalar_fn::fns::between::BetweenKernel; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_error::VortexResult; + use vortex_functions::between::BetweenKernel; + use vortex_functions::between::BetweenOptions; + use vortex_functions::between::StrictComparison; + use vortex_functions::builtins::ArrayFunctions; use vortex_session::VortexSession; use crate::BitPacked; diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index 8203e68a6b7..88d03270002 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -11,8 +11,8 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::match_each_integer_ptype; -use vortex_array::scalar_fn::fns::cast::CastKernel; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastKernel; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_array::validity::Validity; use vortex_error::VortexResult; diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index 68389b6eb2b..d43bf81f05a 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -21,8 +21,8 @@ use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PhysicalPType; use vortex_array::match_each_integer_ptype; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -203,9 +203,9 @@ mod tests { use vortex_array::arrays::slice::SliceKernel; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; - use vortex_array::scalar_fn::fns::binary::CompareKernel; - use vortex_array::scalar_fn::fns::operators::CompareOperator; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::binary::CompareKernel; + use vortex_array::scalar_fn::builtin::operators::CompareOperator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use vortex_session::VortexSession; diff --git a/encodings/fastlanes/src/bitpacking/vtable/kernels.rs b/encodings/fastlanes/src/bitpacking/vtable/kernels.rs index eb0dd9b7a23..33bce64ff86 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/kernels.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/kernels.rs @@ -10,12 +10,12 @@ use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::arrays::slice::SliceExecuteAdaptor; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::between::Between; -use vortex_array::scalar_fn::fns::between::BetweenExecuteAdaptor; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; -use vortex_array::scalar_fn::fns::cast::Cast; -use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::cast::Cast; +use vortex_array::scalar_fn::builtin::cast::CastExecuteAdaptor; +use vortex_functions::between::Between; +use vortex_functions::between::BetweenExecuteAdaptor; use vortex_session::VortexSession; use crate::BitPacked; diff --git a/encodings/fastlanes/src/bitpacking/vtable/rules.rs b/encodings/fastlanes/src/bitpacking/vtable/rules.rs index a26989569b2..f0e101ddc8b 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/rules.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/rules.rs @@ -3,7 +3,7 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; use crate::BitPacked; diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index 4342fd883bd..efa6a127c62 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability::NonNullable; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::delta::Delta; diff --git a/encodings/fastlanes/src/delta/vtable/rules.rs b/encodings/fastlanes/src/delta/vtable/rules.rs index d6892897ab5..c3cc19ef892 100644 --- a/encodings/fastlanes/src/delta/vtable/rules.rs +++ b/encodings/fastlanes/src/delta/vtable/rules.rs @@ -3,7 +3,7 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; use crate::delta::vtable::Delta; diff --git a/encodings/fastlanes/src/for_/compute/cast.rs b/encodings/fastlanes/src/for_/compute/cast.rs index af1ac50af3f..18f78d0c6c1 100644 --- a/encodings/fastlanes/src/for_/compute/cast.rs +++ b/encodings/fastlanes/src/for_/compute/cast.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::FoRReferences; diff --git a/encodings/fastlanes/src/for_/compute/compare.rs b/encodings/fastlanes/src/for_/compute/compare.rs index d9dfcc6d9f8..d5b761e6592 100644 --- a/encodings/fastlanes/src/for_/compute/compare.rs +++ b/encodings/fastlanes/src/for_/compute/compare.rs @@ -15,9 +15,9 @@ use vortex_array::dtype::Nullability; use vortex_array::match_each_integer_ptype; use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::operators::CompareOperator; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexError; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; diff --git a/encodings/fastlanes/src/for_/vtable/kernels.rs b/encodings/fastlanes/src/for_/vtable/kernels.rs index 6c63292bb9f..7687ede7cf1 100644 --- a/encodings/fastlanes/src/for_/vtable/kernels.rs +++ b/encodings/fastlanes/src/for_/vtable/kernels.rs @@ -6,8 +6,8 @@ use vortex_array::arrays::Dict; use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; use vortex_session::VortexSession; use crate::FoR; diff --git a/encodings/fastlanes/src/for_/vtable/rules.rs b/encodings/fastlanes/src/for_/vtable/rules.rs index 541232865e3..5e22819e619 100644 --- a/encodings/fastlanes/src/for_/vtable/rules.rs +++ b/encodings/fastlanes/src/for_/vtable/rules.rs @@ -9,7 +9,7 @@ use vortex_array::arrays::filter::FilterReduceAdaptor; use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; use vortex_error::VortexResult; use crate::FoR; diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index 194c6c36c3e..994a8f0c4d1 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -7,7 +7,7 @@ use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::rle::RLE; diff --git a/encodings/fastlanes/src/rle/probe/tests.rs b/encodings/fastlanes/src/rle/probe/tests.rs index f8523270095..43fa87164a1 100644 --- a/encodings/fastlanes/src/rle/probe/tests.rs +++ b/encodings/fastlanes/src/rle/probe/tests.rs @@ -13,7 +13,7 @@ use vortex_array::builders::builder_with_capacity_in; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::validity::Validity; use vortex_error::VortexResult; diff --git a/encodings/fastlanes/src/rle/vtable/rules.rs b/encodings/fastlanes/src/rle/vtable/rules.rs index 50dd1f16a5a..fb605cabe74 100644 --- a/encodings/fastlanes/src/rle/vtable/rules.rs +++ b/encodings/fastlanes/src/rle/vtable/rules.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; use crate::RLE; diff --git a/encodings/fsst/Cargo.toml b/encodings/fsst/Cargo.toml index 5b15dca85fb..691842af042 100644 --- a/encodings/fsst/Cargo.toml +++ b/encodings/fsst/Cargo.toml @@ -22,6 +22,8 @@ rand = { workspace = true, optional = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-expr = { workspace = true } +vortex-functions = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } @@ -35,6 +37,7 @@ rand = { workspace = true } rstest = { workspace = true } test-with = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-expr = { workspace = true } [[bench]] name = "fsst_compress" diff --git a/encodings/fsst/benches/fsst_compress.rs b/encodings/fsst/benches/fsst_compress.rs index 3a0a4e682e6..69c76be10b3 100644 --- a/encodings/fsst/benches/fsst_compress.rs +++ b/encodings/fsst/benches/fsst_compress.rs @@ -26,7 +26,7 @@ use vortex_array::dtype::Nullability; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::session::ArraySession; use vortex_fsst::fsst_compress; use vortex_fsst::fsst_train_compressor; diff --git a/encodings/fsst/benches/fsst_like.rs b/encodings/fsst/benches/fsst_like.rs index a6336292ef2..fbbab52d57b 100644 --- a/encodings/fsst/benches/fsst_like.rs +++ b/encodings/fsst/benches/fsst_like.rs @@ -14,8 +14,6 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; -use vortex_array::scalar_fn::fns::like::Like; -use vortex_array::scalar_fn::fns::like::LikeOptions; use vortex_array::session::ArraySession; use vortex_fsst::FSSTArray; use vortex_fsst::test_utils::NUM_STRINGS; @@ -26,6 +24,8 @@ use vortex_fsst::test_utils::make_fsst_json_strings; use vortex_fsst::test_utils::make_fsst_log_lines; use vortex_fsst::test_utils::make_fsst_rare_match; use vortex_fsst::test_utils::make_fsst_short_urls; +use vortex_functions::like::Like; +use vortex_functions::like::LikeOptions; use vortex_session::VortexSession; #[global_allocator] diff --git a/encodings/fsst/benches/fsst_url_compare.rs b/encodings/fsst/benches/fsst_url_compare.rs index f62af505f3d..1a4025b1b4d 100644 --- a/encodings/fsst/benches/fsst_url_compare.rs +++ b/encodings/fsst/benches/fsst_url_compare.rs @@ -16,20 +16,21 @@ use vortex_array::arrays::VarBinArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::varbin::VarBinArrayExt; use vortex_array::builtins::ArrayBuiltins; -use vortex_array::expr::like; -use vortex_array::expr::lit; -use vortex_array::expr::root; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::session::ArraySession; +use vortex_expr::ArrayExprExt; +use vortex_expr::lit; +use vortex_expr::root; use vortex_fsst::fsst_compress; use vortex_fsst::fsst_train_compressor; use vortex_fsst::test_utils::HIGH_MATCH_DOMAIN; use vortex_fsst::test_utils::LOW_MATCH_DOMAIN; use vortex_fsst::test_utils::NUM_STRINGS; use vortex_fsst::test_utils::generate_url_data; +use vortex_functions::expr::like; use vortex_session::VortexSession; #[global_allocator] diff --git a/encodings/fsst/src/compute/byte_length.rs b/encodings/fsst/src/compute/byte_length.rs index 05505fe95e5..668fad8e718 100644 --- a/encodings/fsst/src/compute/byte_length.rs +++ b/encodings/fsst/src/compute/byte_length.rs @@ -10,9 +10,9 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::byte_length::ByteLengthKernel; use vortex_array::validity::Validity; use vortex_error::VortexResult; +use vortex_functions::byte_length::ByteLengthKernel; use crate::FSST; use crate::array::FSSTArraySlotsExt; diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs index 63513cd4ba3..a9c7cc6bd57 100644 --- a/encodings/fsst/src/compute/cast.rs +++ b/encodings/fsst/src/compute/cast.rs @@ -8,8 +8,8 @@ use vortex_array::IntoArray; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::varbin::VarBinArraySlotsExt; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastKernel; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastKernel; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_array::validity::Validity; use vortex_error::VortexResult; diff --git a/encodings/fsst/src/compute/compare.rs b/encodings/fsst/src/compute/compare.rs index 53422acebee..dc6699bf26c 100644 --- a/encodings/fsst/src/compute/compare.rs +++ b/encodings/fsst/src/compute/compare.rs @@ -10,9 +10,9 @@ use vortex_array::arrays::ConstantArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::operators::CompareOperator; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_buffer::BitBuffer; use vortex_buffer::ByteBuffer; use vortex_error::VortexExpect; @@ -135,7 +135,7 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use vortex_session::VortexSession; diff --git a/encodings/fsst/src/compute/like.rs b/encodings/fsst/src/compute/like.rs index 380de2f82cc..328675cf777 100644 --- a/encodings/fsst/src/compute/like.rs +++ b/encodings/fsst/src/compute/like.rs @@ -9,9 +9,9 @@ use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::varbin::VarBinArraySlotsExt; use vortex_array::match_each_integer_ptype; -use vortex_array::scalar_fn::fns::like::LikeKernel; -use vortex_array::scalar_fn::fns::like::LikeOptions; use vortex_error::VortexResult; +use vortex_functions::like::LikeKernel; +use vortex_functions::like::LikeOptions; use crate::FSST; use crate::FSSTArrayExt; @@ -89,10 +89,10 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; - use vortex_array::scalar_fn::fns::like::Like; - use vortex_array::scalar_fn::fns::like::LikeKernel; - use vortex_array::scalar_fn::fns::like::LikeOptions; use vortex_error::VortexResult; + use vortex_functions::like::Like; + use vortex_functions::like::LikeKernel; + use vortex_functions::like::LikeOptions; use vortex_session::VortexSession; use crate::FSST; diff --git a/encodings/fsst/src/dfa/tests.rs b/encodings/fsst/src/dfa/tests.rs index c97682e27e3..b53a6a53385 100644 --- a/encodings/fsst/src/dfa/tests.rs +++ b/encodings/fsst/src/dfa/tests.rs @@ -16,9 +16,9 @@ use vortex_array::arrays::VarBinArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::scalar_fn::fns::like::Like; -use vortex_array::scalar_fn::fns::like::LikeOptions; use vortex_error::VortexResult; +use vortex_functions::like::Like; +use vortex_functions::like::LikeOptions; use vortex_session::VortexSession; use super::FsstMatcher; diff --git a/encodings/fsst/src/kernel.rs b/encodings/fsst/src/kernel.rs index 44d9c7b2531..c9184dc5eae 100644 --- a/encodings/fsst/src/kernel.rs +++ b/encodings/fsst/src/kernel.rs @@ -8,14 +8,14 @@ use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; -use vortex_array::scalar_fn::fns::byte_length::ByteLength; -use vortex_array::scalar_fn::fns::byte_length::ByteLengthExecuteAdaptor; -use vortex_array::scalar_fn::fns::cast::Cast; -use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor; -use vortex_array::scalar_fn::fns::like::Like; -use vortex_array::scalar_fn::fns::like::LikeExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::cast::Cast; +use vortex_array::scalar_fn::builtin::cast::CastExecuteAdaptor; +use vortex_functions::byte_length::ByteLength; +use vortex_functions::byte_length::ByteLengthExecuteAdaptor; +use vortex_functions::like::Like; +use vortex_functions::like::LikeExecuteAdaptor; use vortex_session::VortexSession; use crate::FSST; @@ -44,9 +44,10 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; - use vortex_array::expr::byte_length; - use vortex_array::expr::root; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::root; + use vortex_functions::expr::byte_length; use vortex_mask::Mask; use vortex_session::VortexSession; diff --git a/encodings/fsst/src/rules.rs b/encodings/fsst/src/rules.rs index bec81c0fd3a..eb68e392981 100644 --- a/encodings/fsst/src/rules.rs +++ b/encodings/fsst/src/rules.rs @@ -3,7 +3,7 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; use crate::FSST; diff --git a/encodings/onpair/Cargo.toml b/encodings/onpair/Cargo.toml index 4c82ba08aba..f703ca60fb9 100644 --- a/encodings/onpair/Cargo.toml +++ b/encodings/onpair/Cargo.toml @@ -21,6 +21,7 @@ prost = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-functions = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/encodings/onpair/src/compute/byte_length.rs b/encodings/onpair/src/compute/byte_length.rs index 933e8bccd85..3ee181ad551 100644 --- a/encodings/onpair/src/compute/byte_length.rs +++ b/encodings/onpair/src/compute/byte_length.rs @@ -8,8 +8,8 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::byte_length::ByteLengthKernel; use vortex_array::validity::Validity; +use vortex_functions::byte_length::ByteLengthKernel; use crate::OnPair; use crate::OnPairArraySlotsExt; diff --git a/encodings/onpair/src/compute/cast.rs b/encodings/onpair/src/compute/cast.rs index 737b6933f8f..fd9968e8678 100644 --- a/encodings/onpair/src/compute/cast.rs +++ b/encodings/onpair/src/compute/cast.rs @@ -6,8 +6,8 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastKernel; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastKernel; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_array::validity::Validity; use vortex_error::VortexResult; diff --git a/encodings/onpair/src/compute/compare.rs b/encodings/onpair/src/compute/compare.rs index fe8e6a0cec3..2c07f9f0f8a 100644 --- a/encodings/onpair/src/compute/compare.rs +++ b/encodings/onpair/src/compute/compare.rs @@ -11,8 +11,8 @@ use vortex_array::arrays::ConstantArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; use vortex_buffer::BitBuffer; use vortex_error::VortexResult; @@ -102,9 +102,9 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::binary::CompareKernel; - use vortex_array::scalar_fn::fns::operators::CompareOperator; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::binary::CompareKernel; + use vortex_array::scalar_fn::builtin::operators::CompareOperator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_session::VortexSession; diff --git a/encodings/onpair/src/kernel.rs b/encodings/onpair/src/kernel.rs index 6a76b1a5215..6e5c24cb36f 100644 --- a/encodings/onpair/src/kernel.rs +++ b/encodings/onpair/src/kernel.rs @@ -8,12 +8,12 @@ use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; -use vortex_array::scalar_fn::fns::byte_length::ByteLength; -use vortex_array::scalar_fn::fns::byte_length::ByteLengthExecuteAdaptor; -use vortex_array::scalar_fn::fns::cast::Cast; -use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::cast::Cast; +use vortex_array::scalar_fn::builtin::cast::CastExecuteAdaptor; +use vortex_functions::byte_length::ByteLength; +use vortex_functions::byte_length::ByteLengthExecuteAdaptor; use vortex_session::VortexSession; use crate::OnPair; diff --git a/encodings/onpair/src/rules.rs b/encodings/onpair/src/rules.rs index 279c160c1eb..ab18f256604 100644 --- a/encodings/onpair/src/rules.rs +++ b/encodings/onpair/src/rules.rs @@ -3,7 +3,7 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; use crate::OnPair; diff --git a/encodings/onpair/src/tests.rs b/encodings/onpair/src/tests.rs index aa735b0524a..c91ae8e4247 100644 --- a/encodings/onpair/src/tests.rs +++ b/encodings/onpair/src/tests.rs @@ -25,7 +25,7 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::match_each_integer_ptype; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::test_harness::check_metadata; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; diff --git a/encodings/onpair/tests/big_data.rs b/encodings/onpair/tests/big_data.rs index 80cbaeb18d0..37b39ba6e46 100644 --- a/encodings/onpair/tests/big_data.rs +++ b/encodings/onpair/tests/big_data.rs @@ -26,7 +26,7 @@ use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_onpair::DEFAULT_CONFIG; use vortex_onpair::onpair_compress; use vortex_session::VortexSession; diff --git a/encodings/parquet-variant/Cargo.toml b/encodings/parquet-variant/Cargo.toml index bb7e786b5ef..5b5415cc54c 100644 --- a/encodings/parquet-variant/Cargo.toml +++ b/encodings/parquet-variant/Cargo.toml @@ -27,6 +27,7 @@ vortex-array = { workspace = true } vortex-arrow = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-functions = { workspace = true } vortex-json = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } @@ -36,6 +37,7 @@ vortex-edition = { workspace = true } rstest = { workspace = true } tokio = { workspace = true, features = ["full"] } vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-expr = { workspace = true } vortex-file = { workspace = true, features = ["tokio"] } vortex-io = { workspace = true, features = ["tokio"] } vortex-layout = { workspace = true } diff --git a/encodings/parquet-variant/src/json_to_variant_tests.rs b/encodings/parquet-variant/src/json_to_variant_tests.rs index e66ed56c67d..0399a46aa72 100644 --- a/encodings/parquet-variant/src/json_to_variant_tests.rs +++ b/encodings/parquet-variant/src/json_to_variant_tests.rs @@ -26,12 +26,13 @@ use vortex_array::assert_nth_scalar_is_null; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::expr::root; -use vortex_array::expr::variant_get; -use vortex_array::scalar_fn::fns::variant_get::VariantPath; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_expr::ArrayExprExt; +use vortex_expr::root; +use vortex_functions::expr::variant_get; +use vortex_functions::variant_get::VariantPath; use vortex_json::Json; use vortex_session::VortexSession; diff --git a/encodings/parquet-variant/src/kernel.rs b/encodings/parquet-variant/src/kernel.rs index b218d9404f4..f813c9b64d9 100644 --- a/encodings/parquet-variant/src/kernel.rs +++ b/encodings/parquet-variant/src/kernel.rs @@ -40,15 +40,15 @@ use vortex_array::dtype::DType; use vortex_array::kernel::ExecuteParentKernel; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::variant_get::VariantGet; -use vortex_array::scalar_fn::fns::variant_get::VariantPath; -use vortex_array::scalar_fn::fns::variant_get::VariantPathElement; use vortex_arrow::ArrowExportOptions; use vortex_arrow::ArrowSession; use vortex_arrow::ArrowSessionExt; use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; +use vortex_functions::variant_get::VariantGet; +use vortex_functions::variant_get::VariantPath; +use vortex_functions::variant_get::VariantPathElement; use vortex_json::Json; use vortex_json::JsonToVariant; use vortex_mask::Mask; @@ -325,16 +325,17 @@ mod tests { use vortex_array::dtype::DType as VortexDType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::root; - use vortex_array::expr::variant_get; - use vortex_array::scalar_fn::fns::variant_get::VariantPath; - use vortex_array::scalar_fn::fns::variant_get::VariantPathElement; use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; + use vortex_expr::ArrayExprExt; + use vortex_expr::root; + use vortex_functions::expr::variant_get; + use vortex_functions::variant_get::VariantPath; + use vortex_functions::variant_get::VariantPathElement; use vortex_mask::Mask; use vortex_session::VortexSession; diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index af1d003d7f7..4d5e800ff1a 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -5,7 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::Pco; diff --git a/encodings/pco/src/rules.rs b/encodings/pco/src/rules.rs index 4d27392dc01..c7bd63a123b 100644 --- a/encodings/pco/src/rules.rs +++ b/encodings/pco/src/rules.rs @@ -3,7 +3,7 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; use crate::Pco; diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index 0619ae1614f..43fcf66294a 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::RunEnd; diff --git a/encodings/runend/src/compute/compare.rs b/encodings/runend/src/compute/compare.rs index 00ce5a904e4..af6a84f9f9e 100644 --- a/encodings/runend/src/compute/compare.rs +++ b/encodings/runend/src/compute/compare.rs @@ -9,9 +9,9 @@ use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::operators::CompareOperator; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use crate::RunEnd; @@ -59,7 +59,7 @@ mod test { use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_session::VortexSession; use crate::RunEnd; diff --git a/encodings/runend/src/compute/fill_null.rs b/encodings/runend/src/compute/fill_null.rs index 90284be3e51..046d4154300 100644 --- a/encodings/runend/src/compute/fill_null.rs +++ b/encodings/runend/src/compute/fill_null.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::fill_null::FillNullReduce; +use vortex_array::scalar_fn::builtin::fill_null::FillNullReduce; use vortex_error::VortexResult; use crate::RunEnd; diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs index 2eed964e3a9..673826f2478 100644 --- a/encodings/runend/src/kernel.rs +++ b/encodings/runend/src/kernel.rs @@ -17,8 +17,8 @@ use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::kernel::ExecuteParentKernel; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; use vortex_error::VortexResult; use vortex_session::VortexSession; diff --git a/encodings/runend/src/probe/tests.rs b/encodings/runend/src/probe/tests.rs index cedfe8af16d..0e4d7986cb8 100644 --- a/encodings/runend/src/probe/tests.rs +++ b/encodings/runend/src/probe/tests.rs @@ -15,7 +15,7 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; diff --git a/encodings/runend/src/rules.rs b/encodings/runend/src/rules.rs index a6f3d6a4ae3..9c32a36b57b 100644 --- a/encodings/runend/src/rules.rs +++ b/encodings/runend/src/rules.rs @@ -13,8 +13,8 @@ use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::dtype::DType; use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; -use vortex_array::scalar_fn::fns::fill_null::FillNullReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::fill_null::FillNullReduceAdaptor; use vortex_error::VortexResult; use crate::RunEnd; diff --git a/encodings/runend/src/trace_tests.rs b/encodings/runend/src/trace_tests.rs index 302fc9871fd..70e4ff56714 100644 --- a/encodings/runend/src/trace_tests.rs +++ b/encodings/runend/src/trace_tests.rs @@ -15,8 +15,8 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::optimizer::ArrayOptimizer; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::session::ArraySession; use vortex_array::test_harness::trace::trace_op; use vortex_error::VortexResult; diff --git a/encodings/sequence/Cargo.toml b/encodings/sequence/Cargo.toml index a6233c27a7b..71d3bcf67e2 100644 --- a/encodings/sequence/Cargo.toml +++ b/encodings/sequence/Cargo.toml @@ -20,6 +20,8 @@ vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-compute = { workspace = true } vortex-error = { workspace = true } +vortex-expr = { workspace = true } +vortex-functions = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs index 8eca1229f3d..47939ae3379 100644 --- a/encodings/sequence/src/compute/cast.rs +++ b/encodings/sequence/src/compute/cast.rs @@ -5,7 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::Sequence; diff --git a/encodings/sequence/src/compute/compare.rs b/encodings/sequence/src/compute/compare.rs index e2f3a0186be..cac8da99a47 100644 --- a/encodings/sequence/src/compute/compare.rs +++ b/encodings/sequence/src/compute/compare.rs @@ -10,8 +10,8 @@ use vortex_array::arrays::ConstantArray; use vortex_array::dtype::Nullability; use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; use vortex_buffer::BitBufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -135,7 +135,7 @@ mod tests { use vortex_array::dtype::Nullability::Nullable; use vortex_array::dtype::PType; use vortex_array::scalar::PValue; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use vortex_session::VortexSession; diff --git a/encodings/sequence/src/compute/list_contains.rs b/encodings/sequence/src/compute/list_contains.rs index ea966049ea4..1051d2d05fc 100644 --- a/encodings/sequence/src/compute/list_contains.rs +++ b/encodings/sequence/src/compute/list_contains.rs @@ -8,13 +8,13 @@ use vortex_array::arrays::BoolArray; use vortex_array::arrays::Constant; use vortex_array::arrays::ConstantArray; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::list_contains::ListContainsElementReduce; -use vortex_array::scalar_fn::fns::list_contains::ListContainsOptions; -use vortex_array::scalar_fn::fns::list_contains::PreparedSet; -use vortex_array::scalar_fn::fns::list_contains::PreparedSetData; use vortex_array::validity::Validity; use vortex_buffer::BitBufferMut; use vortex_error::VortexResult; +use vortex_functions::list_contains::ListContainsElementReduce; +use vortex_functions::list_contains::ListContainsOptions; +use vortex_functions::list_contains::PreparedSet; +use vortex_functions::list_contains::PreparedSetData; use crate::array::Sequence; use crate::compute::compare::Intersection; @@ -97,13 +97,14 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType::I32; - use vortex_array::expr::list_contains; - use vortex_array::expr::list_contains_opts; - use vortex_array::expr::lit; - use vortex_array::expr::root; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::list_contains::ListContainsOptions; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::lit; + use vortex_expr::root; + use vortex_functions::expr::list_contains; + use vortex_functions::expr::list_contains_opts; + use vortex_functions::list_contains::ListContainsOptions; use vortex_session::VortexSession; use crate::Sequence; diff --git a/encodings/sequence/src/kernel.rs b/encodings/sequence/src/kernel.rs index 37521a261e7..045d34eb0f9 100644 --- a/encodings/sequence/src/kernel.rs +++ b/encodings/sequence/src/kernel.rs @@ -8,8 +8,8 @@ use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; use vortex_session::VortexSession; use crate::Sequence; diff --git a/encodings/sequence/src/model_tests.rs b/encodings/sequence/src/model_tests.rs index 9bfb4624bf0..8c2da4b806d 100644 --- a/encodings/sequence/src/model_tests.rs +++ b/encodings/sequence/src/model_tests.rs @@ -26,7 +26,7 @@ use vortex_array::match_each_integer_ptype; use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_mask::Mask; diff --git a/encodings/sequence/src/rules.rs b/encodings/sequence/src/rules.rs index f87c08e8109..924c06fc5a0 100644 --- a/encodings/sequence/src/rules.rs +++ b/encodings/sequence/src/rules.rs @@ -3,8 +3,8 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; -use vortex_array::scalar_fn::fns::list_contains::ListContainsElementReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; +use vortex_functions::list_contains::ListContainsElementReduceAdaptor; use crate::Sequence; diff --git a/encodings/sparse/Cargo.toml b/encodings/sparse/Cargo.toml index 78f8b9faeb0..84a3eac4756 100644 --- a/encodings/sparse/Cargo.toml +++ b/encodings/sparse/Cargo.toml @@ -22,6 +22,7 @@ prost = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-functions = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/encodings/sparse/benches/sparse_pushdown.rs b/encodings/sparse/benches/sparse_pushdown.rs index f3e6133dccc..1692db3d846 100644 --- a/encodings/sparse/benches/sparse_pushdown.rs +++ b/encodings/sparse/benches/sparse_pushdown.rs @@ -35,7 +35,7 @@ use vortex_array::dtype::PType; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_error::VortexExpect; diff --git a/encodings/sparse/src/compute/between.rs b/encodings/sparse/src/compute/between.rs index 1a01958ac44..bff24fdab71 100644 --- a/encodings/sparse/src/compute/between.rs +++ b/encodings/sparse/src/compute/between.rs @@ -6,10 +6,10 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ConstantArray; -use vortex_array::builtins::ArrayBuiltins; -use vortex_array::scalar_fn::fns::between::BetweenKernel; -use vortex_array::scalar_fn::fns::between::BetweenOptions; use vortex_error::VortexResult; +use vortex_functions::between::BetweenKernel; +use vortex_functions::between::BetweenOptions; +use vortex_functions::builtins::ArrayFunctions; use crate::Sparse; use crate::SparseExt as _; @@ -70,11 +70,11 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::assert_arrays_eq; - use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_buffer::buffer; + use vortex_functions::between::BetweenOptions; + use vortex_functions::between::StrictComparison; + use vortex_functions::builtins::ArrayFunctions; use vortex_session::VortexSession; use crate::Sparse; diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs index 88b07e0d9f4..74b8ad652d9 100644 --- a/encodings/sparse/src/compute/cast.rs +++ b/encodings/sparse/src/compute/cast.rs @@ -7,7 +7,7 @@ use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::Sparse; diff --git a/encodings/sparse/src/compute/compare.rs b/encodings/sparse/src/compute/compare.rs index ad1fad781eb..8e6b5c3ac91 100644 --- a/encodings/sparse/src/compute/compare.rs +++ b/encodings/sparse/src/compute/compare.rs @@ -7,10 +7,10 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ConstantArray; use vortex_array::builtins::ArrayBuiltins; -use vortex_array::scalar_fn::fns::binary::CompareKernel; -use vortex_array::scalar_fn::fns::binary::scalar_cmp; -use vortex_array::scalar_fn::fns::operators::CompareOperator; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::CompareKernel; +use vortex_array::scalar_fn::builtin::binary::scalar_cmp; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use crate::Sparse; @@ -64,7 +64,7 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_buffer::buffer; use vortex_session::VortexSession; diff --git a/encodings/sparse/src/compute/fill_null.rs b/encodings/sparse/src/compute/fill_null.rs index 1ae877068d2..66ab0a3617c 100644 --- a/encodings/sparse/src/compute/fill_null.rs +++ b/encodings/sparse/src/compute/fill_null.rs @@ -7,7 +7,7 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::fill_null::FillNullKernel; +use vortex_array::scalar_fn::builtin::fill_null::FillNullKernel; use vortex_error::VortexResult; use crate::Sparse; diff --git a/encodings/sparse/src/compute/like.rs b/encodings/sparse/src/compute/like.rs index a4cb23a41b3..bc53335512c 100644 --- a/encodings/sparse/src/compute/like.rs +++ b/encodings/sparse/src/compute/like.rs @@ -6,10 +6,10 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ConstantArray; -use vortex_array::scalar_fn::fns::like::Like; -use vortex_array::scalar_fn::fns::like::LikeKernel; -use vortex_array::scalar_fn::fns::like::LikeOptions; use vortex_error::VortexResult; +use vortex_functions::like::Like; +use vortex_functions::like::LikeKernel; +use vortex_functions::like::LikeOptions; use crate::Sparse; use crate::SparseExt as _; @@ -71,10 +71,10 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::like::Like; - use vortex_array::scalar_fn::fns::like::LikeOptions; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_functions::like::Like; + use vortex_functions::like::LikeOptions; use vortex_session::VortexSession; use crate::Sparse; diff --git a/encodings/sparse/src/kernel.rs b/encodings/sparse/src/kernel.rs index 1265a27d678..2b09d2a3758 100644 --- a/encodings/sparse/src/kernel.rs +++ b/encodings/sparse/src/kernel.rs @@ -10,14 +10,14 @@ use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::arrays::slice::SliceExecuteAdaptor; use vortex_array::optimizer::kernels::ArrayKernelsExt; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::between::Between; -use vortex_array::scalar_fn::fns::between::BetweenExecuteAdaptor; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; -use vortex_array::scalar_fn::fns::fill_null::FillNull; -use vortex_array::scalar_fn::fns::fill_null::FillNullExecuteAdaptor; -use vortex_array::scalar_fn::fns::like::Like; -use vortex_array::scalar_fn::fns::like::LikeExecuteAdaptor; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::builtin::fill_null::FillNull; +use vortex_array::scalar_fn::builtin::fill_null::FillNullExecuteAdaptor; +use vortex_functions::between::Between; +use vortex_functions::between::BetweenExecuteAdaptor; +use vortex_functions::like::Like; +use vortex_functions::like::LikeExecuteAdaptor; use vortex_session::VortexSession; use crate::Sparse; diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 55e5ed883e2..d2ecb52dda8 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -43,7 +43,7 @@ use vortex_array::require_child; use vortex_array::require_opt_child; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; use vortex_array::vtable::VTable; diff --git a/encodings/sparse/src/rules.rs b/encodings/sparse/src/rules.rs index d888d64a389..a001a875077 100644 --- a/encodings/sparse/src/rules.rs +++ b/encodings/sparse/src/rules.rs @@ -6,9 +6,9 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; -use vortex_array::scalar_fn::fns::not::NotReduce; -use vortex_array::scalar_fn::fns::not::NotReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::not::NotReduce; +use vortex_array::scalar_fn::builtin::not::NotReduceAdaptor; use vortex_error::VortexResult; use crate::Sparse; diff --git a/encodings/zigzag/src/compute/cast.rs b/encodings/zigzag/src/compute/cast.rs index 1bd582f77f6..76f41106b4c 100644 --- a/encodings/zigzag/src/compute/cast.rs +++ b/encodings/zigzag/src/compute/cast.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_error::VortexResult; use crate::ZigZag; diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 8e301febcc1..0c8fad015f2 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -9,8 +9,8 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::dict::TakeExecute; use vortex_array::arrays::filter::FilterReduce; -use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; -use vortex_array::scalar_fn::fns::mask::MaskReduce; +use vortex_array::scalar_fn::builtin::mask::Mask as MaskExpr; +use vortex_array::scalar_fn::builtin::mask::MaskReduce; use vortex_error::VortexResult; use vortex_mask::Mask; diff --git a/encodings/zigzag/src/rules.rs b/encodings/zigzag/src/rules.rs index c3b612d101d..18f106ddf9a 100644 --- a/encodings/zigzag/src/rules.rs +++ b/encodings/zigzag/src/rules.rs @@ -4,8 +4,8 @@ use vortex_array::arrays::filter::FilterReduceAdaptor; use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; -use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::mask::MaskReduceAdaptor; use crate::ZigZag; diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs index f1bb4ebdcda..e1cd9fcd02c 100644 --- a/encodings/zstd/src/compute/cast.rs +++ b/encodings/zstd/src/compute/cast.rs @@ -6,7 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::scalar_fn::builtin::cast::CastReduce; use vortex_array::vtable::child_to_validity; use vortex_error::VortexResult; diff --git a/encodings/zstd/src/rules.rs b/encodings/zstd/src/rules.rs index cfbf93b09e8..5b547144515 100644 --- a/encodings/zstd/src/rules.rs +++ b/encodings/zstd/src/rules.rs @@ -3,7 +3,7 @@ use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::optimizer::rules::ParentRuleSet; -use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; +use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor; use crate::Zstd; diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index eed6fb69495..fba51cdc2ef 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -35,7 +35,9 @@ vortex-array = { workspace = true, features = ["arbitrary", "_test-harness"] } vortex-btrblocks = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-expr = { workspace = true, features = ["arbitrary"] } vortex-fsst = { workspace = true, features = ["_test-harness"] } +vortex-functions = { workspace = true } vortex-io = { workspace = true, features = ["wasm-bindgen"] } vortex-mask = { workspace = true } vortex-row = { workspace = true } diff --git a/fuzz/fuzz_targets/file_io.rs b/fuzz/fuzz_targets/file_io.rs index 543ea76db0e..511755278a0 100644 --- a/fuzz/fuzz_targets/file_io.rs +++ b/fuzz/fuzz_targets/file_io.rs @@ -6,6 +6,8 @@ use itertools::Itertools; use libfuzzer_sys::Corpus; use libfuzzer_sys::fuzz_target; +use vortex::expr::lit; +use vortex::expr::root; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -15,10 +17,9 @@ use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::StructFields; -use vortex_array::expr::lit; -use vortex_array::expr::root; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_error::VortexExpect; +use vortex_expr::ArrayExprExt; use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder; diff --git a/fuzz/src/array/compare.rs b/fuzz/src/array/compare.rs index 03ce50a7170..ac44571552a 100644 --- a/fuzz/src/array/compare.rs +++ b/fuzz/src/array/compare.rs @@ -15,8 +15,8 @@ use vortex_array::dtype::Nullability; use vortex_array::match_each_decimal_value_type; use vortex_array::match_each_native_ptype; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::binary::scalar_cmp; -use vortex_array::scalar_fn::fns::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::binary::scalar_cmp; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::ByteBuffer; diff --git a/fuzz/src/array/mod.rs b/fuzz/src/array/mod.rs index 638bd1559cf..bdfa0cffdce 100644 --- a/fuzz/src/array/mod.rs +++ b/fuzz/src/array/mod.rs @@ -59,8 +59,8 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; use vortex_array::scalar::arbitrary::random_scalar; -use vortex_array::scalar_fn::fns::operators::CompareOperator; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::search_sorted::SearchResult; use vortex_array::search_sorted::SearchSorted; use vortex_array::search_sorted::SearchSortedArray; diff --git a/fuzz/src/file/mod.rs b/fuzz/src/file/mod.rs index f4fdc6d6ce1..f2254f99d7c 100644 --- a/fuzz/src/file/mod.rs +++ b/fuzz/src/file/mod.rs @@ -3,13 +3,13 @@ use arbitrary::Arbitrary; use arbitrary::Unstructured; +use vortex::expr::Expression; use vortex_array::ArrayRef; use vortex_array::arrays::arbitrary::ArbitraryArray; use vortex_array::arrays::arbitrary::ArbitraryArrayConfig; use vortex_array::arrays::arbitrary::ArbitraryWith; -use vortex_array::expr::Expression; -use vortex_array::expr::arbitrary::filter_expr; -use vortex_array::expr::arbitrary::projection_expr; +use vortex_expr::arbitrary::filter_expr; +use vortex_expr::arbitrary::projection_expr; use crate::FUZZ_FILE_ARRAY_MAX_LEN; use crate::array::CompressorStrategy; diff --git a/fuzz/src/fsst_like.rs b/fuzz/src/fsst_like.rs index 0cd9b4c0cae..168c3d64615 100644 --- a/fuzz/src/fsst_like.rs +++ b/fuzz/src/fsst_like.rs @@ -9,6 +9,8 @@ use std::sync::LazyLock; use arbitrary::Arbitrary; use arbitrary::Unstructured; +use vortex::functions::like::Like; +use vortex::functions::like::LikeOptions; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; @@ -19,8 +21,6 @@ use vortex_array::arrays::VarBinArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::scalar_fn::fns::like::Like; -use vortex_array::scalar_fn::fns::like::LikeOptions; use vortex_error::VortexResult; use vortex_fsst::FSSTArray; use vortex_fsst::fsst_compress; diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index 4beb0672421..ffa61cbe881 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -157,14 +157,6 @@ harness = false name = "kleene_bool" harness = false -[[bench]] -name = "like" -harness = false - -[[bench]] -name = "list_contains_set" -harness = false - [[bench]] name = "interleave" harness = false @@ -197,21 +189,6 @@ harness = false name = "varbinview_compact" harness = false -[[bench]] -name = "expr_large_struct_pack" -path = "benches/expr/large_struct_pack.rs" -harness = false - -[[bench]] -name = "expr_case_when" -path = "benches/expr/case_when_bench.rs" -harness = false - -[[bench]] -name = "expr_optimize" -path = "benches/expr/optimize_bench.rs" -harness = false - [[bench]] name = "chunked_dict_builder" harness = false @@ -243,10 +220,6 @@ harness = false name = "dict_unreferenced_mask" harness = false -[[bench]] -name = "varbinview_zip" -harness = false - [[bench]] name = "varbinview_try_new" harness = false @@ -255,18 +228,6 @@ harness = false name = "varbinview_fill_null" harness = false -[[bench]] -name = "bool_zip" -harness = false - -[[bench]] -name = "primitive_zip" -harness = false - -[[bench]] -name = "listview_zip" -harness = false - [[bench]] name = "take_primitive" harness = false @@ -311,14 +272,6 @@ harness = false name = "filter_list" harness = false -[[bench]] -name = "list_length" -harness = false - -[[bench]] -name = "list_sum" -harness = false - [[bench]] name = "listview_rebuild" harness = false diff --git a/vortex-array/benches/binary_ops.rs b/vortex-array/benches/binary_ops.rs index 1103b6571fb..55dfdb73dbb 100644 --- a/vortex-array/benches/binary_ops.rs +++ b/vortex-array/benches/binary_ops.rs @@ -36,7 +36,7 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DecimalDType; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_session::VortexSession; #[global_allocator] diff --git a/vortex-array/benches/compare.rs b/vortex-array/benches/compare.rs index ddf6343e9ab..e98eb4a91ee 100644 --- a/vortex-array/benches/compare.rs +++ b/vortex-array/benches/compare.rs @@ -46,7 +46,7 @@ use vortex_array::dtype::DecimalDType; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_session::VortexSession; diff --git a/vortex-array/benches/compare_sorted.rs b/vortex-array/benches/compare_sorted.rs index a50a6666949..60175b012cb 100644 --- a/vortex-array/benches/compare_sorted.rs +++ b/vortex-array/benches/compare_sorted.rs @@ -28,7 +28,7 @@ use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_buffer::Buffer; use vortex_session::VortexSession; diff --git a/vortex-array/benches/dict_compare.rs b/vortex-array/benches/dict_compare.rs index f97a12908d2..f973a98d9e9 100644 --- a/vortex-array/benches/dict_compare.rs +++ b/vortex-array/benches/dict_compare.rs @@ -19,12 +19,9 @@ use vortex_array::arrays::dict_test::gen_varbin_words; use vortex_array::arrays::varbin::VarBinArrayExt; use vortex_array::builders::dict::dict_encode; use vortex_array::builtins::ArrayBuiltins; -use vortex_array::expr::eq; -use vortex_array::expr::lit; -use vortex_array::expr::root; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_session::VortexSession; #[global_allocator] @@ -146,7 +143,10 @@ fn bench_compare_sliced_dict_primitive( .with_inputs(|| (&dict, SESSION.create_execution_ctx())) .bench_refs(|(dict, ctx)| { dict.clone() - .apply(&eq(root(), lit(value))) + .binary( + ConstantArray::new(value, codes_len).into_array(), + Operator::Eq, + ) .unwrap() .execute::(ctx) .unwrap() @@ -169,7 +169,10 @@ fn bench_compare_sliced_dict_varbinview( .with_inputs(|| (&dict, SESSION.create_execution_ctx())) .bench_refs(|(dict, ctx)| { dict.clone() - .apply(&eq(root(), lit(value))) + .binary( + ConstantArray::new(value, codes_len).into_array(), + Operator::Eq, + ) .unwrap() .execute::(ctx) .unwrap() diff --git a/vortex-array/benches/kleene_bool.rs b/vortex-array/benches/kleene_bool.rs index b6bfb25e442..970b8344fe4 100644 --- a/vortex-array/benches/kleene_bool.rs +++ b/vortex-array/benches/kleene_bool.rs @@ -20,7 +20,7 @@ use vortex_array::dtype::Nullability; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_session::VortexSession; #[global_allocator] diff --git a/vortex-array/benches/scalar_fn_probe.rs b/vortex-array/benches/scalar_fn_probe.rs index 4ce01520d17..cdf9c19e1d3 100644 --- a/vortex-array/benches/scalar_fn_probe.rs +++ b/vortex-array/benches/scalar_fn_probe.rs @@ -20,8 +20,8 @@ use vortex_array::arrays::ScalarFnArray; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar_fn::TypedScalarFnInstance; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_session::VortexSession; #[global_allocator] diff --git a/vortex-array/benches/scalar_subtract.rs b/vortex-array/benches/scalar_subtract.rs index c882fc9fb76..cf6f0a5a43b 100644 --- a/vortex-array/benches/scalar_subtract.rs +++ b/vortex-array/benches/scalar_subtract.rs @@ -20,7 +20,7 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_buffer::Buffer; use vortex_session::VortexSession; diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs index ff8e526402b..82091fa8772 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs @@ -49,7 +49,7 @@ use crate::expr::stats::Stat; use crate::expr::stats::StatsProvider; use crate::expr::stats::StatsProviderExt; use crate::scalar::Scalar; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::operators::Operator; /// Check if two arrays of the same length have equal values at every position (null-safe). /// diff --git a/vortex-array/src/aggregate_fn/fns/mean/mod.rs b/vortex-array/src/aggregate_fn/fns/mean/mod.rs index b2dcf09fab4..c5e0c2bbb9c 100644 --- a/vortex-array/src/aggregate_fn/fns/mean/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/mean/mod.rs @@ -32,7 +32,7 @@ use crate::dtype::PType; use crate::dtype::i256; use crate::scalar::DecimalValue; use crate::scalar::Scalar; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::operators::Operator; /// Compute the arithmetic mean of an array. /// diff --git a/vortex-array/src/aggregate_fn/fns/sum_v2/mod.rs b/vortex-array/src/aggregate_fn/fns/sum_v2/mod.rs index 46fe681ab0a..9fe62f99a63 100644 --- a/vortex-array/src/aggregate_fn/fns/sum_v2/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/sum_v2/mod.rs @@ -50,7 +50,7 @@ use crate::expr::stats::Stat; use crate::expr::stats::StatsProviderExt; use crate::scalar::Scalar; use crate::scalar::ScalarValue; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; const SUM_FIELD: &str = "sum"; diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index f630d42a4f8..2d9e3eca2fc 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -533,7 +533,7 @@ impl ArrayRef { /// # Safety /// The caller must put back a slot with the same logical dtype and length before exposing the /// parent array, and must only use this for physical rewrites. - pub(crate) unsafe fn take_slot_unchecked( + pub unsafe fn take_slot_unchecked( mut self, slot_idx: usize, ) -> VortexResult<(ArrayRef, ArrayRef)> { @@ -567,7 +567,7 @@ impl ArrayRef { /// # Safety /// The replacement must have the same logical dtype and length as the taken slot, and this /// must only be used for physical rewrites. - pub(crate) unsafe fn put_slot_unchecked( + pub unsafe fn put_slot_unchecked( mut self, slot_idx: usize, replacement: ArrayRef, diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index fae796e1714..4ffee3201cc 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -32,8 +32,6 @@ pub use erased::*; mod plugin; pub use plugin::*; -mod haystack; -pub(crate) use haystack::ArrayHaystack; mod probe; pub use probe::*; diff --git a/vortex-array/src/arrays/bool/compute/boolean.rs b/vortex-array/src/arrays/bool/compute/boolean.rs index 28d1bbb7f52..3980911f2eb 100644 --- a/vortex-array/src/arrays/bool/compute/boolean.rs +++ b/vortex-array/src/arrays/bool/compute/boolean.rs @@ -10,10 +10,10 @@ use crate::array::ArrayView; use crate::arrays::Bool; use crate::arrays::Constant; use crate::arrays::bool::BoolArrayExt; -use crate::scalar_fn::fns::binary::BooleanKernel; -use crate::scalar_fn::fns::binary::kleene_boolean_buffer_scalar; -use crate::scalar_fn::fns::binary::kleene_boolean_buffers; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::binary::BooleanKernel; +use crate::scalar_fn::builtin::binary::kleene_boolean_buffer_scalar; +use crate::scalar_fn::builtin::binary::kleene_boolean_buffers; +use crate::scalar_fn::builtin::operators::Operator; impl BooleanKernel for Bool { fn boolean( diff --git a/vortex-array/src/arrays/bool/compute/cast.rs b/vortex-array/src/arrays/bool/compute/cast.rs index af98c78909c..846e042ea39 100644 --- a/vortex-array/src/arrays/bool/compute/cast.rs +++ b/vortex-array/src/arrays/bool/compute/cast.rs @@ -16,8 +16,8 @@ use crate::arrays::PrimitiveArray; use crate::arrays::bool::BoolArrayExt; use crate::dtype::DType; use crate::match_each_native_ptype; -use crate::scalar_fn::fns::cast::CastKernel; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastKernel; +use crate::scalar_fn::builtin::cast::CastReduce; impl CastReduce for Bool { fn cast(array: ArrayView<'_, Bool>, dtype: &DType) -> VortexResult> { diff --git a/vortex-array/src/arrays/bool/compute/fill_null.rs b/vortex-array/src/arrays/bool/compute/fill_null.rs index 94966f2e829..71358a32590 100644 --- a/vortex-array/src/arrays/bool/compute/fill_null.rs +++ b/vortex-array/src/arrays/bool/compute/fill_null.rs @@ -12,7 +12,7 @@ use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::scalar::Scalar; -use crate::scalar_fn::fns::fill_null::FillNullKernel; +use crate::scalar_fn::builtin::fill_null::FillNullKernel; use crate::validity::Validity; impl FillNullKernel for Bool { diff --git a/vortex-array/src/arrays/bool/compute/mask.rs b/vortex-array/src/arrays/bool/compute/mask.rs index 818dbaebca5..14eea59b24f 100644 --- a/vortex-array/src/arrays/bool/compute/mask.rs +++ b/vortex-array/src/arrays/bool/compute/mask.rs @@ -8,7 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Bool; use crate::arrays::BoolArray; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for Bool { @@ -52,7 +52,7 @@ mod tests { use crate::buffer::BufferHandle; use crate::buffer::DeviceBuffer; use crate::compute::conformance::mask::test_mask_conformance; - use crate::scalar_fn::fns::mask::MaskReduce; + use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/arrays/bool/compute/mod.rs b/vortex-array/src/arrays/bool/compute/mod.rs index da5fa766d85..fc0f25eadfc 100644 --- a/vortex-array/src/arrays/bool/compute/mod.rs +++ b/vortex-array/src/arrays/bool/compute/mod.rs @@ -9,7 +9,6 @@ mod mask; pub mod rules; mod slice; mod take; -mod zip; #[cfg(test)] mod tests { diff --git a/vortex-array/src/arrays/bool/compute/rules.rs b/vortex-array/src/arrays/bool/compute/rules.rs index 43f811785ea..8bfdd6f9146 100644 --- a/vortex-array/src/arrays/bool/compute/rules.rs +++ b/vortex-array/src/arrays/bool/compute/rules.rs @@ -14,8 +14,8 @@ use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&BoolMaskedValidityRule), diff --git a/vortex-array/src/arrays/bool/vtable/kernel.rs b/vortex-array/src/arrays/bool/vtable/kernel.rs index c19d00d5118..4bf52ca0435 100644 --- a/vortex-array/src/arrays/bool/vtable/kernel.rs +++ b/vortex-array/src/arrays/bool/vtable/kernel.rs @@ -9,14 +9,12 @@ use crate::arrays::Dict; use crate::arrays::dict::TakeExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::binary::BooleanExecuteAdaptor; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastExecuteAdaptor; -use crate::scalar_fn::fns::fill_null::FillNull; -use crate::scalar_fn::fns::fill_null::FillNullExecuteAdaptor; -use crate::scalar_fn::fns::zip::Zip; -use crate::scalar_fn::fns::zip::ZipExecuteAdaptor; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::binary::BooleanExecuteAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastExecuteAdaptor; +use crate::scalar_fn::builtin::fill_null::FillNull; +use crate::scalar_fn::builtin::fill_null::FillNullExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); @@ -24,5 +22,4 @@ pub(crate) fn initialize(session: &VortexSession) { kernels.register_execute_parent_kernel(Cast.id(), Bool, CastExecuteAdaptor(Bool)); kernels.register_execute_parent_kernel(FillNull.id(), Bool, FillNullExecuteAdaptor(Bool)); kernels.register_execute_parent_kernel(Dict.id(), Bool, TakeExecuteAdaptor(Bool)); - kernels.register_execute_parent_kernel(Zip.id(), Bool, ZipExecuteAdaptor(Bool)); } diff --git a/vortex-array/src/arrays/chunked/compute/cast.rs b/vortex-array/src/arrays/chunked/compute/cast.rs index 6f52c792f91..de718143fe1 100644 --- a/vortex-array/src/arrays/chunked/compute/cast.rs +++ b/vortex-array/src/arrays/chunked/compute/cast.rs @@ -11,7 +11,7 @@ use crate::arrays::ChunkedArray; use crate::arrays::chunked::ChunkedArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastReduce; impl CastReduce for Chunked { fn cast(array: ArrayView<'_, Chunked>, dtype: &DType) -> VortexResult> { diff --git a/vortex-array/src/arrays/chunked/compute/fill_null.rs b/vortex-array/src/arrays/chunked/compute/fill_null.rs index 7d05a2914e6..9d2680f2701 100644 --- a/vortex-array/src/arrays/chunked/compute/fill_null.rs +++ b/vortex-array/src/arrays/chunked/compute/fill_null.rs @@ -11,7 +11,7 @@ use crate::arrays::ChunkedArray; use crate::arrays::chunked::ChunkedArrayExt; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; -use crate::scalar_fn::fns::fill_null::FillNullReduce; +use crate::scalar_fn::builtin::fill_null::FillNullReduce; impl FillNullReduce for Chunked { fn fill_null( diff --git a/vortex-array/src/arrays/chunked/compute/kernel.rs b/vortex-array/src/arrays/chunked/compute/kernel.rs index db0042105cd..dc074dc4775 100644 --- a/vortex-array/src/arrays/chunked/compute/kernel.rs +++ b/vortex-array/src/arrays/chunked/compute/kernel.rs @@ -13,10 +13,8 @@ use crate::arrays::filter::FilterExecuteAdaptor; use crate::arrays::slice::SliceExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::mask::Mask; -use crate::scalar_fn::fns::mask::MaskExecuteAdaptor; -use crate::scalar_fn::fns::zip::Zip; -use crate::scalar_fn::fns::zip::ZipExecuteAdaptor; +use crate::scalar_fn::builtin::mask::Mask; +use crate::scalar_fn::builtin::mask::MaskExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); @@ -24,5 +22,4 @@ pub(crate) fn initialize(session: &VortexSession) { kernels.register_execute_parent_kernel(Mask.id(), Chunked, MaskExecuteAdaptor(Chunked)); kernels.register_execute_parent_kernel(Slice.id(), Chunked, SliceExecuteAdaptor(Chunked)); kernels.register_execute_parent_kernel(Dict.id(), Chunked, TakeExecuteAdaptor(Chunked)); - kernels.register_execute_parent_kernel(Zip.id(), Chunked, ZipExecuteAdaptor(Chunked)); } diff --git a/vortex-array/src/arrays/chunked/compute/mask.rs b/vortex-array/src/arrays/chunked/compute/mask.rs index f99d4254780..215d27bd9c0 100644 --- a/vortex-array/src/arrays/chunked/compute/mask.rs +++ b/vortex-array/src/arrays/chunked/compute/mask.rs @@ -10,8 +10,8 @@ use crate::array::ArrayView; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; use crate::arrays::chunked::ChunkedArrayExt; -use crate::scalar_fn::fns::mask::Mask as MaskExpr; -use crate::scalar_fn::fns::mask::MaskKernel; +use crate::scalar_fn::builtin::mask::Mask as MaskExpr; +use crate::scalar_fn::builtin::mask::MaskKernel; impl MaskKernel for Chunked { fn mask( diff --git a/vortex-array/src/arrays/chunked/compute/mod.rs b/vortex-array/src/arrays/chunked/compute/mod.rs index e7c27e00361..cf7c79425fe 100644 --- a/vortex-array/src/arrays/chunked/compute/mod.rs +++ b/vortex-array/src/arrays/chunked/compute/mod.rs @@ -6,12 +6,10 @@ mod cast; mod fill_null; mod filter; pub(crate) mod kernel; -mod list_contains; mod mask; pub(crate) mod rules; mod slice; mod take; -mod zip; #[cfg(test)] mod tests { diff --git a/vortex-array/src/arrays/chunked/compute/rules.rs b/vortex-array/src/arrays/chunked/compute/rules.rs index c5f904910f2..2036b4d6200 100644 --- a/vortex-array/src/arrays/chunked/compute/rules.rs +++ b/vortex-array/src/arrays/chunked/compute/rules.rs @@ -18,14 +18,12 @@ use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::optimizer::ArrayOptimizer; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::fill_null::FillNullReduceAdaptor; -use crate::scalar_fn::fns::list_contains::ListContainsElementReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::fill_null::FillNullReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(Chunked)), ParentRuleSet::lift(&ChunkedUnaryScalarFnPushDownRule), - ParentRuleSet::lift(&ListContainsElementReduceAdaptor(Chunked)), ParentRuleSet::lift(&ChunkedConstantScalarFnPushDownRule), ParentRuleSet::lift(&FillNullReduceAdaptor(Chunked)), ]); diff --git a/vortex-array/src/arrays/chunked/mod.rs b/vortex-array/src/arrays/chunked/mod.rs index 3bd361d66a1..3f0cd0b2c74 100644 --- a/vortex-array/src/arrays/chunked/mod.rs +++ b/vortex-array/src/arrays/chunked/mod.rs @@ -9,7 +9,6 @@ pub use array::ChunkedSlotsView; pub use vtable::ChunkedArray; pub(crate) mod compute; -pub(crate) mod paired_chunks; mod vtable; pub use vtable::Chunked; diff --git a/vortex-array/src/arrays/constant/compute/cast.rs b/vortex-array/src/arrays/constant/compute/cast.rs index ea26f632bb8..60c7ecd8d3c 100644 --- a/vortex-array/src/arrays/constant/compute/cast.rs +++ b/vortex-array/src/arrays/constant/compute/cast.rs @@ -9,7 +9,7 @@ use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::dtype::DType; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastReduce; impl CastReduce for Constant { fn cast(array: ArrayView<'_, Constant>, dtype: &DType) -> VortexResult> { diff --git a/vortex-array/src/arrays/constant/compute/fill_null.rs b/vortex-array/src/arrays/constant/compute/fill_null.rs index a559bbb8a70..ee34bec0dc2 100644 --- a/vortex-array/src/arrays/constant/compute/fill_null.rs +++ b/vortex-array/src/arrays/constant/compute/fill_null.rs @@ -7,8 +7,8 @@ use crate::ArrayRef; use crate::array::ArrayView; use crate::arrays::Constant; use crate::scalar::Scalar; -use crate::scalar_fn::fns::fill_null::FillNullReduce; -use crate::scalar_fn::fns::fill_null::fill_null_constant; +use crate::scalar_fn::builtin::fill_null::FillNullReduce; +use crate::scalar_fn::builtin::fill_null::fill_null_constant; impl FillNullReduce for Constant { fn fill_null( diff --git a/vortex-array/src/arrays/constant/compute/mod.rs b/vortex-array/src/arrays/constant/compute/mod.rs index 2a6f0bb0ce5..1303992bea0 100644 --- a/vortex-array/src/arrays/constant/compute/mod.rs +++ b/vortex-array/src/arrays/constant/compute/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -mod between; mod cast; mod fill_null; mod filter; diff --git a/vortex-array/src/arrays/constant/compute/not.rs b/vortex-array/src/arrays/constant/compute/not.rs index 1eb912d0a91..3e068a56735 100644 --- a/vortex-array/src/arrays/constant/compute/not.rs +++ b/vortex-array/src/arrays/constant/compute/not.rs @@ -9,7 +9,7 @@ use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::scalar::Scalar; -use crate::scalar_fn::fns::not::NotReduce; +use crate::scalar_fn::builtin::not::NotReduce; impl NotReduce for Constant { fn invert(array: ArrayView<'_, Constant>) -> VortexResult> { diff --git a/vortex-array/src/arrays/constant/compute/rules.rs b/vortex-array/src/arrays/constant/compute/rules.rs index f97e6c596c7..b7424dd493b 100644 --- a/vortex-array/src/arrays/constant/compute/rules.rs +++ b/vortex-array/src/arrays/constant/compute/rules.rs @@ -14,13 +14,11 @@ use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::between::BetweenReduceAdaptor; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::fill_null::FillNullReduceAdaptor; -use crate::scalar_fn::fns::not::NotReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::fill_null::FillNullReduceAdaptor; +use crate::scalar_fn::builtin::not::NotReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ - ParentRuleSet::lift(&BetweenReduceAdaptor(Constant)), ParentRuleSet::lift(&CastReduceAdaptor(Constant)), ParentRuleSet::lift(&ConstantFilterRule), ParentRuleSet::lift(&FillNullReduceAdaptor(Constant)), diff --git a/vortex-array/src/arrays/datetime/test.rs b/vortex-array/src/arrays/datetime/test.rs index 2e641e262d4..10c32a566f9 100644 --- a/vortex-array/src/arrays/datetime/test.rs +++ b/vortex-array/src/arrays/datetime/test.rs @@ -13,18 +13,18 @@ use crate::EqMode; use crate::IntoArray; use crate::VortexSessionExecute; use crate::arrays::BoolArray; +use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; use crate::arrays::datetime::TemporalData; use crate::assert_arrays_eq; -use crate::expr::gt; -use crate::expr::lit; -use crate::expr::root; +use crate::builtins::ArrayBuiltins; use crate::extension::datetime::TemporalMetadata; use crate::extension::datetime::TimeUnit; use crate::extension::datetime::Timestamp; use crate::extension::datetime::TimestampOptions; use crate::hash::ArrayEq; use crate::scalar::Scalar; +use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); @@ -223,20 +223,21 @@ fn comparison_rejects_mismatched_timestamp_units() -> VortexResult<()> { .into_array(); let temporal = TemporalData::new_timestamp(ts_array, TimeUnit::Milliseconds, None); - let filter_expr = gt( - root(), - lit(Scalar::extension::( + let temporal = temporal.into_array(); + let threshold = ConstantArray::new( + Scalar::extension::( TimestampOptions { unit: TimeUnit::Seconds, tz: None, }, Scalar::from(1704153600i64), - )), - ); + ), + temporal.len(), + ) + .into_array(); let error = temporal - .into_array() - .apply(&filter_expr)? + .binary(threshold, Operator::Gt)? .execute::(&mut SESSION.create_execution_ctx()) .unwrap_err(); assert!( diff --git a/vortex-array/src/arrays/decimal/compute/cast.rs b/vortex-array/src/arrays/decimal/compute/cast.rs index 29b38ce9406..ac61556451a 100644 --- a/vortex-array/src/arrays/decimal/compute/cast.rs +++ b/vortex-array/src/arrays/decimal/compute/cast.rs @@ -38,8 +38,8 @@ use crate::scalar::DecimalToFloatCast; use crate::scalar::DecimalToIntegerCast; use crate::scalar::DecimalValue; use crate::scalar::Scalar; -use crate::scalar_fn::fns::cast::CastKernel; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastKernel; +use crate::scalar_fn::builtin::cast::CastReduce; use crate::validity::Validity; impl CastReduce for Decimal { diff --git a/vortex-array/src/arrays/decimal/compute/fill_null.rs b/vortex-array/src/arrays/decimal/compute/fill_null.rs index 3755f6c6d1f..d817bcf8706 100644 --- a/vortex-array/src/arrays/decimal/compute/fill_null.rs +++ b/vortex-array/src/arrays/decimal/compute/fill_null.rs @@ -20,7 +20,7 @@ use crate::dtype::NativeDecimalType; use crate::match_each_decimal_value_type; use crate::scalar::DecimalValue; use crate::scalar::Scalar; -use crate::scalar_fn::fns::fill_null::FillNullKernel; +use crate::scalar_fn::builtin::fill_null::FillNullKernel; use crate::validity::Validity; impl FillNullKernel for Decimal { diff --git a/vortex-array/src/arrays/decimal/compute/mask.rs b/vortex-array/src/arrays/decimal/compute/mask.rs index 91490a288ad..dc51edb9e31 100644 --- a/vortex-array/src/arrays/decimal/compute/mask.rs +++ b/vortex-array/src/arrays/decimal/compute/mask.rs @@ -8,7 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Decimal; use crate::arrays::DecimalArray; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for Decimal { diff --git a/vortex-array/src/arrays/decimal/compute/mod.rs b/vortex-array/src/arrays/decimal/compute/mod.rs index 88a8713c2dd..62fe5f496cf 100644 --- a/vortex-array/src/arrays/decimal/compute/mod.rs +++ b/vortex-array/src/arrays/decimal/compute/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -mod between; mod cast; mod fill_null; mod fixed_width; diff --git a/vortex-array/src/arrays/decimal/compute/rules.rs b/vortex-array/src/arrays/decimal/compute/rules.rs index 8f1d09ea92a..9d1e2a26186 100644 --- a/vortex-array/src/arrays/decimal/compute/rules.rs +++ b/vortex-array/src/arrays/decimal/compute/rules.rs @@ -16,8 +16,8 @@ use crate::arrays::slice::SliceReduceAdaptor; use crate::match_each_decimal_value_type; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&DecimalMaskedValidityRule), diff --git a/vortex-array/src/arrays/decimal/vtable/kernel.rs b/vortex-array/src/arrays/decimal/vtable/kernel.rs index 19bb30d8c95..0f2ea03d7be 100644 --- a/vortex-array/src/arrays/decimal/vtable/kernel.rs +++ b/vortex-array/src/arrays/decimal/vtable/kernel.rs @@ -9,16 +9,13 @@ use crate::arrays::Dict; use crate::arrays::dict::TakeExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::between::Between; -use crate::scalar_fn::fns::between::BetweenExecuteAdaptor; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastExecuteAdaptor; -use crate::scalar_fn::fns::fill_null::FillNull; -use crate::scalar_fn::fns::fill_null::FillNullExecuteAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastExecuteAdaptor; +use crate::scalar_fn::builtin::fill_null::FillNull; +use crate::scalar_fn::builtin::fill_null::FillNullExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); - kernels.register_execute_parent_kernel(Between.id(), Decimal, BetweenExecuteAdaptor(Decimal)); kernels.register_execute_parent_kernel(Cast.id(), Decimal, CastExecuteAdaptor(Decimal)); kernels.register_execute_parent_kernel(FillNull.id(), Decimal, FillNullExecuteAdaptor(Decimal)); kernels.register_execute_parent_kernel(Dict.id(), Decimal, TakeExecuteAdaptor(Decimal)); diff --git a/vortex-array/src/arrays/dict/compute/cast.rs b/vortex-array/src/arrays/dict/compute/cast.rs index 2c705464d53..2585aa82cc8 100644 --- a/vortex-array/src/arrays/dict/compute/cast.rs +++ b/vortex-array/src/arrays/dict/compute/cast.rs @@ -12,7 +12,7 @@ use crate::arrays::dict::DictArrayExt; use crate::arrays::dict::DictArraySlotsExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastReduce; impl CastReduce for Dict { fn cast(array: ArrayView<'_, Dict>, dtype: &DType) -> VortexResult> { diff --git a/vortex-array/src/arrays/dict/compute/compare.rs b/vortex-array/src/arrays/dict/compute/compare.rs index d59be4bf963..7fdd149d7ea 100644 --- a/vortex-array/src/arrays/dict/compute/compare.rs +++ b/vortex-array/src/arrays/dict/compute/compare.rs @@ -14,9 +14,9 @@ use crate::arrays::ConstantArray; use crate::arrays::dict::DictArrayExt; use crate::arrays::dict::DictArraySlotsExt; use crate::builtins::ArrayBuiltins; -use crate::scalar_fn::fns::binary::CompareKernel; -use crate::scalar_fn::fns::operators::CompareOperator; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::binary::CompareKernel; +use crate::scalar_fn::builtin::operators::CompareOperator; +use crate::scalar_fn::builtin::operators::Operator; impl CompareKernel for Dict { fn compare( diff --git a/vortex-array/src/arrays/dict/compute/fill_null.rs b/vortex-array/src/arrays/dict/compute/fill_null.rs index 077ee865bda..3e65be51599 100644 --- a/vortex-array/src/arrays/dict/compute/fill_null.rs +++ b/vortex-array/src/arrays/dict/compute/fill_null.rs @@ -19,8 +19,8 @@ use crate::builtins::ArrayBuiltins; use crate::match_each_integer_ptype; use crate::scalar::Scalar; use crate::scalar::ScalarValue; -use crate::scalar_fn::fns::fill_null::FillNullKernel; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::fill_null::FillNullKernel; +use crate::scalar_fn::builtin::operators::Operator; impl FillNullKernel for Dict { fn fill_null( diff --git a/vortex-array/src/arrays/dict/compute/mask.rs b/vortex-array/src/arrays/dict/compute/mask.rs index b129fdd19c5..f56e1ae2ddc 100644 --- a/vortex-array/src/arrays/dict/compute/mask.rs +++ b/vortex-array/src/arrays/dict/compute/mask.rs @@ -9,8 +9,8 @@ use crate::array::ArrayView; use crate::arrays::Dict; use crate::arrays::DictArray; use crate::arrays::dict::DictArraySlotsExt; -use crate::scalar_fn::fns::mask::Mask as MaskExpr; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::Mask as MaskExpr; +use crate::scalar_fn::builtin::mask::MaskReduce; impl MaskReduce for Dict { fn mask(array: ArrayView<'_, Dict>, mask: &ArrayRef) -> VortexResult> { diff --git a/vortex-array/src/arrays/dict/compute/mod.rs b/vortex-array/src/arrays/dict/compute/mod.rs index 25eb0c5e442..163da7f63de 100644 --- a/vortex-array/src/arrays/dict/compute/mod.rs +++ b/vortex-array/src/arrays/dict/compute/mod.rs @@ -7,7 +7,6 @@ mod fill_null; mod filter; pub(crate) mod is_constant; pub(crate) mod is_sorted; -mod like; mod mask; pub(crate) mod min_max; pub(crate) mod rules; @@ -65,7 +64,7 @@ mod test { use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType::I32; - use crate::scalar_fn::fns::operators::Operator; + use crate::scalar_fn::builtin::operators::Operator; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/arrays/dict/compute/rules.rs b/vortex-array/src/arrays/dict/compute/rules.rs index 273d2b8562d..3e86c7bf8af 100644 --- a/vortex-array/src/arrays/dict/compute/rules.rs +++ b/vortex-array/src/arrays/dict/compute/rules.rs @@ -28,18 +28,16 @@ use crate::builtins::ArrayBuiltins; use crate::optimizer::ArrayOptimizer; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::like::LikeReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; -use crate::scalar_fn::fns::pack::Pack; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::pack::Pack; use crate::validity::Validity; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&FilterReduceAdaptor(Dict)), ParentRuleSet::lift(&CastReduceAdaptor(Dict)), ParentRuleSet::lift(&MaskReduceAdaptor(Dict)), - ParentRuleSet::lift(&LikeReduceAdaptor(Dict)), ParentRuleSet::lift(&DictionaryChunkedValuesPullUpRule), ParentRuleSet::lift(&DictionaryScalarFnValuesPushDownRule), ParentRuleSet::lift(&DictionaryScalarFnCodesPullUpRule), @@ -288,9 +286,9 @@ mod tests { use crate::executor::VortexSessionExecute; use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; - use crate::scalar_fn::fns::binary::Binary; - use crate::scalar_fn::fns::not::Not; - use crate::scalar_fn::fns::operators::Operator; + use crate::scalar_fn::builtin::binary::Binary; + use crate::scalar_fn::builtin::not::Not; + use crate::scalar_fn::builtin::operators::Operator; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/arrays/dict/vtable/kernel.rs b/vortex-array/src/arrays/dict/vtable/kernel.rs index 71786f63f66..77838761180 100644 --- a/vortex-array/src/arrays/dict/vtable/kernel.rs +++ b/vortex-array/src/arrays/dict/vtable/kernel.rs @@ -11,10 +11,10 @@ use crate::arrays::dict::TakeExecuteAdaptor; use crate::arrays::dict::take::DictConstantCodesKernel; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::binary::CompareExecuteAdaptor; -use crate::scalar_fn::fns::fill_null::FillNull; -use crate::scalar_fn::fns::fill_null::FillNullExecuteAdaptor; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::binary::CompareExecuteAdaptor; +use crate::scalar_fn::builtin::fill_null::FillNull; +use crate::scalar_fn::builtin::fill_null::FillNullExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index 5328e897225..2fc73d875e9 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -11,7 +11,7 @@ use crate::arrays::ExtensionArray; use crate::arrays::extension::ExtensionArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastReduce; impl CastReduce for Extension { fn cast(array: ArrayView<'_, Extension>, dtype: &DType) -> VortexResult> { diff --git a/vortex-array/src/arrays/extension/compute/compare.rs b/vortex-array/src/arrays/extension/compute/compare.rs index b3f978ef503..48d4b7d6c8c 100644 --- a/vortex-array/src/arrays/extension/compute/compare.rs +++ b/vortex-array/src/arrays/extension/compute/compare.rs @@ -11,9 +11,9 @@ use crate::arrays::ConstantArray; use crate::arrays::Extension; use crate::arrays::extension::ExtensionArrayExt; use crate::builtins::ArrayBuiltins; -use crate::scalar_fn::fns::binary::CompareKernel; -use crate::scalar_fn::fns::operators::CompareOperator; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::binary::CompareKernel; +use crate::scalar_fn::builtin::operators::CompareOperator; +use crate::scalar_fn::builtin::operators::Operator; impl CompareKernel for Extension { fn compare( diff --git a/vortex-array/src/arrays/extension/compute/mask.rs b/vortex-array/src/arrays/extension/compute/mask.rs index 10ae5e904bc..16e0db13a90 100644 --- a/vortex-array/src/arrays/extension/compute/mask.rs +++ b/vortex-array/src/arrays/extension/compute/mask.rs @@ -9,8 +9,8 @@ use crate::array::ArrayView; use crate::arrays::Extension; use crate::arrays::ExtensionArray; use crate::arrays::extension::ExtensionArrayExt; -use crate::scalar_fn::fns::mask::Mask as MaskExpr; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::Mask as MaskExpr; +use crate::scalar_fn::builtin::mask::MaskReduce; impl MaskReduce for Extension { fn mask(array: ArrayView<'_, Extension>, mask: &ArrayRef) -> VortexResult> { diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index fd92940ac6d..214af5e14cf 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -20,8 +20,8 @@ use crate::optimizer::rules::ArrayReduceRule; use crate::optimizer::rules::ParentRuleSet; use crate::optimizer::rules::ReduceRuleSet; use crate::scalar::Scalar; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const RULES: ReduceRuleSet = ReduceRuleSet::new(&[&ExtensionConstantRule]); @@ -110,8 +110,8 @@ mod tests { use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; use crate::scalar::ScalarValue; - use crate::scalar_fn::fns::binary::Binary; - use crate::scalar_fn::fns::operators::Operator; + use crate::scalar_fn::builtin::binary::Binary; + use crate::scalar_fn::builtin::operators::Operator; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/arrays/extension/vtable/kernel.rs b/vortex-array/src/arrays/extension/vtable/kernel.rs index c90458f1d5f..c88f321025a 100644 --- a/vortex-array/src/arrays/extension/vtable/kernel.rs +++ b/vortex-array/src/arrays/extension/vtable/kernel.rs @@ -9,8 +9,8 @@ use crate::arrays::Extension; use crate::arrays::dict::TakeExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::binary::CompareExecuteAdaptor; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::binary::CompareExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); diff --git a/vortex-array/src/arrays/filter/rules.rs b/vortex-array/src/arrays/filter/rules.rs index 5535d2cb52e..6fea1d672d3 100644 --- a/vortex-array/src/arrays/filter/rules.rs +++ b/vortex-array/src/arrays/filter/rules.rs @@ -24,7 +24,7 @@ use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; use crate::optimizer::rules::ParentRuleSet; use crate::optimizer::rules::ReduceRuleSet; -use crate::scalar_fn::fns::get_item::GetItem; +use crate::scalar_fn::builtin::get_item::GetItem; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&FilterReduceAdaptor(Filter)), diff --git a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs index 8e813209a5b..d11e75263a6 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs @@ -13,8 +13,8 @@ use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; -use crate::scalar_fn::fns::cast::CastKernel; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastKernel; +use crate::scalar_fn::builtin::cast::CastReduce; use crate::validity::Validity; fn build_with_validity( diff --git a/vortex-array/src/arrays/fixed_size_list/compute/mask.rs b/vortex-array/src/arrays/fixed_size_list/compute/mask.rs index 983c848c85a..9fc948dc3d4 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/mask.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/mask.rs @@ -10,7 +10,7 @@ use crate::arrays::FixedSizeList; use crate::arrays::FixedSizeListArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for FixedSizeList { diff --git a/vortex-array/src/arrays/fixed_size_list/compute/rules.rs b/vortex-array/src/arrays/fixed_size_list/compute/rules.rs index da1d91423e2..69173dde685 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/rules.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/rules.rs @@ -4,8 +4,8 @@ use crate::arrays::FixedSizeList; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(FixedSizeList)), diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs b/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs index 47dc3ec0651..2095334a940 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/kernel.rs @@ -9,8 +9,8 @@ use crate::arrays::FixedSizeList; use crate::arrays::dict::TakeExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastExecuteAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 377499faea6..6b4eb479939 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -35,7 +35,7 @@ use crate::dtype::DType; use crate::dtype::NativePType; use crate::match_each_integer_ptype; use crate::match_each_native_ptype; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; #[array_slots(List)] diff --git a/vortex-array/src/arrays/list/compute/cast.rs b/vortex-array/src/arrays/list/compute/cast.rs index 8ecd998dbd4..2ec444dcafb 100644 --- a/vortex-array/src/arrays/list/compute/cast.rs +++ b/vortex-array/src/arrays/list/compute/cast.rs @@ -12,8 +12,8 @@ use crate::arrays::ListArray; use crate::arrays::list::ListArraySlotsExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; -use crate::scalar_fn::fns::cast::CastKernel; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastKernel; +use crate::scalar_fn::builtin::cast::CastReduce; impl CastReduce for List { fn cast(array: ArrayView<'_, List>, dtype: &DType) -> VortexResult> { diff --git a/vortex-array/src/arrays/list/compute/kernels.rs b/vortex-array/src/arrays/list/compute/kernels.rs index 6af858dbf49..426657a7099 100644 --- a/vortex-array/src/arrays/list/compute/kernels.rs +++ b/vortex-array/src/arrays/list/compute/kernels.rs @@ -11,8 +11,8 @@ use crate::arrays::dict::TakeExecuteAdaptor; use crate::arrays::filter::FilterExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastExecuteAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); diff --git a/vortex-array/src/arrays/list/compute/mask.rs b/vortex-array/src/arrays/list/compute/mask.rs index 40737eb6583..0ad7bddf2f2 100644 --- a/vortex-array/src/arrays/list/compute/mask.rs +++ b/vortex-array/src/arrays/list/compute/mask.rs @@ -9,7 +9,7 @@ use crate::array::ArrayView; use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::list::ListArraySlotsExt; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for List { diff --git a/vortex-array/src/arrays/list/compute/rules.rs b/vortex-array/src/arrays/list/compute/rules.rs index d2aca066f7f..afc2aaf1c80 100644 --- a/vortex-array/src/arrays/list/compute/rules.rs +++ b/vortex-array/src/arrays/list/compute/rules.rs @@ -4,8 +4,8 @@ use crate::arrays::List; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(List)), diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index bebd5df30fe..8b485fe5f26 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -37,7 +37,7 @@ use crate::dtype::PType; use crate::expr::stats::Stat; use crate::match_each_integer_ptype; use crate::match_each_unsigned_integer_ptype; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; #[array_slots(ListView)] diff --git a/vortex-array/src/arrays/listview/compute/cast.rs b/vortex-array/src/arrays/listview/compute/cast.rs index 1c5e1fdfd84..b0f555c8ac9 100644 --- a/vortex-array/src/arrays/listview/compute/cast.rs +++ b/vortex-array/src/arrays/listview/compute/cast.rs @@ -12,8 +12,8 @@ use crate::arrays::ListViewArray; use crate::arrays::listview::ListViewArraySlotsExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; -use crate::scalar_fn::fns::cast::CastKernel; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastKernel; +use crate::scalar_fn::builtin::cast::CastReduce; use crate::validity::Validity; fn build_with_validity( diff --git a/vortex-array/src/arrays/listview/compute/mask.rs b/vortex-array/src/arrays/listview/compute/mask.rs index b9d12f39772..ba44cfb2f78 100644 --- a/vortex-array/src/arrays/listview/compute/mask.rs +++ b/vortex-array/src/arrays/listview/compute/mask.rs @@ -11,7 +11,7 @@ use crate::arrays::ListView; use crate::arrays::ListViewArray; use crate::arrays::listview::ListViewArraySlotsExt; use crate::arrays::listview::ListViewData; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for ListView { diff --git a/vortex-array/src/arrays/listview/compute/mod.rs b/vortex-array/src/arrays/listview/compute/mod.rs index 87587495f7a..9a43503c4b5 100644 --- a/vortex-array/src/arrays/listview/compute/mod.rs +++ b/vortex-array/src/arrays/listview/compute/mod.rs @@ -6,4 +6,3 @@ mod mask; pub(crate) mod rules; mod slice; mod take; -mod zip; diff --git a/vortex-array/src/arrays/listview/compute/rules.rs b/vortex-array/src/arrays/listview/compute/rules.rs index 233533202a8..8997d00d255 100644 --- a/vortex-array/src/arrays/listview/compute/rules.rs +++ b/vortex-array/src/arrays/listview/compute/rules.rs @@ -14,8 +14,8 @@ use crate::arrays::listview::ListViewArraySlotsExt; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&ListViewFilterPushDown), diff --git a/vortex-array/src/arrays/listview/rebuild.rs b/vortex-array/src/arrays/listview/rebuild.rs index 0fe73d4f0ad..a2297ea02e0 100644 --- a/vortex-array/src/arrays/listview/rebuild.rs +++ b/vortex-array/src/arrays/listview/rebuild.rs @@ -26,7 +26,7 @@ use crate::dtype::PType; use crate::match_each_integer_ptype; use crate::match_each_unsigned_integer_ptype; use crate::scalar::Scalar; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; /// The widened offset type used for a rebuilt, zero-copy-to-list array. diff --git a/vortex-array/src/arrays/listview/tests/operations.rs b/vortex-array/src/arrays/listview/tests/operations.rs index 45cace9c945..92a1b29dc0f 100644 --- a/vortex-array/src/arrays/listview/tests/operations.rs +++ b/vortex-array/src/arrays/listview/tests/operations.rs @@ -6,14 +6,12 @@ use std::sync::LazyLock; use rstest::rstest; use vortex_buffer::buffer; -use vortex_error::VortexResult; use vortex_mask::Mask; use vortex_session::VortexSession; use super::common::create_basic_listview; use super::common::create_large_listview; use super::common::create_nullable_listview; -use crate::ArrayRef; use crate::IntoArray; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::is_constant::is_constant; @@ -507,108 +505,6 @@ fn test_cast_large_dataset() { assert_arrays_eq!(result_list, expected, &mut SESSION.create_execution_ctx()); } -//////////////////////////////////////////////////////////////////////////////////////////////////// -// Zip tests -//////////////////////////////////////////////////////////////////////////////////////////////////// - -#[test] -fn test_zip_widens_false_element_nullability() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - // [[1, 2], [3], [4]] - let if_true = ListViewArray::new( - buffer![1i32, 2, 3, 4].into_array(), - buffer![0u32, 2, 3].into_array(), - buffer![2u32, 1, 1].into_array(), - Validity::NonNullable, - &mut ctx, - ) - .into_array(); - // [[10, null], [30], [40]] - let if_false = ListViewArray::new( - PrimitiveArray::from_option_iter([Some(10i32), None, Some(30), Some(40)]).into_array(), - buffer![0u32, 2, 3].into_array(), - buffer![2u32, 1, 1].into_array(), - Validity::NonNullable, - &mut ctx, - ) - .into_array(); - let mask = Mask::from_iter([false, true, false]); - - let result = mask - .into_array() - .zip(if_true, if_false)? - .execute::(&mut SESSION.create_execution_ctx())?; - assert!(result.is::()); - assert_eq!( - result.dtype(), - &DType::List( - Arc::new(DType::Primitive(PType::I32, Nullability::Nullable)), - Nullability::NonNullable, - ) - ); - - // [[10, null], [3], [40]] - let expected = ListViewArray::new( - PrimitiveArray::from_option_iter([Some(10i32), None, Some(3), Some(40)]).into_array(), - buffer![0u32, 2, 3].into_array(), - buffer![2u32, 1, 1].into_array(), - Validity::NonNullable, - &mut ctx, - ) - .into_array(); - assert_arrays_eq!(result, expected, &mut ctx); - Ok(()) -} - -#[test] -fn test_zip_widens_true_element_nullability() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - // [[1, null], [3], [4]] - let if_true = ListViewArray::new( - PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), Some(4)]).into_array(), - buffer![0u32, 2, 3].into_array(), - buffer![2u32, 1, 1].into_array(), - Validity::NonNullable, - &mut ctx, - ) - .into_array(); - // [[10], [20], [30]] - let if_false = ListViewArray::new( - buffer![10i32, 20, 30].into_array(), - buffer![0u32, 1, 2].into_array(), - buffer![1u32, 1, 1].into_array(), - Validity::NonNullable, - &mut ctx, - ) - .into_array(); - let mask = Mask::from_iter([true, false, true]); - - let result = mask - .into_array() - .zip(if_true, if_false)? - .execute::(&mut SESSION.create_execution_ctx())?; - assert!(result.is::()); - assert_eq!( - result.dtype(), - &DType::List( - Arc::new(DType::Primitive(PType::I32, Nullability::Nullable)), - Nullability::NonNullable, - ) - ); - - // [[1, null], [20], [4]] - let expected = ListViewArray::new( - PrimitiveArray::from_option_iter([Some(1i32), None, Some(20), Some(4)]).into_array(), - buffer![0u32, 2, 3].into_array(), - buffer![2u32, 1, 1].into_array(), - Validity::NonNullable, - &mut ctx, - ) - .into_array(); - assert_arrays_eq!(result, expected, &mut ctx); - Ok(()) -} - //////////////////////////////////////////////////////////////////////////////////////////////////// // Constant tests //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/vortex-array/src/arrays/listview/vtable/kernel.rs b/vortex-array/src/arrays/listview/vtable/kernel.rs index 5621b23e79e..80107f2c769 100644 --- a/vortex-array/src/arrays/listview/vtable/kernel.rs +++ b/vortex-array/src/arrays/listview/vtable/kernel.rs @@ -6,13 +6,10 @@ use vortex_session::VortexSession; use crate::arrays::ListView; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastExecuteAdaptor; -use crate::scalar_fn::fns::zip::Zip; -use crate::scalar_fn::fns::zip::ZipExecuteAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); kernels.register_execute_parent_kernel(Cast.id(), ListView, CastExecuteAdaptor(ListView)); - kernels.register_execute_parent_kernel(Zip.id(), ListView, ZipExecuteAdaptor(ListView)); } diff --git a/vortex-array/src/arrays/map/compute/cast.rs b/vortex-array/src/arrays/map/compute/cast.rs index bfbab6e17e9..0fb3622d990 100644 --- a/vortex-array/src/arrays/map/compute/cast.rs +++ b/vortex-array/src/arrays/map/compute/cast.rs @@ -16,8 +16,8 @@ use crate::arrays::map::compute::rebuild_map_from_array; use crate::dtype::DType; use crate::dtype::MapDType; use crate::executor::ExecutionCtx; -use crate::scalar_fn::fns::cast::CastKernel; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastKernel; +use crate::scalar_fn::builtin::cast::CastReduce; fn prepare_map_cast_target( array: ArrayView<'_, Map>, diff --git a/vortex-array/src/arrays/map/compute/mask.rs b/vortex-array/src/arrays/map/compute/mask.rs index 97218d08454..a56c453ffd3 100644 --- a/vortex-array/src/arrays/map/compute/mask.rs +++ b/vortex-array/src/arrays/map/compute/mask.rs @@ -11,8 +11,8 @@ use crate::arrays::map::MapArrayExt; use crate::arrays::map::MapArraySlotsExt; use crate::arrays::map::compute::rebuild_map_from_array; use crate::executor::ExecutionCtx; -use crate::scalar_fn::fns::mask::MaskKernel; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskKernel; +use crate::scalar_fn::builtin::mask::MaskReduce; impl MaskReduce for Map { const VALIDITY_IS_METADATA_ONLY: bool = true; diff --git a/vortex-array/src/arrays/map/compute/rules.rs b/vortex-array/src/arrays/map/compute/rules.rs index 27622ff1c5f..518d00bf2f6 100644 --- a/vortex-array/src/arrays/map/compute/rules.rs +++ b/vortex-array/src/arrays/map/compute/rules.rs @@ -6,8 +6,8 @@ use crate::arrays::dict::TakeReduceAdaptor; use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&FilterReduceAdaptor(Map)), diff --git a/vortex-array/src/arrays/map/vtable/kernel.rs b/vortex-array/src/arrays/map/vtable/kernel.rs index 860496834d6..d2c1665c542 100644 --- a/vortex-array/src/arrays/map/vtable/kernel.rs +++ b/vortex-array/src/arrays/map/vtable/kernel.rs @@ -13,10 +13,10 @@ use crate::arrays::filter::FilterExecuteAdaptor; use crate::arrays::slice::SliceExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastExecuteAdaptor; -use crate::scalar_fn::fns::mask::Mask; -use crate::scalar_fn::fns::mask::MaskExecuteAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastExecuteAdaptor; +use crate::scalar_fn::builtin::mask::Mask; +use crate::scalar_fn::builtin::mask::MaskExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); diff --git a/vortex-array/src/arrays/masked/compute/mask.rs b/vortex-array/src/arrays/masked/compute/mask.rs index 0de2dea5cd9..b92b9c4f921 100644 --- a/vortex-array/src/arrays/masked/compute/mask.rs +++ b/vortex-array/src/arrays/masked/compute/mask.rs @@ -8,8 +8,8 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; use crate::arrays::masked::MaskedArraySlotsExt; -use crate::scalar_fn::fns::mask::Mask as MaskExpr; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::Mask as MaskExpr; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for Masked { diff --git a/vortex-array/src/arrays/masked/compute/rules.rs b/vortex-array/src/arrays/masked/compute/rules.rs index ac2123aedc8..e213452d38e 100644 --- a/vortex-array/src/arrays/masked/compute/rules.rs +++ b/vortex-array/src/arrays/masked/compute/rules.rs @@ -16,8 +16,8 @@ use crate::arrays::slice::SliceReduceAdaptor; use crate::builtins::ArrayBuiltins; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::get_item::GetItem; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::get_item::GetItem; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&FilterReduceAdaptor(Masked)), diff --git a/vortex-array/src/arrays/mod.rs b/vortex-array/src/arrays/mod.rs index ed6e272b2b1..9208f83a863 100644 --- a/vortex-array/src/arrays/mod.rs +++ b/vortex-array/src/arrays/mod.rs @@ -148,7 +148,6 @@ pub(crate) fn initialize(session: &VortexSession) { struct_::initialize(session); varbin::initialize(session); varbinview::initialize(session); - variant::initialize(session); } #[cfg(feature = "arbitrary")] diff --git a/vortex-array/src/arrays/null/compute/cast.rs b/vortex-array/src/arrays/null/compute/cast.rs index be9da91131e..78b94467042 100644 --- a/vortex-array/src/arrays/null/compute/cast.rs +++ b/vortex-array/src/arrays/null/compute/cast.rs @@ -11,7 +11,7 @@ use crate::arrays::ConstantArray; use crate::arrays::Null; use crate::dtype::DType; use crate::scalar::Scalar; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastReduce; impl CastReduce for Null { fn cast(array: ArrayView<'_, Null>, dtype: &DType) -> VortexResult> { diff --git a/vortex-array/src/arrays/null/compute/mask.rs b/vortex-array/src/arrays/null/compute/mask.rs index dbb3afaad1c..8205ee6dbb0 100644 --- a/vortex-array/src/arrays/null/compute/mask.rs +++ b/vortex-array/src/arrays/null/compute/mask.rs @@ -6,7 +6,7 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::array::ArrayView; use crate::arrays::Null; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; impl MaskReduce for Null { fn mask(array: ArrayView<'_, Null>, _mask: &ArrayRef) -> VortexResult> { diff --git a/vortex-array/src/arrays/null/compute/rules.rs b/vortex-array/src/arrays/null/compute/rules.rs index b2ee44a19dd..996d7d869e6 100644 --- a/vortex-array/src/arrays/null/compute/rules.rs +++ b/vortex-array/src/arrays/null/compute/rules.rs @@ -5,8 +5,8 @@ use crate::arrays::Null; use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&FilterReduceAdaptor(Null)), diff --git a/vortex-array/src/arrays/patched/compute/compare.rs b/vortex-array/src/arrays/patched/compute/compare.rs index 800783b11c5..1e043ef6cbe 100644 --- a/vortex-array/src/arrays/patched/compute/compare.rs +++ b/vortex-array/src/arrays/patched/compute/compare.rs @@ -22,8 +22,8 @@ use crate::arrays::primitive::NativeValue; use crate::builtins::ArrayBuiltins; use crate::dtype::NativePType; use crate::match_each_native_ptype; -use crate::scalar_fn::fns::binary::CompareKernel; -use crate::scalar_fn::fns::operators::CompareOperator; +use crate::scalar_fn::builtin::binary::CompareKernel; +use crate::scalar_fn::builtin::operators::CompareOperator; impl CompareKernel for Patched { fn compare( @@ -175,8 +175,8 @@ mod tests { use crate::assert_arrays_eq; use crate::optimizer::ArrayOptimizer; use crate::patches::Patches; - use crate::scalar_fn::fns::binary::CompareKernel; - use crate::scalar_fn::fns::operators::CompareOperator; + use crate::scalar_fn::builtin::binary::CompareKernel; + use crate::scalar_fn::builtin::operators::CompareOperator; use crate::validity::Validity; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/arrays/patched/vtable/kernels.rs b/vortex-array/src/arrays/patched/vtable/kernels.rs index 51f53d15625..6ce98942c61 100644 --- a/vortex-array/src/arrays/patched/vtable/kernels.rs +++ b/vortex-array/src/arrays/patched/vtable/kernels.rs @@ -9,8 +9,8 @@ use crate::arrays::Patched; use crate::arrays::dict::TakeExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::binary::CompareExecuteAdaptor; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::binary::CompareExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index c937d68d611..dad36d912a0 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -44,8 +44,8 @@ use crate::match_each_native_ptype; use crate::match_each_signed_integer_ptype; use crate::scalar::DecimalValue; use crate::scalar::Scalar; -use crate::scalar_fn::fns::cast::CastKernel; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastKernel; +use crate::scalar_fn::builtin::cast::CastReduce; use crate::validity::Validity; impl CastReduce for Primitive { diff --git a/vortex-array/src/arrays/primitive/compute/fill_null.rs b/vortex-array/src/arrays/primitive/compute/fill_null.rs index ea657448327..cc105683e7c 100644 --- a/vortex-array/src/arrays/primitive/compute/fill_null.rs +++ b/vortex-array/src/arrays/primitive/compute/fill_null.rs @@ -15,7 +15,7 @@ use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; use crate::match_each_native_ptype; use crate::scalar::Scalar; -use crate::scalar_fn::fns::fill_null::FillNullKernel; +use crate::scalar_fn::builtin::fill_null::FillNullKernel; use crate::validity::Validity; impl FillNullKernel for Primitive { diff --git a/vortex-array/src/arrays/primitive/compute/mask.rs b/vortex-array/src/arrays/primitive/compute/mask.rs index 4cd307f0909..1965997abfb 100644 --- a/vortex-array/src/arrays/primitive/compute/mask.rs +++ b/vortex-array/src/arrays/primitive/compute/mask.rs @@ -8,7 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for Primitive { diff --git a/vortex-array/src/arrays/primitive/compute/mod.rs b/vortex-array/src/arrays/primitive/compute/mod.rs index c12771d3da1..142d42e6887 100644 --- a/vortex-array/src/arrays/primitive/compute/mod.rs +++ b/vortex-array/src/arrays/primitive/compute/mod.rs @@ -1,14 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -mod between; mod cast; mod fill_null; mod fixed_width; mod mask; pub(crate) mod rules; mod slice; -mod zip; #[cfg(test)] mod tests { diff --git a/vortex-array/src/arrays/primitive/compute/rules.rs b/vortex-array/src/arrays/primitive/compute/rules.rs index 528e5e91520..cf8192f8a22 100644 --- a/vortex-array/src/arrays/primitive/compute/rules.rs +++ b/vortex-array/src/arrays/primitive/compute/rules.rs @@ -12,8 +12,8 @@ use crate::arrays::PrimitiveArray; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&PrimitiveMaskedValidityRule), diff --git a/vortex-array/src/arrays/primitive/vtable/kernel.rs b/vortex-array/src/arrays/primitive/vtable/kernel.rs index 6382ea73794..26589e42c08 100644 --- a/vortex-array/src/arrays/primitive/vtable/kernel.rs +++ b/vortex-array/src/arrays/primitive/vtable/kernel.rs @@ -9,22 +9,13 @@ use crate::arrays::Primitive; use crate::arrays::dict::TakeExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::between::Between; -use crate::scalar_fn::fns::between::BetweenExecuteAdaptor; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastExecuteAdaptor; -use crate::scalar_fn::fns::fill_null::FillNull; -use crate::scalar_fn::fns::fill_null::FillNullExecuteAdaptor; -use crate::scalar_fn::fns::zip::Zip; -use crate::scalar_fn::fns::zip::ZipExecuteAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastExecuteAdaptor; +use crate::scalar_fn::builtin::fill_null::FillNull; +use crate::scalar_fn::builtin::fill_null::FillNullExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); - kernels.register_execute_parent_kernel( - Between.id(), - Primitive, - BetweenExecuteAdaptor(Primitive), - ); kernels.register_execute_parent_kernel(Cast.id(), Primitive, CastExecuteAdaptor(Primitive)); kernels.register_execute_parent_kernel( FillNull.id(), @@ -32,5 +23,4 @@ pub(crate) fn initialize(session: &VortexSession) { FillNullExecuteAdaptor(Primitive), ); kernels.register_execute_parent_kernel(Dict.id(), Primitive, TakeExecuteAdaptor(Primitive)); - kernels.register_execute_parent_kernel(Zip.id(), Primitive, ZipExecuteAdaptor(Primitive)); } diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 391ea9a03cd..c36f39e929e 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -18,7 +18,7 @@ use crate::array::TypedArrayRef; use crate::arrays::ScalarFn; use crate::dtype::DType; use crate::scalar_fn::ScalarFnRef; -use crate::scalar_fn::fns::literal::Literal; +use crate::scalar_fn::builtin::literal::Literal; // ScalarFnArray has a variable number of slots (one per child) diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 5383a614bd6..78858b69a9f 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -21,7 +21,7 @@ use crate::optimizer::rules::ArrayReduceRule; use crate::optimizer::rules::ParentRuleSet; use crate::optimizer::rules::ReduceRuleSet; use crate::scalar_fn::ArrayReduceNode; -use crate::scalar_fn::fns::pack::Pack; +use crate::scalar_fn::builtin::pack::Pack; use crate::validity::Validity; pub(super) const RULES: ReduceRuleSet = @@ -165,16 +165,10 @@ mod tests { use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; - use crate::expr::cast; - use crate::expr::is_not_null; - use crate::expr::is_null; - use crate::expr::root; use crate::optimizer::rules::ArrayParentReduceRule; - use crate::scalar::Scalar; use crate::scalar_fn::TypedScalarFnInstance; - use crate::scalar_fn::fns::binary::Binary; - use crate::scalar_fn::fns::list_contains::PreparedSet; - use crate::scalar_fn::fns::operators::Operator; + use crate::scalar_fn::builtin::binary::Binary; + use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); @@ -227,30 +221,6 @@ mod tests { Ok(()) } - #[test] - fn test_filter_pushdown_without_children() -> VortexResult<()> { - let list = Scalar::list( - DType::Primitive(PType::I32, Nullability::NonNullable), - vec![Scalar::from(7i32)], - Nullability::NonNullable, - ); - let array = PreparedSet::try_new_array(list.clone(), 3)?; - let parent = FilterArray::try_new(array.clone(), Mask::from_iter([true, false, true]))?; - - let result = ScalarFilterPushdownRule - .reduce_parent(array.as_::(), parent.as_view(), 0)? - .ok_or_else(|| vortex_err!("expected filter pushdown"))?; - - assert_eq!(result.as_::().nchildren(), 0); - assert_eq!(result.len(), 2); - assert_arrays_eq!( - result, - ConstantArray::new(list, 2), - &mut SESSION.create_execution_ctx() - ); - Ok(()) - } - #[test] fn test_empty_constants() { let array = ChunkedArray::try_new( @@ -258,10 +228,7 @@ mod tests { ConstantArray::new(Some(1u64), 0).into_array(), PrimitiveArray::from_iter(vec![2u64]) .into_array() - .apply(&cast( - root(), - DType::Primitive(PType::U64, Nullability::Nullable), - )) + .cast(DType::Primitive(PType::U64, Nullability::Nullable)) .vortex_expect("casted"), ], DType::Primitive(PType::U64, Nullability::Nullable), @@ -269,8 +236,7 @@ mod tests { .vortex_expect("construction") .into_array(); - let expr = is_null(root()); - let result = array.apply(&expr).vortex_expect("expr evaluation"); + let result = array.is_null().vortex_expect("is_null"); assert_arrays_eq!( result, BoolArray::from_iter([false]), @@ -283,22 +249,14 @@ mod tests { let ctx = &mut SESSION.create_execution_ctx(); let validity = BoolArray::from_iter([true, false, true]).into_array(); - assert_arrays_eq!( - validity.clone().apply(&is_null(root()))?, - ConstantArray::new(false, 3), - ctx - ); - assert_arrays_eq!( - validity.clone().apply(&is_not_null(root()))?, - ConstantArray::new(true, 3), - ctx - ); + assert_arrays_eq!(validity.is_null()?, ConstantArray::new(false, 3), ctx); + assert_arrays_eq!(validity.is_not_null()?, ConstantArray::new(true, 3), ctx); let buffer = buffer![1, 2, 3]; let nullable = PrimitiveArray::new(buffer, Validity::Array(validity.clone())).into_array(); - assert_arrays_eq!(nullable.clone().apply(&is_not_null(root()))?, validity, ctx); - assert_arrays_eq!(nullable.apply(&is_null(root()))?, validity.not()?, ctx); + assert_arrays_eq!(nullable.is_not_null()?, validity, ctx); + assert_arrays_eq!(nullable.is_null()?, validity.not()?, ctx); Ok(()) } diff --git a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs index 943d7a2a041..c38213bad02 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs @@ -80,23 +80,17 @@ mod tests { use crate::array::Array; use crate::array::ArrayParts; use crate::arrays::BoolArray; - use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; use crate::arrays::ScalarFnArray; - use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::array::ScalarFnData; use crate::arrays::scalar_fn::vtable::ScalarFn; use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability; use crate::dtype::PType; use crate::scalar::Scalar; use crate::scalar_fn::TypedScalarFnInstance; - use crate::scalar_fn::fns::binary::Binary; - use crate::scalar_fn::fns::list_contains::PreparedSet; - use crate::scalar_fn::fns::list_contains::PreparedSetData; - use crate::scalar_fn::fns::literal::Literal; - use crate::scalar_fn::fns::operators::Operator; + use crate::scalar_fn::builtin::binary::Binary; + use crate::scalar_fn::builtin::literal::Literal; + use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); @@ -177,37 +171,6 @@ mod tests { ); } - #[test] - fn test_scalar_fn_without_children_requires_explicit_len() -> VortexResult<()> { - let list = Scalar::list( - DType::Primitive(PType::I32, Nullability::NonNullable), - vec![Scalar::from(1i32)], - Nullability::NonNullable, - ); - let scalar_fn = - TypedScalarFnInstance::new(PreparedSet, PreparedSetData::try_new(list.clone())?) - .erased(); - - let Err(err) = ScalarFnArray::try_new(scalar_fn.clone(), vec![]) else { - panic!("ScalarFnArray::try_new should reject zero children"); - }; - assert!( - err.to_string() - .contains("ScalarFnArray length cannot be inferred without children") - ); - - let scalar_fn_array = ScalarFnArray::try_new_with_len(scalar_fn, vec![], 3)?; - assert_eq!(scalar_fn_array.len(), 3); - assert_eq!(scalar_fn_array.child_count(), 0); - assert_arrays_eq!( - scalar_fn_array, - ConstantArray::new(list, 3), - &mut SESSION.create_execution_ctx() - ); - - Ok(()) - } - #[test] fn rejects_literal() { let scalar_fn = TypedScalarFnInstance::new(Literal, Scalar::from(1i32)).erased(); diff --git a/vortex-array/src/arrays/slice/rules.rs b/vortex-array/src/arrays/slice/rules.rs index 62ed8892541..25e96b5c15b 100644 --- a/vortex-array/src/arrays/slice/rules.rs +++ b/vortex-array/src/arrays/slice/rules.rs @@ -13,7 +13,7 @@ use crate::arrays::slice::SliceReduceAdaptor; use crate::builtins::ArrayBuiltins; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::get_item::GetItem; +use crate::scalar_fn::builtin::get_item::GetItem; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&SliceReduceAdaptor(Slice)), diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index e73ea26446b..8daf31428ef 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -23,7 +23,7 @@ use crate::kernel::ExecuteParentKernel; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar::Scalar; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::cast::Cast; +use crate::scalar_fn::builtin::cast::Cast; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); @@ -153,7 +153,7 @@ mod tests { use crate::optimizer::kernels::ExecuteParentFn; use crate::optimizer::kernels::KernelSession; use crate::scalar::Scalar; - use crate::scalar_fn::fns::cast::Cast; + use crate::scalar_fn::builtin::cast::Cast; use crate::validity::Validity; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/arrays/struct_/compute/mask.rs b/vortex-array/src/arrays/struct_/compute/mask.rs index 68db6166455..758ede1bbf0 100644 --- a/vortex-array/src/arrays/struct_/compute/mask.rs +++ b/vortex-array/src/arrays/struct_/compute/mask.rs @@ -9,7 +9,7 @@ use crate::array::ArrayView; use crate::arrays::Struct; use crate::arrays::StructArray; use crate::arrays::struct_::StructArrayExt; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for Struct { diff --git a/vortex-array/src/arrays/struct_/compute/mod.rs b/vortex-array/src/arrays/struct_/compute/mod.rs index 1d528448ad3..6edc2c92e19 100644 --- a/vortex-array/src/arrays/struct_/compute/mod.rs +++ b/vortex-array/src/arrays/struct_/compute/mod.rs @@ -6,7 +6,6 @@ mod mask; pub(crate) mod rules; mod slice; mod take; -mod zip; #[cfg(test)] mod tests { diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs index f7bcd750030..7ca9671d527 100644 --- a/vortex-array/src/arrays/struct_/compute/rules.rs +++ b/vortex-array/src/arrays/struct_/compute/rules.rs @@ -21,9 +21,9 @@ use crate::matcher::Matcher; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; use crate::scalar::Scalar; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::get_item::GetItem; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::get_item::GetItem; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; use crate::validity::Validity; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ diff --git a/vortex-array/src/arrays/struct_/mod.rs b/vortex-array/src/arrays/struct_/mod.rs index ab533dd8cdb..faaf03dc23c 100644 --- a/vortex-array/src/arrays/struct_/mod.rs +++ b/vortex-array/src/arrays/struct_/mod.rs @@ -15,7 +15,6 @@ pub use vtable::Struct; pub(crate) fn initialize(session: &vortex_session::VortexSession) { compute::cast::initialize(session); - vtable::initialize(session); } #[cfg(test)] diff --git a/vortex-array/src/arrays/struct_/vtable/kernel.rs b/vortex-array/src/arrays/struct_/vtable/kernel.rs deleted file mode 100644 index 1e58da2d0d0..00000000000 --- a/vortex-array/src/arrays/struct_/vtable/kernel.rs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex_session::VortexSession; - -use crate::arrays::Struct; -use crate::optimizer::kernels::ArrayKernelsExt; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::zip::Zip; -use crate::scalar_fn::fns::zip::ZipExecuteAdaptor; - -pub(crate) fn initialize(session: &VortexSession) { - let kernels = session.kernels(); - kernels.register_execute_parent_kernel(Zip.id(), Struct, ZipExecuteAdaptor(Struct)); -} diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 15cf128b8e9..398b5db53a4 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -26,7 +26,6 @@ use crate::builders::StructBuilder; use crate::dtype::DType; use crate::serde::ArrayChildren; use crate::validity::Validity; -mod kernel; mod operations; mod validity; @@ -37,10 +36,6 @@ use crate::array::ArrayId; /// A [`Struct`]-encoded Vortex array. pub type StructArray = Array; -pub(crate) fn initialize(session: &VortexSession) { - kernel::initialize(session); -} - impl VTable for Struct { type TypedArrayData = EmptyArrayData; diff --git a/vortex-array/src/arrays/union/compute/mask.rs b/vortex-array/src/arrays/union/compute/mask.rs index 63bb9ae60f7..18f74b5daf8 100644 --- a/vortex-array/src/arrays/union/compute/mask.rs +++ b/vortex-array/src/arrays/union/compute/mask.rs @@ -11,7 +11,7 @@ use crate::arrays::UnionArray; use crate::arrays::union::UnionArrayExt; use crate::arrays::union::UnionArraySlotsExt; use crate::builtins::ArrayBuiltins; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; impl MaskReduce for Union { fn mask(array: ArrayView<'_, Union>, mask: &ArrayRef) -> VortexResult> { diff --git a/vortex-array/src/arrays/union/compute/rules.rs b/vortex-array/src/arrays/union/compute/rules.rs index 921d66a24d3..33123cc7793 100644 --- a/vortex-array/src/arrays/union/compute/rules.rs +++ b/vortex-array/src/arrays/union/compute/rules.rs @@ -5,7 +5,7 @@ use crate::arrays::Union; use crate::arrays::dict::TakeReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&MaskReduceAdaptor(Union)), diff --git a/vortex-array/src/arrays/varbin/compute/cast.rs b/vortex-array/src/arrays/varbin/compute/cast.rs index 09b68449103..c8449d8b2be 100644 --- a/vortex-array/src/arrays/varbin/compute/cast.rs +++ b/vortex-array/src/arrays/varbin/compute/cast.rs @@ -11,8 +11,8 @@ use crate::arrays::VarBin; use crate::arrays::VarBinArray; use crate::arrays::varbin::VarBinArraySlotsExt; use crate::dtype::DType; -use crate::scalar_fn::fns::cast::CastKernel; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastKernel; +use crate::scalar_fn::builtin::cast::CastReduce; use crate::validity::Validity; fn build_with_validity( diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index ab2abce91c9..7559b1a30b3 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -19,8 +19,8 @@ use crate::arrays::varbin::VarBinArraySlotsExt; use crate::dtype::DType; use crate::dtype::IntegerPType; use crate::match_each_integer_ptype; -use crate::scalar_fn::fns::binary::CompareKernel; -use crate::scalar_fn::fns::operators::CompareOperator; +use crate::scalar_fn::builtin::binary::CompareKernel; +use crate::scalar_fn::builtin::operators::CompareOperator; // This implementation exists so we can compare against a constant in encoded space, without // canonicalizing the VarBin array to VarBinView. @@ -189,7 +189,7 @@ mod test { use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; - use crate::scalar_fn::fns::operators::Operator; + use crate::scalar_fn::builtin::operators::Operator; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); @@ -261,7 +261,7 @@ mod tests { use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; - use crate::scalar_fn::fns::operators::Operator; + use crate::scalar_fn::builtin::operators::Operator; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/arrays/varbin/compute/mask.rs b/vortex-array/src/arrays/varbin/compute/mask.rs index 5f75ccc3839..14a48924f71 100644 --- a/vortex-array/src/arrays/varbin/compute/mask.rs +++ b/vortex-array/src/arrays/varbin/compute/mask.rs @@ -9,7 +9,7 @@ use crate::array::ArrayView; use crate::arrays::VarBin; use crate::arrays::VarBinArray; use crate::arrays::varbin::VarBinArraySlotsExt; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for VarBin { diff --git a/vortex-array/src/arrays/varbin/compute/rules.rs b/vortex-array/src/arrays/varbin/compute/rules.rs index 89e22ce0d20..75c3bd6086a 100644 --- a/vortex-array/src/arrays/varbin/compute/rules.rs +++ b/vortex-array/src/arrays/varbin/compute/rules.rs @@ -4,8 +4,8 @@ use crate::arrays::VarBin; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(VarBin)), diff --git a/vortex-array/src/arrays/varbin/vtable/kernel.rs b/vortex-array/src/arrays/varbin/vtable/kernel.rs index 9e80abd1037..c23d5e2af69 100644 --- a/vortex-array/src/arrays/varbin/vtable/kernel.rs +++ b/vortex-array/src/arrays/varbin/vtable/kernel.rs @@ -11,10 +11,10 @@ use crate::arrays::dict::TakeExecuteAdaptor; use crate::arrays::filter::FilterExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::binary::CompareExecuteAdaptor; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastExecuteAdaptor; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::binary::CompareExecuteAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); diff --git a/vortex-array/src/arrays/varbinview/compute/cast.rs b/vortex-array/src/arrays/varbinview/compute/cast.rs index 19b669f2eac..3ee2f80e3a3 100644 --- a/vortex-array/src/arrays/varbinview/compute/cast.rs +++ b/vortex-array/src/arrays/varbinview/compute/cast.rs @@ -12,8 +12,8 @@ use crate::array::ArrayView; use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::dtype::DType; -use crate::scalar_fn::fns::cast::CastKernel; -use crate::scalar_fn::fns::cast::CastReduce; +use crate::scalar_fn::builtin::cast::CastKernel; +use crate::scalar_fn::builtin::cast::CastReduce; use crate::validity::Validity; fn build_with_validity( diff --git a/vortex-array/src/arrays/varbinview/compute/fill_null.rs b/vortex-array/src/arrays/varbinview/compute/fill_null.rs index cd3c9bfd10e..322f68cd0ff 100644 --- a/vortex-array/src/arrays/varbinview/compute/fill_null.rs +++ b/vortex-array/src/arrays/varbinview/compute/fill_null.rs @@ -20,7 +20,7 @@ use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::BinaryView; use crate::dtype::DType; use crate::scalar::Scalar; -use crate::scalar_fn::fns::fill_null::FillNullKernel; +use crate::scalar_fn::builtin::fill_null::FillNullKernel; use crate::validity::Validity; // Points every null row at one view of the fill value. The data buffers are reused as they are, diff --git a/vortex-array/src/arrays/varbinview/compute/mask.rs b/vortex-array/src/arrays/varbinview/compute/mask.rs index 8869f816600..d71b2b3b48c 100644 --- a/vortex-array/src/arrays/varbinview/compute/mask.rs +++ b/vortex-array/src/arrays/varbinview/compute/mask.rs @@ -10,7 +10,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; -use crate::scalar_fn::fns::mask::MaskReduce; +use crate::scalar_fn::builtin::mask::MaskReduce; use crate::validity::Validity; impl MaskReduce for VarBinView { diff --git a/vortex-array/src/arrays/varbinview/compute/mod.rs b/vortex-array/src/arrays/varbinview/compute/mod.rs index dab35219b5c..d55c77f8e4a 100644 --- a/vortex-array/src/arrays/varbinview/compute/mod.rs +++ b/vortex-array/src/arrays/varbinview/compute/mod.rs @@ -7,7 +7,6 @@ mod mask; pub(crate) mod rules; mod slice; mod take; -mod zip; #[cfg(test)] mod tests { diff --git a/vortex-array/src/arrays/varbinview/compute/rules.rs b/vortex-array/src/arrays/varbinview/compute/rules.rs index 5ec24dca7de..8aa16def538 100644 --- a/vortex-array/src/arrays/varbinview/compute/rules.rs +++ b/vortex-array/src/arrays/varbinview/compute/rules.rs @@ -3,8 +3,8 @@ use crate::arrays::VarBinView; use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ParentRuleSet; -use crate::scalar_fn::fns::cast::CastReduceAdaptor; -use crate::scalar_fn::fns::mask::MaskReduceAdaptor; +use crate::scalar_fn::builtin::cast::CastReduceAdaptor; +use crate::scalar_fn::builtin::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(VarBinView)), diff --git a/vortex-array/src/arrays/varbinview/vtable/kernel.rs b/vortex-array/src/arrays/varbinview/vtable/kernel.rs index 44dfcf5c028..0b22dbe73c3 100644 --- a/vortex-array/src/arrays/varbinview/vtable/kernel.rs +++ b/vortex-array/src/arrays/varbinview/vtable/kernel.rs @@ -9,12 +9,10 @@ use crate::arrays::VarBinView; use crate::arrays::dict::TakeExecuteAdaptor; use crate::optimizer::kernels::ArrayKernelsExt; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::cast::CastExecuteAdaptor; -use crate::scalar_fn::fns::fill_null::FillNull; -use crate::scalar_fn::fns::fill_null::FillNullExecuteAdaptor; -use crate::scalar_fn::fns::zip::Zip; -use crate::scalar_fn::fns::zip::ZipExecuteAdaptor; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::cast::CastExecuteAdaptor; +use crate::scalar_fn::builtin::fill_null::FillNull; +use crate::scalar_fn::builtin::fill_null::FillNullExecuteAdaptor; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); @@ -25,5 +23,4 @@ pub(crate) fn initialize(session: &VortexSession) { FillNullExecuteAdaptor(VarBinView), ); kernels.register_execute_parent_kernel(Dict.id(), VarBinView, TakeExecuteAdaptor(VarBinView)); - kernels.register_execute_parent_kernel(Zip.id(), VarBinView, ZipExecuteAdaptor(VarBinView)); } diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 3a44a11019d..e92449c15bf 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -10,11 +10,7 @@ use vortex_error::vortex_ensure; pub use self::vtable::Variant; pub use self::vtable::VariantArray; - -pub(crate) fn initialize(session: &vortex_session::VortexSession) { - vtable::initialize(session); -} - +pub use self::vtable::merge_typed_scalar_as_variant; use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; @@ -92,18 +88,13 @@ mod tests { use crate::arrays::ChunkedArray; use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; - use crate::arrays::StructArray; use crate::arrays::VariantArray; use crate::arrays::variant::VariantArraySlotsExt; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::expr::root; - use crate::expr::variant_get; use crate::scalar::Scalar; - use crate::scalar_fn::fns::variant_get::VariantPath; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); @@ -376,71 +367,4 @@ mod tests { assert_variant_core_rows(&masked, &[Some(1), None, None, Some(4)]) } - - #[test] - fn variant_get_keeps_valid_shredded_rows_for_matching_dtype() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - let core_storage = row_storage([1, 2, 3])?; - let shredded = StructArray::try_from_iter([( - "a", - PrimitiveArray::from_iter([10i32, 20, 30]).into_array(), - )])?; - let variant = VariantArray::try_new(core_storage, Some(shredded.into_array()))?; - let expr = variant_get( - root(), - VariantPath::field("a"), - Some(DType::Primitive(PType::I32, Nullability::NonNullable)), - ); - - let result = variant.into_array().apply(&expr)?; - - assert_arrays_eq!( - result, - PrimitiveArray::from_option_iter([Some(10i32), Some(20), Some(30)]), - &mut ctx - ); - Ok(()) - } - - #[test] - fn variant_get_treats_value_and_typed_value_as_logical_field_names() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - let core_storage = row_storage([1, 2, 3])?; - let shredded = StructArray::try_from_iter([ - ( - "value", - PrimitiveArray::from_iter([10i32, 20, 30]).into_array(), - ), - ( - "typed_value", - PrimitiveArray::from_iter([40i32, 50, 60]).into_array(), - ), - ])?; - let variant = VariantArray::try_new(core_storage, Some(shredded.into_array()))?; - - let value_expr = variant_get( - root(), - VariantPath::field("value"), - Some(DType::Primitive(PType::I32, Nullability::NonNullable)), - ); - let value_result = variant.clone().into_array().apply(&value_expr)?; - assert_arrays_eq!( - value_result, - PrimitiveArray::from_option_iter([Some(10i32), Some(20), Some(30)]), - &mut ctx - ); - - let typed_value_expr = variant_get( - root(), - VariantPath::field("typed_value"), - Some(DType::Primitive(PType::I32, Nullability::NonNullable)), - ); - let typed_value_result = variant.into_array().apply(&typed_value_expr)?; - assert_arrays_eq!( - typed_value_result, - PrimitiveArray::from_option_iter([Some(40i32), Some(50), Some(60)]), - &mut ctx - ); - Ok(()) - } } diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 9e07169012a..a1f7724915e 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -mod kernel; mod operations; mod validity; @@ -41,10 +40,6 @@ use crate::serde::ArrayChildren; /// A [`Variant`]-encoded Vortex array. pub type VariantArray = Array; -pub(crate) fn initialize(session: &VortexSession) { - kernel::initialize(session); -} - #[derive(Clone, Debug)] pub struct Variant; @@ -198,7 +193,9 @@ impl VTable for Variant { } } -fn merge_typed_scalar_as_variant( +/// Merges a shredded `typed_scalar` with the unshredded `fallback_scalar` of the same row into a +/// variant scalar of `dtype`. +pub fn merge_typed_scalar_as_variant( typed_scalar: Scalar, fallback_scalar: Option, dtype: &DType, diff --git a/vortex-array/src/builders/lazy_null_builder.rs b/vortex-array/src/builders/lazy_null_builder.rs index d46d731cf25..59684a4d101 100644 --- a/vortex-array/src/builders/lazy_null_builder.rs +++ b/vortex-array/src/builders/lazy_null_builder.rs @@ -106,12 +106,18 @@ impl LazyBitBufferBuilder { } /// Returns the current length of the builder. + /// Whether no entries have been appended. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + pub fn len(&self) -> usize { // self.len is the length of the builder if the inner buffer is not materialized self.inner.as_ref().map(|i| i.len()).unwrap_or(self.len) } - fn finish(&mut self) -> Option { + /// The validity bits appended so far, or `None` when every entry is valid. + pub fn finish(&mut self) -> Option { self.len = 0; self.inner.take().map(|b| b.freeze()) } diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs index 480c3f6a701..9590ae6fe56 100644 --- a/vortex-array/src/builders/mod.rs +++ b/vortex-array/src/builders/mod.rs @@ -54,7 +54,7 @@ use crate::memory::BufferAllocatorRef; use crate::scalar::Scalar; mod lazy_null_builder; -pub(crate) use lazy_null_builder::LazyBitBufferBuilder; +pub use lazy_null_builder::LazyBitBufferBuilder; mod bool; mod child; diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index 8751802aeb0..475fdc1497a 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -1074,14 +1074,13 @@ impl DeduplicatedBuffers { .collect() } - pub(crate) fn extend_from_iter( - &mut self, - buffers: impl Iterator, - ) -> Vec { + /// Pushes each of `buffers`, returning the index of each in the deduplicated buffers. + pub fn extend_from_iter(&mut self, buffers: impl Iterator) -> Vec { buffers.map(|buffer| self.push(buffer)).collect() } - pub(crate) fn finish(self) -> Arc<[ByteBuffer]> { + /// The deduplicated buffers, in the order they were first pushed. + pub fn finish(self) -> Arc<[ByteBuffer]> { Arc::from(self.buffers) } } diff --git a/vortex-array/src/builtins.rs b/vortex-array/src/builtins.rs index 42b696c2f47..bab182156a1 100644 --- a/vortex-array/src/builtins.rs +++ b/vortex-array/src/builtins.rs @@ -17,105 +17,19 @@ use crate::arrays::ConstantArray; use crate::arrays::InterleaveArray; use crate::dtype::DType; use crate::dtype::FieldName; -use crate::expr::Expression; use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; -use crate::scalar_fn::EmptyOptions; -use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::between::Between; -use crate::scalar_fn::fns::between::BetweenOptions; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::fill_null::FillNull; -use crate::scalar_fn::fns::get_item::GetItem; -use crate::scalar_fn::fns::is_not_null::IsNotNull; -use crate::scalar_fn::fns::is_null::IsNull; -use crate::scalar_fn::fns::list_contains::ListContains; -use crate::scalar_fn::fns::list_contains::ListContainsOptions; -use crate::scalar_fn::fns::mask::Mask; -use crate::scalar_fn::fns::not::Not; -use crate::scalar_fn::fns::operators::Operator; -use crate::scalar_fn::fns::zip::Zip; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::fill_null::FillNull; +use crate::scalar_fn::builtin::get_item::GetItem; +use crate::scalar_fn::builtin::is_not_null::IsNotNull; +use crate::scalar_fn::builtin::is_null::IsNull; +use crate::scalar_fn::builtin::mask::Mask; +use crate::scalar_fn::builtin::not::Not; +use crate::scalar_fn::builtin::operators::Operator; /// A collection of built-in scalar functions that can be applied to expressions or arrays. -pub trait ExprBuiltins: Sized { - /// Cast to the given data type. - fn cast(&self, dtype: DType) -> VortexResult; - - /// Replace null values with the given fill value. - fn fill_null(&self, fill_value: Expression) -> VortexResult; - - /// Get item by field name (for struct types). - fn get_item(&self, field_name: impl Into) -> VortexResult; - - /// Is null check. - fn is_null(&self) -> VortexResult; - - /// Is not null check. - fn is_not_null(&self) -> VortexResult; - - /// Mask the expression using the given boolean mask. - /// The resulting expression's validity is the intersection of the original expression's - /// validity. - fn mask(&self, mask: Expression) -> VortexResult; - - /// Boolean negation. - fn not(&self) -> VortexResult; - - /// Check if a list contains a value. - fn list_contains(&self, value: Expression) -> VortexResult; - - /// Conditional selection: `result[i] = if mask[i] then if_true[i] else if_false[i]`. - fn zip(&self, if_true: Expression, if_false: Expression) -> VortexResult; - - // TODO(joe): add an `interleave` expression builtin mirroring `ArrayBuiltins::interleave`. - - /// Apply a binary operator to this expression and another. - fn binary(&self, rhs: Expression, op: Operator) -> VortexResult; -} - -impl ExprBuiltins for Expression { - fn cast(&self, dtype: DType) -> VortexResult { - Cast.try_new_expr(dtype, [self.clone()]) - } - - fn fill_null(&self, fill_value: Expression) -> VortexResult { - FillNull.try_new_expr(EmptyOptions, [self.clone(), fill_value]) - } - - fn get_item(&self, field_name: impl Into) -> VortexResult { - GetItem.try_new_expr(field_name.into(), [self.clone()]) - } - - fn is_null(&self) -> VortexResult { - IsNull.try_new_expr(EmptyOptions, [self.clone()]) - } - - fn is_not_null(&self) -> VortexResult { - IsNotNull.try_new_expr(EmptyOptions, [self.clone()]) - } - - fn mask(&self, mask: Expression) -> VortexResult { - Mask.try_new_expr(EmptyOptions, [self.clone(), mask]) - } - - fn not(&self) -> VortexResult { - Not.try_new_expr(EmptyOptions, [self.clone()]) - } - - fn list_contains(&self, value: Expression) -> VortexResult { - ListContains.try_new_expr(ListContainsOptions::default(), [self.clone(), value]) - } - - fn zip(&self, if_true: Expression, if_false: Expression) -> VortexResult { - Zip.try_new_expr(EmptyOptions, [if_true, if_false, self.clone()]) - } - - fn binary(&self, rhs: Expression, op: Operator) -> VortexResult { - Binary.try_new_expr(op, [self.clone(), rhs]) - } -} - pub trait ArrayBuiltins: Sized { /// Cast to the given data type. fn cast(&self, dtype: DType) -> VortexResult; @@ -140,9 +54,6 @@ pub trait ArrayBuiltins: Sized { /// Boolean negation. fn not(&self) -> VortexResult; - /// Conditional selection: `result[i] = if mask[i] then if_true[i] else if_false[i]`. - fn zip(&self, if_true: ArrayRef, if_false: ArrayRef) -> VortexResult; - /// Random-access gather by `(array_index, row_index)`: output row `i` is taken from /// `values[array_indices[i]][row_indices[i]]`, where `self` is the (non-nullable) /// `array_indices` selector and `row_indices` names the position within the selected value. @@ -153,19 +64,8 @@ pub trait ArrayBuiltins: Sized { row_indices: ArrayRef, ) -> VortexResult; - /// Check if a list contains a value. - fn list_contains(&self, value: ArrayRef) -> VortexResult; - /// Apply a binary operator to this array and another. fn binary(&self, rhs: ArrayRef, op: Operator) -> VortexResult; - - /// Compare a values between lower VortexResult; } impl ArrayBuiltins for ArrayRef { @@ -211,10 +111,6 @@ impl ArrayBuiltins for ArrayRef { Not::try_new(self.clone())?.into_array().optimize() } - fn zip(&self, if_true: ArrayRef, if_false: ArrayRef) -> VortexResult { - Ok(Zip::try_new(if_true, if_false, self.clone())?.into_array()) - } - fn interleave( &self, values: impl IntoIterator, @@ -226,26 +122,9 @@ impl ArrayBuiltins for ArrayRef { ) } - fn list_contains(&self, value: ArrayRef) -> VortexResult { - ListContains::try_new(self.clone(), value)? - .into_array() - .optimize() - } - fn binary(&self, rhs: ArrayRef, op: Operator) -> VortexResult { Binary::try_new(self.clone(), rhs, op)? .into_array() .optimize() } - - fn between( - self, - lower: ArrayRef, - upper: ArrayRef, - options: BetweenOptions, - ) -> VortexResult { - Between::try_new(self, lower, upper, options)? - .into_array() - .optimize() - } } diff --git a/vortex-array/src/compute/conformance/binary_numeric.rs b/vortex-array/src/compute/conformance/binary_numeric.rs index 29f3c3a6531..aa6847902e9 100644 --- a/vortex-array/src/compute/conformance/binary_numeric.rs +++ b/vortex-array/src/compute/conformance/binary_numeric.rs @@ -52,7 +52,7 @@ use crate::scalar::DecimalValue; use crate::scalar::NumericOperator; use crate::scalar::PrimitiveScalar; use crate::scalar::Scalar; -use crate::scalar_fn::fns::binary::numeric_op_result_decimal_dtype; +use crate::scalar_fn::builtin::binary::numeric_op_result_decimal_dtype; fn to_vec_of_scalar(array: &ArrayRef, ctx: &mut ExecutionCtx) -> Vec { // Not fast, but obviously correct diff --git a/vortex-array/src/compute/conformance/consistency.rs b/vortex-array/src/compute/conformance/consistency.rs index 6d1b919e51d..ca8099f8011 100644 --- a/vortex-array/src/compute/conformance/consistency.rs +++ b/vortex-array/src/compute/conformance/consistency.rs @@ -43,7 +43,7 @@ use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::operators::Operator; /// Tests that filter and take operations produce consistent results. /// diff --git a/vortex-array/src/expr/aliases.rs b/vortex-array/src/expr/aliases.rs deleted file mode 100644 index bf4694355d0..00000000000 --- a/vortex-array/src/expr/aliases.rs +++ /dev/null @@ -1,7 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -pub mod paste { - //! Re-export of [`paste`](https://docs.rs/paste/latest/paste/). - pub use paste::paste; -} diff --git a/vortex-array/src/expr/bound_expression.rs b/vortex-array/src/expr/bound_expression.rs index 035ca91f404..d82cfb0a027 100644 --- a/vortex-array/src/expr/bound_expression.rs +++ b/vortex-array/src/expr/bound_expression.rs @@ -14,12 +14,8 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use crate::dtype::DType; -use crate::expr::Expression; use crate::expr::display::DisplayTreeExpr; -use crate::expr::expression::unbound_variable; -use crate::expr::scope::Scope; -use crate::expr::traversal::TraversalOrder; -use crate::expr::traversal::pre_order_visit_down; +use crate::scalar::Scalar; use crate::scalar_fn::EmptyOptions; use crate::scalar_fn::ExpressionReduceNode; use crate::scalar_fn::ReduceNode; @@ -27,9 +23,11 @@ use crate::scalar_fn::ReduceNodeValidity; use crate::scalar_fn::ScalarFnRef; use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::is_not_null::IsNotNull; +use crate::scalar_fn::builtin::is_not_null::IsNotNull; +use crate::scalar_fn::builtin::literal::Literal; +use crate::scalar_fn::display::ExprDisplay; -/// An [`Expression`] that has been type-checked against a [`Scope`]. +/// An expression that has been type-checked against a scope's root dtype. /// /// Every node carries its own dtype, so reading one is a field access rather than a walk of the /// subtree. Holding a `BoundExpression` is proof that the whole tree type-checked. @@ -199,15 +197,25 @@ impl BoundExpression { /// Return whether this expression tree contains a node using the given scalar-function vtable. pub fn contains(&self) -> VortexResult { - let mut contains = false; - pre_order_visit_down(self, |node| { - if node.is::() { - contains = true; - return Ok(TraversalOrder::Stop); + Ok(self.any_node(|node| node.is::())) + } + + /// Whether any node of this tree, visited depth-first, satisfies `predicate`. + fn any_node(&self, mut predicate: impl FnMut(&BoundExpression) -> bool) -> bool { + let mut stack = vec![self]; + while let Some(node) = stack.pop() { + if predicate(node) { + return true; } - Ok(TraversalOrder::Continue) - })?; - Ok(contains) + stack.extend(node.children().iter().rev()); + } + false + } + + /// Create a bound literal node. + pub fn literal(value: Scalar) -> Self { + Self::try_new(Literal.bind(value), []) + .vortex_expect("literal expressions are always well-typed") } /// Return the typed scalar-function options when this node uses the given vtable. @@ -234,16 +242,7 @@ impl BoundExpression { /// /// Expressions without a scope root, such as literals, match every dtype. pub fn is_root_bound_to(&self, dtype: &DType) -> bool { - let mut is_bound_to = true; - pre_order_visit_down(self, |node| { - if node.is_root() && node.dtype() != dtype { - is_bound_to = false; - return Ok(TraversalOrder::Stop); - } - Ok(TraversalOrder::Continue) - }) - .vortex_expect("bound expression traversal cannot not fail"); - is_bound_to + !self.any_node(|node| node.is_root() && node.dtype() != dtype) } /// Returns a new expression representing the validity mask output of this expression. @@ -262,7 +261,7 @@ impl BoundExpression { // IsNotNull(x) -> x.validity() symbolic reduction rule works only // when node's.validity is Reduced to avoid infinite recursion. ReduceNodeValidity::Irreducible => { - IsNotNull.try_new_bound_expr(EmptyOptions, [self.clone()])? + BoundExpression::try_new(IsNotNull.bind(EmptyOptions), [self.clone()])? } }) } @@ -284,37 +283,6 @@ impl Display for BoundExpression { } } -impl Expression { - /// Bind this expression against a root dtype, type-checking every node in a single walk. - /// - /// The returned tree carries a dtype on each node, so callers needing types at more than one - /// node should bind once and read fields rather than calling - /// [`return_dtype`](Expression::return_dtype) repeatedly. - pub fn bind(&self, dtype: &DType) -> VortexResult { - self.bind_scope(&Scope::new(dtype.clone())) - } - - /// Bind this expression against an explicit [`Scope`]. - pub fn bind_scope(&self, scope: &Scope) -> VortexResult { - if self.is_root() { - return Ok(BoundExpression::new_root(scope.root().clone())); - } - if let Expression::Variable(name) = self { - return Err(unbound_variable(name)); - } - - let children: Vec<_> = self - .children() - .iter() - .map(|child| child.bind_scope(scope)) - .try_collect()?; - let scalar_fn = self - .as_scalar() - .vortex_expect("leaves were handled above, so this is a scalar node"); - BoundExpression::try_new(scalar_fn.clone(), children) - } -} - /// Iterative drop to avoid stack overflows on deep trees. impl Drop for BoundExpression { fn drop(&mut self) { @@ -336,158 +304,12 @@ impl Drop for BoundExpression { } } -#[cfg(test)] -mod tests { - use vortex_error::VortexResult; - - use super::*; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::expr::col; - use crate::expr::eq; - use crate::expr::lit; - use crate::expr::root; - use crate::expr::test_harness::struct_dtype; - use crate::scalar_fn::fns::literal::Literal; - - fn scope() -> Scope { - Scope::new(struct_dtype()) - } - - #[test] - fn root_binds_to_the_scope() -> VortexResult<()> { - let bound = root().bind_scope(&scope())?; - assert!(bound.is_root()); - assert_eq!(bound.dtype(), &struct_dtype()); - assert_eq!(bound, BoundExpression::new_root(struct_dtype())); - Ok(()) - } - - #[test] - fn every_node_carries_its_dtype() -> VortexResult<()> { - let expr = eq(col("a"), lit(1_i32)); - let bound = expr.bind_scope(&scope())?; - - assert_eq!(bound.dtype(), &DType::Bool(Nullability::NonNullable)); - - let lhs = &bound.children()[0]; - assert_eq!( - lhs.dtype(), - &DType::Primitive(PType::I32, Nullability::NonNullable) - ); - assert_eq!(lhs.children()[0].dtype(), &struct_dtype()); - Ok(()) - } - - #[test] - fn bind_agrees_with_return_dtype() -> VortexResult<()> { - for (expr, expected) in [ - (root(), struct_dtype()), - ( - col("a"), - DType::Primitive(PType::I32, Nullability::NonNullable), - ), - ( - eq(col("a"), lit(1_i32)), - DType::Bool(Nullability::NonNullable), - ), - (lit(true), DType::Bool(Nullability::NonNullable)), - ] { - assert_eq!( - expr.bind(&struct_dtype())?.dtype(), - &expected, - "binding {expr}" - ); - assert_eq!( - expr.return_dtype(&struct_dtype())?, - expected, - "return dtype of {expr}" - ); - } - Ok(()) - } - - #[test] - fn contains_scalar_function() -> VortexResult<()> { - let bound = eq(col("a"), lit(1_i32)).bind_scope(&scope())?; - assert!(bound.contains::()?); - assert!(!root().bind_scope(&scope())?.contains::()?); - Ok(()) - } - - #[test] - fn bound_to_checks_every_root() -> VortexResult<()> { - let dtype = struct_dtype(); - let bound = eq(col("a"), col("a")).bind(&dtype)?; - assert!(bound.is_root_bound_to(&dtype)); - assert!(!bound.is_root_bound_to(&DType::Bool(Nullability::NonNullable))); - assert!( - lit(true) - .bind(&dtype)? - .is_root_bound_to(&DType::Bool(Nullability::NonNullable)) - ); - Ok(()) - } - - #[test] - fn bound_display_matches_unbound() -> VortexResult<()> { - for expr in [root(), col("a"), eq(col("a"), lit(1_i32)), lit(true)] { - let bound = expr.bind_scope(&scope())?; - assert_eq!(bound.to_string(), expr.to_string()); - assert_eq!( - bound.display_tree().to_string(), - expr.display_tree().to_string() - ); - } - Ok(()) - } - - #[test] - fn clone_shares_children() -> VortexResult<()> { - let bound = eq(col("a"), lit(1_i32)).bind_scope(&scope())?; - let cloned = bound.clone(); - - let ( - BoundExpression::Scalar { children: a, .. }, - BoundExpression::Scalar { children: b, .. }, - ) = (&bound, &cloned) - else { - unreachable!("eq is a scalar node") - }; - assert!(Arc::ptr_eq(a, b)); - Ok(()) - } - - #[test] - fn repeated_subtree_is_bound_per_occurrence() -> VortexResult<()> { - let shared = col("a"); - let bound = eq(shared.clone(), shared).bind_scope(&scope())?; - let children = bound.children(); - assert_eq!(children.len(), 2); - for child in children { - assert_eq!( - child.dtype(), - &DType::Primitive(PType::I32, Nullability::NonNullable) - ); - } - Ok(()) - } - - #[test] - fn structural_and_exact_equality_are_distinct() -> VortexResult<()> { - let expr = eq(col("a"), lit(1_i32)); - let bound = expr.bind_scope(&scope())?; - let independently_bound = expr.bind_scope(&scope())?; - - assert_eq!(bound, independently_bound); - assert_eq!(ExactBoundExpr(bound.clone()), ExactBoundExpr(bound.clone())); - assert_ne!(ExactBoundExpr(bound), ExactBoundExpr(independently_bound)); - Ok(()) +impl ExprDisplay for BoundExpression { + fn display_child(&self, index: usize) -> &dyn ExprDisplay { + &self.children()[index] } - #[test] - fn binding_reports_a_type_error() { - let expr = eq(col("a"), lit("nope")); - assert!(expr.bind_scope(&scope()).is_err()); + fn display_children_count(&self) -> usize { + self.children().len() } } diff --git a/vortex-array/src/expr/display.rs b/vortex-array/src/expr/display.rs index 8de2a7d112d..013b4d197ed 100644 --- a/vortex-array/src/expr/display.rs +++ b/vortex-array/src/expr/display.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +//! Tree rendering shared by bound and unbound expressions. + use std::fmt; use std::fmt::Display; use std::fmt::Formatter; @@ -9,78 +11,22 @@ use vortex_utils::tree::TreeDisplayAdapter; use vortex_utils::tree::write_branch_tree; use crate::expr::BoundExpression; -use crate::expr::Expression; use crate::scalar_fn::ChildName; -pub enum DisplayFormat { - Compact, - Tree, -} - -/// Read-only expression-tree interface used by scalar functions for SQL-style formatting. -/// -/// Both [`Expression`] and [`BoundExpression`] implement this interface, allowing scalar -/// functions to format either representation without converting between them. -pub trait ExprDisplay: Display { - /// Return the child at `index`. - fn display_child(&self, index: usize) -> &dyn ExprDisplay; - - /// Return the number of children in this node. - fn display_children_count(&self) -> usize; -} - -impl ExprDisplay for Expression { - fn display_child(&self, index: usize) -> &dyn ExprDisplay { - Expression::child(self, index) - } - - fn display_children_count(&self) -> usize { - self.children().len() - } -} - -impl ExprDisplay for BoundExpression { - fn display_child(&self, index: usize) -> &dyn ExprDisplay { - &self.children()[index] - } - - fn display_children_count(&self) -> usize { - self.children().len() - } -} - -trait DisplayTreeNode: Sized { +/// A node of an expression tree that [`DisplayTreeExpr`] can render. +pub trait DisplayTreeNode: Sized { + /// The children of this node, in argument order. fn tree_children(&self) -> &[Self]; + /// The display name of the child at `index`. fn tree_child_name(&self, index: usize) -> ChildName; + /// Write this node's own label. fn fmt_tree_node(&self, f: &mut Formatter<'_>) -> fmt::Result; } /// Tree-display label for the scope root. -const ROOT_DISPLAY: &str = "vortex.root()"; - -impl DisplayTreeNode for Expression { - fn tree_children(&self) -> &[Self] { - Expression::children(self) - } - - fn tree_child_name(&self, index: usize) -> ChildName { - match self { - Expression::Scalar { scalar_fn, .. } => scalar_fn.signature().child_name(index), - Expression::Root => unreachable!("the scope root has no children"), - Expression::Variable(_) => unreachable!("a variable has no children"), - } - } - - fn fmt_tree_node(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - Expression::Scalar { scalar_fn, .. } => Display::fmt(scalar_fn, f), - Expression::Root => write!(f, "{ROOT_DISPLAY}"), - Expression::Variable(name) => write!(f, "@{name}"), - } - } -} +pub const ROOT_DISPLAY: &str = "vortex.root()"; impl DisplayTreeNode for BoundExpression { fn tree_children(&self) -> &[Self] { @@ -102,7 +48,8 @@ impl DisplayTreeNode for BoundExpression { } } -pub struct DisplayTreeExpr<'a, T: ?Sized = Expression>(pub &'a T); +/// Renders an expression tree with one node per line. +pub struct DisplayTreeExpr<'a, T: ?Sized>(pub &'a T); impl TreeDisplayAdapter for DisplayTreeExpr<'_, T> { type Context = (); @@ -136,203 +83,3 @@ impl Display for DisplayTreeExpr<'_, T> { write_branch_tree(self, self.0, &mut (), f) } } - -#[cfg(test)] -mod tests { - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::expr::and; - use crate::expr::between; - use crate::expr::cast; - use crate::expr::eq; - use crate::expr::get_item; - use crate::expr::gt; - use crate::expr::lit; - use crate::expr::not; - use crate::expr::pack; - use crate::expr::root; - use crate::expr::select; - use crate::expr::select_exclude; - use crate::scalar_fn::fns::between::BetweenOptions; - use crate::scalar_fn::fns::between::StrictComparison; - - #[test] - fn test_display_tree_root() { - use insta::assert_snapshot; - let root_expr = root(); - assert_snapshot!(root_expr.display_tree().to_string(), @"vortex.root()"); - } - - #[test] - fn test_display_tree_literal() { - use insta::assert_snapshot; - let lit_expr = lit(42); - assert_snapshot!(lit_expr.display_tree().to_string(), @"vortex.literal(42i32)"); - } - - #[test] - fn test_display_tree_get_item() { - use insta::assert_snapshot; - let get_item_expr = get_item("my_field", root()); - assert_snapshot!(get_item_expr.display_tree().to_string(), @r" - vortex.get_item(my_field) - └── input: vortex.root() - "); - } - - #[test] - fn test_display_tree_binary() { - use insta::assert_snapshot; - let binary_expr = gt(get_item("x", root()), lit(10)); - assert_snapshot!(binary_expr.display_tree().to_string(), @r" - vortex.binary(>) - ├── lhs: vortex.get_item(x) - │ └── input: vortex.root() - └── rhs: vortex.literal(10i32) - "); - } - - #[test] - fn test_display_tree_complex_binary() { - use insta::assert_snapshot; - let complex_binary = and( - eq(get_item("name", root()), lit("alice")), - gt(get_item("age", root()), lit(18)), - ); - assert_snapshot!(complex_binary.display_tree().to_string(), @r#" - vortex.binary(and) - ├── lhs: vortex.binary(=) - │ ├── lhs: vortex.get_item(name) - │ │ └── input: vortex.root() - │ └── rhs: vortex.literal("alice") - └── rhs: vortex.binary(>) - ├── lhs: vortex.get_item(age) - │ └── input: vortex.root() - └── rhs: vortex.literal(18i32) - "#); - } - - #[test] - fn test_display_tree_select() { - use insta::assert_snapshot; - let select_expr = select(["name", "age"], root()); - assert_snapshot!(select_expr.display_tree().to_string(), @r" - vortex.select({name, age}) - └── child: vortex.root() - "); - } - - #[test] - fn test_display_tree_select_exclude() { - use insta::assert_snapshot; - let select_exclude_expr = select_exclude(["internal_id", "metadata"], root()); - assert_snapshot!(select_exclude_expr.display_tree().to_string(), @r" - vortex.select(~{internal_id, metadata}) - └── child: vortex.root() - "); - } - - #[test] - fn test_display_tree_cast() { - use insta::assert_snapshot; - let cast_expr = cast( - get_item("value", root()), - DType::Primitive(PType::I64, Nullability::NonNullable), - ); - assert_snapshot!(cast_expr.display_tree().to_string(), @r" - vortex.cast(i64) - └── input: vortex.get_item(value) - └── input: vortex.root() - "); - } - - #[test] - fn test_display_tree_not() { - use insta::assert_snapshot; - let not_expr = not(eq(get_item("active", root()), lit(true))); - assert_snapshot!(not_expr.display_tree().to_string(), @r" - vortex.not() - └── input: vortex.binary(=) - ├── lhs: vortex.get_item(active) - │ └── input: vortex.root() - └── rhs: vortex.literal(true) - "); - } - - #[test] - fn test_display_tree_between() { - use insta::assert_snapshot; - let between_expr = between( - get_item("score", root()), - lit(0), - lit(100), - BetweenOptions { - lower_strict: StrictComparison::NonStrict, - upper_strict: StrictComparison::NonStrict, - }, - ); - assert_snapshot!(between_expr.display_tree().to_string(), @r" - vortex.between(lower_strict: <=, upper_strict: <=) - ├── array: vortex.get_item(score) - │ └── input: vortex.root() - ├── lower: vortex.literal(0i32) - └── upper: vortex.literal(100i32) - "); - } - - #[test] - fn test_display_tree_nested() { - use insta::assert_snapshot; - let nested_expr = select( - ["result"], - cast( - between( - get_item("score", root()), - lit(50), - lit(100), - BetweenOptions { - lower_strict: StrictComparison::Strict, - upper_strict: StrictComparison::NonStrict, - }, - ), - DType::Bool(Nullability::NonNullable), - ), - ); - assert_snapshot!(nested_expr.display_tree().to_string(), @r" - vortex.select({result}) - └── child: vortex.cast(bool) - └── input: vortex.between(lower_strict: <, upper_strict: <=) - ├── array: vortex.get_item(score) - │ └── input: vortex.root() - ├── lower: vortex.literal(50i32) - └── upper: vortex.literal(100i32) - "); - } - - #[test] - fn test_display_tree_pack() { - use insta::assert_snapshot; - let select_from_pack_expr = select( - ["fizz", "buzz"], - pack( - [ - ("fizz", root()), - ("bar", lit(5)), - ("buzz", eq(lit(42), get_item("answer", root()))), - ], - Nullability::Nullable, - ), - ); - assert_snapshot!(select_from_pack_expr.display_tree().to_string(), @r" - vortex.select({fizz, buzz}) - └── child: vortex.pack(names: [fizz, bar, buzz], nullability: Nullable) - ├── fizz: vortex.root() - ├── bar: vortex.literal(5i32) - └── buzz: vortex.binary(=) - ├── lhs: vortex.literal(42i32) - └── rhs: vortex.get_item(answer) - └── input: vortex.root() - "); - } -} diff --git a/vortex-array/src/expr/exprs.rs b/vortex-array/src/expr/exprs.rs deleted file mode 100644 index 8634ae901b3..00000000000 --- a/vortex-array/src/expr/exprs.rs +++ /dev/null @@ -1,1471 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Factory functions for creating [`Expression`]s from scalar function vtables. - -use std::sync::Arc; - -use vortex_error::VortexExpect; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_error::vortex_panic; -use vortex_utils::iter::ReduceBalancedIterExt; - -use crate::aggregate_fn::NumericalAggregateOpts; -use crate::dtype::DType; -use crate::dtype::FieldName; -use crate::dtype::FieldNames; -use crate::dtype::Nullability; -use crate::expr::BoundExpression; -use crate::expr::Expression; -use crate::scalar::Scalar; -use crate::scalar::ScalarValue; -use crate::scalar_fn::EmptyOptions; -use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::between::Between; -use crate::scalar_fn::fns::between::BetweenOptions; -use crate::scalar_fn::fns::between::StrictComparison; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::byte_length::ByteLength; -use crate::scalar_fn::fns::case_when::CaseWhen; -use crate::scalar_fn::fns::case_when::CaseWhenOptions; -use crate::scalar_fn::fns::cast::Cast; -use crate::scalar_fn::fns::dynamic::DynamicComparison; -use crate::scalar_fn::fns::dynamic::DynamicComparisonExpr; -use crate::scalar_fn::fns::dynamic::Rhs; -use crate::scalar_fn::fns::ext_storage::ExtStorage; -use crate::scalar_fn::fns::fill_null::FillNull; -use crate::scalar_fn::fns::get_item::GetItem; -use crate::scalar_fn::fns::is_nan::IsNan; -use crate::scalar_fn::fns::is_not_null::IsNotNull; -use crate::scalar_fn::fns::is_null::IsNull; -use crate::scalar_fn::fns::like::Like; -use crate::scalar_fn::fns::like::LikeOptions; -use crate::scalar_fn::fns::list_contains::ListContains; -use crate::scalar_fn::fns::list_contains::ListContainsOptions; -use crate::scalar_fn::fns::list_length::ListLength; -use crate::scalar_fn::fns::list_sum::ListSum; -use crate::scalar_fn::fns::literal::Literal; -use crate::scalar_fn::fns::mask::Mask; -use crate::scalar_fn::fns::merge::DuplicateHandling; -use crate::scalar_fn::fns::merge::Merge; -use crate::scalar_fn::fns::not::Not; -use crate::scalar_fn::fns::operators::CompareOperator; -use crate::scalar_fn::fns::operators::Operator; -use crate::scalar_fn::fns::pack::Pack; -use crate::scalar_fn::fns::pack::PackOptions; -use crate::scalar_fn::fns::replace_time_zone::ReplaceTimeZone; -use crate::scalar_fn::fns::replace_time_zone::ReplaceTimeZoneOptions; -use crate::scalar_fn::fns::select::FieldSelection; -use crate::scalar_fn::fns::select::Select; -use crate::scalar_fn::fns::variant_get::VariantGet; -use crate::scalar_fn::fns::variant_get::VariantGetOptions; -use crate::scalar_fn::fns::variant_get::VariantPath; -use crate::scalar_fn::fns::zip::Zip; - -/// Creates an expression that references the root scope. -/// -/// Returns the entire input array as passed to the expression evaluator. -/// This is commonly used as the starting point for field access and other operations. -pub fn root() -> Expression { - Expression::Root -} - -/// Creates a bound expression that references a root scope with the given dtype. -pub fn bound_root(dtype: DType) -> BoundExpression { - BoundExpression::new_root(dtype) -} - -/// Return whether the expression is a root expression. -pub fn is_root(expr: &Expression) -> bool { - expr.is_root() -} - -// ---- Literal ---- - -/// Create a new `Literal` expression from a type that coerces to `Scalar`. -/// -/// -/// ## Example usage -/// -/// ``` -/// use vortex_array::arrays::PrimitiveArray; -/// use vortex_array::dtype::Nullability; -/// use vortex_array::expr::lit; -/// use vortex_array::scalar_fn::fns::literal::Literal; -/// use vortex_array::scalar::Scalar; -/// -/// let number = lit(34i32); -/// -/// let scalar = number.as_::(); -/// assert_eq!(scalar, &Scalar::primitive(34i32, Nullability::NonNullable)); -/// ``` -pub fn lit(value: impl Into) -> Expression { - Literal.new_expr(value.into(), []) -} - -/// Creates a bound literal expression. -pub fn bound_lit(value: impl Into) -> BoundExpression { - Literal - .try_new_bound_expr(value.into(), []) - .vortex_expect("literal expressions are always well-typed") -} - -// ---- GetItem / Col ---- - -/// Creates an expression that accesses a field from the root array. -/// -/// Equivalent to `get_item(field, root())` - extracts a named field from the input array. -/// -/// ```rust -/// # use vortex_array::expr::col; -/// let expr = col("name"); -/// ``` -pub fn col(field: impl Into) -> Expression { - GetItem.new_expr(field.into(), vec![root()]) -} - -/// Creates a bound expression that accesses a field from a root scope with the given dtype. -pub fn bound_col(field: impl Into, scope: DType) -> BoundExpression { - bound_get_item(field, bound_root(scope)) -} - -/// Creates an expression that extracts a named field from a struct expression. -/// -/// Accesses the specified field from the result of the child expression. -/// -/// ```rust -/// # use vortex_array::expr::{get_item, root}; -/// let expr = get_item("user_id", root()); -/// ``` -pub fn get_item(field: impl Into, child: Expression) -> Expression { - GetItem.new_expr(field.into(), vec![child]) -} - -/// Creates a bound expression that extracts a named field from a struct expression. -pub fn bound_get_item(field: impl Into, child: BoundExpression) -> BoundExpression { - GetItem - .try_new_bound_expr(field.into(), [child]) - .vortex_expect("get-item expressions must reference a field in the child dtype") -} - -// ---- Variable ---- - -/// Creates an expression that reads the parameter `name` of the enclosing lambda. -/// -/// It is only meaningful in the body of a [`Lambda`](crate::expr::Lambda). Anywhere else it fails -/// to bind. -/// -/// ```rust -/// # use vortex_array::dtype::{DType, Nullability, PType}; -/// # use vortex_array::expr::{checked_add, lit, var}; -/// let expr = checked_add(var("x"), lit(1_i32)); -/// assert_eq!(expr.to_string(), "(@x + 1i32)"); -/// // Outside a lambda body, nothing gives `x` a value. -/// assert!(expr.bind(&DType::Primitive(PType::I32, Nullability::NonNullable)).is_err()); -/// ``` -pub fn var(name: impl Into) -> Expression { - Expression::Variable(name.into()) -} - -// ---- VariantGet ---- - -/// Creates an expression that extracts a path from a Variant expression. -/// -/// Missing paths, traversal mismatches, and failed casts return null. When `dtype` is `None`, -/// results are nullable Variant values; otherwise results are nullable values of `dtype`. -pub fn variant_get( - child: Expression, - path: impl Into, - dtype: Option, -) -> Expression { - VariantGet.new_expr(VariantGetOptions::new(path.into(), dtype), vec![child]) -} - -/// Creates a bound expression that extracts a path from a Variant expression. -pub fn bound_variant_get( - child: BoundExpression, - path: impl Into, - dtype: Option, -) -> BoundExpression { - VariantGet - .try_new_bound_expr(VariantGetOptions::new(path.into(), dtype), [child]) - .vortex_expect("variant-get expressions require a Variant child") -} - -// ---- CaseWhen ---- - -/// Creates a CASE WHEN expression with one WHEN/THEN pair and an ELSE value. -pub fn case_when( - condition: Expression, - then_value: Expression, - else_value: Expression, -) -> Expression { - let options = CaseWhenOptions { - num_when_then_pairs: 1, - has_else: true, - }; - CaseWhen.new_expr(options, [condition, then_value, else_value]) -} - -/// Creates a bound CASE WHEN expression with one WHEN/THEN pair and an ELSE value. -pub fn bound_case_when( - condition: BoundExpression, - then_value: BoundExpression, - else_value: BoundExpression, -) -> BoundExpression { - let options = CaseWhenOptions { - num_when_then_pairs: 1, - has_else: true, - }; - CaseWhen - .try_new_bound_expr(options, [condition, then_value, else_value]) - .vortex_expect("case expressions must have boolean conditions and matching branch dtypes") -} - -/// Creates a CASE WHEN expression with one WHEN/THEN pair and no ELSE value. -pub fn case_when_no_else(condition: Expression, then_value: Expression) -> Expression { - let options = CaseWhenOptions { - num_when_then_pairs: 1, - has_else: false, - }; - CaseWhen.new_expr(options, [condition, then_value]) -} - -/// Creates a bound CASE WHEN expression with one WHEN/THEN pair and no ELSE value. -pub fn bound_case_when_no_else( - condition: BoundExpression, - then_value: BoundExpression, -) -> BoundExpression { - let options = CaseWhenOptions { - num_when_then_pairs: 1, - has_else: false, - }; - CaseWhen - .try_new_bound_expr(options, [condition, then_value]) - .vortex_expect("case expressions must have boolean conditions") -} - -/// Creates an n-ary CASE WHEN expression from WHEN/THEN pairs and an optional ELSE value. -pub fn nested_case_when( - when_then_pairs: Vec<(Expression, Expression)>, - else_value: Option, -) -> Expression { - assert!( - !when_then_pairs.is_empty(), - "nested_case_when requires at least one when/then pair" - ); - - let has_else = else_value.is_some(); - let mut children = Vec::with_capacity(when_then_pairs.len() * 2 + usize::from(has_else)); - for (condition, then_value) in &when_then_pairs { - children.push(condition.clone()); - children.push(then_value.clone()); - } - if let Some(else_expr) = else_value { - children.push(else_expr); - } - - let Ok(num_when_then_pairs) = u32::try_from(when_then_pairs.len()) else { - vortex_panic!("nested_case_when has too many when/then pairs"); - }; - let options = CaseWhenOptions { - num_when_then_pairs, - has_else, - }; - CaseWhen.new_expr(options, children) -} - -/// Creates a bound n-ary CASE WHEN expression from WHEN/THEN pairs and an optional ELSE value. -pub fn bound_nested_case_when( - when_then_pairs: Vec<(BoundExpression, BoundExpression)>, - else_value: Option, -) -> BoundExpression { - assert!( - !when_then_pairs.is_empty(), - "nested_case_when requires at least one when/then pair" - ); - - let Ok(num_when_then_pairs) = u32::try_from(when_then_pairs.len()) else { - vortex_panic!("nested_case_when has too many when/then pairs"); - }; - let has_else = else_value.is_some(); - let mut children = Vec::with_capacity(when_then_pairs.len() * 2 + usize::from(has_else)); - for (condition, then_value) in when_then_pairs { - children.push(condition); - children.push(then_value); - } - if let Some(else_expr) = else_value { - children.push(else_expr); - } - - let options = CaseWhenOptions { - num_when_then_pairs, - has_else, - }; - CaseWhen - .try_new_bound_expr(options, children) - .vortex_expect("case expressions must have boolean conditions and matching branch dtypes") -} - -// ---- Binary operators ---- - -/// Creates a binary expression with the given operator. -pub fn binary(operator: Operator, lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(operator, [lhs, rhs]) - .vortex_expect("Failed to create binary expression") -} - -/// Creates a bound binary expression with the given operator. -pub fn bound_binary( - operator: Operator, - lhs: BoundExpression, - rhs: BoundExpression, -) -> BoundExpression { - Binary - .try_new_bound_expr(operator, [lhs, rhs]) - .vortex_expect("binary expressions must have compatible operand dtypes") -} - -/// Create a new [`Binary`] using the [`Eq`](Operator::Eq) operator. -/// -/// ## Example usage -/// -/// ``` -/// # use vortex_array::arrays::{BoolArray, PrimitiveArray}; -/// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::IntoArray; -/// # use vortex_array::{VortexSessionExecute}; -/// # use vortex_session::VortexSession; -/// # use vortex_array::validity::Validity; -/// # use vortex_buffer::buffer; -/// # use vortex_array::expr::{eq, root, lit}; -/// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); -/// let result = xs.into_array().apply(&eq(root(), lit(3))).unwrap(); -/// let mut ctx = VortexSession::empty().create_execution_ctx(); -/// -/// assert_eq!( -/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), -/// BoolArray::from_iter(vec![false, false, true]).to_bit_buffer(), -/// ); -/// ``` -pub fn eq(lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(Operator::Eq, [lhs, rhs]) - .vortex_expect("Failed to create Eq binary expression") -} - -/// Creates a bound equality expression. -pub fn bound_eq(lhs: BoundExpression, rhs: BoundExpression) -> BoundExpression { - bound_binary(Operator::Eq, lhs, rhs) -} - -/// Create a new [`Binary`] using the [`NotEq`](Operator::NotEq) operator. -/// -/// ## Example usage -/// -/// ``` -/// # use vortex_array::arrays::{BoolArray, PrimitiveArray}; -/// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::IntoArray; -/// # use vortex_array::{VortexSessionExecute}; -/// # use vortex_session::VortexSession; -/// # use vortex_array::validity::Validity; -/// # use vortex_buffer::buffer; -/// # use vortex_array::expr::{root, lit, not_eq}; -/// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); -/// let result = xs.into_array().apply(¬_eq(root(), lit(3))).unwrap(); -/// let mut ctx = VortexSession::empty().create_execution_ctx(); -/// -/// assert_eq!( -/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), -/// BoolArray::from_iter(vec![true, true, false]).to_bit_buffer(), -/// ); -/// ``` -pub fn not_eq(lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(Operator::NotEq, [lhs, rhs]) - .vortex_expect("Failed to create NotEq binary expression") -} - -/// Creates a bound inequality expression. -pub fn bound_not_eq(lhs: BoundExpression, rhs: BoundExpression) -> BoundExpression { - bound_binary(Operator::NotEq, lhs, rhs) -} - -/// Create a new [`Binary`] using the [`Gte`](Operator::Gte) operator. -/// -/// ## Example usage -/// -/// ``` -/// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; -/// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::IntoArray; -/// # use vortex_array::{VortexSessionExecute}; -/// # use vortex_session::VortexSession; -/// # use vortex_array::validity::Validity; -/// # use vortex_buffer::buffer; -/// # use vortex_array::expr::{gt_eq, root, lit}; -/// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); -/// let result = xs.into_array().apply(>_eq(root(), lit(3))).unwrap(); -/// let mut ctx = VortexSession::empty().create_execution_ctx(); -/// -/// assert_eq!( -/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), -/// BoolArray::from_iter(vec![false, false, true]).to_bit_buffer(), -/// ); -/// ``` -pub fn gt_eq(lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(Operator::Gte, [lhs, rhs]) - .vortex_expect("Failed to create Gte binary expression") -} - -/// Creates a bound greater-than-or-equal expression. -pub fn bound_gt_eq(lhs: BoundExpression, rhs: BoundExpression) -> BoundExpression { - bound_binary(Operator::Gte, lhs, rhs) -} - -/// Create a new [`Binary`] using the [`Gt`](Operator::Gt) operator. -/// -/// ## Example usage -/// -/// ``` -/// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; -/// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::IntoArray; -/// # use vortex_array::{VortexSessionExecute}; -/// # use vortex_session::VortexSession; -/// # use vortex_array::validity::Validity; -/// # use vortex_buffer::buffer; -/// # use vortex_array::expr::{gt, root, lit}; -/// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); -/// let result = xs.into_array().apply(>(root(), lit(2))).unwrap(); -/// let mut ctx = VortexSession::empty().create_execution_ctx(); -/// -/// assert_eq!( -/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), -/// BoolArray::from_iter(vec![false, false, true]).to_bit_buffer(), -/// ); -/// ``` -pub fn gt(lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(Operator::Gt, [lhs, rhs]) - .vortex_expect("Failed to create Gt binary expression") -} - -/// Creates a bound greater-than expression. -pub fn bound_gt(lhs: BoundExpression, rhs: BoundExpression) -> BoundExpression { - bound_binary(Operator::Gt, lhs, rhs) -} - -/// Create a new [`Binary`] using the [`Lte`](Operator::Lte) operator. -/// -/// ## Example usage -/// -/// ``` -/// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; -/// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::IntoArray; -/// # use vortex_array::{VortexSessionExecute}; -/// # use vortex_session::VortexSession; -/// # use vortex_array::validity::Validity; -/// # use vortex_buffer::buffer; -/// # use vortex_array::expr::{root, lit, lt_eq}; -/// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); -/// let result = xs.into_array().apply(<_eq(root(), lit(2))).unwrap(); -/// let mut ctx = VortexSession::empty().create_execution_ctx(); -/// -/// assert_eq!( -/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), -/// BoolArray::from_iter(vec![true, true, false]).to_bit_buffer(), -/// ); -/// ``` -pub fn lt_eq(lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(Operator::Lte, [lhs, rhs]) - .vortex_expect("Failed to create Lte binary expression") -} - -/// Creates a bound less-than-or-equal expression. -pub fn bound_lt_eq(lhs: BoundExpression, rhs: BoundExpression) -> BoundExpression { - bound_binary(Operator::Lte, lhs, rhs) -} - -/// Create a new [`Binary`] using the [`Lt`](Operator::Lt) operator. -/// -/// ## Example usage -/// -/// ``` -/// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; -/// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::IntoArray; -/// # use vortex_array::{VortexSessionExecute}; -/// # use vortex_session::VortexSession; -/// # use vortex_array::validity::Validity; -/// # use vortex_buffer::buffer; -/// # use vortex_array::expr::{root, lit, lt}; -/// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); -/// let result = xs.into_array().apply(<(root(), lit(3))).unwrap(); -/// let mut ctx = VortexSession::empty().create_execution_ctx(); -/// -/// assert_eq!( -/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), -/// BoolArray::from_iter(vec![true, true, false]).to_bit_buffer(), -/// ); -/// ``` -pub fn lt(lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(Operator::Lt, [lhs, rhs]) - .vortex_expect("Failed to create Lt binary expression") -} - -/// Creates a bound less-than expression. -pub fn bound_lt(lhs: BoundExpression, rhs: BoundExpression) -> BoundExpression { - bound_binary(Operator::Lt, lhs, rhs) -} - -/// Create a new [`Binary`] using the [`Or`](Operator::Or) operator. -/// -/// ## Example usage -/// -/// ``` -/// # use vortex_array::arrays::BoolArray; -/// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::IntoArray; -/// # use vortex_array::{VortexSessionExecute}; -/// # use vortex_session::VortexSession; -/// # use vortex_array::expr::{root, lit, or}; -/// let xs = BoolArray::from_iter(vec![true, false, true]); -/// let result = xs.into_array().apply(&or(root(), lit(false))).unwrap(); -/// let mut ctx = VortexSession::empty().create_execution_ctx(); -/// -/// assert_eq!( -/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), -/// BoolArray::from_iter(vec![true, false, true]).to_bit_buffer(), -/// ); -/// ``` -pub fn or(lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(Operator::Or, [lhs, rhs]) - .vortex_expect("Failed to create Or binary expression") -} - -/// Creates a bound boolean OR expression. -pub fn bound_or(lhs: BoundExpression, rhs: BoundExpression) -> BoundExpression { - bound_binary(Operator::Or, lhs, rhs) -} - -/// Collects a list of `or`ed values into a single expression using a balanced tree. -/// -/// This creates a balanced binary tree to avoid deep nesting that could cause -/// stack overflow during drop or evaluation. -/// -/// [a, b, c, d] => or(or(a, b), or(c, d)) -pub fn or_collect(iter: I) -> Option -where - I: IntoIterator, -{ - iter.into_iter().reduce_balanced(or) -} - -/// Collects bound expressions into a balanced tree of boolean OR expressions. -pub fn bound_or_collect(iter: I) -> Option -where - I: IntoIterator, -{ - iter.into_iter().reduce_balanced(bound_or) -} - -/// Create a new [`Binary`] using the [`And`](Operator::And) operator. -/// -/// ## Example usage -/// -/// ``` -/// # use vortex_array::arrays::BoolArray; -/// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::IntoArray; -/// # use vortex_array::{VortexSessionExecute}; -/// # use vortex_session::VortexSession; -/// # use vortex_array::expr::{and, root, lit}; -/// let xs = BoolArray::from_iter(vec![true, false, true]).into_array(); -/// let result = xs.apply(&and(root(), lit(true))).unwrap(); -/// let mut ctx = VortexSession::empty().create_execution_ctx(); -/// -/// assert_eq!( -/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), -/// BoolArray::from_iter(vec![true, false, true]).to_bit_buffer(), -/// ); -/// ``` -pub fn and(lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(Operator::And, [lhs, rhs]) - .vortex_expect("Failed to create And binary expression") -} - -/// Creates a bound boolean AND expression. -pub fn bound_and(lhs: BoundExpression, rhs: BoundExpression) -> BoundExpression { - bound_binary(Operator::And, lhs, rhs) -} - -/// Collects a list of `and`ed values into a single expression using a balanced tree. -/// -/// This creates a balanced binary tree to avoid deep nesting that could cause -/// stack overflow during drop or evaluation. -/// -/// [a, b, c, d] => and(and(a, b), and(c, d)) -pub fn and_collect(iter: I) -> Option -where - I: IntoIterator, -{ - iter.into_iter().reduce_balanced(and) -} - -/// Collects bound expressions into a balanced tree of boolean AND expressions. -pub fn bound_and_collect(iter: I) -> Option -where - I: IntoIterator, -{ - iter.into_iter().reduce_balanced(bound_and) -} - -/// Create a new [`Binary`] using the [`Add`](Operator::Add) operator. -/// -/// ## Example usage -/// -/// ``` -/// # use vortex_array::IntoArray; -/// # use vortex_array::arrays::PrimitiveArray; -/// # use vortex_array::builtins::ArrayBuiltins; -/// # use vortex_array::{VortexSessionExecute}; -/// # use vortex_session::VortexSession; -/// # use vortex_buffer::buffer; -/// # use vortex_array::expr::{checked_add, lit, root}; -/// let xs = buffer![1, 2, 3].into_array(); -/// let result = xs.apply(&checked_add(root(), lit(5))).unwrap(); -/// -/// let mut ctx = VortexSession::empty().create_execution_ctx(); -/// let result = result.execute::(&mut ctx).unwrap(); -/// assert_eq!(result.as_slice::(), [6, 7, 8]); -/// ``` -pub fn checked_add(lhs: Expression, rhs: Expression) -> Expression { - Binary - .try_new_expr(Operator::Add, [lhs, rhs]) - .vortex_expect("Failed to create Add binary expression") -} - -/// Creates a bound checked-add expression. -pub fn bound_checked_add(lhs: BoundExpression, rhs: BoundExpression) -> BoundExpression { - bound_binary(Operator::Add, lhs, rhs) -} - -// ---- Not ---- - -/// Creates an expression that logically inverts boolean values. -/// -/// Returns the logical negation of the input boolean expression. -/// -/// ```rust -/// # use vortex_array::expr::{not, root}; -/// let expr = not(root()); -/// ``` -pub fn not(operand: Expression) -> Expression { - Not.new_expr(EmptyOptions, vec![operand]) -} - -/// Creates a bound expression that logically inverts boolean values. -pub fn bound_not(operand: BoundExpression) -> BoundExpression { - Not.try_new_bound_expr(EmptyOptions, [operand]) - .vortex_expect("not expressions require a boolean operand") -} - -// ---- Between ---- - -/// Creates an expression that checks if values are between two bounds. -/// -/// Returns a boolean array indicating which values fall within the specified range. -/// The comparison strictness is controlled by the options parameter. -/// -/// ```rust -/// # use vortex_array::scalar_fn::fns::between::BetweenOptions; -/// # use vortex_array::scalar_fn::fns::between::StrictComparison; -/// # use vortex_array::expr::{between, lit, root}; -/// let opts = BetweenOptions { -/// lower_strict: StrictComparison::NonStrict, -/// upper_strict: StrictComparison::NonStrict, -/// }; -/// let expr = between(root(), lit(10), lit(20), opts); -/// ``` -pub fn between( - arr: Expression, - lower: Expression, - upper: Expression, - options: BetweenOptions, -) -> Expression { - Between - .try_new_expr(options, [arr, lower, upper]) - .vortex_expect("Failed to create Between expression") -} - -/// Creates a bound expression that checks if values are between two bounds. -pub fn bound_between( - arr: BoundExpression, - lower: BoundExpression, - upper: BoundExpression, - options: BetweenOptions, -) -> BoundExpression { - Between - .try_new_bound_expr(options, [arr, lower, upper]) - .vortex_expect("between expressions require compatible operand dtypes") -} - -// ---- Select ---- - -/// Creates an expression that selects (includes) specific fields from an array. -/// -/// Projects only the specified fields from the child expression, which must be of DType struct. -/// ```rust -/// # use vortex_array::expr::{select, root}; -/// let expr = select(["name", "age"], root()); -/// ``` -pub fn select(field_names: impl Into, child: Expression) -> Expression { - Select - .try_new_expr(FieldSelection::Include(field_names.into()), [child]) - .vortex_expect("Failed to create Select expression") -} - -/// Creates a bound expression that selects specific fields from a struct expression. -pub fn bound_select(field_names: impl Into, child: BoundExpression) -> BoundExpression { - Select - .try_new_bound_expr(FieldSelection::Include(field_names.into()), [child]) - .vortex_expect("select expressions require fields from a struct child") -} - -/// Creates an expression that excludes specific fields from an array. -/// -/// Projects all fields except the specified ones from the input struct expression. -/// -/// ```rust -/// # use vortex_array::expr::{select_exclude, root}; -/// let expr = select_exclude(["internal_id", "metadata"], root()); -/// ``` -pub fn select_exclude(fields: impl Into, child: Expression) -> Expression { - Select - .try_new_expr(FieldSelection::Exclude(fields.into()), [child]) - .vortex_expect("Failed to create Select expression") -} - -/// Creates a bound expression that excludes specific fields from a struct expression. -pub fn bound_select_exclude( - fields: impl Into, - child: BoundExpression, -) -> BoundExpression { - Select - .try_new_bound_expr(FieldSelection::Exclude(fields.into()), [child]) - .vortex_expect("select expressions require fields from a struct child") -} - -// ---- Pack ---- - -/// Creates an expression that packs values into a struct with named fields. -/// -/// ```rust -/// # use vortex_array::dtype::Nullability; -/// # use vortex_array::expr::{pack, col, lit}; -/// let expr = pack([("id", col("user_id")), ("constant", lit(42))], Nullability::NonNullable); -/// ``` -pub fn pack( - elements: impl IntoIterator, Expression)>, - nullability: Nullability, -) -> Expression { - let (names, values): (Vec<_>, Vec<_>) = elements - .into_iter() - .map(|(name, value)| (name.into(), value)) - .unzip(); - Pack.new_expr( - PackOptions { - names: names.into(), - nullability, - }, - values, - ) -} - -/// Creates a bound expression that packs values into a struct with named fields. -pub fn bound_pack( - elements: impl IntoIterator, BoundExpression)>, - nullability: Nullability, -) -> BoundExpression { - let (names, values): (Vec<_>, Vec<_>) = elements - .into_iter() - .map(|(name, value)| (name.into(), value)) - .unzip(); - Pack.try_new_bound_expr( - PackOptions { - names: names.into(), - nullability, - }, - values, - ) - .vortex_expect("pack expressions must have one name per child") -} - -// ---- Cast ---- - -/// Creates an expression that casts values to a target data type. -/// -/// Converts the input expression's values to the specified target type. -/// -/// ```rust -/// # use vortex_array::dtype::{DType, Nullability, PType}; -/// # use vortex_array::expr::{cast, root}; -/// let expr = cast(root(), DType::Primitive(PType::I64, Nullability::NonNullable)); -/// ``` -pub fn cast(child: Expression, target: DType) -> Expression { - Cast.try_new_expr(target, [child]) - .vortex_expect("Failed to create Cast expression") -} - -/// Creates a bound expression that casts values to a target dtype. -pub fn bound_cast(child: BoundExpression, target: DType) -> BoundExpression { - Cast.try_new_bound_expr(target, [child]) - .vortex_expect("cast expressions require a supported source and target dtype") -} - -// ---- FillNull ---- - -/// Creates an expression that replaces null values with a fill value. -/// -/// ```rust -/// # use vortex_array::expr::{fill_null, root, lit}; -/// let expr = fill_null(root(), lit(0i32)); -/// ``` -pub fn fill_null(child: Expression, fill_value: Expression) -> Expression { - FillNull.new_expr(EmptyOptions, [child, fill_value]) -} - -/// Creates a bound expression that replaces null values with a fill value. -pub fn bound_fill_null(child: BoundExpression, fill_value: BoundExpression) -> BoundExpression { - FillNull - .try_new_bound_expr(EmptyOptions, [child, fill_value]) - .vortex_expect("fill-null expressions require compatible child and fill dtypes") -} - -// ---- IsNull ---- - -/// Creates an expression that checks for null values. -/// -/// Returns a boolean array indicating which positions contain null values. -/// -/// ```rust -/// # use vortex_array::expr::{is_null, root}; -/// let expr = is_null(root()); -/// ``` -pub fn is_null(child: Expression) -> Expression { - IsNull.new_expr(EmptyOptions, vec![child]) -} - -/// Creates a bound expression that checks for null values. -pub fn bound_is_null(child: BoundExpression) -> BoundExpression { - IsNull - .try_new_bound_expr(EmptyOptions, [child]) - .vortex_expect("is-null expressions are always well-typed") -} - -// ---- IsNan ---- - -/// Creates an expression that checks for NaN values. -/// -/// The expression is strict: null inputs produce null outputs, so the output nullability -/// follows the input. Only primitive float inputs are supported. -/// -/// ```rust -/// # use vortex_array::expr::{is_nan, root}; -/// let expr = is_nan(root()); -/// ``` -pub fn is_nan(child: Expression) -> Expression { - IsNan.new_expr(EmptyOptions, vec![child]) -} - -/// Creates a bound expression that checks for NaN values. -pub fn bound_is_nan(child: BoundExpression) -> BoundExpression { - IsNan - .try_new_bound_expr(EmptyOptions, [child]) - .vortex_expect("is-nan expressions are always well-typed") -} - -// ---- IsNotNull ---- - -/// Creates an expression that checks for non-null values. -/// -/// Returns a boolean array indicating which positions contain non-null values. -/// -/// ```rust -/// # use vortex_array::expr::{is_not_null, root}; -/// let expr = is_not_null(root()); -/// ``` -pub fn is_not_null(child: Expression) -> Expression { - IsNotNull.new_expr(EmptyOptions, vec![child]) -} - -/// Creates a bound expression that checks for non-null values. -pub fn bound_is_not_null(child: BoundExpression) -> BoundExpression { - IsNotNull - .try_new_bound_expr(EmptyOptions, [child]) - .vortex_expect("is-not-null expressions are always well-typed") -} - -// ---- Like ---- - -/// Creates a SQL LIKE expression. -pub fn like(child: Expression, pattern: Expression) -> Expression { - Like.new_expr( - LikeOptions { - negated: false, - case_insensitive: false, - }, - [child, pattern], - ) -} - -/// Creates a bound SQL LIKE expression. -pub fn bound_like(child: BoundExpression, pattern: BoundExpression) -> BoundExpression { - bound_like_with_options(child, pattern, false, false) -} - -/// Creates a case-insensitive SQL ILIKE expression. -pub fn ilike(child: Expression, pattern: Expression) -> Expression { - Like.new_expr( - LikeOptions { - negated: false, - case_insensitive: true, - }, - [child, pattern], - ) -} - -/// Creates a bound case-insensitive SQL ILIKE expression. -pub fn bound_ilike(child: BoundExpression, pattern: BoundExpression) -> BoundExpression { - bound_like_with_options(child, pattern, false, true) -} - -/// Creates a negated SQL NOT LIKE expression. -pub fn not_like(child: Expression, pattern: Expression) -> Expression { - Like.new_expr( - LikeOptions { - negated: true, - case_insensitive: false, - }, - [child, pattern], - ) -} - -/// Creates a bound negated SQL NOT LIKE expression. -pub fn bound_not_like(child: BoundExpression, pattern: BoundExpression) -> BoundExpression { - bound_like_with_options(child, pattern, true, false) -} - -/// Creates a negated case-insensitive SQL NOT ILIKE expression. -pub fn not_ilike(child: Expression, pattern: Expression) -> Expression { - Like.new_expr( - LikeOptions { - negated: true, - case_insensitive: true, - }, - [child, pattern], - ) -} - -/// Creates a bound negated case-insensitive SQL NOT ILIKE expression. -pub fn bound_not_ilike(child: BoundExpression, pattern: BoundExpression) -> BoundExpression { - bound_like_with_options(child, pattern, true, true) -} - -fn bound_like_with_options( - child: BoundExpression, - pattern: BoundExpression, - negated: bool, - case_insensitive: bool, -) -> BoundExpression { - Like.try_new_bound_expr( - LikeOptions { - negated, - case_insensitive, - }, - [child, pattern], - ) - .vortex_expect("like expressions require UTF-8 or binary operands") -} - -// ---- Mask ---- - -/// Creates a mask expression that applies the given boolean mask to the input array. -pub fn mask(array: Expression, mask: Expression) -> Expression { - Mask.new_expr(EmptyOptions, [array, mask]) -} - -/// Creates a bound mask expression. -pub fn bound_mask(array: BoundExpression, mask: BoundExpression) -> BoundExpression { - Mask.try_new_bound_expr(EmptyOptions, [array, mask]) - .vortex_expect("mask expressions require a boolean mask") -} - -// ---- Merge ---- - -/// Creates an expression that merges struct expressions into a single struct. -/// -/// Combines fields from all input expressions. If field names are duplicated, -/// later expressions win. Fields are not recursively merged. -/// -/// ```rust -/// # use vortex_array::dtype::Nullability; -/// # use vortex_array::expr::{merge, get_item, root}; -/// let expr = merge([get_item("a", root()), get_item("b", root())]); -/// ``` -pub fn merge(elements: impl IntoIterator>) -> Expression { - use itertools::Itertools as _; - let values = elements.into_iter().map(|value| value.into()).collect_vec(); - Merge.new_expr(DuplicateHandling::default(), values) -} - -/// Creates a bound expression that merges struct expressions. -pub fn bound_merge(elements: impl IntoIterator) -> BoundExpression { - bound_merge_opts(elements, DuplicateHandling::default()) -} - -/// Creates a merge expression with explicit duplicate handling. -pub fn merge_opts( - elements: impl IntoIterator>, - duplicate_handling: DuplicateHandling, -) -> Expression { - use itertools::Itertools as _; - let values = elements.into_iter().map(|value| value.into()).collect_vec(); - Merge.new_expr(duplicate_handling, values) -} - -/// Creates a bound merge expression with explicit duplicate handling. -pub fn bound_merge_opts( - elements: impl IntoIterator, - duplicate_handling: DuplicateHandling, -) -> BoundExpression { - Merge - .try_new_bound_expr(duplicate_handling, elements) - .vortex_expect("merge expressions require non-nullable struct children") -} - -// ---- Zip ---- - -/// Creates a zip expression that conditionally selects between two arrays. -/// -/// ```rust -/// # use vortex_array::expr::{zip_expr, root, lit}; -/// let expr = zip_expr(lit(true), root(), lit(0i32)); -/// ``` -pub fn zip_expr(mask: Expression, if_true: Expression, if_false: Expression) -> Expression { - Zip.new_expr(EmptyOptions, [if_true, if_false, mask]) -} - -/// Creates a bound zip expression that conditionally selects between two arrays. -pub fn bound_zip_expr( - mask: BoundExpression, - if_true: BoundExpression, - if_false: BoundExpression, -) -> BoundExpression { - Zip.try_new_bound_expr(EmptyOptions, [if_true, if_false, mask]) - .vortex_expect("zip expressions require a boolean mask and compatible value dtypes") -} - -// ---- Dynamic ---- - -/// Creates a dynamic comparison expression from its complete options. -pub fn dynamic_with_options(options: DynamicComparisonExpr, lhs: Expression) -> Expression { - DynamicComparison.new_expr(options, [lhs]) -} - -/// Creates a bound dynamic comparison expression from its complete options. -pub fn bound_dynamic_with_options( - options: DynamicComparisonExpr, - lhs: BoundExpression, -) -> BoundExpression { - DynamicComparison - .try_new_bound_expr(options, [lhs]) - .vortex_expect("dynamic comparisons require a compatible left-hand dtype") -} - -/// Creates a dynamic comparison expression. -pub fn dynamic( - operator: CompareOperator, - rhs_value: impl Fn() -> Option + Send + Sync + 'static, - rhs_dtype: DType, - default: bool, - lhs: Expression, -) -> Expression { - dynamic_with_options( - DynamicComparisonExpr { - operator, - rhs: Arc::new(Rhs { - value: Arc::new(rhs_value), - dtype: rhs_dtype, - }), - default, - }, - lhs, - ) -} - -/// Creates a bound dynamic comparison expression. -pub fn bound_dynamic( - operator: CompareOperator, - rhs_value: impl Fn() -> Option + Send + Sync + 'static, - rhs_dtype: DType, - default: bool, - lhs: BoundExpression, -) -> BoundExpression { - bound_dynamic_with_options( - DynamicComparisonExpr { - operator, - rhs: Arc::new(Rhs { - value: Arc::new(rhs_value), - dtype: rhs_dtype, - }), - default, - }, - lhs, - ) -} - -// ---- ListContains ---- - -/// Creates an expression that checks whether a list contains a value. -/// -/// A null element never matches anything: a needle that matches no element yields `false`. For -/// SQL's three-valued `IN`, where a null element makes that `null`, use [`in_list`]. -/// -/// ```rust -/// # use vortex_array::expr::{list_contains, lit, root}; -/// let expr = list_contains(root(), lit(42)); -/// ``` -pub fn list_contains(list: Expression, value: Expression) -> Expression { - list_contains_opts(list, value, ListContainsOptions::default()) -} - -/// Creates an expression that checks whether a list contains a value, with explicit -/// [`ListContainsOptions`]. -pub fn list_contains_opts( - list: Expression, - value: Expression, - options: ListContainsOptions, -) -> Expression { - ListContains.new_expr(options, [list, value]) -} - -/// Creates `needle IN (list)` with SQL null semantics: a null element is an unknown value, so a -/// needle that matches no element yields `null` rather than `false` when the list holds one, and -/// `not(in_list(..))` never admits such a row. -/// -/// ```rust -/// # use vortex_array::expr::{in_list, lit, root}; -/// let expr = in_list(root(), lit(vec![1, 2, 3])); -/// ``` -pub fn in_list(needle: Expression, list: Expression) -> Expression { - list_contains_opts( - list, - needle, - ListContainsOptions { - sql_null_semantics: true, - }, - ) -} - -/// Creates `needle IN (elements)` with SQL null semantics for a list of constant elements of -/// `element_dtype`, ignoring nullability. An element can also be an untyped null. -/// -/// The nullability of `element_dtype` is not used: the list elements are nullable only when a -/// null element is present. -/// -/// Non-null integers that hold every value between their smallest and largest become a range -/// check, which is cheaper to evaluate and prunes with tighter bounds than a set lookup, as for -/// the keys of a small dimension table that a join pushes down. -/// -/// # Errors -/// -/// Returns an error if an element is not a null and not of `element_dtype`. -/// -/// ```rust -/// # use vortex_array::dtype::{DType, Nullability, PType}; -/// # use vortex_array::expr::{in_list_of, root}; -/// let dtype = DType::Primitive(PType::I32, Nullability::NonNullable); -/// let expr = in_list_of(root(), dtype, vec![3i32.into(), 1i32.into(), 2i32.into()])?; -/// assert_eq!(expr.to_string(), "(1i32 <= $ <= 3i32)"); -/// # Ok::<(), vortex_error::VortexError>(()) -/// ``` -pub fn in_list_of( - needle: Expression, - element_dtype: DType, - elements: Vec, -) -> VortexResult { - let element_dtype = if elements.iter().any(Scalar::is_null) { - element_dtype.as_nullable() - } else { - element_dtype.as_nonnullable() - }; - let mut elements = elements - .iter() - .map(|element| { - vortex_ensure!( - element.dtype() == &DType::Null - || element.dtype().eq_ignore_nullability(&element_dtype), - "IN list element {element} is not of type {element_dtype}" - ); - element.cast(&element_dtype) - }) - .collect::>>()?; - - if let Some((low, high)) = contiguous_integer_bounds(&elements) { - let options = BetweenOptions { - lower_strict: StrictComparison::NonStrict, - upper_strict: StrictComparison::NonStrict, - }; - return Ok(between( - needle, - lit(low.clone()), - lit(high.clone()), - options, - )); - } - - // Ordered elements make equal lists from the same set, so that equal filters deduplicate. - if elements - .windows(2) - .all(|pair| pair[0].partial_cmp(&pair[1]).is_some()) - { - elements.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - } - let list = Scalar::list(Arc::new(element_dtype), elements, Nullability::NonNullable); - Ok(in_list(needle, lit(list))) -} - -/// The smallest and largest of non-null integer `scalars` when they hold every integer between -/// those two. -fn contiguous_integer_bounds(scalars: &[Scalar]) -> Option<(&Scalar, &Scalar)> { - let mut values = scalars - .iter() - .map(|scalar| { - let primitive = scalar.as_primitive_opt()?; - if !primitive.ptype().is_int() { - return None; - } - let value = if primitive.ptype().is_unsigned_int() { - primitive.as_opt::()??.into() - } else { - i128::from(primitive.as_opt::()??) - }; - Some((value, scalar)) - }) - .collect::>>()?; - values.sort_unstable_by_key(|(value, _)| *value); - values.dedup_by_key(|(value, _)| *value); - let (low, low_scalar) = values.first()?; - let (high, high_scalar) = values.last()?; - (high - low + 1 == i128::try_from(values.len()).ok()?).then_some((*low_scalar, *high_scalar)) -} - -/// Creates a bound expression that checks whether a list contains a value. -pub fn bound_list_contains(list: BoundExpression, value: BoundExpression) -> BoundExpression { - ListContains - .try_new_bound_expr(ListContainsOptions::default(), [list, value]) - .vortex_expect("list-contains expressions require a list child") -} - -/// Creates a bound `needle IN (list)` with SQL null semantics. See [`in_list`]. -pub fn bound_in_list(needle: BoundExpression, list: BoundExpression) -> BoundExpression { - ListContains - .try_new_bound_expr( - ListContainsOptions { - sql_null_semantics: true, - }, - [list, needle], - ) - .vortex_expect("list-contains expressions require a list child") -} - -// ---- ByteLength ---- - -/// Creates an expression that computes the byte length of each element. -/// This is akin to ANSI SQL OCTET_LENGTH(), or DuckDB's strlen(). -/// -/// ```rust -/// # use vortex_array::expr::{byte_length, root}; -/// let expr = byte_length(root()); -/// ``` -pub fn byte_length(input: Expression) -> Expression { - ByteLength.new_expr(EmptyOptions, [input]) -} - -/// Creates a bound expression that computes each element's byte length. -pub fn bound_byte_length(input: BoundExpression) -> BoundExpression { - ByteLength - .try_new_bound_expr(EmptyOptions, [input]) - .vortex_expect("byte-length expressions require a variable-length binary child") -} - -// ---- ExtStorage ---- - -/// Creates an expression that extracts the storage values from an extension array. -/// -/// ```rust -/// # use vortex_array::expr::{ext_storage, root}; -/// let expr = ext_storage(root()); -/// ``` -pub fn ext_storage(input: Expression) -> Expression { - ExtStorage.new_expr(EmptyOptions, [input]) -} - -/// Creates a bound expression that extracts an extension array's storage values. -pub fn bound_ext_storage(input: BoundExpression) -> BoundExpression { - ExtStorage - .try_new_bound_expr(EmptyOptions, [input]) - .vortex_expect("extension-storage expressions require an extension child") -} - -// ---- ListLength ---- - -/// Creates an expression that computes the number of elements in each list -/// for `List` and `FixedSizeList` inputs. This is akin to ANSI SQL `CARDINALITY()`, -/// or DuckDB's `len()`/`array_length()`. -/// -/// ```rust -/// # use vortex_array::expr::{list_length, root}; -/// let expr = list_length(root()); -/// ``` -pub fn list_length(input: Expression) -> Expression { - ListLength.new_expr(EmptyOptions, [input]) -} - -/// Creates a bound expression that computes the number of elements in each list. -pub fn bound_list_length(input: BoundExpression) -> BoundExpression { - ListLength - .try_new_bound_expr(EmptyOptions, [input]) - .vortex_expect("list-length expressions require a list child") -} - -// ---- ListSum ---- - -/// Creates an expression that sums the elements of each list for `List` and -/// `FixedSizeList` inputs, akin to DuckDB's `list_sum()`. -/// -/// Follows SQL `SUM` semantics per list: null lists, empty lists, and lists whose elements are -/// all null yield null; null elements are skipped; integer and decimal overflow yields a null -/// value. The result dtype follows `sum`'s widening rules and is always nullable. NaN float -/// elements are skipped by default; see [`list_sum_opts`] for the NaN-including variant. -/// -/// ```rust -/// # use vortex_array::expr::{list_sum, root}; -/// let expr = list_sum(root()); -/// ``` -pub fn list_sum(input: Expression) -> Expression { - ListSum.new_expr(NumericalAggregateOpts::default(), [input]) -} - -/// Creates a bound expression that sums the elements of each list. -pub fn bound_list_sum(input: BoundExpression) -> BoundExpression { - ListSum - .try_new_bound_expr(NumericalAggregateOpts::default(), [input]) - .vortex_expect("list-sum expressions require a numeric list child") -} - -/// Creates a [`list_sum`] expression with explicit [`NumericalAggregateOpts`], controlling -/// whether NaN float elements are skipped (the default) or poison the list's sum to NaN. -pub fn list_sum_opts(input: Expression, options: NumericalAggregateOpts) -> Expression { - ListSum.new_expr(options, [input]) -} - -/// Creates a bound list-sum expression with explicit aggregate options. -pub fn bound_list_sum_opts( - input: BoundExpression, - options: NumericalAggregateOpts, -) -> BoundExpression { - ListSum - .try_new_bound_expr(options, [input]) - .vortex_expect("list-sum expressions require a numeric list child") -} - -/// Reinterpret timestamp wall times in `options.time_zone`, preserving the time unit. -/// -/// `ambiguous` is a UTF-8 expression containing `raise`, `earliest`, `latest`, or `null`. -/// Removing the timezone preserves wall time in a timezone-naive timestamp. -pub fn replace_time_zone( - input: Expression, - ambiguous: Expression, - options: ReplaceTimeZoneOptions, -) -> Expression { - ReplaceTimeZone.new_expr(options, [input, ambiguous]) -} - -/// Create a bound timezone replacement expression. -pub fn bound_replace_time_zone( - input: BoundExpression, - ambiguous: BoundExpression, - options: ReplaceTimeZoneOptions, -) -> BoundExpression { - ReplaceTimeZone - .try_new_bound_expr(options, [input, ambiguous]) - .vortex_expect("timezone replacement requires a timestamp and a UTF-8 ambiguity policy") -} - -/// Constructors for expressions whose children have already been bound and type-checked. -/// -/// These mirror the constructors in [`crate::expr`] and panic when the supplied children do not -/// form a well-typed expression. Use [`BoundExpression::try_new`] when construction must be -/// fallible. -pub mod bound { - pub use super::bound_and as and; - pub use super::bound_and_collect as and_collect; - pub use super::bound_between as between; - pub use super::bound_binary as binary; - pub use super::bound_byte_length as byte_length; - pub use super::bound_case_when as case_when; - pub use super::bound_case_when_no_else as case_when_no_else; - pub use super::bound_cast as cast; - pub use super::bound_checked_add as checked_add; - pub use super::bound_col as col; - pub use super::bound_dynamic as dynamic; - pub use super::bound_dynamic_with_options as dynamic_with_options; - pub use super::bound_eq as eq; - pub use super::bound_ext_storage as ext_storage; - pub use super::bound_fill_null as fill_null; - pub use super::bound_get_item as get_item; - pub use super::bound_gt as gt; - pub use super::bound_gt_eq as gt_eq; - pub use super::bound_ilike as ilike; - pub use super::bound_in_list as in_list; - pub use super::bound_is_nan as is_nan; - pub use super::bound_is_not_null as is_not_null; - pub use super::bound_is_null as is_null; - pub use super::bound_like as like; - pub use super::bound_list_contains as list_contains; - pub use super::bound_list_length as list_length; - pub use super::bound_list_sum as list_sum; - pub use super::bound_list_sum_opts as list_sum_opts; - pub use super::bound_lit as lit; - pub use super::bound_lt as lt; - pub use super::bound_lt_eq as lt_eq; - pub use super::bound_mask as mask; - pub use super::bound_merge as merge; - pub use super::bound_merge_opts as merge_opts; - pub use super::bound_nested_case_when as nested_case_when; - pub use super::bound_not as not; - pub use super::bound_not_eq as not_eq; - pub use super::bound_not_ilike as not_ilike; - pub use super::bound_not_like as not_like; - pub use super::bound_or as or; - pub use super::bound_or_collect as or_collect; - pub use super::bound_pack as pack; - pub use super::bound_replace_time_zone as replace_time_zone; - pub use super::bound_root as root; - pub use super::bound_select as select; - pub use super::bound_select_exclude as select_exclude; - pub use super::bound_variant_get as variant_get; - pub use super::bound_zip_expr as zip_expr; -} diff --git a/vortex-array/src/expr/mod.rs b/vortex-array/src/expr/mod.rs index bbdbbfdf01a..a9c7e0de407 100644 --- a/vortex-array/src/expr/mod.rs +++ b/vortex-array/src/expr/mod.rs @@ -1,314 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Vortex's expression language: scalar operations over [arrays](crate::ArrayRef). +//! Bound expression trees and their optimizer, and the statistics vocabulary arrays expose. //! -//! An [`Expression`] is a tree of scalar operations rooted at a scope (see [`root`]). Expressions -//! are the common currency of scans: a scan takes a *filter* expression that resolves to a boolean -//! and a *projection* expression that shapes the output. All expressions are serializable and own -//! their own wire format, so they can be pushed down to remote sources and reconstructed on workers. -//! -//! # Scalar functions -//! -//! Each node references a scalar function defined by a -//! [`ScalarFnVTable`](crate::scalar_fn::ScalarFnVTable). The vtable declares the function signature, -//! properties such as strictness, and the logic that executes it over input arrays. Built-in -//! functions live in [`crate::scalar_fn`]; integration and plugin crates supply additional, -//! use-case-specific functions. -//! -//! # Deferred execution -//! -//! Applying an expression to an array does not compute the result eagerly. Instead it builds a -//! [`ScalarFnArray`](crate::arrays::ScalarFnArray) representing the deferred application, letting -//! downstream encodings push the computation into compressed data, or fuse several expressions -//! together, before any data is materialized. The deferred tree is executed toward canonical form -//! only when a result is actually required. -//! -//! # Type checking -//! -//! Expressions are strictly typed: an input array's dtype must match the function signature exactly, -//! so callers perform any required casts themselves before building the expression. The one -//! relaxation is nullability—for example, equality may compare a `u32` against a `u32?`, but never -//! a `u32` against an `i32`. -//! -//! Scans decompose bound filter expressions into independent conjuncts so that they can evaluate -//! and reorder the most selective predicates first. -//! -//! The implementation takes inspiration from [Postgres] and [Apache Datafusion]. -//! -//! [Postgres]: https://www.postgresql.org/docs/current/sql-expressions.html -//! [Apache Datafusion]: https://github.com/apache/datafusion/tree/5fac581efbaffd0e6a9edf931182517524526afd/datafusion/expr +//! [`BoundExpression`] lives here because the scalar function vtable reduces over it: keeping it +//! next to [`ScalarFnVTable`](crate::scalar_fn::ScalarFnVTable) lets every reduction rule be +//! monomorphized for expression trees as well as array trees. Unbound expressions, binding, +//! constructors and the analyses built on them live in the `vortex-expr` crate. -pub mod aliases; -#[cfg(feature = "arbitrary")] -pub mod arbitrary; -pub mod bound_expression; +mod bound_expression; pub mod display; -pub(crate) mod expression; -mod exprs; -pub(crate) mod field; -pub mod lambda; mod optimize; -pub mod proto; -pub mod scope; pub mod stats; -pub mod traversal; pub use bound_expression::*; -pub use expression::*; -pub use exprs::and; -pub use exprs::and_collect; -pub use exprs::between; -pub use exprs::binary; -pub use exprs::bound; -pub use exprs::byte_length; -pub use exprs::case_when; -pub use exprs::case_when_no_else; -pub use exprs::cast; -pub use exprs::checked_add; -pub use exprs::col; -pub use exprs::dynamic; -pub use exprs::dynamic_with_options; -pub use exprs::eq; -pub use exprs::ext_storage; -pub use exprs::fill_null; -pub use exprs::get_item; -pub use exprs::gt; -pub use exprs::gt_eq; -pub use exprs::ilike; -pub use exprs::in_list; -pub use exprs::in_list_of; -pub use exprs::is_nan; -pub use exprs::is_not_null; -pub use exprs::is_null; -pub use exprs::is_root; -pub use exprs::like; -pub use exprs::list_contains; -pub use exprs::list_contains_opts; -pub use exprs::list_length; -pub use exprs::list_sum; -pub use exprs::list_sum_opts; -pub use exprs::lit; -pub use exprs::lt; -pub use exprs::lt_eq; -pub use exprs::mask; -pub use exprs::merge; -pub use exprs::merge_opts; -pub use exprs::nested_case_when; -pub use exprs::not; -pub use exprs::not_eq; -pub use exprs::not_ilike; -pub use exprs::not_like; -pub use exprs::or; -pub use exprs::or_collect; -pub use exprs::pack; -pub use exprs::replace_time_zone; -pub use exprs::root; -pub use exprs::select; -pub use exprs::select_exclude; -pub use exprs::var; -pub use exprs::variant_get; -pub use exprs::zip_expr; -pub use lambda::*; -pub use scope::*; - -#[cfg(feature = "_test-harness")] -pub mod test_harness { - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::StructFields; - - pub fn struct_dtype() -> DType { - DType::Struct( - StructFields::new( - ["a", "col1", "col2", "bool1", "bool2"].into(), - vec![ - DType::Primitive(PType::I32, Nullability::NonNullable), - DType::Primitive(PType::U16, Nullability::Nullable), - DType::Primitive(PType::U16, Nullability::Nullable), - DType::Bool(Nullability::NonNullable), - DType::Bool(Nullability::NonNullable), - ], - ), - Nullability::NonNullable, - ) - } -} - -#[cfg(test)] -mod tests { - use vortex_array::expr::eq; - use vortex_array::expr::lit; - use vortex_array::expr::root; - - use super::*; - use crate::dtype::DType; - use crate::dtype::FieldName; - use crate::dtype::FieldNames; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::StructFields; - use crate::expr::and; - use crate::expr::bound; - use crate::expr::case_when; - use crate::expr::col; - use crate::expr::gt; - use crate::expr::gt_eq; - use crate::expr::lt; - use crate::expr::lt_eq; - use crate::expr::not; - use crate::expr::not_eq; - use crate::expr::or; - use crate::expr::select; - use crate::expr::select_exclude; - use crate::scalar::Scalar; - use crate::scalar_fn::fns::literal::Literal; - - #[test] - fn bound_constructors_preserve_order_and_types() -> vortex_error::VortexResult<()> { - let value_dtype = DType::Primitive(PType::I32, Nullability::NonNullable); - let scope = DType::Struct( - StructFields::from_iter([("value", value_dtype.clone())]), - Nullability::NonNullable, - ); - - let root = bound::root(scope.clone()); - let value = bound::get_item("value", root); - let literal = bound::lit(5i32); - let condition = bound::gt(value.clone(), literal.clone()); - assert_eq!(condition.dtype(), &DType::Bool(Nullability::NonNullable)); - assert_eq!(condition.children(), &[value.clone(), literal.clone()]); - - let case = bound::case_when(condition.clone(), value.clone(), literal.clone()); - assert_eq!(case.dtype(), &value_dtype); - assert_eq!(case.children(), &[condition.clone(), value, literal]); - - let packed = bound::pack( - [("condition", condition.clone()), ("value", case.clone())], - Nullability::NonNullable, - ); - assert_eq!(packed.children(), &[condition, case.clone()]); - assert_eq!( - packed.dtype(), - &DType::Struct( - StructFields::from_iter([ - ("condition", DType::Bool(Nullability::NonNullable)), - ("value", value_dtype), - ]), - Nullability::NonNullable, - ) - ); - - let unbound = case_when(gt(col("value"), lit(5i32)), col("value"), lit(5i32)); - assert_eq!(unbound.bind(&scope)?, case); - Ok(()) - } - - #[test] - fn expr_display() { - assert_eq!(col("a").to_string(), "$.a"); - assert_eq!(root().to_string(), "$"); - - let col1: Expression = col("col1"); - let col2: Expression = col("col2"); - assert_eq!( - and(col1.clone(), col2.clone()).to_string(), - "($.col1 and $.col2)" - ); - assert_eq!( - or(col1.clone(), col2.clone()).to_string(), - "($.col1 or $.col2)" - ); - assert_eq!( - eq(col1.clone(), col2.clone()).to_string(), - "($.col1 = $.col2)" - ); - assert_eq!( - not_eq(col1.clone(), col2.clone()).to_string(), - "($.col1 != $.col2)" - ); - assert_eq!( - gt(col1.clone(), col2.clone()).to_string(), - "($.col1 > $.col2)" - ); - assert_eq!( - gt_eq(col1.clone(), col2.clone()).to_string(), - "($.col1 >= $.col2)" - ); - assert_eq!( - lt(col1.clone(), col2.clone()).to_string(), - "($.col1 < $.col2)" - ); - assert_eq!( - lt_eq(col1.clone(), col2.clone()).to_string(), - "($.col1 <= $.col2)" - ); - - assert_eq!( - or(lt(col1.clone(), col2.clone()), not_eq(col1.clone(), col2),).to_string(), - "(($.col1 < $.col2) or ($.col1 != $.col2))" - ); - - assert_eq!(not(col1).to_string(), "vortex.not($.col1)"); - - assert_eq!( - select(vec![FieldName::from("col1")], root()).to_string(), - "${col1}" - ); - assert_eq!( - select( - vec![FieldName::from("col1"), FieldName::from("col2")], - root() - ) - .to_string(), - "${col1, col2}" - ); - assert_eq!( - select_exclude( - vec![FieldName::from("col1"), FieldName::from("col2")], - root() - ) - .to_string(), - "${~ col1, col2}" - ); - - assert_eq!(lit(Scalar::from(0u8)).to_string(), "0u8"); - assert_eq!(lit(Scalar::from(0.0f32)).to_string(), "0f32"); - assert_eq!( - lit(Scalar::from(i64::MAX)).to_string(), - "9223372036854775807i64" - ); - assert_eq!(lit(Scalar::from(true)).to_string(), "true"); - assert_eq!( - lit(Scalar::null(DType::Bool(Nullability::Nullable))).to_string(), - "null" - ); - - assert_eq!( - lit(Scalar::struct_( - DType::Struct( - StructFields::new( - FieldNames::from(["dog", "cat"]), - vec![ - DType::Primitive(PType::U32, Nullability::NonNullable), - DType::Utf8(Nullability::NonNullable) - ], - ), - Nullability::NonNullable - ), - vec![Scalar::from(32_u32), Scalar::from("rufus".to_string())] - )) - .to_string(), - "{dog: 32u32, cat: \"rufus\"}" - ); - } - - #[test] - fn expr_contains() { - let expression = &eq(root(), lit(3u64)); - assert!(expression.contains::().unwrap()); - let expression = root(); - assert!(!expression.contains::().unwrap()); - } -} diff --git a/vortex-array/src/expr/optimize.rs b/vortex-array/src/expr/optimize.rs index ac4356892b6..411a8bc01a0 100644 --- a/vortex-array/src/expr/optimize.rs +++ b/vortex-array/src/expr/optimize.rs @@ -10,7 +10,7 @@ impl BoundExpression { /// Optimize the root expression node only, iterating to convergence. /// /// This applies optimization rules repeatedly until no more changes occur: - /// 1. `simplify` - scalar-function simplifications over the bound node + /// 1. `simplify` - scalar-function simplifications over the node /// 2. `reduce` - abstract reduction rules via `ReduceNode` pub fn optimize(&self) -> VortexResult { Ok(self.try_optimize()?.unwrap_or_else(|| self.clone())) @@ -21,18 +21,19 @@ impl BoundExpression { /// Non-scalar nodes carry no rules, so they never simplify. fn simplify_node(&self) -> VortexResult> { match self { - BoundExpression::Scalar { scalar_fn, .. } => scalar_fn.simplify(self), + BoundExpression::Scalar { scalar_fn, .. } => Ok(scalar_fn + .simplify_expression(&ExpressionReduceNode::new(self))? + .map(ExpressionReduceNode::into_expression)), BoundExpression::Root { .. } => Ok(None), } } /// Apply this node's own abstract reduction rule, if it has one. - fn reduce_node<'a>( - &self, - node: &ExpressionReduceNode<'a>, - ) -> VortexResult>> { + fn reduce_node(&self) -> VortexResult> { match self { - BoundExpression::Scalar { scalar_fn, .. } => scalar_fn.reduce_expression(node), + BoundExpression::Scalar { scalar_fn, .. } => Ok(scalar_fn + .reduce_expression(&ExpressionReduceNode::new(self))? + .map(ExpressionReduceNode::into_expression)), BoundExpression::Root { .. } => Ok(None), } } @@ -60,15 +61,9 @@ impl BoundExpression { changed = true; } - // Try reduce via ReduceNode. The node borrows the expression, so constructing it is - // free; the block scopes the borrow so `current` can be updated. - let reduced = { - let expr = current.as_ref().unwrap_or(self); - expr.reduce_node(&ExpressionReduceNode::new(expr))? - .map(ExpressionReduceNode::into_expression) - }; - if let Some(reduced_expr) = reduced { - current = Some(reduced_expr); + let expr = current.as_ref().unwrap_or(self); + if let Some(reduced) = expr.reduce_node()? { + current = Some(reduced); changed = true; } @@ -79,7 +74,6 @@ impl BoundExpression { Ok(current) } - /// Optimize the entire expression tree recursively. /// /// Optimizes children first (bottom-up), then optimizes the root. @@ -119,210 +113,3 @@ impl BoundExpression { Ok(current) } } - -#[cfg(test)] -mod tests { - use std::sync::LazyLock; - - use rstest::rstest; - use vortex_buffer::buffer; - use vortex_error::VortexResult; - use vortex_error::vortex_err; - use vortex_session::VortexSession; - - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::BoolArray; - use crate::arrays::StructArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::FieldName; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::StructFields; - use crate::expr::Expression; - use crate::expr::and; - use crate::expr::cast; - use crate::expr::eq; - use crate::expr::get_item; - use crate::expr::lit; - use crate::expr::lt_eq; - use crate::expr::merge; - use crate::expr::or; - use crate::expr::pack; - use crate::expr::root; - use crate::expr::select; - use crate::expr::zip_expr; - use crate::scalar::Scalar; - use crate::scalar_fn::fns::literal::Literal; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); - - #[test] - fn optimize_or_chain_correctness() -> VortexResult<()> { - let expr = or( - eq(get_item("x", root()), lit(1i32)), - eq(get_item("x", root()), lit(2i32)), - ); - let scope = DType::Struct( - StructFields::new( - ["x"].into(), - vec![DType::Primitive(PType::I32, Nullability::NonNullable)], - ), - Nullability::NonNullable, - ); - let optimized = expr.bind(&scope)?.optimize_recursive()?; - - let data = - StructArray::from_fields(&[("x", buffer![0i32, 1, 2, 3].into_array())])?.into_array(); - let actual = data.apply_bound(&optimized)?; - assert_arrays_eq!( - actual, - BoolArray::from_iter([false, true, true, false]), - &mut SESSION.create_execution_ctx() - ); - Ok(()) - } - - #[test] - fn optimize_folds_cast_of_literal_in_comparison() -> VortexResult<()> { - let expr = lt_eq( - get_item("x", root()), - cast( - lit(3i32), - DType::Primitive(PType::F64, Nullability::NonNullable), - ), - ); - let scope = DType::Struct( - StructFields::new( - ["x"].into(), - vec![DType::Primitive(PType::F64, Nullability::NonNullable)], - ), - Nullability::NonNullable, - ); - let optimized = expr.bind(&scope)?.optimize_recursive()?; - - // Prune rules pattern-match a bare Literal on the comparison RHS; a cast wrapper - // silently disables pruning. - let rhs = optimized - .child(1) - .as_opt::() - .ok_or_else(|| vortex_err!("expected a bare literal RHS, got {optimized}"))?; - assert_eq!(rhs, &Scalar::primitive(3.0f64, Nullability::NonNullable)); - Ok(()) - } - - #[rstest] - #[case::and_annihilator(and(root(), lit(false)), DType::Bool(Nullability::Nullable))] - #[case::or_annihilator(or(root(), lit(true)), DType::Bool(Nullability::Nullable))] - #[case::and_nullable_identity( - and(root(), lit(Scalar::from(Some(true)))), - DType::Bool(Nullability::NonNullable) - )] - #[case::or_nullable_identity( - or(root(), lit(Scalar::from(Some(false)))), - DType::Bool(Nullability::NonNullable) - )] - #[case::zip_true( - zip_expr(lit(true), lit(1i32), root()), - DType::Primitive(PType::I32, Nullability::Nullable) - )] - #[case::zip_false( - zip_expr(lit(false), root(), lit(1i32)), - DType::Primitive(PType::I32, Nullability::Nullable) - )] - fn literal_simplification_preserves_result_dtype( - #[case] expr: Expression, - #[case] scope: DType, - ) -> VortexResult<()> { - let original = expr.bind(&scope)?; - let optimized = expr.bind(&scope)?.optimize_recursive()?; - assert_eq!(optimized.dtype(), original.dtype()); - - Ok(()) - } - - /// `Select` over a `pack` rewrites to a `pack` of the selected child fields. It must read - /// those fields out of the child `pack` itself, because the optimizer visits children before - /// their parent and so never revisits the children a rule introduces. - #[test] - fn optimize_select_of_pack_reads_pack_fields() -> VortexResult<()> { - let field_names = (0..4) - .map(|idx| FieldName::from(format!("field_{idx}"))) - .collect::>(); - let fields = StructFields::new( - field_names.clone().into(), - vec![DType::Primitive(PType::U64, Nullability::NonNullable); field_names.len()], - ); - let scope = DType::Struct(fields, Nullability::NonNullable); - - let root_pack = pack( - field_names - .iter() - .map(|name| (name.clone(), get_item(name.clone(), root()))) - .collect::>(), - Nullability::NonNullable, - ); - let expr = select( - field_names - .iter() - .cloned() - .chain(["input_row_idx".into(), "input_file_idx".into()]) - .collect::>(), - merge([ - root_pack, - pack( - [ - ("input_row_idx", lit(0_u64)), - ("input_file_idx", lit(1_u64)), - ], - Nullability::NonNullable, - ), - ]), - ); - - let expected = pack( - field_names - .into_iter() - .map(|name| (name.clone(), get_item(name, root()))) - .chain([ - ("input_row_idx".into(), lit(0_u64)), - ("input_file_idx".into(), lit(1_u64)), - ]) - .collect::>(), - Nullability::NonNullable, - ); - - let expected = expected.bind(&scope)?; - - let optimized = expr.bind(&scope)?.optimize_recursive()?; - assert_eq!(optimized, expected, "got {optimized}"); - assert_eq!(optimized.optimize_recursive()?, expected); - - Ok(()) - } - - /// `Merge` reduces to a `pack` that reads each field out of its children, so it must also - /// read straight out of a `pack` child. - #[test] - fn optimize_merge_of_packs_reads_pack_fields() -> VortexResult<()> { - let scope = DType::Struct(StructFields::empty(), Nullability::NonNullable); - let expr = merge([ - pack([("a", lit(0_u64))], Nullability::NonNullable), - pack([("b", lit(1_u64))], Nullability::NonNullable), - ]); - - let expected = pack( - [("a", lit(0_u64)), ("b", lit(1_u64))], - Nullability::NonNullable, - ); - - let expected = expected.bind(&scope)?; - - let optimized = expr.bind(&scope)?.optimize_recursive()?; - assert_eq!(optimized, expected, "got {optimized}"); - assert_eq!(optimized.optimize_recursive()?, expected); - - Ok(()) - } -} diff --git a/vortex-array/src/extension/datetime/timestamp.rs b/vortex-array/src/extension/datetime/timestamp.rs index 7c0cdaeb1bb..f43f8f23cef 100644 --- a/vortex-array/src/extension/datetime/timestamp.rs +++ b/vortex-array/src/extension/datetime/timestamp.rs @@ -78,7 +78,7 @@ impl fmt::Display for TimestampOptions { } /// Resolve an Arrow timestamp timezone, accepting IANA names and fixed offsets. -pub(crate) fn resolve_time_zone(name: Option<&str>) -> VortexResult { +pub fn resolve_time_zone(name: Option<&str>) -> VortexResult { Ok(match name { Some(name) if name.starts_with('+') || name.starts_with('-') => { DateTimeParser::new().parse_time_zone(name)? diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index 53ccf53bb52..395a3ffb50a 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -111,7 +111,6 @@ pub mod display; pub mod dtype; mod executor; pub mod expr; -mod expression; pub mod extension; pub mod flatbuffers; mod hash; diff --git a/vortex-array/src/optimizer/kernels.rs b/vortex-array/src/optimizer/kernels.rs index 6a7eca5d6d6..ab83191863b 100644 --- a/vortex-array/src/optimizer/kernels.rs +++ b/vortex-array/src/optimizer/kernels.rs @@ -48,8 +48,9 @@ use crate::arrays::Struct; use crate::arrays::struct_::compute::rules::struct_cast_reduce_parent; use crate::kernel::ExecuteParentKernel; use crate::matcher::Matcher; +use crate::optimizer::rules::ArrayParentReduceRule; use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::cast::Cast; +use crate::scalar_fn::builtin::cast::Cast; /// Shared hasher used to combine `(outer, child)` tuples into registry keys. static FN_HASHER: LazyLock = LazyLock::new(DefaultHashBuilder::default); @@ -65,6 +66,37 @@ static FN_HASHER: LazyLock = LazyLock::new(DefaultHashBuilde pub type ReduceParentFn = fn(child: &ArrayRef, parent: &ArrayRef, child_idx: usize) -> VortexResult>; +/// Applies a typed [`ArrayParentReduceRule`] as a [`ReduceParentFn`] would, so a crate that defines +/// a scalar function can register parent-reduce rules for encodings it does not own. +/// +/// The rule applies when `child` is a `V` array and `parent` matches the rule's +/// [`Parent`](ArrayParentReduceRule::Parent) matcher. Wrap it in a function to register it: +/// +/// ```ignore +/// fn constant_between(child: &ArrayRef, parent: &ArrayRef, idx: usize) -> VortexResult> { +/// apply_reduce_parent_rule(&BetweenReduceAdaptor(Constant), child, parent, idx) +/// } +/// session.kernels().register_reduce_parent(Between.id(), Constant.id(), &[constant_between]); +/// ``` +pub fn apply_reduce_parent_rule( + rule: &R, + child: &ArrayRef, + parent: &ArrayRef, + child_idx: usize, +) -> VortexResult> +where + V: VTable, + R: ArrayParentReduceRule, +{ + let Some(array) = child.as_opt::() else { + return Ok(None); + }; + let Some(parent) = ::try_match(parent) else { + return Ok(None); + }; + rule.reduce_parent(array, parent, child_idx) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] #[repr(transparent)] struct ReduceParentFnId(u64); @@ -386,7 +418,7 @@ mod tests { use crate::ArrayVTable; use crate::arrays::Bool; use crate::scalar_fn::ScalarFnVTable; - use crate::scalar_fn::fns::binary::Binary; + use crate::scalar_fn::builtin::binary::Binary; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 3dab7b37c3d..832f3239c9b 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -662,7 +662,7 @@ impl Patches { /// Unlike filter, this preserves the patch indices. /// Unlike mask on a single array, this does not set masked values to null. /// - /// Masking an array always yields a nullable result (see [`crate::scalar_fn::fns::mask`]), + /// Masking an array always yields a nullable result (see [`crate::scalar_fn::builtin::mask`]), /// so the surviving patch values are widened to nullable to match the masked parent. Callers /// therefore receive patches with the correct dtype and do not need to re-cast them. // TODO(joe): make this lazy and remove the ctx. diff --git a/vortex-array/src/scalar/typed_view/list.rs b/vortex-array/src/scalar/typed_view/list.rs index 01e91800432..35b24012771 100644 --- a/vortex-array/src/scalar/typed_view/list.rs +++ b/vortex-array/src/scalar/typed_view/list.rs @@ -164,7 +164,7 @@ impl<'a> ListScalar<'a> { /// scalar for each. /// /// Returns None if the list is null. - pub(crate) fn element_values(&self) -> Option<&'a [Option]> { + pub fn element_values(&self) -> Option<&'a [Option]> { self.elements } diff --git a/vortex-array/src/scalar/typed_view/primitive/numeric_operator.rs b/vortex-array/src/scalar/typed_view/primitive/numeric_operator.rs index d7bcc8d0b4c..bdb33005ec0 100644 --- a/vortex-array/src/scalar/typed_view/primitive/numeric_operator.rs +++ b/vortex-array/src/scalar/typed_view/primitive/numeric_operator.rs @@ -8,7 +8,7 @@ use std::fmt; use vortex_error::VortexError; use vortex_error::vortex_err; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::operators::Operator; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] /// Binary element-wise operations. diff --git a/vortex-array/src/scalar_fn/fns/binary/boolean.rs b/vortex-array/src/scalar_fn/builtin/binary/boolean.rs similarity index 99% rename from vortex-array/src/scalar_fn/fns/binary/boolean.rs rename to vortex-array/src/scalar_fn/builtin/binary/boolean.rs index df23e5a87cb..20b6dd7204a 100644 --- a/vortex-array/src/scalar_fn/fns/binary/boolean.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/boolean.rs @@ -33,8 +33,8 @@ use crate::dtype::Nullability; use crate::kernel::ExecuteParentKernel; use crate::scalar::BoolScalar; use crate::scalar::Scalar; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; /// Trait for encoding-specific boolean kernels that operate in encoded space. @@ -764,7 +764,7 @@ mod tests { use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; - use crate::scalar_fn::fns::operators::Operator; + use crate::scalar_fn::builtin::operators::Operator; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/scalar_fn/fns/binary/compare/boolean.rs b/vortex-array/src/scalar_fn/builtin/binary/compare/boolean.rs similarity index 97% rename from vortex-array/src/scalar_fn/fns/binary/compare/boolean.rs rename to vortex-array/src/scalar_fn/builtin/binary/compare/boolean.rs index b569e791f32..816b471e2fa 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare/boolean.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/compare/boolean.rs @@ -14,8 +14,8 @@ use crate::IntoArray; use crate::arrays::BoolArray; use crate::arrays::Constant; use crate::dtype::Nullability; -use crate::scalar_fn::fns::binary::compare::compare_validity; -use crate::scalar_fn::fns::operators::CompareOperator; +use crate::scalar_fn::builtin::binary::compare::compare_validity; +use crate::scalar_fn::builtin::operators::CompareOperator; use crate::validity::Validity; enum BoolOperand { diff --git a/vortex-array/src/scalar_fn/fns/binary/compare/bytes.rs b/vortex-array/src/scalar_fn/builtin/binary/compare/bytes.rs similarity index 98% rename from vortex-array/src/scalar_fn/fns/binary/compare/bytes.rs rename to vortex-array/src/scalar_fn/builtin/binary/compare/bytes.rs index ef833adf6c4..0287cc2882b 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare/bytes.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/compare/bytes.rs @@ -27,9 +27,9 @@ use crate::arrays::varbinview::ResolvedViews; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; -use crate::scalar_fn::fns::binary::compare::compare_validity; -use crate::scalar_fn::fns::binary::compare::ordering_predicate; -use crate::scalar_fn::fns::operators::CompareOperator; +use crate::scalar_fn::builtin::binary::compare::compare_validity; +use crate::scalar_fn::builtin::binary::compare::ordering_predicate; +use crate::scalar_fn::builtin::operators::CompareOperator; use crate::validity::Validity; enum BytesOperand { diff --git a/vortex-array/src/scalar_fn/fns/binary/compare/decimal.rs b/vortex-array/src/scalar_fn/builtin/binary/compare/decimal.rs similarity index 98% rename from vortex-array/src/scalar_fn/fns/binary/compare/decimal.rs rename to vortex-array/src/scalar_fn/builtin/binary/compare/decimal.rs index e88e12f11d0..bac46c35d70 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare/decimal.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/compare/decimal.rs @@ -28,8 +28,8 @@ use crate::dtype::Nullability; use crate::dtype::i256; use crate::match_each_decimal_value_type; use crate::scalar::DecimalValue; -use crate::scalar_fn::fns::binary::compare::compare_validity; -use crate::scalar_fn::fns::operators::CompareOperator; +use crate::scalar_fn::builtin::binary::compare::compare_validity; +use crate::scalar_fn::builtin::operators::CompareOperator; use crate::validity::Validity; enum DecimalOperand { diff --git a/vortex-array/src/scalar_fn/fns/binary/compare/mod.rs b/vortex-array/src/scalar_fn/builtin/binary/compare/mod.rs similarity index 98% rename from vortex-array/src/scalar_fn/fns/binary/compare/mod.rs rename to vortex-array/src/scalar_fn/builtin/binary/compare/mod.rs index 6593af28c6e..36041f1f7d3 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare/mod.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/compare/mod.rs @@ -35,15 +35,16 @@ use crate::dtype::DType; use crate::dtype::Nullability; use crate::kernel::ExecuteParentKernel; use crate::scalar::Scalar; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::operators::CompareOperator; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::operators::CompareOperator; use crate::validity::Validity; mod boolean; mod bytes; mod decimal; mod nested; -pub(crate) use nested::build_comparator as build_row_comparator; +pub use nested::RowComparator; +pub use nested::build_comparator as build_row_comparator; mod primitive; #[cfg(test)] mod tests; diff --git a/vortex-array/src/scalar_fn/fns/binary/compare/nested.rs b/vortex-array/src/scalar_fn/builtin/binary/compare/nested.rs similarity index 97% rename from vortex-array/src/scalar_fn/fns/binary/compare/nested.rs rename to vortex-array/src/scalar_fn/builtin/binary/compare/nested.rs index ccccc3ca955..b95f8b55bde 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare/nested.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/compare/nested.rs @@ -45,11 +45,12 @@ use crate::dtype::NativePType; use crate::dtype::Nullability; use crate::match_each_integer_ptype; use crate::match_each_native_ptype; -use crate::scalar_fn::fns::binary::compare::compare_validity; -use crate::scalar_fn::fns::operators::CompareOperator; +use crate::scalar_fn::builtin::binary::compare::compare_validity; +use crate::scalar_fn::builtin::operators::CompareOperator; /// A row comparator: compares row `i` of the left operand against row `j` of the right operand. -pub(crate) type RowComparator = Box Ordering>; +/// Compares row `i` of the left array with row `j` of the right array. +pub type RowComparator = Box Ordering>; /// Compare two nested arrays row by row. pub(super) fn compare_nested( @@ -116,7 +117,7 @@ fn validity_mask(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult /// Build a row comparator over two recursively canonical arrays of the same logical dtype /// (ignoring nullability). Null values order before all non-null values at every level. -pub(crate) fn build_comparator( +pub fn build_comparator( lhs: &ArrayRef, rhs: &ArrayRef, ctx: &mut ExecutionCtx, diff --git a/vortex-array/src/scalar_fn/fns/binary/compare/primitive.rs b/vortex-array/src/scalar_fn/builtin/binary/compare/primitive.rs similarity index 97% rename from vortex-array/src/scalar_fn/fns/binary/compare/primitive.rs rename to vortex-array/src/scalar_fn/builtin/binary/compare/primitive.rs index a1da5a84097..64e836387b7 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare/primitive.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/compare/primitive.rs @@ -24,9 +24,9 @@ use crate::dtype::Nullability; use crate::dtype::PType; use crate::match_each_native_ptype; use crate::scalar::Scalar; -use crate::scalar_fn::fns::binary::compare::compare_validity; -use crate::scalar_fn::fns::binary::primitive_operand::PrimitiveOperand; -use crate::scalar_fn::fns::operators::CompareOperator; +use crate::scalar_fn::builtin::binary::compare::compare_validity; +use crate::scalar_fn::builtin::binary::primitive_operand::PrimitiveOperand; +use crate::scalar_fn::builtin::operators::CompareOperator; /// Compare two primitive arrays of the same [`PType`]. /// diff --git a/vortex-array/src/scalar_fn/fns/binary/compare/tests.rs b/vortex-array/src/scalar_fn/builtin/binary/compare/tests.rs similarity index 99% rename from vortex-array/src/scalar_fn/fns/binary/compare/tests.rs rename to vortex-array/src/scalar_fn/builtin/binary/compare/tests.rs index 83d01e2d831..cc9ad9a6348 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare/tests.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/compare/tests.rs @@ -43,9 +43,9 @@ use crate::memory::MemorySessionExt; use crate::memory::test_allocator::counting_allocator; use crate::scalar::DecimalValue; use crate::scalar::Scalar; -use crate::scalar_fn::fns::binary::scalar_cmp; -use crate::scalar_fn::fns::operators::CompareOperator; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::binary::scalar_cmp; +use crate::scalar_fn::builtin::operators::CompareOperator; +use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/scalar_fn/builtin/binary/mod.rs b/vortex-array/src/scalar_fn/builtin/binary/mod.rs new file mode 100644 index 00000000000..49dcb576642 --- /dev/null +++ b/vortex-array/src/scalar_fn/builtin/binary/mod.rs @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; + +#[expect(deprecated)] +pub use boolean::and_kleene; +#[expect(deprecated)] +pub use boolean::or_kleene; +use prost::Message; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::ArrayRef; +use crate::ExecutionCtx; +use crate::arrays::ScalarFnArray; +use crate::dtype::DType; +use crate::dtype::Nullability; +use crate::proto::expr as pb; +use crate::scalar_fn::Arity; +use crate::scalar_fn::ChildName; +use crate::scalar_fn::ExecutionArgs; +use crate::scalar_fn::ReduceNode; +use crate::scalar_fn::ScalarFnId; +use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::ScalarFnVTableExt; +use crate::scalar_fn::builtin::cast::Cast; +use crate::scalar_fn::builtin::operators::CompareOperator; +use crate::scalar_fn::builtin::operators::Operator; +use crate::scalar_fn::display::ExprDisplay; + +pub mod boolean; +pub use boolean::BooleanExecuteAdaptor; +pub use boolean::BooleanKernel; +pub(crate) use boolean::execute_boolean; +pub use boolean::kleene_boolean_buffer_scalar; +pub use boolean::kleene_boolean_buffers; +mod compare; +pub use compare::*; +mod numeric; +pub(crate) use numeric::*; +mod primitive_operand; + +use crate::scalar::NumericOperator; +use crate::scalar::Scalar; + +#[derive(Clone)] +pub struct Binary; + +impl Binary { + /// Creates a lazy binary operation over `lhs` and `rhs`. + /// + /// # Errors + /// + /// Returns an error if the children have different lengths or incompatible dtypes. + pub fn try_new( + lhs: ArrayRef, + rhs: ArrayRef, + operator: Operator, + ) -> VortexResult { + ScalarFnArray::try_new(Binary.bind(operator), [lhs, rhs]) + } +} + +/// Kleene and/or lookup table where both arguments are constant and non-NULL. +/// is_and, left, right +const KLEENE_LUT: [[[bool; 2]; 2]; 2] = [ + [[false, true], [true, true]], // or + [[false, false], [false, true]], // and +]; + +/// Kleene and/or reduction where one argument is constant non-NULL and other +/// is non-constant. +fn kleene_one_const(node: T, constant: bool, is_and: bool) -> T { + match (is_and, constant) { + (true, true) | (false, false) => node, + (is_and, constant) => node.new_constant((constant && !is_and).into()), + } +} + +impl ScalarFnVTable for Binary { + type Options = Operator; + + fn id(&self) -> ScalarFnId { + static ID: CachedId = CachedId::new("vortex.binary"); + *ID + } + + fn serialize(&self, instance: &Self::Options) -> VortexResult>> { + Ok(Some( + pb::BinaryOpts { + op: (*instance).into(), + } + .encode_to_vec(), + )) + } + + fn deserialize( + &self, + _metadata: &[u8], + _session: &VortexSession, + ) -> VortexResult { + let opts = pb::BinaryOpts::decode(_metadata)?; + Operator::try_from(opts.op) + } + + fn arity(&self, _options: &Self::Options) -> Arity { + Arity::Exact(2) + } + + fn child_name(&self, _instance: &Self::Options, child_idx: usize) -> ChildName { + match child_idx { + 0 => ChildName::from("lhs"), + 1 => ChildName::from("rhs"), + _ => unreachable!("Binary has only two children"), + } + } + + fn fmt_sql( + &self, + operator: &Operator, + expr: &dyn ExprDisplay, + f: &mut Formatter<'_>, + ) -> std::fmt::Result { + write!(f, "(")?; + Display::fmt(expr.display_child(0), f)?; + write!(f, " {} ", operator)?; + Display::fmt(expr.display_child(1), f)?; + write!(f, ")") + } + + fn return_dtype(&self, operator: &Operator, arg_dtypes: &[DType]) -> VortexResult { + let lhs = &arg_dtypes[0]; + let rhs = &arg_dtypes[1]; + + if operator.is_arithmetic() { + if lhs.is_primitive() && lhs.eq_ignore_nullability(rhs) { + return Ok(lhs.with_nullability(lhs.nullability() | rhs.nullability())); + } + + if let DType::Decimal(decimal_dtype, _) = lhs + && lhs.eq_ignore_nullability(rhs) + { + let numeric_op = NumericOperator::try_from(*operator)?; + return Ok(DType::Decimal( + numeric_op_result_decimal_dtype(*decimal_dtype, numeric_op)?, + lhs.nullability() | rhs.nullability(), + )); + } + vortex_bail!( + "incompatible types for arithmetic operation: {} {}", + lhs, + rhs + ); + } + + if operator.is_comparison() + && !lhs.eq_ignore_nullability(rhs) + && !lhs.is_extension() + && !rhs.is_extension() + { + vortex_bail!("Cannot compare different DTypes {} and {}", lhs, rhs); + } + + if matches!(operator, Operator::And | Operator::Or) + && !(lhs.is_boolean() && rhs.is_boolean()) + { + vortex_bail!("'{operator}' requires bool operands, got {lhs} and {rhs}",); + } + + Ok(DType::Bool((lhs.is_nullable() || rhs.is_nullable()).into())) + } + + fn execute( + &self, + op: &Operator, + args: &dyn ExecutionArgs, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let lhs = args.get(0)?; + let rhs = args.get(1)?; + + match op { + Operator::Eq => execute_compare(&lhs, &rhs, CompareOperator::Eq, ctx), + Operator::NotEq => execute_compare(&lhs, &rhs, CompareOperator::NotEq, ctx), + Operator::Lt => execute_compare(&lhs, &rhs, CompareOperator::Lt, ctx), + Operator::Lte => execute_compare(&lhs, &rhs, CompareOperator::Lte, ctx), + Operator::Gt => execute_compare(&lhs, &rhs, CompareOperator::Gt, ctx), + Operator::Gte => execute_compare(&lhs, &rhs, CompareOperator::Gte, ctx), + Operator::And => execute_boolean(lhs, rhs, Operator::And, ctx), + Operator::Or => execute_boolean(lhs, rhs, Operator::Or, ctx), + Operator::Add => execute_numeric(&lhs, &rhs, NumericOperator::Add, ctx), + Operator::Sub => execute_numeric(&lhs, &rhs, NumericOperator::Sub, ctx), + Operator::Mul => execute_numeric(&lhs, &rhs, NumericOperator::Mul, ctx), + Operator::Div => execute_numeric(&lhs, &rhs, NumericOperator::Div, ctx), + } + } + + fn simplify(&self, operator: &Operator, node: &T) -> VortexResult> { + let lhs = node.child(0); + let rhs = node.child(1); + + let is_constant_null = |node: &T| node.as_constant().is_some_and(|s| s.is_null()); + + // Binding already rejected ill-typed comparisons such as `int_col = null_utf8`, so a + // comparison against a null constant always evaluates to null. + if operator.is_comparison() && (is_constant_null(&lhs) || is_constant_null(&rhs)) { + return Ok(Some( + node.new_constant(Scalar::null(DType::Bool(Nullability::Nullable))), + )); + } + + if !matches!(operator, Operator::And | Operator::Or) { + return Ok(None); + } + + let bool_constant = |node: &T| node.as_constant()?.as_bool_opt().map(|value| value.value()); + + // AND/OR use Kleene three-valued logic. `None` below is a boolean null. + // + // AND: + // - false AND x => false + // - true AND x => x + // - null AND null => null + // + // OR: + // - true OR x => true + // - false OR x => x + // - null OR null => null + // + // Other null cases either fall out of the identity/annihilator rules + // above (`null AND true`, `null OR false`) or cannot be reduced under + // Kleene semantics (`null AND x`, `null OR x` for non-constant `x`). + // Reduction must keep the result nullability binding inferred: the annihilator + // constant and the identity child can both be less nullable than `lhs OP rhs`. + let result_nullable = lhs.node_dtype()?.is_nullable() || rhs.node_dtype()?.is_nullable(); + let nullability = Nullability::from(result_nullable); + let bool_constant_node = |value: bool| node.new_constant(Scalar::bool(value, nullability)); + let widen = |child: T| -> VortexResult { + if child.node_dtype()?.is_nullable() == result_nullable { + Ok(child) + } else { + child.new_node( + Cast.bind(DType::Bool(nullability)), + std::slice::from_ref(&child), + ) + } + }; + + Ok(match operator { + Operator::And => match (bool_constant(&lhs), bool_constant(&rhs)) { + (Some(Some(false)), _) | (_, Some(Some(false))) => Some(bool_constant_node(false)), + (Some(Some(true)), _) => Some(widen(rhs)?), + (_, Some(Some(true))) => Some(widen(lhs)?), + (Some(None), Some(None)) => Some(widen(lhs)?), + _ => None, + }, + Operator::Or => match (bool_constant(&lhs), bool_constant(&rhs)) { + (Some(Some(true)), _) | (_, Some(Some(true))) => Some(bool_constant_node(true)), + (Some(Some(false)), _) => Some(widen(rhs)?), + (_, Some(Some(false))) => Some(widen(lhs)?), + (Some(None), Some(None)) => Some(widen(lhs)?), + _ => None, + }, + _ => None, + }) + } + + fn reduce(&self, operator: &Operator, node: &T) -> VortexResult> { + if !matches!(operator, Operator::And | Operator::Or) { + return Ok(None); + } + let left = node.child(0); + let right = node.child(1); + + let left_const = left.as_constant(); + let right_const = right.as_constant(); + + // We don't handle Kleene NULL reduction here (.value() returns None + // for NULL). This will be reduced in the boolean kernel during + // execution. Not handling the case keeps the code much simpler. + let left_const = left_const + .and_then(|s| s.value().cloned()) + .map(|v| v.as_bool()); + let right_const = right_const + .and_then(|s| s.value().cloned()) + .map(|v| v.as_bool()); + let is_and = *operator == Operator::And; + + Ok(Some(match (left_const, right_const) { + (None, None) => return Ok(None), + (Some(left_const), Some(right_const)) => left.new_constant( + KLEENE_LUT[is_and as usize][left_const as usize][right_const as usize].into(), + ), + (Some(constant), None) => kleene_one_const(right, constant, is_and), + (None, Some(constant)) => kleene_one_const(left, constant, is_and), + })) + } + + fn is_strict(&self, operator: &Operator) -> bool { + !matches!(operator, Operator::And | Operator::Or) + } + + fn is_infallible(&self, operator: &Operator) -> bool { + // Arithmetic operations could be better modelled here. + matches!( + operator, + Operator::Eq + | Operator::NotEq + | Operator::Gt + | Operator::Gte + | Operator::Lt + | Operator::Lte + | Operator::And + | Operator::Or + ) + } +} diff --git a/vortex-array/src/scalar_fn/fns/binary/numeric/checked.rs b/vortex-array/src/scalar_fn/builtin/binary/numeric/checked.rs similarity index 100% rename from vortex-array/src/scalar_fn/fns/binary/numeric/checked.rs rename to vortex-array/src/scalar_fn/builtin/binary/numeric/checked.rs diff --git a/vortex-array/src/scalar_fn/fns/binary/numeric/decimal.rs b/vortex-array/src/scalar_fn/builtin/binary/numeric/decimal.rs similarity index 100% rename from vortex-array/src/scalar_fn/fns/binary/numeric/decimal.rs rename to vortex-array/src/scalar_fn/builtin/binary/numeric/decimal.rs diff --git a/vortex-array/src/scalar_fn/fns/binary/numeric/mod.rs b/vortex-array/src/scalar_fn/builtin/binary/numeric/mod.rs similarity index 100% rename from vortex-array/src/scalar_fn/fns/binary/numeric/mod.rs rename to vortex-array/src/scalar_fn/builtin/binary/numeric/mod.rs diff --git a/vortex-array/src/scalar_fn/fns/binary/numeric/primitive.rs b/vortex-array/src/scalar_fn/builtin/binary/numeric/primitive.rs similarity index 100% rename from vortex-array/src/scalar_fn/fns/binary/numeric/primitive.rs rename to vortex-array/src/scalar_fn/builtin/binary/numeric/primitive.rs diff --git a/vortex-array/src/scalar_fn/fns/binary/numeric/row.rs b/vortex-array/src/scalar_fn/builtin/binary/numeric/row.rs similarity index 98% rename from vortex-array/src/scalar_fn/fns/binary/numeric/row.rs rename to vortex-array/src/scalar_fn/builtin/binary/numeric/row.rs index ff7fbe7ab81..8ca4f49edaa 100644 --- a/vortex-array/src/scalar_fn/fns/binary/numeric/row.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/numeric/row.rs @@ -26,7 +26,7 @@ use crate::scalar::NumericOperator; use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::VecExecutionArgs; -use crate::scalar_fn::fns::binary::Binary; +use crate::scalar_fn::builtin::binary::Binary; use crate::scalar_fn::unstable::row::InitializedElement; use crate::scalar_fn::unstable::row::RowFn; use crate::scalar_fn::unstable::row::RowVisitor; diff --git a/vortex-array/src/scalar_fn/fns/binary/numeric/tests.rs b/vortex-array/src/scalar_fn/builtin/binary/numeric/tests.rs similarity index 99% rename from vortex-array/src/scalar_fn/fns/binary/numeric/tests.rs rename to vortex-array/src/scalar_fn/builtin/binary/numeric/tests.rs index 325e55665f8..f6eb344908a 100644 --- a/vortex-array/src/scalar_fn/fns/binary/numeric/tests.rs +++ b/vortex-array/src/scalar_fn/builtin/binary/numeric/tests.rs @@ -31,7 +31,7 @@ use crate::dtype::i256; use crate::scalar::DecimalValue; use crate::scalar::NumericOperator; use crate::scalar::Scalar; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/scalar_fn/fns/binary/primitive_operand.rs b/vortex-array/src/scalar_fn/builtin/binary/primitive_operand.rs similarity index 100% rename from vortex-array/src/scalar_fn/fns/binary/primitive_operand.rs rename to vortex-array/src/scalar_fn/builtin/binary/primitive_operand.rs diff --git a/vortex-array/src/scalar_fn/fns/cast/kernel.rs b/vortex-array/src/scalar_fn/builtin/cast/kernel.rs similarity index 98% rename from vortex-array/src/scalar_fn/fns/cast/kernel.rs rename to vortex-array/src/scalar_fn/builtin/cast/kernel.rs index ca9783cc22e..3f9d6f45eb4 100644 --- a/vortex-array/src/scalar_fn/fns/cast/kernel.rs +++ b/vortex-array/src/scalar_fn/builtin/cast/kernel.rs @@ -13,7 +13,7 @@ use crate::dtype::DType; use crate::kernel::ExecuteParentKernel; use crate::matcher::Matcher; use crate::optimizer::rules::ArrayParentReduceRule; -use crate::scalar_fn::fns::cast::Cast; +use crate::scalar_fn::builtin::cast::Cast; /// Reduce rule for cast: restructure the array without reading buffers. /// diff --git a/vortex-array/src/scalar_fn/fns/cast/mod.rs b/vortex-array/src/scalar_fn/builtin/cast/mod.rs similarity index 60% rename from vortex-array/src/scalar_fn/fns/cast/mod.rs rename to vortex-array/src/scalar_fn/builtin/cast/mod.rs index f3d90cf05fc..13b39e98e2c 100644 --- a/vortex-array/src/scalar_fn/fns/cast/mod.rs +++ b/vortex-array/src/scalar_fn/builtin/cast/mod.rs @@ -37,9 +37,6 @@ use crate::arrays::VarBinView; use crate::arrays::struct_::compute::cast::struct_cast; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; -use crate::expr::BoundExpression; -use crate::expr::bound; -use crate::expr::display::ExprDisplay; use crate::proto::expr as pb; use crate::scalar_fn::Arity; use crate::scalar_fn::ChildName; @@ -49,7 +46,7 @@ use crate::scalar_fn::ReduceNodeValidity; use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::literal::Literal; +use crate::scalar_fn::display::ExprDisplay; use crate::scalar_fn::is_not_null_node; /// A cast expression that converts values to a target data type. @@ -167,17 +164,21 @@ impl ScalarFnVTable for Cast { Ok(None) } - fn simplify( - &self, - target_dtype: &DType, - expr: &BoundExpression, - ) -> VortexResult> { - let Some(scalar) = expr.child(0).as_opt::() else { - return Ok(None); - }; - // A failing cast (e.g. null to a non-nullable dtype) is left in place so the error - // surfaces at execution time rather than during optimization. - Ok(scalar.cast(target_dtype).ok().map(bound::lit)) + fn simplify(&self, target_dtype: &DType, node: &T) -> VortexResult> { + let child = node.child(0); + if let Some(scalar) = child.as_constant() { + // A failing cast (e.g. null to a non-nullable dtype) is left in place so the error + // surfaces at execution time rather than during optimization. + return Ok(scalar + .cast(target_dtype) + .ok() + .map(|scalar| node.new_constant(scalar))); + } + // Collapse node if child is already the target type + if &child.node_dtype()? == target_dtype { + return Ok(Some(child)); + } + Ok(None) } fn validity( @@ -238,147 +239,3 @@ fn cast_canonical( fn cast_constant(array: ArrayView, dtype: &DType) -> VortexResult> { ::cast(array, dtype) } - -#[cfg(test)] -mod tests { - use std::sync::LazyLock; - - use vortex_buffer::buffer; - use vortex_error::VortexResult; - use vortex_error::vortex_err; - use vortex_session::VortexSession; - - use super::Cast; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::StructArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::DecimalDType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::expr::Expression; - use crate::expr::cast; - use crate::expr::get_item; - use crate::expr::lit; - use crate::expr::root; - use crate::expr::test_harness; - use crate::scalar::DecimalValue; - use crate::scalar::Scalar; - use crate::scalar_fn::fns::literal::Literal; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); - - #[test] - fn dtype() { - let dtype = test_harness::struct_dtype(); - assert_eq!( - cast(root(), DType::Bool(Nullability::NonNullable)) - .return_dtype(&dtype) - .unwrap(), - DType::Bool(Nullability::NonNullable) - ); - } - - #[test] - fn replace_children() -> VortexResult<()> { - let dtype = DType::Bool(Nullability::Nullable); - let expr = cast(root(), dtype.clone()); - let replaced = expr.with_children(vec![get_item("a", root())])?; - assert_eq!(replaced, cast(get_item("a", root()), dtype)); - Ok(()) - } - - #[test] - fn evaluate() { - let test_array = StructArray::from_fields(&[ - ("a", buffer![0i32, 1, 2].into_array()), - ("b", buffer![4i64, 5, 6].into_array()), - ]) - .unwrap() - .into_array(); - - let expr: Expression = cast( - get_item("a", root()), - DType::Primitive(PType::I64, Nullability::NonNullable), - ); - let result = test_array.apply(&expr).unwrap(); - - assert_eq!( - result.dtype(), - &DType::Primitive(PType::I64, Nullability::NonNullable) - ); - assert_arrays_eq!( - result, - buffer![0i64, 1, 2].into_array(), - &mut SESSION.create_execution_ctx() - ); - } - - #[test] - fn simplify_folds_cast_of_literal() -> VortexResult<()> { - let expr = cast( - lit(3i32), - DType::Primitive(PType::F64, Nullability::NonNullable), - ); - let optimized = expr.bind(&test_harness::struct_dtype())?.optimize()?; - - let scalar = optimized - .as_opt::() - .ok_or_else(|| vortex_err!("expected a bare literal, got {optimized}"))?; - assert_eq!(scalar, &Scalar::primitive(3.0f64, Nullability::NonNullable)); - Ok(()) - } - - #[test] - fn simplify_folds_cast_of_decimal_literal() -> VortexResult<()> { - let decimal = Scalar::decimal( - DecimalValue::I128(319), - DecimalDType::new(3, 2), - Nullability::NonNullable, - ); - let expr = cast( - lit(decimal), - DType::Primitive(PType::F64, Nullability::NonNullable), - ); - let optimized = expr.bind(&test_harness::struct_dtype())?.optimize()?; - - let scalar = optimized - .as_opt::() - .ok_or_else(|| vortex_err!("expected a bare literal, got {optimized}"))?; - assert_eq!( - scalar, - &Scalar::primitive(3.19f64, Nullability::NonNullable) - ); - Ok(()) - } - - #[test] - fn simplify_leaves_failing_cast_unchanged() -> VortexResult<()> { - let target = DType::Primitive(PType::F64, Nullability::NonNullable); - let expr = cast( - lit(Scalar::null(DType::Primitive( - PType::I32, - Nullability::Nullable, - ))), - target.clone(), - ); - let optimized = expr.bind(&test_harness::struct_dtype())?.optimize()?; - - assert!(optimized.as_opt::().is_none()); - assert_eq!(optimized.as_opt::(), Some(&target)); - Ok(()) - } - - #[test] - fn test_display() { - let expr = cast( - get_item("value", root()), - DType::Primitive(PType::I64, Nullability::NonNullable), - ); - assert_eq!(expr.to_string(), "cast($.value as i64)"); - - let expr2 = cast(root(), DType::Bool(Nullability::Nullable)); - assert_eq!(expr2.to_string(), "cast($ as bool?)"); - } -} diff --git a/vortex-array/src/expr/field.rs b/vortex-array/src/scalar_fn/builtin/field.rs similarity index 91% rename from vortex-array/src/expr/field.rs rename to vortex-array/src/scalar_fn/builtin/field.rs index 7a96f949ad5..e0125adfc8f 100644 --- a/vortex-array/src/expr/field.rs +++ b/vortex-array/src/scalar_fn/builtin/field.rs @@ -4,8 +4,7 @@ use std::fmt::Display; use itertools::Itertools; - -use crate::dtype::FieldNames; +use vortex_array::dtype::FieldNames; pub struct DisplayFieldNames<'a>(pub &'a FieldNames); diff --git a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs b/vortex-array/src/scalar_fn/builtin/fill_null/kernel.rs similarity index 98% rename from vortex-array/src/scalar_fn/fns/fill_null/kernel.rs rename to vortex-array/src/scalar_fn/builtin/fill_null/kernel.rs index 1fdcf1c2d76..a37ce62a70c 100644 --- a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs +++ b/vortex-array/src/scalar_fn/builtin/fill_null/kernel.rs @@ -20,7 +20,7 @@ use crate::builtins::ArrayBuiltins; use crate::kernel::ExecuteParentKernel; use crate::optimizer::rules::ArrayParentReduceRule; use crate::scalar::Scalar; -use crate::scalar_fn::fns::fill_null::FillNull as FillNullExpr; +use crate::scalar_fn::builtin::fill_null::FillNull as FillNullExpr; use crate::validity::Validity; /// Fill nulls in an array with a scalar value without reading buffers. diff --git a/vortex-array/src/scalar_fn/builtin/fill_null/mod.rs b/vortex-array/src/scalar_fn/builtin/fill_null/mod.rs new file mode 100644 index 00000000000..d60b3470c9f --- /dev/null +++ b/vortex-array/src/scalar_fn/builtin/fill_null/mod.rs @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +mod kernel; + +pub use kernel::*; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_error::vortex_err; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::AnyColumnar; +use crate::ArrayRef; +use crate::Canonical; +use crate::CanonicalView; +use crate::ColumnarView; +use crate::ExecutionCtx; +use crate::IntoArray; +use crate::arrays::Bool; +use crate::arrays::Constant; +use crate::arrays::Decimal; +use crate::arrays::Primitive; +use crate::arrays::ScalarFnArray; +use crate::arrays::VarBinView; +use crate::dtype::DType; +use crate::optimizer::ArrayOptimizer; +use crate::scalar::Scalar; +use crate::scalar_fn::Arity; +use crate::scalar_fn::ChildName; +use crate::scalar_fn::EmptyOptions; +use crate::scalar_fn::ExecutionArgs; +use crate::scalar_fn::ReduceNode; +use crate::scalar_fn::ReduceNodeValidity; +use crate::scalar_fn::ScalarFnId; +use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::ScalarFnVTableExt; +use crate::scalar_fn::is_not_null_node; + +/// An expression that replaces null values in the input with a fill value. +#[derive(Clone)] +pub struct FillNull; + +impl FillNull { + /// Creates a lazy operation that replaces null input values with `fill_value`. + /// + /// # Errors + /// + /// Returns an error if the children have different lengths or incompatible dtypes. + pub fn try_new(input: ArrayRef, fill_value: ArrayRef) -> VortexResult { + ScalarFnArray::try_new(FillNull.bind(EmptyOptions), [input, fill_value]) + } + + /// Returns whether fill_null can execute inputs of `dtype`. + /// + /// These are the dtypes whose canonical arrays have a [`FillNullKernel`], so update this list + /// whenever `fill_null_canonical` gains or loses one. Rewrites into fill_null, such as the + /// CASE WHEN simplification, check it first because other dtypes fail at execution time. + pub fn supports_dtype(dtype: &DType) -> bool { + matches!( + dtype, + DType::Bool(_) + | DType::Primitive(..) + | DType::Decimal(..) + | DType::Utf8(_) + | DType::Binary(_) + ) + } +} + +impl ScalarFnVTable for FillNull { + type Options = EmptyOptions; + + fn id(&self) -> ScalarFnId { + static ID: CachedId = CachedId::new("vortex.fill_null"); + *ID + } + + fn serialize(&self, _options: &Self::Options) -> VortexResult>> { + Ok(Some(vec![])) + } + + fn deserialize( + &self, + _metadata: &[u8], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyOptions) + } + + fn arity(&self, _options: &Self::Options) -> Arity { + Arity::Exact(2) + } + + fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName { + match child_idx { + 0 => ChildName::from("input"), + 1 => ChildName::from("fill_value"), + _ => unreachable!("Invalid child index {} for FillNull expression", child_idx), + } + } + + fn return_dtype(&self, _options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult { + vortex_ensure!( + arg_dtypes[0].eq_ignore_nullability(&arg_dtypes[1]), + "fill_null requires input and fill value to have the same base type, got {} and {}", + arg_dtypes[0], + arg_dtypes[1] + ); + // The result dtype takes the nullability of the fill value. + Ok(arg_dtypes[0] + .clone() + .with_nullability(arg_dtypes[1].nullability())) + } + + fn execute( + &self, + _options: &Self::Options, + args: &dyn ExecutionArgs, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let input = args.get(0)?; + let fill_value = args.get(1)?; + // Parent kernels decline a fill that is not already constant. Materialize it here so a + // lazy scalar, such as a literal `ScalarFnArray`, can still fill nulls. + let fill_was_lazy = !fill_value.is::(); + let fill_value = if fill_was_lazy { + fill_value.execute_until::(ctx)? + } else { + fill_value + }; + + let Some(fill_scalar) = fill_value.as_constant() else { + // Slicing a constant fill to an empty range yields an empty canonical array rather + // than a constant. An empty input has no nulls to fill, so there is nothing to do. + vortex_ensure!( + input.is_empty(), + "fill_null fill_value must be a constant/scalar" + ); + let dtype = input + .dtype() + .with_nullability(fill_value.dtype().nullability()); + return Ok(Canonical::empty(&dtype).into_array()); + }; + + vortex_ensure!( + !fill_scalar.is_null(), + "fill_null requires a non-null fill value" + ); + + let Some(columnar) = input.as_opt::() else { + if fill_was_lazy { + // Hand the rebuilt node back with a constant fill so the input's FillNull + // kernels get a chance before we decode the input. + return FillNull::try_new(input, fill_value)? + .into_array() + .optimize_ctx(ctx.session()); + } + let input = input.execute::(ctx)?; + return Ok(FillNull::try_new(input, fill_value)?.into_array()); + }; + + match columnar { + ColumnarView::Canonical(canonical) => fill_null_canonical(canonical, &fill_scalar, ctx), + ColumnarView::Constant(constant) => fill_null_constant(constant, &fill_scalar), + } + } + + fn simplify( + &self, + _options: &Self::Options, + node: &T, + ) -> VortexResult> { + let child = node.child(0); + if !child.node_dtype()?.is_nullable() { + return Ok(Some(child)); + } + Ok(None) + } + + fn validity( + &self, + _options: &Self::Options, + node: &T, + ) -> VortexResult> { + // After fill_null, the result validity depends on the fill value's nullability. + // If fill_value is non-nullable, the result is always valid. + Ok(ReduceNodeValidity::Reduced(is_not_null_node( + &node.child(1), + )?)) + } + + fn is_strict(&self, _options: &Self::Options) -> bool { + // This function replaces null input values instead of propagating them. + false + } + + fn is_infallible(&self, _options: &Self::Options) -> bool { + true + } +} + +/// Fill nulls on a canonical array by directly dispatching to the appropriate kernel. +/// +/// Returns the filled array, or bails if no kernel is registered for the canonical type. +fn fill_null_canonical( + canonical: CanonicalView<'_>, + fill_value: &Scalar, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let arr = canonical.to_array_ref(); + if let Some(result) = short_circuit(&arr, fill_value)? { + return Ok(result); + } + match canonical { + CanonicalView::Bool(a) => ::fill_null(a, fill_value, ctx)? + .ok_or_else(|| vortex_err!("FillNullKernel for BoolArray returned None")), + CanonicalView::Primitive(a) => { + ::fill_null(a, fill_value, ctx)? + .ok_or_else(|| vortex_err!("FillNullKernel for PrimitiveArray returned None")) + } + CanonicalView::Decimal(a) => ::fill_null(a, fill_value, ctx)? + .ok_or_else(|| vortex_err!("FillNullKernel for DecimalArray returned None")), + CanonicalView::VarBinView(a) => { + ::fill_null(a, fill_value, ctx)? + .ok_or_else(|| vortex_err!("FillNullKernel for VarBinViewArray returned None")) + } + other => vortex_bail!( + "No FillNullKernel for canonical array {}", + other.to_array_ref().encoding_id() + ), + } +} diff --git a/vortex-array/src/scalar_fn/builtin/get_item.rs b/vortex-array/src/scalar_fn/builtin/get_item.rs new file mode 100644 index 00000000000..b04e83c30d4 --- /dev/null +++ b/vortex-array/src/scalar_fn/builtin/get_item.rs @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::slice; + +use prost::Message; +use vortex_error::VortexResult; +use vortex_error::vortex_err; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::ArrayRef; +use crate::ExecutionCtx; +use crate::IntoArray; +use crate::arrays::Constant; +use crate::arrays::ConstantArray; +use crate::arrays::ScalarFnArray; +use crate::arrays::StructArray; +use crate::arrays::struct_::StructArrayExt; +use crate::builtins::ArrayBuiltins; +use crate::dtype::DType; +use crate::dtype::FieldName; +use crate::dtype::Nullability; +use crate::proto::expr as pb; +use crate::scalar::Scalar; +use crate::scalar_fn::Arity; +use crate::scalar_fn::ChildName; +use crate::scalar_fn::EmptyOptions; +use crate::scalar_fn::ExecutionArgs; +use crate::scalar_fn::ReduceNode; +use crate::scalar_fn::ScalarFnId; +use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::ScalarFnVTableExt; +use crate::scalar_fn::builtin::mask::Mask; +use crate::scalar_fn::builtin::pack::Pack; +use crate::scalar_fn::display::ExprDisplay; + +#[derive(Clone)] +pub struct GetItem; + +impl GetItem { + /// Creates a lazy projection of `field_name` from `input`. + /// + /// # Errors + /// + /// Returns an error if `input` is not a struct or does not contain `field_name`. + pub fn try_new( + input: ArrayRef, + field_name: impl Into, + ) -> VortexResult { + ScalarFnArray::try_new(GetItem.bind(field_name.into()), [input]) + } + + /// Read `field_name` out of `struct_node`, without a `get_item` node where possible. + /// + /// Rules that read a field out of their own child use this instead of building the + /// `get_item` node themselves. The optimizer visits children before their parent, so a + /// `get_item` that a rule introduces is never simplified again. Removing it here keeps the + /// output of the rule a fixed point. + pub fn apply_now(struct_node: &T, field_name: &FieldName) -> VortexResult { + if let Some(field) = Self::maybe_apply_now(struct_node, field_name)? { + return Ok(field); + } + + struct_node.new_node( + GetItem.bind(field_name.clone()), + slice::from_ref(struct_node), + ) + } + + /// Read `field_name` straight out of a `pack` node, which holds each field as a child. + /// + /// Returns `None` if `struct_node` is not a `pack`, or does not hold `field_name`. + fn maybe_apply_now( + struct_node: &T, + field_name: &FieldName, + ) -> VortexResult> { + let Some(pack) = struct_node + .scalar_fn() + .and_then(|scalar_fn| scalar_fn.as_opt::()) + else { + return Ok(None); + }; + let Some(idx) = pack.names.find(field_name) else { + return Ok(None); + }; + + let field = struct_node.child(idx); + if !pack.nullability.is_nullable() { + return Ok(Some(field)); + } + + // A nullable pack intersects its validity into the field, so mask the field with an + // all-true constant to make its dtype nullable. + Ok(Some(struct_node.new_node( + Mask.bind(EmptyOptions), + &[field, struct_node.new_constant(true.into())], + )?)) + } +} + +impl ScalarFnVTable for GetItem { + type Options = FieldName; + + fn id(&self) -> ScalarFnId { + static ID: CachedId = CachedId::new("vortex.get_item"); + *ID + } + + fn serialize(&self, instance: &Self::Options) -> VortexResult>> { + Ok(Some( + pb::GetItemOpts { + path: instance.to_string(), + } + .encode_to_vec(), + )) + } + + fn deserialize( + &self, + _metadata: &[u8], + _session: &VortexSession, + ) -> VortexResult { + let opts = pb::GetItemOpts::decode(_metadata)?; + Ok(FieldName::from(opts.path)) + } + + fn arity(&self, _field_name: &FieldName) -> Arity { + Arity::Exact(1) + } + + fn child_name(&self, _instance: &Self::Options, child_idx: usize) -> ChildName { + match child_idx { + 0 => ChildName::from("input"), + _ => unreachable!("Invalid child index {} for GetItem expression", child_idx), + } + } + + fn fmt_sql( + &self, + field_name: &FieldName, + expr: &dyn ExprDisplay, + f: &mut Formatter<'_>, + ) -> std::fmt::Result { + Display::fmt(expr.display_child(0), f)?; + write!(f, ".{}", field_name) + } + + fn return_dtype(&self, field_name: &FieldName, arg_dtypes: &[DType]) -> VortexResult { + let struct_dtype = &arg_dtypes[0]; + let field_dtype = struct_dtype + .as_struct_fields_opt() + .and_then(|st| st.field(field_name)) + .ok_or_else(|| { + vortex_err!("Couldn't find the {} field in the input scope", field_name) + })?; + + // Match here to avoid cloning the dtype if nullability doesn't need to change + if matches!( + (struct_dtype.nullability(), field_dtype.nullability()), + (Nullability::Nullable, Nullability::NonNullable) + ) { + return Ok(field_dtype.with_nullability(Nullability::Nullable)); + } + + Ok(field_dtype) + } + + fn execute( + &self, + field_name: &FieldName, + args: &dyn ExecutionArgs, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let input = args.get(0)?; + if let Some(constant) = input.as_opt::() { + let dtype = self.return_dtype(field_name, &[input.dtype().clone()])?; + let scalar = if constant.scalar().is_null() { + Scalar::null(dtype) + } else { + constant + .scalar() + .as_struct() + .field(field_name) + .ok_or_else(|| { + vortex_err!( + "Field '{}' missing from constant struct array {}", + field_name, + input.dtype() + ) + })? + .cast(&dtype)? + }; + + return Ok(ConstantArray::new(scalar, input.len()).into_array()); + } + + let input = input.execute::(ctx)?; + let field = input.unmasked_field_by_name(field_name).cloned()?; + + match input.dtype().nullability() { + Nullability::NonNullable => Ok(field), + Nullability::Nullable => field.mask(input.validity()?.to_array(input.len())), + } + } + + fn reduce(&self, field_name: &FieldName, node: &T) -> VortexResult> { + Self::maybe_apply_now(&node.child(0), field_name) + } + + fn is_strict(&self, _field_name: &FieldName) -> bool { + true + } + + fn is_infallible(&self, _field_name: &FieldName) -> bool { + // If this type-checks, it is infallible. + true + } +} diff --git a/vortex-array/src/scalar_fn/builtin/is_not_null.rs b/vortex-array/src/scalar_fn/builtin/is_not_null.rs new file mode 100644 index 00000000000..85eba3eacbd --- /dev/null +++ b/vortex-array/src/scalar_fn/builtin/is_not_null.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; + +use vortex_error::VortexExpect as _; +use vortex_error::VortexResult; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::ArrayRef; +use crate::ExecutionCtx; +use crate::arrays::ScalarFnArray; +use crate::dtype::DType; +use crate::dtype::Nullability; +use crate::scalar_fn::Arity; +use crate::scalar_fn::ChildName; +use crate::scalar_fn::EmptyOptions; +use crate::scalar_fn::ExecutionArgs; +use crate::scalar_fn::ReduceNode; +use crate::scalar_fn::ReduceNodeValidity; +use crate::scalar_fn::ScalarFnId; +use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::ScalarFnVTableExt; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::is_null::IsNull; +use crate::scalar_fn::builtin::not::Not; +use crate::scalar_fn::builtin::operators::Operator; +use crate::scalar_fn::display::ExprDisplay; +use crate::scalar_fn::is_not_null_node; + +/// In array context, reduce +/// +/// IsNull(x) -> if !x.nullable lit(false) else not(x.validity()) +/// IsNotNull(x) -> if !x.nullable lit(true) or x.validity() +/// +/// In expression and array contexts for x, y where x is nullable but y is not, +/// reduce +/// +/// 1. IsNull(and(x, y)) -> and(IsNull(x), y) +/// 2. IsNull(or(x, y)) -> and(IsNull(x), not(y)) +/// 3. IsNotNull(and(x, y)) -> or(IsNotNull(x), not(y)) +/// 4. IsNotNull(or(x, y)) -> or(IsNotNull(x), y) +/// +/// Latter optimizations make sense because calculating IsNull(x)/IsNotNull(x) +/// is at most expensive as calculating x, but usually much cheaper. Although +/// in two cases you exchange 4 computations to 4 computations, the latter +/// four are cheaper. +pub(crate) fn reduce_null(is_null: bool, node: &T) -> VortexResult> { + let child = node.child(0); + if !child.node_dtype()?.is_nullable() { + return Ok(Some(node.new_constant((!is_null).into()))); + } + + if let ReduceNodeValidity::Reduced(validity) = child.validity()? { + return Ok(Some(if is_null { + validity.new_node(Not.bind(EmptyOptions), std::slice::from_ref(&validity))? + } else { + validity + })); + } + + let Some(child_fn) = child.scalar_fn() else { + return Ok(None); + }; + let Some(operator) = child_fn.as_opt::() else { + return Ok(None); + }; + let is_and = match operator { + Operator::And => true, + Operator::Or => false, + _ => return Ok(None), + }; + + let left = child.child(0); + let right = child.child(1); + let (nullable, non_nullable) = match ( + left.node_dtype()?.is_nullable(), + right.node_dtype()?.is_nullable(), + ) { + (true, false) => (left, right), + (false, true) => (right, left), + // (false, false) already rewritten by constant folding + _ => return Ok(None), + }; + + // is_null(nullable) (rule 1, 2) + // is_not_null(nullable) (3, 4) + let left = if is_null { + nullable.new_node(IsNull.bind(EmptyOptions), std::slice::from_ref(&nullable))? + } else { + is_not_null_node(&nullable)? + }; + // non_nullable (1, 4) + // not(non_nullable) (2, 3) + let right = if is_null == is_and { + non_nullable + } else { + non_nullable.new_node(Not.bind(EmptyOptions), std::slice::from_ref(&non_nullable))? + }; + let combine = if is_null { Operator::And } else { Operator::Or }; + Ok(Some(node.new_node(Binary.bind(combine), &[left, right])?)) +} + +/// Expression that checks for non-null values. +#[derive(Clone)] +pub struct IsNotNull; + +impl IsNotNull { + /// Creates a lazy non-null check over `input`. + #[expect(clippy::new_ret_no_self, reason = "constructs the lazy result array")] + pub fn new(input: ArrayRef) -> ScalarFnArray { + ScalarFnArray::try_new(IsNotNull.bind(EmptyOptions), [input]) + .vortex_expect("IsNotNull has one child and an infallible return dtype") + } +} + +impl ScalarFnVTable for IsNotNull { + type Options = EmptyOptions; + + fn id(&self) -> ScalarFnId { + static ID: CachedId = CachedId::new("vortex.is_not_null"); + *ID + } + + fn serialize(&self, _instance: &Self::Options) -> VortexResult>> { + Ok(Some(vec![])) + } + + fn deserialize( + &self, + _metadata: &[u8], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyOptions) + } + + fn arity(&self, _options: &Self::Options) -> Arity { + Arity::Exact(1) + } + + fn child_name(&self, _instance: &Self::Options, child_idx: usize) -> ChildName { + match child_idx { + 0 => ChildName::from("input"), + _ => unreachable!("Invalid child index {} for IsNotNull expression", child_idx), + } + } + + fn fmt_sql( + &self, + _options: &Self::Options, + expr: &dyn ExprDisplay, + f: &mut Formatter<'_>, + ) -> std::fmt::Result { + write!(f, "is_not_null(")?; + Display::fmt(expr.display_child(0), f)?; + write!(f, ")") + } + + fn return_dtype(&self, _options: &Self::Options, _arg_dtypes: &[DType]) -> VortexResult { + Ok(DType::Bool(Nullability::NonNullable)) + } + + fn execute( + &self, + _data: &Self::Options, + args: &dyn ExecutionArgs, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(args.get(0)?.validity()?.to_array(args.row_count())) + } + + fn reduce(&self, _options: &Self::Options, node: &T) -> VortexResult> { + reduce_null(false, node) + } + + fn is_strict(&self, _instance: &Self::Options) -> bool { + // Null input produces the non-null boolean value `false`. + false + } + + fn is_infallible(&self, _instance: &Self::Options) -> bool { + true + } +} diff --git a/vortex-array/src/scalar_fn/builtin/is_null.rs b/vortex-array/src/scalar_fn/builtin/is_null.rs new file mode 100644 index 00000000000..693f0519a64 --- /dev/null +++ b/vortex-array/src/scalar_fn/builtin/is_null.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_error::VortexExpect as _; +use vortex_error::VortexResult; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::ArrayRef; +use crate::ExecutionCtx; +use crate::IntoArray; +use crate::arrays::ConstantArray; +use crate::arrays::ScalarFnArray; +use crate::builtins::ArrayBuiltins; +use crate::dtype::DType; +use crate::dtype::Nullability; +use crate::scalar_fn::Arity; +use crate::scalar_fn::ChildName; +use crate::scalar_fn::EmptyOptions; +use crate::scalar_fn::ExecutionArgs; +use crate::scalar_fn::ReduceNode; +use crate::scalar_fn::ScalarFnId; +use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::ScalarFnVTableExt; +use crate::scalar_fn::builtin::is_not_null::reduce_null; +use crate::validity::Validity; + +/// Expression that checks for null values. +#[derive(Clone)] +pub struct IsNull; + +impl IsNull { + /// Creates a lazy null check over `input`. + #[expect(clippy::new_ret_no_self, reason = "constructs the lazy result array")] + pub fn new(input: ArrayRef) -> ScalarFnArray { + ScalarFnArray::try_new(IsNull.bind(EmptyOptions), [input]) + .vortex_expect("IsNull has one child and an infallible return dtype") + } +} + +impl ScalarFnVTable for IsNull { + type Options = EmptyOptions; + + fn id(&self) -> ScalarFnId { + static ID: CachedId = CachedId::new("vortex.is_null"); + *ID + } + + fn serialize(&self, _instance: &Self::Options) -> VortexResult>> { + Ok(Some(vec![])) + } + + fn deserialize( + &self, + _metadata: &[u8], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyOptions) + } + + fn arity(&self, _options: &Self::Options) -> Arity { + Arity::Exact(1) + } + + fn child_name(&self, _instance: &Self::Options, child_idx: usize) -> ChildName { + match child_idx { + 0 => ChildName::from("input"), + _ => unreachable!("Invalid child index {} for IsNull expression", child_idx), + } + } + + fn return_dtype(&self, _options: &Self::Options, _arg_dtypes: &[DType]) -> VortexResult { + Ok(DType::Bool(Nullability::NonNullable)) + } + + fn execute( + &self, + _data: &Self::Options, + args: &dyn ExecutionArgs, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + let child = args.get(0)?; + if let Some(scalar) = child.as_constant() { + return Ok(ConstantArray::new(scalar.is_null(), args.row_count()).into_array()); + } + + match child.validity()? { + Validity::NonNullable | Validity::AllValid => { + Ok(ConstantArray::new(false, args.row_count()).into_array()) + } + Validity::AllInvalid => Ok(ConstantArray::new(true, args.row_count()).into_array()), + Validity::Array(a) => a.not(), + } + } + + fn reduce(&self, _options: &Self::Options, node: &T) -> VortexResult> { + reduce_null(true, node) + } + + fn is_strict(&self, _instance: &Self::Options) -> bool { + // Null input produces the non-null boolean value `true`. + false + } + + fn is_infallible(&self, _instance: &Self::Options) -> bool { + true + } +} diff --git a/vortex-array/src/scalar_fn/fns/literal.rs b/vortex-array/src/scalar_fn/builtin/literal.rs similarity index 63% rename from vortex-array/src/scalar_fn/fns/literal.rs rename to vortex-array/src/scalar_fn/builtin/literal.rs index cb6c3284523..22e8e599a2f 100644 --- a/vortex-array/src/scalar_fn/fns/literal.rs +++ b/vortex-array/src/scalar_fn/builtin/literal.rs @@ -14,7 +14,6 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::ConstantArray; use crate::dtype::DType; -use crate::expr::display::ExprDisplay; use crate::proto::expr as pb; use crate::scalar::Scalar; use crate::scalar_fn::Arity; @@ -24,6 +23,7 @@ use crate::scalar_fn::ReduceNode; use crate::scalar_fn::ReduceNodeValidity; use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::display::ExprDisplay; /// Expression that represents a literal scalar value. #[derive(Clone)] @@ -108,58 +108,3 @@ impl ScalarFnVTable for Literal { true } } - -#[cfg(test)] -mod tests { - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::StructFields; - use crate::expr::lit; - use crate::expr::test_harness; - use crate::scalar::Scalar; - - #[test] - fn dtype() { - let dtype = test_harness::struct_dtype(); - - assert_eq!( - lit(10).return_dtype(&dtype).unwrap(), - DType::Primitive(PType::I32, Nullability::NonNullable) - ); - assert_eq!( - lit(i64::MAX).return_dtype(&dtype).unwrap(), - DType::Primitive(PType::I64, Nullability::NonNullable) - ); - assert_eq!( - lit(true).return_dtype(&dtype).unwrap(), - DType::Bool(Nullability::NonNullable) - ); - assert_eq!( - lit(Scalar::null(DType::Bool(Nullability::Nullable))) - .return_dtype(&dtype) - .unwrap(), - DType::Bool(Nullability::Nullable) - ); - - let sdtype = DType::Struct( - StructFields::new( - ["dog", "cat"].into(), - vec![ - DType::Primitive(PType::U32, Nullability::NonNullable), - DType::Utf8(Nullability::NonNullable), - ], - ), - Nullability::NonNullable, - ); - assert_eq!( - lit(Scalar::struct_( - sdtype.clone(), - vec![Scalar::from(32_u32), Scalar::from("rufus".to_string())] - )) - .return_dtype(&dtype) - .unwrap(), - sdtype - ); - } -} diff --git a/vortex-array/src/scalar_fn/fns/mask/kernel.rs b/vortex-array/src/scalar_fn/builtin/mask/kernel.rs similarity index 97% rename from vortex-array/src/scalar_fn/fns/mask/kernel.rs rename to vortex-array/src/scalar_fn/builtin/mask/kernel.rs index b03c2dd5509..2c25d0ce4c3 100644 --- a/vortex-array/src/scalar_fn/fns/mask/kernel.rs +++ b/vortex-array/src/scalar_fn/builtin/mask/kernel.rs @@ -16,7 +16,7 @@ use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayView; use crate::kernel::ExecuteParentKernel; use crate::optimizer::rules::ArrayParentReduceRule; -use crate::scalar_fn::fns::mask::Mask as MaskExpr; +use crate::scalar_fn::builtin::mask::Mask as MaskExpr; /// Mask an array without reading buffers. /// @@ -150,9 +150,9 @@ mod tests { use crate::executor::VortexSessionExecute; use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; - use crate::scalar_fn::fns::binary::Binary; - use crate::scalar_fn::fns::mask::Mask as MaskExpr; - use crate::scalar_fn::fns::operators::Operator; + use crate::scalar_fn::builtin::binary::Binary; + use crate::scalar_fn::builtin::mask::Mask as MaskExpr; + use crate::scalar_fn::builtin::operators::Operator; use crate::validity::Validity; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/scalar_fn/fns/mask/mod.rs b/vortex-array/src/scalar_fn/builtin/mask/mod.rs similarity index 75% rename from vortex-array/src/scalar_fn/fns/mask/mod.rs rename to vortex-array/src/scalar_fn/builtin/mask/mod.rs index 994c409adc3..39a72a0e451 100644 --- a/vortex-array/src/scalar_fn/fns/mask/mod.rs +++ b/vortex-array/src/scalar_fn/builtin/mask/mod.rs @@ -22,8 +22,6 @@ use crate::builtins::ArrayBuiltins; use crate::child_to_validity; use crate::dtype::DType; use crate::dtype::Nullability; -use crate::expr::BoundExpression; -use crate::expr::bound; use crate::scalar::Scalar; use crate::scalar_fn::Arity; use crate::scalar_fn::ChildName; @@ -34,9 +32,8 @@ use crate::scalar_fn::ReduceNodeValidity; use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::literal::Literal; -use crate::scalar_fn::fns::operators::Operator; +use crate::scalar_fn::builtin::binary::Binary; +use crate::scalar_fn::builtin::operators::Operator; use crate::scalar_fn::is_not_null_node; /// An expression that masks an input based on a boolean mask. @@ -115,27 +112,29 @@ impl ScalarFnVTable for Mask { execute_canonical(input, mask_array, ctx) } - fn simplify( + fn simplify( &self, _options: &Self::Options, - expr: &BoundExpression, - ) -> VortexResult> { - let Some(mask_lit) = expr.child(1).as_opt::() else { + node: &T, + ) -> VortexResult> { + let Some(mask_constant) = node.child(1).as_constant() else { return Ok(None); }; - let mask_lit = mask_lit + let mask_constant = mask_constant .as_bool() .value() .vortex_expect("Mask must be non-nullable"); - if mask_lit { + if mask_constant { // Mask is all true, so the output is just the input. - Ok(Some(expr.child(0).clone())) + Ok(Some(node.child(0))) } else { // Mask is all false, so the output is all nulls. - let input_dtype = expr.child(0).dtype(); - Ok(Some(bound::lit(Scalar::null(input_dtype.as_nullable())))) + let input_dtype = node.child(0).node_dtype()?; + Ok(Some( + node.new_constant(Scalar::null(input_dtype.as_nullable())), + )) } } @@ -194,41 +193,3 @@ fn execute_canonical( let canonical = input.execute::(ctx)?; Ok(mask_validity_canonical(canonical, validity, ctx)?.into_array()) } - -#[cfg(test)] -mod test { - use vortex_error::VortexExpect; - - use crate::dtype::DType; - use crate::dtype::Nullability::Nullable; - use crate::dtype::PType; - use crate::expr::bound; - use crate::expr::lit; - use crate::expr::mask; - use crate::scalar::Scalar; - - #[test] - fn test_simplify() { - let input_expr = lit(42u32); - let true_mask_expr = lit(true); - let false_mask_expr = lit(false); - - let mask_true_expr = mask(input_expr.clone(), true_mask_expr); - let simplified_true = mask_true_expr - .bind(&DType::Null) - .and_then(|expr| expr.optimize()) - .vortex_expect("Simplification"); - assert_eq!( - &simplified_true, - &input_expr.bind(&DType::Null).vortex_expect("bind") - ); - - let mask_false_expr = mask(input_expr, false_mask_expr); - let simplified_false = mask_false_expr - .bind(&DType::Null) - .and_then(|expr| expr.optimize()) - .vortex_expect("Simplification"); - let expected_null_expr = bound::lit(Scalar::null(DType::Primitive(PType::U32, Nullable))); - assert_eq!(&simplified_false, &expected_null_expr); - } -} diff --git a/vortex-array/src/scalar_fn/builtin/mod.rs b/vortex-array/src/scalar_fn/builtin/mod.rs new file mode 100644 index 00000000000..f6cf7c38296 --- /dev/null +++ b/vortex-array/src/scalar_fn/builtin/mod.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The scalar functions that vortex-array itself is built on. +//! +//! These are the functions that the core crate's own arrays, validity and expression machinery +//! construct: literals and struct projection, selection and packing for expressions, `is_null`, +//! `is_not_null`, `not` and the boolean operators for validity reduction, and `cast`, `mask`, +//! `fill_null` and the comparison and arithmetic operators, which canonical arrays apply lazily to +//! their own children (offsets, indices, codes and validity). [`ArrayBuiltins`] exposes them as +//! methods. +//! +//! Every other scalar function lives in the `vortex-functions` crate, which registers them, and +//! their kernels for vortex-array's encodings, with a session. +//! +//! [`ArrayBuiltins`]: crate::builtins::ArrayBuiltins + +pub mod binary; +pub mod cast; +mod field; +pub mod fill_null; +pub mod get_item; +pub mod is_not_null; +pub mod is_null; +pub mod literal; +pub mod mask; +pub mod not; +pub mod operators; +pub mod pack; +pub mod select; diff --git a/vortex-array/src/scalar_fn/fns/not/kernel.rs b/vortex-array/src/scalar_fn/builtin/not/kernel.rs similarity index 96% rename from vortex-array/src/scalar_fn/fns/not/kernel.rs rename to vortex-array/src/scalar_fn/builtin/not/kernel.rs index 3da8f66e17f..9e601eb8008 100644 --- a/vortex-array/src/scalar_fn/fns/not/kernel.rs +++ b/vortex-array/src/scalar_fn/builtin/not/kernel.rs @@ -9,7 +9,7 @@ use crate::array::VTable; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayView; use crate::optimizer::rules::ArrayParentReduceRule; -use crate::scalar_fn::fns::not::Not as NotExpr; +use crate::scalar_fn::builtin::not::Not as NotExpr; /// Invert a boolean array without reading buffers. /// diff --git a/vortex-array/src/scalar_fn/fns/not/mod.rs b/vortex-array/src/scalar_fn/builtin/not/mod.rs similarity index 59% rename from vortex-array/src/scalar_fn/fns/not/mod.rs rename to vortex-array/src/scalar_fn/builtin/not/mod.rs index b4c0c1b3bb5..d4f08156b74 100644 --- a/vortex-array/src/scalar_fn/fns/not/mod.rs +++ b/vortex-array/src/scalar_fn/builtin/not/mod.rs @@ -119,87 +119,3 @@ impl ScalarFnVTable for Not { true } } - -#[cfg(test)] -mod tests { - use std::sync::LazyLock; - - use vortex_error::VortexResult; - use vortex_session::VortexSession; - - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::expr::col; - use crate::expr::get_item; - use crate::expr::not; - use crate::expr::root; - use crate::expr::test_harness; - use crate::scalar_fn::fns::not::BoolArray; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); - - #[test] - fn is_strict() { - assert!( - not(root()) - .as_scalar() - .is_some_and(|f| f.signature().is_strict()) - ); - } - - #[test] - fn preserves_nulls() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - let input = BoolArray::from_iter([Some(false), None, Some(true)]).into_array(); - - let result = input.apply(¬(root()))?; - - assert_arrays_eq!( - result, - BoolArray::from_iter([Some(true), None, Some(false)]), - &mut ctx - ); - Ok(()) - } - - #[test] - fn invert_booleans() { - let mut ctx = SESSION.create_execution_ctx(); - let not_expr = not(root()); - let bools = BoolArray::from_iter([false, true, false, false, true, true]); - let result = bools.into_array().apply(¬_expr).unwrap(); - assert_arrays_eq!( - result, - BoolArray::from_iter([true, false, true, true, false, false]), - &mut ctx - ); - } - - #[test] - fn test_display_order_of_operations() { - let a = not(get_item("a", root())); - let b = get_item("a", not(root())); - assert_ne!(a.to_string(), b.to_string()); - assert_eq!(a.to_string(), "vortex.not($.a)"); - assert_eq!(b.to_string(), "vortex.not($).a"); - } - - #[test] - fn dtype() { - let not_expr = not(root()); - let dtype = DType::Bool(Nullability::NonNullable); - assert_eq!( - not_expr.return_dtype(&dtype).unwrap(), - DType::Bool(Nullability::NonNullable) - ); - - let dtype = test_harness::struct_dtype(); - assert_eq!( - not(col("bool1")).return_dtype(&dtype).unwrap(), - DType::Bool(Nullability::NonNullable) - ); - } -} diff --git a/vortex-array/src/scalar_fn/fns/operators.rs b/vortex-array/src/scalar_fn/builtin/operators.rs similarity index 100% rename from vortex-array/src/scalar_fn/fns/operators.rs rename to vortex-array/src/scalar_fn/builtin/operators.rs diff --git a/vortex-array/src/scalar_fn/builtin/pack.rs b/vortex-array/src/scalar_fn/builtin/pack.rs new file mode 100644 index 00000000000..ed54a251982 --- /dev/null +++ b/vortex-array/src/scalar_fn/builtin/pack.rs @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::hash::Hash; +use std::sync::Arc; + +use itertools::Itertools as _; +use prost::Message; +use vortex_error::VortexResult; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::ArrayRef; +use crate::ExecutionCtx; +use crate::IntoArray; +use crate::arrays::StructArray; +use crate::dtype::DType; +use crate::dtype::FieldName; +use crate::dtype::FieldNames; +use crate::dtype::Nullability; +use crate::dtype::StructFields; +use crate::proto::expr as pb; +use crate::scalar_fn::Arity; +use crate::scalar_fn::ChildName; +use crate::scalar_fn::ExecutionArgs; +use crate::scalar_fn::ReduceNode; +use crate::scalar_fn::ReduceNodeValidity; +use crate::scalar_fn::ScalarFnId; +use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::display::ExprDisplay; +use crate::validity::Validity; + +/// Pack zero or more expressions into a structure with named fields. +#[derive(Clone)] +pub struct Pack; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PackOptions { + pub names: FieldNames, + pub nullability: Nullability, +} + +impl Display for PackOptions { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "names: [{}], nullability: {:#}", + self.names.iter().join(", "), + self.nullability + ) + } +} + +impl ScalarFnVTable for Pack { + type Options = PackOptions; + + fn id(&self) -> ScalarFnId { + static ID: CachedId = CachedId::new("vortex.pack"); + *ID + } + + fn serialize(&self, instance: &Self::Options) -> VortexResult>> { + Ok(Some( + pb::PackOpts { + paths: instance.names.iter().map(|n| n.to_string()).collect(), + nullable: instance.nullability.into(), + } + .encode_to_vec(), + )) + } + + fn deserialize( + &self, + _metadata: &[u8], + _session: &VortexSession, + ) -> VortexResult { + let opts = pb::PackOpts::decode(_metadata)?; + let names: FieldNames = opts + .paths + .iter() + .map(|name| FieldName::from(name.as_str())) + .collect(); + Ok(PackOptions { + names, + nullability: opts.nullable.into(), + }) + } + + fn arity(&self, options: &Self::Options) -> Arity { + Arity::Exact(options.names.len()) + } + + fn child_name(&self, instance: &Self::Options, child_idx: usize) -> ChildName { + match instance.names.get(child_idx) { + Some(name) => ChildName::from(Arc::clone(name.inner())), + None => unreachable!( + "Invalid child index {} for Pack expression with {} fields", + child_idx, + instance.names.len() + ), + } + } + + fn fmt_sql( + &self, + options: &Self::Options, + expr: &dyn ExprDisplay, + f: &mut Formatter<'_>, + ) -> std::fmt::Result { + write!(f, "pack(")?; + for (i, name) in options.names.iter().enumerate() { + write!(f, "{}: ", name)?; + Display::fmt(expr.display_child(i), f)?; + if i + 1 < options.names.len() { + write!(f, ", ")?; + } + } + write!(f, "){}", options.nullability) + } + + fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult { + Ok(DType::Struct( + StructFields::new(options.names.clone(), arg_dtypes.to_vec()), + options.nullability, + )) + } + + fn validity( + &self, + _options: &Self::Options, + node: &T, + ) -> VortexResult> { + Ok(ReduceNodeValidity::Reduced(node.new_constant(true.into()))) + } + + fn execute( + &self, + options: &Self::Options, + args: &dyn ExecutionArgs, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let len = args.row_count(); + let value_arrays: Vec = (0..args.num_inputs()) + .map(|i| args.get(i)) + .collect::>()?; + let validity: Validity = options.nullability.into(); + StructArray::try_new(options.names.clone(), value_arrays, len, validity)? + .into_array() + .execute(ctx) + } + + fn is_strict(&self, _instance: &Self::Options) -> bool { + // A null field value does not force the packed struct row to be null. + false + } + + fn is_infallible(&self, _instance: &Self::Options) -> bool { + true + } +} diff --git a/vortex-array/src/scalar_fn/fns/select.rs b/vortex-array/src/scalar_fn/builtin/select.rs similarity index 57% rename from vortex-array/src/scalar_fn/fns/select.rs rename to vortex-array/src/scalar_fn/builtin/select.rs index e410105d1b4..884dd9f5981 100644 --- a/vortex-array/src/scalar_fn/fns/select.rs +++ b/vortex-array/src/scalar_fn/builtin/select.rs @@ -25,20 +25,20 @@ use crate::arrays::struct_::StructArrayExt; use crate::dtype::DType; use crate::dtype::FieldName; use crate::dtype::FieldNames; -use crate::expr::BoundExpression; -use crate::expr::display::ExprDisplay; -use crate::expr::field::DisplayFieldNames; use crate::proto::expr::FieldNames as ProtoFieldNames; use crate::proto::expr::SelectOpts; use crate::proto::expr::select_opts::Opts; use crate::scalar_fn::Arity; use crate::scalar_fn::ChildName; use crate::scalar_fn::ExecutionArgs; +use crate::scalar_fn::ReduceNode; use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::pack::Pack; -use crate::scalar_fn::fns::pack::PackOptions; +use crate::scalar_fn::builtin::field::DisplayFieldNames; +use crate::scalar_fn::builtin::pack::Pack; +use crate::scalar_fn::builtin::pack::PackOptions; +use crate::scalar_fn::display::ExprDisplay; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum FieldSelection { @@ -182,13 +182,13 @@ impl ScalarFnVTable for Select { result.into_array().execute(ctx) } - fn simplify( + fn simplify( &self, selection: &FieldSelection, - expr: &BoundExpression, - ) -> VortexResult> { - let child_struct = expr.child(0); - let struct_dtype = child_struct.dtype(); + node: &T, + ) -> VortexResult> { + let child_struct = node.child(0); + let struct_dtype = child_struct.node_dtype()?; let struct_nullability = struct_dtype.nullability(); let struct_fields = struct_dtype.as_struct_fields_opt().ok_or_else(|| { @@ -210,17 +210,17 @@ impl ScalarFnVTable for Select { .is_nullable() }); - // If no fields are included, we can trivially simplify to a pack expression. + // If no fields are included, we can trivially reduce to a pack node. // NOTE(ngates): we do this knowing that our layout expression partitioning logic has // special-casing for pack, but not for select. We will fix this up when we revisit the // layout APIs. if included_fields.is_empty() { - return Ok(Some(Pack.try_new_bound_expr( - PackOptions { + return Ok(Some(node.new_node( + Pack.bind(PackOptions { names: FieldNames::empty(), nullability: struct_nullability, - }, - [], + }), + &[], )?)); } @@ -228,22 +228,22 @@ impl ScalarFnVTable for Select { // because reading a field does a validity intersection of the struct validity with the // field, which is not the same as just "masking" out the unwanted fields (a selection). // - // We can, however, make this simplification when the child of the `select` is already a + // We can, however, make this reduction when the child of the `select` is already a // `pack` and we know that reading a field will do no validity intersections. // // The intersection only happens when the struct is nullable but the field is not. That - // would change the semantics of a `select`, so we can only simplify when it won't happen. + // would change the semantics of a `select`, so we can only reduce when it won't happen. let would_intersect_validity = struct_nullability.is_nullable() && !all_included_fields_are_nullable; - if let Some(child_pack) = child_struct.as_opt::() + if let Some(child_pack) = child_struct.scalar_fn().and_then(|f| f.as_opt::()) && !would_intersect_validity { - // Take the field expressions straight out of the child `pack` rather than wrap the + // Take the field nodes straight out of the child `pack` rather than wrap the // `pack` in `get_item`. The guard above proves the two are equivalent here, and the // optimizer visits children before their parent, so a `get_item` introduced now - // would never be simplified away. - let fields: Vec = included_fields + // would never be reduced away. + let fields: Vec = included_fields .iter() .map(|name| { let idx = child_pack.names.find(name).ok_or_else(|| { @@ -254,16 +254,16 @@ impl ScalarFnVTable for Select { ) })?; - Ok(child_struct.child(idx).clone()) + Ok(child_struct.child(idx)) }) .try_collect::<_, _, VortexError>()?; - return Ok(Some(Pack.try_new_bound_expr( - PackOptions { + return Ok(Some(node.new_node( + Pack.bind(PackOptions { names: included_fields, nullability: struct_nullability, - }, - fields, + }), + &fields, )?)); } @@ -343,214 +343,3 @@ impl Display for FieldSelection { } } } - -#[cfg(test)] -mod tests { - use std::sync::LazyLock; - - use vortex_buffer::buffer; - use vortex_error::VortexResult; - use vortex_session::VortexSession; - - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::Constant; - use crate::arrays::ConstantArray; - use crate::arrays::ScalarFnArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::FieldName; - use crate::dtype::FieldNames; - use crate::dtype::Nullability; - use crate::dtype::Nullability::Nullable; - use crate::dtype::PType::I32; - use crate::dtype::StructFields; - use crate::expr::root; - use crate::expr::select; - use crate::expr::select_exclude; - use crate::expr::test_harness; - use crate::scalar::Scalar; - use crate::scalar_fn::ScalarFnVTableExt; - use crate::scalar_fn::fns::select::FieldSelection; - use crate::scalar_fn::fns::select::Select; - use crate::scalar_fn::fns::select::StructArray; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); - - fn test_array() -> StructArray { - StructArray::from_fields(&[ - ("a", buffer![0, 1, 2].into_array()), - ("b", buffer![4, 5, 6].into_array()), - ]) - .unwrap() - } - - #[test] - pub fn include_columns() { - let mut ctx = SESSION.create_execution_ctx(); - let st = test_array(); - let select = select(vec![FieldName::from("a")], root()); - let selected = st.into_array().apply(&select).unwrap(); - let selected_names = selected.dtype().as_struct_fields().names(); - assert_eq!(selected_names.as_ref(), &["a"]); - assert_arrays_eq!( - selected, - StructArray::from_fields(&[("a", buffer![0i32, 1, 2].into_array())]).unwrap(), - &mut ctx - ); - } - - #[test] - pub fn exclude_columns() { - let mut ctx = SESSION.create_execution_ctx(); - let st = test_array(); - let select = select_exclude(vec![FieldName::from("a")], root()); - let selected = st.into_array().apply(&select).unwrap(); - let selected_names = selected.dtype().as_struct_fields().names(); - assert_eq!(selected_names.as_ref(), &["b"]); - assert_arrays_eq!( - selected, - StructArray::from_fields(&[("b", buffer![4i32, 5, 6].into_array())]).unwrap(), - &mut ctx - ); - } - - #[test] - fn dtype() { - let dtype = test_harness::struct_dtype(); - - let select_expr = select(vec![FieldName::from("a")], root()); - let expected_dtype = DType::Struct( - dtype - .as_struct_fields_opt() - .unwrap() - .project(&["a".into()]) - .unwrap(), - Nullability::NonNullable, - ); - assert_eq!(select_expr.return_dtype(&dtype).unwrap(), expected_dtype); - - let select_expr_exclude = select_exclude( - vec![ - FieldName::from("col1"), - FieldName::from("col2"), - FieldName::from("bool1"), - FieldName::from("bool2"), - ], - root(), - ); - assert_eq!( - select_expr_exclude.return_dtype(&dtype).unwrap(), - expected_dtype - ); - - let select_expr_exclude = select_exclude( - vec![FieldName::from("col1"), FieldName::from("col2")], - root(), - ); - assert_eq!( - select_expr_exclude.return_dtype(&dtype).unwrap(), - DType::Struct( - dtype - .as_struct_fields_opt() - .unwrap() - .project(&["a".into(), "bool1".into(), "bool2".into()]) - .unwrap(), - Nullability::NonNullable - ) - ); - } - - #[test] - fn test_as_include_names() { - let field_names = FieldNames::from(["a", "b", "c"]); - for expr in [select(["a"], root()), select_exclude(["b", "c"], root())] { - assert_eq!( - expr.as_::() + .normalize_to_included_fields(&field_names) + .unwrap(), + FieldNames::from(["a"]) + ); + } +} + +#[test] +fn execute_constant_struct_stays_constant() -> VortexResult<()> { + let field_count = 128usize; + let names = (0..field_count) + .map(|idx| FieldName::from(format!("f{idx}"))) + .collect::>(); + let dtypes = vec![I32.into(); field_count]; + let fields = StructFields::new(FieldNames::from(names), dtypes); + let scalar = Scalar::struct_( + DType::Struct(fields, Nullability::NonNullable), + (0..128).map(Scalar::from), + ); + let array = ConstantArray::new(scalar, 1_000_000).into_array(); + let mut ctx = SESSION.create_execution_ctx(); + + let item: ArrayRef = ScalarFnArray::try_new( + Select.bind(FieldSelection::Include(FieldNames::from(["f97", "f3"]))), + vec![array], + )? + .into_array() + .execute(&mut ctx)?; + + let constant = item.as_::(); + assert_eq!(constant.len(), 1_000_000); + assert_eq!( + constant.scalar(), + &Scalar::struct_( + DType::Struct( + StructFields::new( + FieldNames::from(["f97", "f3"]), + vec![I32.into(), I32.into()], + ), + Nullability::NonNullable, + ), + [Scalar::from(97), Scalar::from(3)], + ) + ); + Ok(()) +} + +#[test] +fn test_remove_select_rule() { + let dtype = DType::Struct( + StructFields::new(["a", "b"].into(), vec![I32.into(), I32.into()]), + Nullable, + ); + let e = select(["a", "b"], root()); + + let result = e.bind(&dtype).unwrap().optimize_recursive().unwrap(); + + assert_eq!(result.dtype(), &dtype); +} + +#[test] +fn test_remove_select_rule_exclude_fields() { + use crate::select_exclude; + + let dtype = DType::Struct( + StructFields::new( + ["a", "b", "c"].into(), + vec![I32.into(), I32.into(), I32.into()], + ), + Nullable, + ); + let e = select_exclude(["c"], root()); + + let result = e.bind(&dtype).unwrap().optimize_recursive().unwrap(); + + // Should exclude "c" and include "a" and "b" + let result_dtype = result.dtype(); + assert!(result_dtype.is_nullable()); + let fields = result_dtype.as_struct_fields_opt().unwrap(); + assert_eq!(fields.names().as_ref(), &["a", "b"]); + assert_eq!( + result_dtype, + &DType::struct_([("a", I32), ("b", I32)], Nullable) + ); +} diff --git a/vortex-expr/src/transform/bound_partition.rs b/vortex-expr/src/transform/bound_partition.rs index 88ef5287062..d7d63ca803c 100644 --- a/vortex-expr/src/transform/bound_partition.rs +++ b/vortex-expr/src/transform/bound_partition.rs @@ -11,23 +11,23 @@ use vortex_array::dtype::FieldName; use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability; use vortex_array::dtype::StructFields; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::ExactBoundExpr; -use vortex_array::expr::bound::get_item; -use vortex_array::expr::bound::pack; -use vortex_array::expr::traversal::NodeExt; -use vortex_array::expr::traversal::NodeRewriter; -use vortex_array::expr::traversal::Transformed; -use vortex_array::expr::traversal::TraversalOrder; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; use vortex_utils::aliases::hash_map::HashMap; +use crate::BoundExpression; +use crate::ExactBoundExpr; use crate::analysis::Annotation; use crate::analysis::AnnotationFn; use crate::analysis::BoundAnnotations; use crate::analysis::descendent_bound_annotations; +use crate::bound::get_item; +use crate::bound::pack; +use crate::traversal::NodeExt; +use crate::traversal::NodeRewriter; +use crate::traversal::Transformed; +use crate::traversal::TraversalOrder; /// Partition an expression into sub-expressions that are uniquely associated with an annotation. /// A root expression is also returned that can be used to recombine the results of the partitions @@ -295,16 +295,15 @@ mod tests { use vortex_array::dtype::Nullability::Nullable; use vortex_array::dtype::PType::I32; use vortex_array::dtype::StructFields; - use vortex_array::expr::checked_add; - use vortex_array::expr::col; - use vortex_array::expr::get_item; - use vortex_array::expr::lit; - use vortex_array::expr::merge; - use vortex_array::expr::pack; - use vortex_array::expr::root; use super::*; use crate::analysis::make_bound_free_field_annotator; + use crate::checked_add; + use crate::col; + use crate::get_item; + use crate::lit; + use crate::pack; + use crate::root; #[fixture] fn dtype() -> DType { @@ -442,41 +441,6 @@ mod tests { assert_eq!(partitioned.root, expected_root); } - #[rstest] - fn test_expr_merge(dtype: DType) { - let expr = merge([col("a"), pack([("b", col("b"))], NonNullable)]); - - let partitioned = partition_by_field(expr.bind(&dtype).unwrap(), &dtype).unwrap(); - let expected = merge([get_item("a_0", col("a")), get_item("b_0", col("b"))]); - let root_dtype = - partition_root_dtype(&partitioned.partition_names, &partitioned.partitions); - assert_eq!( - partitioned.root, - expected.bind(&root_dtype).unwrap(), - "{} {}", - partitioned.root, - expected - ); - - assert_eq!(partitioned.partitions.len(), 2); - - let part_a = partitioned.find_partition(&"a".into()).unwrap(); - let expected_a = pack([("a_0", col("a"))], NonNullable); - assert_eq!( - part_a, - &expected_a.bind(&dtype).unwrap(), - "{part_a} {expected_a}" - ); - - let part_b = partitioned.find_partition(&"b".into()).unwrap(); - let expected_b = pack([("b_0", pack([("b", col("b"))], NonNullable))], NonNullable); - assert_eq!( - part_b, - &expected_b.bind(&dtype).unwrap(), - "{part_b} {expected_b}" - ); - } - #[rstest] fn replacing_partitions_refreshes_root_dtype(dtype: DType) -> VortexResult<()> { let mut partitioned = partition_by_field(col("b").bind(&dtype)?, &dtype)?; diff --git a/vortex-expr/src/transform/mod.rs b/vortex-expr/src/transform/mod.rs index 2043ebdebf2..151a8c8d5c5 100644 --- a/vortex-expr/src/transform/mod.rs +++ b/vortex-expr/src/transform/mod.rs @@ -1,12 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Transformations for [`Expression`](vortex_array::expr::Expression) and -//! [`BoundExpression`](vortex_array::expr::BoundExpression) trees. +//! Transformations for [`Expression`](crate::Expression) and +//! [`BoundExpression`](crate::BoundExpression) trees. mod bound_partition; -mod match_between; mod replace; pub use bound_partition::*; -pub use match_between::find_between; pub use replace::*; diff --git a/vortex-expr/src/transform/replace.rs b/vortex-expr/src/transform/replace.rs index a990b601d0f..914cd58023c 100644 --- a/vortex-expr/src/transform/replace.rs +++ b/vortex-expr/src/transform/replace.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_array::expr::Expression; -use vortex_array::expr::traversal::NodeExt; -use vortex_array::expr::traversal::Transformed; -use vortex_array::expr::traversal::TraversalOrder; use vortex_error::VortexExpect; +use crate::Expression; +use crate::traversal::NodeExt; +use crate::traversal::Transformed; +use crate::traversal::TraversalOrder; + /// Replaces all occurrences of `needle` in the expression `expr` with `replacement`. pub fn replace(expr: Expression, needle: &Expression, replacement: Expression) -> Expression { expr.transform_down(|node| { @@ -28,11 +29,11 @@ pub fn replace(expr: Expression, needle: &Expression, replacement: Expression) - #[cfg(test)] mod test { use vortex_array::dtype::Nullability::NonNullable; - use vortex_array::expr::get_item; - use vortex_array::expr::lit; - use vortex_array::expr::pack; use super::replace; + use crate::get_item; + use crate::lit; + use crate::pack; #[test] fn test_replace_full_tree() { diff --git a/vortex-array/src/expr/traversal/fold.rs b/vortex-expr/src/traversal/fold.rs similarity index 93% rename from vortex-array/src/expr/traversal/fold.rs rename to vortex-expr/src/traversal/fold.rs index 5b6c7c94944..bbfa6a3a3bb 100644 --- a/vortex-array/src/expr/traversal/fold.rs +++ b/vortex-expr/src/traversal/fold.rs @@ -3,7 +3,7 @@ use vortex_error::VortexResult; -use crate::expr::traversal::Node; +use crate::traversal::Node; /// Use to indicate the control flow of the fold on the downwards pass. /// In the case of Continue, the context is passed on to the children nodes. @@ -72,18 +72,18 @@ pub trait NodeFolderContext { #[cfg(test)] mod tests { + use vortex_array::scalar_fn::builtin::binary::Binary; + use vortex_array::scalar_fn::builtin::literal::Literal; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexExpect; use vortex_error::vortex_bail; use super::*; - use crate::expr::Expression; - use crate::expr::checked_add; - use crate::expr::gt; - use crate::expr::lit; - use crate::expr::traversal::NodeExt; - use crate::scalar_fn::fns::binary::Binary; - use crate::scalar_fn::fns::literal::Literal; - use crate::scalar_fn::fns::operators::Operator; + use crate::Expression; + use crate::checked_add; + use crate::gt; + use crate::lit; + use crate::traversal::NodeExt; struct AddFold; impl NodeFolderContext for AddFold { diff --git a/vortex-array/src/expr/traversal/mod.rs b/vortex-expr/src/traversal/mod.rs similarity index 97% rename from vortex-array/src/expr/traversal/mod.rs rename to vortex-expr/src/traversal/mod.rs index 5b80816c30a..56739fa8c93 100644 --- a/vortex-array/src/expr/traversal/mod.rs +++ b/vortex-expr/src/traversal/mod.rs @@ -17,8 +17,8 @@ pub use fold::NodeFolderContext; pub use visitor::pre_order_visit_down; use vortex_error::VortexResult; -use crate::expr::BoundExpression; -use crate::expr::Expression; +use crate::BoundExpression; +use crate::Expression; /// Signal to control a traversal's flow #[derive(Debug, Clone, PartialEq, Eq)] @@ -549,6 +549,10 @@ impl Node for BoundExpression { #[cfg(test)] mod tests { + use vortex_array::scalar_fn::builtin::binary::Binary; + use vortex_array::scalar_fn::builtin::get_item::GetItem; + use vortex_array::scalar_fn::builtin::literal::Literal; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use vortex_utils::aliases::hash_set::HashSet; @@ -558,18 +562,14 @@ mod tests { use super::Transformed; use super::TraversalOrder; use super::visitor::pre_order_visit_down; - use crate::expr::Expression; - use crate::expr::and; - use crate::expr::col; - use crate::expr::eq; - use crate::expr::is_root; - use crate::expr::lit; - use crate::expr::not_eq; - use crate::expr::root; - use crate::scalar_fn::fns::binary::Binary; - use crate::scalar_fn::fns::get_item::GetItem; - use crate::scalar_fn::fns::literal::Literal; - use crate::scalar_fn::fns::operators::Operator; + use crate::Expression; + use crate::and; + use crate::col; + use crate::eq; + use crate::is_root; + use crate::lit; + use crate::not_eq; + use crate::root; #[derive(Default)] pub struct ExprLitCollector<'a>(pub Vec<&'a Expression>); diff --git a/vortex-array/src/expr/traversal/visitor.rs b/vortex-expr/src/traversal/visitor.rs similarity index 85% rename from vortex-array/src/expr/traversal/visitor.rs rename to vortex-expr/src/traversal/visitor.rs index d5f6c669e87..1aa330df18f 100644 --- a/vortex-array/src/expr/traversal/visitor.rs +++ b/vortex-expr/src/traversal/visitor.rs @@ -5,10 +5,10 @@ use std::marker::PhantomData; use vortex_error::VortexResult; -use crate::expr::traversal::Node; -use crate::expr::traversal::NodeExt; -use crate::expr::traversal::NodeVisitor; -use crate::expr::traversal::TraversalOrder; +use crate::traversal::Node; +use crate::traversal::NodeExt; +use crate::traversal::NodeVisitor; +use crate::traversal::TraversalOrder; struct FnVisitor<'a, F, T: 'a> where diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index e339215c2e0..fd5ad916c8f 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -36,6 +36,7 @@ use vortex::error::vortex_ensure; use vortex::error::vortex_ensure_eq; use vortex::error::vortex_err; use vortex::error::vortex_panic; +use vortex::expr::ArrayExprExt; use vortex_arrow::ArrowSessionExt; use crate::box_wrapper; diff --git a/vortex-ffi/src/expression.rs b/vortex-ffi/src/expression.rs index 970e81fa126..4126b390f59 100644 --- a/vortex-ffi/src/expression.rs +++ b/vortex-ffi/src/expression.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use vortex::dtype::FieldName; use vortex::error::VortexExpect; use vortex::expr::Expression; +use vortex::expr::ScalarFnExprExt; use vortex::expr::and_collect; use vortex::expr::get_item; use vortex::expr::is_null; @@ -18,9 +19,8 @@ use vortex::expr::not; use vortex::expr::or_collect; use vortex::expr::root; use vortex::expr::select; -use vortex::scalar_fn::ScalarFnVTableExt; -use vortex::scalar_fn::fns::binary::Binary; -use vortex::scalar_fn::fns::operators::Operator; +use vortex::scalar_fn::builtin::binary::Binary; +use vortex::scalar_fn::builtin::operators::Operator; use crate::box_wrapper; use crate::error::try_or; diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index 5d3dc2c346b..93d78f57344 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -41,6 +41,7 @@ vortex-error = { workspace = true } vortex-expr = { workspace = true } vortex-fastlanes = { workspace = true } vortex-fsst = { workspace = true } +vortex-functions = { workspace = true } vortex-io = { workspace = true } vortex-layout = { workspace = true } vortex-mask = { workspace = true } diff --git a/vortex-file/src/file.rs b/vortex-file/src/file.rs index cb6772a80ad..1e19ca86f11 100644 --- a/vortex-file/src/file.rs +++ b/vortex-file/src/file.rs @@ -14,9 +14,9 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; -use vortex_array::expr::Expression; use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; +use vortex_expr::Expression; use vortex_layout::LayoutReader; use vortex_layout::scan::layout::LayoutReaderDataSource; use vortex_layout::scan::scan_builder::ScanBuilder; diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 1c309112078..a13fed66743 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -169,6 +169,8 @@ mod forever_constant { /// Registration covers reading: a session can decode every encoding registered here. The /// writer is gated separately by the editions enabled on its session. pub fn register_default_encodings(session: &VortexSession) { + vortex_expr::initialize(session); + vortex_functions::initialize(session); vortex_bytebool::initialize(session); vortex_fsst::initialize(session); vortex_onpair::initialize(session); diff --git a/vortex-file/src/pruning.rs b/vortex-file/src/pruning.rs index dd68b78bc0e..0e845d3ae69 100644 --- a/vortex-file/src/pruning.rs +++ b/vortex-file/src/pruning.rs @@ -10,15 +10,16 @@ use vortex_array::arrays::NullArray; use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; use vortex_array::dtype::StructFields; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::bound::lit; use vortex_array::expr::stats::Stat; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::cast::Cast; -use vortex_array::scalar_fn::fns::get_item::GetItem; -use vortex_array::scalar_fn::fns::literal::Literal; -use vortex_array::scalar_fn::internal::row_count::substitute_row_count; +use vortex_array::scalar_fn::builtin::cast::Cast; +use vortex_array::scalar_fn::builtin::get_item::GetItem; +use vortex_array::scalar_fn::builtin::literal::Literal; use vortex_error::VortexResult; +use vortex_expr::ArrayExprExt; +use vortex_expr::BoundExpression; +use vortex_expr::bound::lit; +use vortex_expr::lambda::row_count::substitute_row_count; use vortex_expr::stats::bind::StatBinder; use vortex_expr::stats::bind::bind_stats; use vortex_expr::stats::rewrite::BoundExpressionPruneExt; diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index 13746255c03..8ae004fdfc0 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -43,21 +43,6 @@ use vortex_array::dtype::PType; use vortex_array::dtype::PType::I32; use vortex_array::dtype::StructFields; use vortex_array::dtype::i256; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::Expression; -use vortex_array::expr::and; -use vortex_array::expr::cast; -use vortex_array::expr::col; -use vortex_array::expr::eq; -use vortex_array::expr::get_item; -use vortex_array::expr::gt; -use vortex_array::expr::gt_eq; -use vortex_array::expr::lit; -use vortex_array::expr::lt; -use vortex_array::expr::lt_eq; -use vortex_array::expr::or; -use vortex_array::expr::root; -use vortex_array::expr::select; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; use vortex_array::extension::datetime::TimeUnit; @@ -66,9 +51,8 @@ use vortex_array::extension::datetime::TimestampOptions; use vortex_array::field_path; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; -use vortex_array::scalar_fn::ScalarFnVTableExt; -use vortex_array::scalar_fn::fns::pack::Pack; -use vortex_array::scalar_fn::fns::pack::PackOptions; +use vortex_array::scalar_fn::builtin::pack::Pack; +use vortex_array::scalar_fn::builtin::pack::PackOptions; use vortex_array::stats::PRUNING_STATS; use vortex_array::stream::ArrayStreamAdapter; use vortex_array::stream::ArrayStreamExt; @@ -88,6 +72,22 @@ use vortex_edition::EditionSessionExt; use vortex_edition::declarations::core::CORE_2026_08_3; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; +use vortex_expr::Expression; +use vortex_expr::ScalarFnExprExt; +use vortex_expr::and; +use vortex_expr::cast; +use vortex_expr::col; +use vortex_expr::eq; +use vortex_expr::get_item; +use vortex_expr::gt; +use vortex_expr::gt_eq; +use vortex_expr::lit; +use vortex_expr::lt; +use vortex_expr::lt_eq; +use vortex_expr::or; +use vortex_expr::root; +use vortex_expr::select; use vortex_io::VortexWrite; use vortex_io::runtime::tokio::TokioRuntime; use vortex_io::session::RuntimeSession; diff --git a/vortex-file/src/v2/file_stats_reader.rs b/vortex-file/src/v2/file_stats_reader.rs index d971542cc43..11cb26b2949 100644 --- a/vortex-file/src/v2/file_stats_reader.rs +++ b/vortex-file/src/v2/file_stats_reader.rs @@ -14,9 +14,9 @@ use vortex_array::MaskFuture; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::dtype::StructFields; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::ExactBoundExpr; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; +use vortex_expr::ExactBoundExpr; use vortex_layout::ArrayFuture; use vortex_layout::LayoutReader; use vortex_layout::LayoutReaderRef; @@ -174,13 +174,6 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::checked_add; - use vortex_array::expr::get_item; - use vortex_array::expr::gt; - use vortex_array::expr::is_not_null; - use vortex_array::expr::is_null; - use vortex_array::expr::lit; - use vortex_array::expr::root; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; use vortex_array::extension::datetime::TimeUnit; @@ -188,6 +181,13 @@ mod tests { use vortex_array::stats::StatsSet; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::checked_add; + use vortex_expr::get_item; + use vortex_expr::gt; + use vortex_expr::is_not_null; + use vortex_expr::is_null; + use vortex_expr::lit; + use vortex_expr::root; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSession; use vortex_io::session::RuntimeSessionExt; diff --git a/vortex-functions/Cargo.toml b/vortex-functions/Cargo.toml new file mode 100644 index 00000000000..dbccb4e87b5 --- /dev/null +++ b/vortex-functions/Cargo.toml @@ -0,0 +1,83 @@ +[package] +name = "vortex-functions" +authors = { workspace = true } +categories = { workspace = true } +description = "Vortex scalar functions beyond those the core array crate relies on" +edition = { workspace = true } +include = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[dependencies] +arrow-buffer = { workspace = true } +itertools = { workspace = true } +jiff = { workspace = true } +memchr = { workspace = true } +num-traits = { workspace = true } +parking_lot = { workspace = true } +prost = { workspace = true } +regex = { workspace = true } +regex-syntax = { workspace = true } +tracing = { workspace = true } +vortex-array = { workspace = true, features = ["unstable_row_fns"] } +vortex-buffer = { workspace = true } +vortex-compute = { workspace = true } +vortex-error = { workspace = true } +vortex-expr = { workspace = true } +vortex-mask = { workspace = true } +vortex-session = { workspace = true } +vortex-utils = { workspace = true } + +[lints] +workspace = true + +[dev-dependencies] +divan = { workspace = true } +insta = { workspace = true } +mimalloc = { workspace = true } +rand = { workspace = true } +rstest = { workspace = true } +vortex-array = { workspace = true, features = [ + "_test-harness", + "table-display", +] } + +[[bench]] +name = "bool_zip" +harness = false + +[[bench]] +name = "varbinview_zip" +harness = false + +[[bench]] +name = "case_when" +harness = false + +[[bench]] +name = "like" +harness = false + +[[bench]] +name = "list_contains_set" +harness = false + +[[bench]] +name = "list_length" +harness = false + +[[bench]] +name = "list_sum" +harness = false + +[[bench]] +name = "listview_zip" +harness = false + +[[bench]] +name = "primitive_zip" +harness = false diff --git a/vortex-array/benches/bool_zip.rs b/vortex-functions/benches/bool_zip.rs similarity index 89% rename from vortex-array/benches/bool_zip.rs rename to vortex-functions/benches/bool_zip.rs index 40994da33d7..c6423b0d0d1 100644 --- a/vortex-array/benches/bool_zip.rs +++ b/vortex-functions/benches/bool_zip.rs @@ -12,8 +12,8 @@ use vortex_array::IntoArray; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; -use vortex_array::builtins::ArrayBuiltins; use vortex_array::optimizer::kernels::KernelSession; +use vortex_functions::builtins::ArrayFunctions; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -25,8 +25,11 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty().with::(); + vortex_functions::initialize(&session); + session +}); const LEN: usize = 65_536; diff --git a/vortex-array/benches/expr/case_when_bench.rs b/vortex-functions/benches/case_when.rs similarity index 92% rename from vortex-array/benches/expr/case_when_bench.rs rename to vortex-functions/benches/case_when.rs index 01f721dffc3..299585aa690 100644 --- a/vortex-array/benches/expr/case_when_bench.rs +++ b/vortex-functions/benches/case_when.rs @@ -12,6 +12,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; +use vortex_array::aggregate_fn::session::AggregateFnSession; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::DecimalArray; @@ -20,33 +21,43 @@ use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DecimalDType; -use vortex_array::expr::case_when; -use vortex_array::expr::case_when_no_else; -use vortex_array::expr::eq; -use vortex_array::expr::get_item; -use vortex_array::expr::gt; -use vortex_array::expr::lit; -use vortex_array::expr::lt; -use vortex_array::expr::nested_case_when; -use vortex_array::expr::root; +use vortex_array::dtype::session::DTypeSession; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::ScalarFnVTableExt; -use vortex_array::scalar_fn::fns::case_when::CaseWhen; -use vortex_array::scalar_fn::fns::case_when::CaseWhenOptions; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; +use vortex_array::scalar_fn::session::ScalarFnSession; +use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; +use vortex_expr::ArrayExprExt; +use vortex_expr::eq; +use vortex_expr::get_item; +use vortex_expr::gt; +use vortex_expr::lit; +use vortex_expr::lt; +use vortex_expr::root; +use vortex_functions::case_when::CaseWhen; +use vortex_functions::case_when::CaseWhenOptions; +use vortex_functions::expr::case_when; +use vortex_functions::expr::case_when_no_else; +use vortex_functions::expr::nested_case_when; use vortex_session::VortexSession; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; static SESSION: LazyLock = LazyLock::new(|| { - VortexSession::empty() + let session = VortexSession::empty() + .with::() .with::() - .with::() + .with::() + .with::() + .with::() + .with::(); + vortex_functions::initialize(&session); + session }); fn main() { diff --git a/vortex-array/benches/like.rs b/vortex-functions/benches/like.rs similarity index 94% rename from vortex-array/benches/like.rs rename to vortex-functions/benches/like.rs index bce9e8fa5fb..b14a04edfbe 100644 --- a/vortex-array/benches/like.rs +++ b/vortex-functions/benches/like.rs @@ -19,14 +19,16 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; -use vortex_array::scalar_fn::fns::like::Like; -use vortex_array::scalar_fn::fns::like::LikeOptions; +use vortex_functions::like::Like; +use vortex_functions::like::LikeOptions; use vortex_session::VortexSession; static SESSION: LazyLock = LazyLock::new(|| { - VortexSession::empty() + let session = VortexSession::empty() .with::() - .with::() + .with::(); + vortex_functions::initialize(&session); + session }); #[global_allocator] diff --git a/vortex-array/benches/list_contains_set.rs b/vortex-functions/benches/list_contains_set.rs similarity index 95% rename from vortex-array/benches/list_contains_set.rs rename to vortex-functions/benches/list_contains_set.rs index 56b486dd404..9311d84484f 100644 --- a/vortex-array/benches/list_contains_set.rs +++ b/vortex-functions/benches/list_contains_set.rs @@ -28,18 +28,21 @@ use vortex_array::arrays::VarBinViewArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::expr::list_contains; -use vortex_array::expr::lit; -use vortex_array::expr::root; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::scalar::Scalar; +use vortex_expr::ArrayExprExt; +use vortex_expr::lit; +use vortex_expr::root; +use vortex_functions::expr::list_contains; use vortex_session::VortexSession; static SESSION: LazyLock = LazyLock::new(|| { - VortexSession::empty() + let session = VortexSession::empty() .with::() - .with::() + .with::(); + vortex_functions::initialize(&session); + session }); #[global_allocator] diff --git a/vortex-array/benches/list_length.rs b/vortex-functions/benches/list_length.rs similarity index 91% rename from vortex-array/benches/list_length.rs rename to vortex-functions/benches/list_length.rs index 238f8ab3be2..dcbdc110d03 100644 --- a/vortex-array/benches/list_length.rs +++ b/vortex-functions/benches/list_length.rs @@ -26,12 +26,16 @@ use vortex_array::arrays::BoolArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::expr::list_length; -use vortex_array::expr::root; +use vortex_array::dtype::session::DTypeSession; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; +use vortex_array::scalar_fn::session::ScalarFnSession; +use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; +use vortex_expr::ArrayExprExt; +use vortex_expr::root; +use vortex_functions::expr::list_length; use vortex_session::VortexSession; #[global_allocator] @@ -43,10 +47,15 @@ fn main() { } static SESSION: LazyLock = LazyLock::new(|| { - VortexSession::empty() + let session = VortexSession::empty() + .with::() .with::() + .with::() + .with::() .with::() - .with::() + .with::(); + vortex_functions::initialize(&session); + session }); const BASE_LIST_SIZE: usize = 8; diff --git a/vortex-array/benches/list_sum.rs b/vortex-functions/benches/list_sum.rs similarity index 93% rename from vortex-array/benches/list_sum.rs rename to vortex-functions/benches/list_sum.rs index fe366820140..5e17449b6b6 100644 --- a/vortex-array/benches/list_sum.rs +++ b/vortex-functions/benches/list_sum.rs @@ -25,12 +25,16 @@ use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::expr::list_sum; -use vortex_array::expr::root; +use vortex_array::dtype::session::DTypeSession; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; +use vortex_array::scalar_fn::session::ScalarFnSession; +use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; +use vortex_expr::ArrayExprExt; +use vortex_expr::root; +use vortex_functions::expr::list_sum; use vortex_session::VortexSession; #[global_allocator] @@ -42,10 +46,15 @@ fn main() { } static SESSION: LazyLock = LazyLock::new(|| { - VortexSession::empty() + let session = VortexSession::empty() + .with::() .with::() + .with::() + .with::() .with::() - .with::() + .with::(); + vortex_functions::initialize(&session); + session }); const BASE_LIST_SIZE: usize = 8; diff --git a/vortex-array/benches/listview_zip.rs b/vortex-functions/benches/listview_zip.rs similarity index 94% rename from vortex-array/benches/listview_zip.rs rename to vortex-functions/benches/listview_zip.rs index eafde33ea8e..8f8d4039bb2 100644 --- a/vortex-array/benches/listview_zip.rs +++ b/vortex-functions/benches/listview_zip.rs @@ -13,11 +13,11 @@ use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ListViewArray; -use vortex_array::builtins::ArrayBuiltins; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; +use vortex_functions::builtins::ArrayFunctions; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -30,9 +30,11 @@ fn main() { } static SESSION: LazyLock = LazyLock::new(|| { - VortexSession::empty() + let session = VortexSession::empty() .with::() - .with::() + .with::(); + vortex_functions::initialize(&session); + session }); // Smaller than the value-path benches: listview zip cost is dominated by element concatenation and diff --git a/vortex-array/benches/primitive_zip.rs b/vortex-functions/benches/primitive_zip.rs similarity index 91% rename from vortex-array/benches/primitive_zip.rs rename to vortex-functions/benches/primitive_zip.rs index 0df9a15c6b5..6629955b86e 100644 --- a/vortex-array/benches/primitive_zip.rs +++ b/vortex-functions/benches/primitive_zip.rs @@ -16,10 +16,10 @@ use vortex_array::IntoArray; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; -use vortex_array::builtins::ArrayBuiltins; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; +use vortex_functions::builtins::ArrayFunctions; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -31,8 +31,11 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty().with::(); + vortex_functions::initialize(&session); + session +}); // Sized so the bench stays well under a few hundred microseconds under CodSpeed's instruction-count // simulation, which runs ~10x the local walltime; the branchless value blend is still exercised. diff --git a/vortex-array/benches/varbinview_zip.rs b/vortex-functions/benches/varbinview_zip.rs similarity index 94% rename from vortex-array/benches/varbinview_zip.rs rename to vortex-functions/benches/varbinview_zip.rs index 1a39b83ddfc..44e52723359 100644 --- a/vortex-array/benches/varbinview_zip.rs +++ b/vortex-functions/benches/varbinview_zip.rs @@ -11,11 +11,11 @@ use vortex_array::IntoArray; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBinViewArray; -use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; +use vortex_functions::builtins::ArrayFunctions; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -28,9 +28,11 @@ fn main() { } static SESSION: LazyLock = LazyLock::new(|| { - VortexSession::empty() + let session = VortexSession::empty() .with::() - .with::() + .with::(); + vortex_functions::initialize(&session); + session }); /// Benchmarks zip on VarBinView arrays with a highly fragmented mask (worst case for per-slice lookup paths). diff --git a/vortex-array/src/arrays/constant/compute/between.rs b/vortex-functions/src/between/constant.rs similarity index 84% rename from vortex-array/src/arrays/constant/compute/between.rs rename to vortex-functions/src/between/constant.rs index d79dff2fc17..1e53d4d1389 100644 --- a/vortex-array/src/arrays/constant/compute/between.rs +++ b/vortex-functions/src/between/constant.rs @@ -1,16 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::IntoArray; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; +use vortex_array::scalar::Scalar; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::Constant; -use crate::arrays::ConstantArray; -use crate::scalar::Scalar; -use crate::scalar_fn::fns::between::BetweenOptions; -use crate::scalar_fn::fns::between::BetweenReduce; +use crate::between::BetweenOptions; +use crate::between::BetweenReduce; impl BetweenReduce for Constant { fn between( diff --git a/vortex-array/src/arrays/decimal/compute/between.rs b/vortex-functions/src/between/decimal.rs similarity index 91% rename from vortex-array/src/arrays/decimal/compute/between.rs rename to vortex-functions/src/between/decimal.rs index c3c647cc96e..5e29323c1bc 100644 --- a/vortex-array/src/arrays/decimal/compute/between.rs +++ b/vortex-functions/src/between/decimal.rs @@ -1,24 +1,24 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::Decimal; +use vortex_array::dtype::NativeDecimalType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::i256; +use vortex_array::match_each_decimal_value_type; +use vortex_array::scalar::Scalar; use vortex_buffer::BitBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::BoolArray; -use crate::arrays::Decimal; -use crate::dtype::NativeDecimalType; -use crate::dtype::Nullability; -use crate::dtype::i256; -use crate::match_each_decimal_value_type; -use crate::scalar::Scalar; -use crate::scalar_fn::fns::between::BetweenKernel; -use crate::scalar_fn::fns::between::BetweenOptions; -use crate::scalar_fn::fns::between::StrictComparison; +use crate::between::BetweenKernel; +use crate::between::BetweenOptions; +use crate::between::StrictComparison; impl BetweenKernel for Decimal { fn between( diff --git a/vortex-expr/src/transform/match_between.rs b/vortex-functions/src/between/find.rs similarity index 93% rename from vortex-expr/src/transform/match_between.rs rename to vortex-functions/src/between/find.rs index 71a5f2b6c30..6e66580ffd6 100644 --- a/vortex-expr/src/transform/match_between.rs +++ b/vortex-functions/src/between/find.rs @@ -5,17 +5,18 @@ use std::ptr; -use vortex_array::expr::BoundExpression; -use vortex_array::scalar_fn::ScalarFnVTableExt; -use vortex_array::scalar_fn::fns::between::Between; -use vortex_array::scalar_fn::fns::between::BetweenOptions; -use vortex_array::scalar_fn::fns::between::StrictComparison; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::get_item::GetItem; -use vortex_array::scalar_fn::fns::literal::Literal; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::get_item::GetItem; +use vortex_array::scalar_fn::builtin::literal::Literal; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; +use vortex_expr::ScalarFnExprExt; + +use crate::between::Between; +use crate::between::BetweenOptions; +use crate::between::StrictComparison; /// Merge pairs of range comparisons on the same column within any conjunction of `expr` into a /// single [`Between`]. @@ -226,24 +227,25 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; - use vortex_array::expr::Expression; - use vortex_array::expr::and; - use vortex_array::expr::between; - use vortex_array::expr::col; - use vortex_array::expr::gt; - use vortex_array::expr::gt_eq; - use vortex_array::expr::lit; - use vortex_array::expr::lt; - use vortex_array::expr::lt_eq; - use vortex_array::expr::or; use vortex_array::scalar::Scalar; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::Expression; + use vortex_expr::and; + use vortex_expr::col; + use vortex_expr::gt; + use vortex_expr::gt_eq; + use vortex_expr::lit; + use vortex_expr::lt; + use vortex_expr::lt_eq; + use vortex_expr::or; use vortex_session::VortexSession; use super::find_between; + use crate::between::BetweenOptions; + use crate::between::StrictComparison; + use crate::expr::between; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); diff --git a/vortex-array/src/scalar_fn/fns/between/kernel.rs b/vortex-functions/src/between/kernel.rs similarity index 89% rename from vortex-array/src/scalar_fn/fns/between/kernel.rs rename to vortex-functions/src/between/kernel.rs index f7a42252164..9027030d35a 100644 --- a/vortex-array/src/scalar_fn/fns/between/kernel.rs +++ b/vortex-functions/src/between/kernel.rs @@ -1,22 +1,22 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::VTable; +use vortex_array::arrays::ScalarFn; +use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::kernel::ExecuteParentKernel; +use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_error::VortexExpect; use vortex_error::VortexResult; use super::Between; use super::BetweenOptions; use super::short_circuit; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::array::ArrayView; -use crate::array::VTable; -use crate::arrays::ScalarFn; -use crate::arrays::scalar_fn::ExactScalarFn; -use crate::arrays::scalar_fn::ScalarFnArrayExt; -use crate::arrays::scalar_fn::ScalarFnArrayView; -use crate::kernel::ExecuteParentKernel; -use crate::optimizer::rules::ArrayParentReduceRule; /// Reduce rule for between: restructure the array without reading buffers. /// diff --git a/vortex-array/src/scalar_fn/fns/between/mod.rs b/vortex-functions/src/between/mod.rs similarity index 84% rename from vortex-array/src/scalar_fn/fns/between/mod.rs rename to vortex-functions/src/between/mod.rs index 9ba0173975d..8324aa3319f 100644 --- a/vortex-array/src/scalar_fn/fns/between/mod.rs +++ b/vortex-functions/src/between/mod.rs @@ -1,39 +1,71 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod constant; +mod decimal; +mod find; mod kernel; +mod primitive; use std::fmt::Display; use std::fmt::Formatter; +pub use find::find_between; pub use kernel::*; use prost::Message; +use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Decimal; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::ScalarFnArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::DType::Bool; +use vortex_array::optimizer::kernels::ArrayKernelsExt; +use vortex_array::optimizer::kernels::apply_reduce_parent_rule; +use vortex_array::proto::expr as pb; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::builtin::operators::Operator; +use vortex_array::scalar_fn::display::ExprDisplay; +use vortex_array::scalar_fn::session::ScalarFnSessionExt; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::Canonical; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::ConstantArray; -use crate::arrays::Decimal; -use crate::arrays::Primitive; -use crate::arrays::ScalarFnArray; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::DType::Bool; -use crate::expr::display::ExprDisplay; -use crate::proto::expr as pb; -use crate::scalar::Scalar; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::operators::Operator; +use crate::builtins::ArrayFunctions; + +/// Registers [`Between`] and its kernels for vortex-array's canonical encodings. +pub(crate) fn initialize(session: &VortexSession) { + session.scalar_fns().register(Between); + let kernels = session.kernels(); + kernels.register_execute_parent_kernel( + Between.id(), + Primitive, + BetweenExecuteAdaptor(Primitive), + ); + kernels.register_execute_parent_kernel(Between.id(), Decimal, BetweenExecuteAdaptor(Decimal)); + kernels.register_reduce_parent(Between.id(), Constant.id(), &[constant_between]); +} + +fn constant_between( + child: &ArrayRef, + parent: &ArrayRef, + child_idx: usize, +) -> VortexResult> { + apply_reduce_parent_rule(&BetweenReduceAdaptor(Constant), child, parent, child_idx) +} #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct BetweenOptions { @@ -88,7 +120,7 @@ impl StrictComparison { /// The result can be a lazy [`ScalarFn`] array, so a caller that needs a computed array /// **must** execute it. /// -/// [`ScalarFn`]: crate::arrays::ScalarFn +/// [`ScalarFn`]: vortex_array::arrays::ScalarFn pub(super) fn short_circuit( arr: &ArrayRef, lower: &ArrayRef, @@ -340,31 +372,40 @@ mod tests { use std::sync::LazyLock; use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::DecimalArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::StructArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::DecimalDType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::scalar::DecimalValue; + use vortex_array::scalar::Scalar; + use vortex_array::test_harness::to_int_indices; + use vortex_array::validity::Validity; use vortex_buffer::buffer; + use vortex_expr::ArrayExprExt; + use vortex_expr::Expression; + use vortex_expr::col; + use vortex_expr::get_item; + use vortex_expr::lit; + use vortex_expr::proto::ExprSerializeProtoExt; + use vortex_expr::root; use super::*; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::BoolArray; - use crate::arrays::DecimalArray; - use crate::arrays::PrimitiveArray; - use crate::arrays::StructArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::DecimalDType; - use crate::dtype::Nullability; - use crate::dtype::PType; use crate::expr::between; - use crate::expr::col; - use crate::expr::get_item; - use crate::expr::lit; - use crate::expr::root; - use crate::scalar::DecimalValue; - use crate::scalar::Scalar; - use crate::test_harness::to_int_indices; - use crate::validity::Validity; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); const NON_STRICT: BetweenOptions = BetweenOptions { lower_strict: StrictComparison::NonStrict, @@ -738,4 +779,41 @@ mod tests { assert_eq!(to_int_indices(result, ctx).unwrap(), expected_indices); } + + #[test] + fn test_display_tree_between() { + use insta::assert_snapshot; + let between_expr = between( + get_item("score", root()), + lit(0), + lit(100), + BetweenOptions { + lower_strict: StrictComparison::NonStrict, + upper_strict: StrictComparison::NonStrict, + }, + ); + assert_snapshot!(between_expr.display_tree().to_string(), @r" + vortex.between(lower_strict: <=, upper_strict: <=) + ├── array: vortex.get_item(score) + │ └── input: vortex.root() + ├── lower: vortex.literal(0i32) + └── upper: vortex.literal(100i32) + "); + } + + #[test] + fn between_serde() -> VortexResult<()> { + let expr = between( + lit(1), + root(), + get_item("a", root()), + BetweenOptions { + lower_strict: StrictComparison::Strict, + upper_strict: StrictComparison::Strict, + }, + ); + let proto = expr.serialize_proto()?; + assert_eq!(Expression::from_proto(&proto, &SESSION)?, expr); + Ok(()) + } } diff --git a/vortex-array/src/arrays/primitive/compute/between.rs b/vortex-functions/src/between/primitive.rs similarity index 88% rename from vortex-array/src/arrays/primitive/compute/between.rs rename to vortex-functions/src/between/primitive.rs index 7012cb1be5f..adc5488eb1d 100644 --- a/vortex-array/src/arrays/primitive/compute/between.rs +++ b/vortex-functions/src/between/primitive.rs @@ -1,22 +1,22 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::Primitive; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; +use vortex_array::match_each_native_ptype; use vortex_buffer::BitBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::BoolArray; -use crate::arrays::Primitive; -use crate::dtype::NativePType; -use crate::dtype::Nullability; -use crate::match_each_native_ptype; -use crate::scalar_fn::fns::between::BetweenKernel; -use crate::scalar_fn::fns::between::BetweenOptions; -use crate::scalar_fn::fns::between::StrictComparison; +use crate::between::BetweenKernel; +use crate::between::BetweenOptions; +use crate::between::StrictComparison; impl BetweenKernel for Primitive { fn between( diff --git a/vortex-functions/src/builtins.rs b/vortex-functions/src/builtins.rs new file mode 100644 index 00000000000..c9d09c08883 --- /dev/null +++ b/vortex-functions/src/builtins.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Builder methods for this crate's scalar functions, complementing +//! [`ExprBuiltins`](vortex_expr::ExprBuiltins) and +//! [`ArrayBuiltins`](vortex_array::builtins::ArrayBuiltins). + +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::optimizer::ArrayOptimizer; +use vortex_array::scalar_fn::EmptyOptions; +use vortex_error::VortexResult; +use vortex_expr::Expression; +use vortex_expr::ScalarFnExprExt; + +use crate::between::Between; +use crate::between::BetweenOptions; +use crate::list_contains::ListContains; +use crate::list_contains::ListContainsOptions; +use crate::zip::Zip; + +/// Builder methods on [`Expression`] for this crate's scalar functions. +pub trait ExprFunctions: Sized { + /// Conditional selection: `result[i] = if mask[i] then if_true[i] else if_false[i]`. + fn zip(&self, if_true: Expression, if_false: Expression) -> VortexResult; + + /// Check if a list contains a value. + fn list_contains(&self, value: Expression) -> VortexResult; +} + +impl ExprFunctions for Expression { + fn zip(&self, if_true: Expression, if_false: Expression) -> VortexResult { + Zip.try_new_expr(EmptyOptions, [if_true, if_false, self.clone()]) + } + + fn list_contains(&self, value: Expression) -> VortexResult { + ListContains.try_new_expr(ListContainsOptions::default(), [self.clone(), value]) + } +} + +/// Builder methods on [`ArrayRef`] for this crate's scalar functions. +pub trait ArrayFunctions: Sized { + /// Conditional selection: `result[i] = if mask[i] then if_true[i] else if_false[i]`. + fn zip(&self, if_true: ArrayRef, if_false: ArrayRef) -> VortexResult; + + /// Check if a list contains a value. + fn list_contains(&self, value: ArrayRef) -> VortexResult; + + /// Compare a values between lower VortexResult; +} + +impl ArrayFunctions for ArrayRef { + fn zip(&self, if_true: ArrayRef, if_false: ArrayRef) -> VortexResult { + Ok(Zip::try_new(if_true, if_false, self.clone())?.into_array()) + } + + fn list_contains(&self, value: ArrayRef) -> VortexResult { + ListContains::try_new(self.clone(), value)? + .into_array() + .optimize() + } + + fn between( + self, + lower: ArrayRef, + upper: ArrayRef, + options: BetweenOptions, + ) -> VortexResult { + Between::try_new(self, lower, upper, options)? + .into_array() + .optimize() + } +} diff --git a/vortex-array/src/scalar_fn/fns/byte_length.rs b/vortex-functions/src/byte_length.rs similarity index 82% rename from vortex-array/src/scalar_fn/fns/byte_length.rs rename to vortex-functions/src/byte_length.rs index afdf77d118d..81574c4b3a3 100644 --- a/vortex-array/src/scalar_fn/fns/byte_length.rs +++ b/vortex-functions/src/byte_length.rs @@ -2,6 +2,28 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use num_traits::AsPrimitive; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::arrays::varbinview::VarBinViewArrayExt; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::kernel::ExecuteParentKernel; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::EmptyOptions; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; use vortex_buffer::Buffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -10,29 +32,6 @@ use vortex_error::vortex_ensure_eq; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::array::VTable; -use crate::arrays::ConstantArray; -use crate::arrays::PrimitiveArray; -use crate::arrays::VarBinViewArray; -use crate::arrays::scalar_fn::ExactScalarFn; -use crate::arrays::scalar_fn::ScalarFnArrayView; -use crate::arrays::varbinview::VarBinViewArrayExt; -use crate::dtype::DType; -use crate::dtype::Nullability; -use crate::dtype::PType; -use crate::kernel::ExecuteParentKernel; -use crate::scalar::Scalar; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::EmptyOptions; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; - pub trait ByteLengthKernel: VTable { fn byte_length( array: ArrayView<'_, Self>, @@ -169,24 +168,31 @@ mod tests { use std::sync::LazyLock; use rstest::rstest; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::VarBinArray; + use vortex_array::arrays::VarBinViewArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::scalar::Scalar; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::root; use vortex_session::VortexSession; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::ConstantArray; - use crate::arrays::PrimitiveArray; - use crate::arrays::VarBinArray; - use crate::arrays::VarBinViewArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability; use crate::expr::byte_length; - use crate::expr::root; - use crate::scalar::Scalar; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); #[rstest] #[case(VarBinArray::from_strs(vec!["hello", "world", ""]).into_array(), vec![5u64, 5, 0])] diff --git a/vortex-array/src/scalar_fn/fns/case_when.rs b/vortex-functions/src/case_when.rs similarity index 95% rename from vortex-array/src/scalar_fn/fns/case_when.rs rename to vortex-functions/src/case_when.rs index 1a2e7f6abb5..fce061326ec 100644 --- a/vortex-array/src/scalar_fn/fns/case_when.rs +++ b/vortex-functions/src/case_when.rs @@ -19,6 +19,30 @@ use std::hash::Hash; use std::sync::Arc; use prost::Message; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::builders::builder_with_capacity_in; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::proto::expr as pb; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::EmptyOptions; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ReduceNode; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnRef; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::builtin::fill_null::FillNull; +use vortex_array::scalar_fn::builtin::is_not_null::IsNotNull; +use vortex_array::scalar_fn::builtin::is_null::IsNull; +use vortex_array::scalar_fn::display::ExprDisplay; use vortex_buffer::BufferMut; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -28,31 +52,7 @@ use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::BoolArray; -use crate::arrays::ConstantArray; -use crate::arrays::bool::BoolArrayExt; -use crate::builders::builder_with_capacity_in; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::expr::BoundExpression; -use crate::expr::display::ExprDisplay; -use crate::proto::expr as pb; -use crate::scalar::Scalar; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::EmptyOptions; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::fill_null::FillNull; -use crate::scalar_fn::fns::is_not_null::IsNotNull; -use crate::scalar_fn::fns::is_null::IsNull; -use crate::scalar_fn::fns::literal::Literal; -use crate::scalar_fn::fns::zip::zip_nullability_union; +use crate::zip::zip_nullability_union; /// Options for the n-ary CaseWhen expression. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -269,11 +269,11 @@ impl ScalarFnVTable for CaseWhen { merge_case_branches(&branches, &remaining, &else_value, &output_dtype, ctx) } - fn simplify( + fn simplify( &self, options: &Self::Options, - expr: &BoundExpression, - ) -> VortexResult> { + node: &T, + ) -> VortexResult> { // Rewrite the COALESCE-shaped CASE WHEN into `fill_null`, which references `x` // once and lowers to a single fill kernel instead of a `zip`/merge that resolves // `x` twice (once for the `is_null` predicate, once for the value branch). @@ -281,43 +281,44 @@ impl ScalarFnVTable for CaseWhen { // CASE WHEN is_null(x) THEN c ELSE x END ==> fill_null(x, c) // CASE WHEN is_not_null(x) THEN x ELSE c END ==> fill_null(x, c) // - // The fill `c` must be a `Literal`: `fill_null`'s kernel reads the fill value via - // `as_constant()`, so a non-constant fill would produce an unexecutable expression. + // The fill `c` must be a constant: `fill_null`'s kernel reads the fill value via + // `as_constant()`, so a non-constant fill would produce an unexecutable node. if options.num_when_then_pairs != 1 || !options.has_else { return Ok(None); } - let when = expr.child(0); - let then = expr.child(1); - let els = expr.child(2); + let when = node.child(0); + let then = node.child(1); + let els = node.child(2); + + let when_is = |f: &dyn Fn(&ScalarFnRef) -> bool| when.scalar_fn().is_some_and(f); // `is_null(x) ? c : x` — predicate operand and ELSE are the same `x`, fill is THEN. - let (x, fill) = if when.is::() && when.child(0) == els { + let (x, fill) = if when_is(&|f| f.is::()) && when.child(0).is_same(&els) { (els, then) // `is_not_null(x) ? x : c` — predicate operand and THEN are the same `x`, fill is ELSE. - } else if when.is::() && when.child(0) == then { + } else if when_is(&|f| f.is::()) && when.child(0).is_same(&then) { (then, els) } else { return Ok(None); }; - let Some(scalar) = fill.as_opt::() else { + let Some(scalar) = fill.as_constant() else { return Ok(None); }; if scalar.is_null() { // Filling the nulls of `x` with NULL is a no-op - return Ok(Some(x.clone())); + return Ok(Some(x)); } - if !FillNull::supports_dtype(x.dtype()) { + if !FillNull::supports_dtype(&x.node_dtype()?) { return Ok(None); } - Ok(Some(FillNull.try_new_bound_expr( - EmptyOptions, - [x.clone(), fill.clone()], - )?)) + Ok(Some( + node.new_node(FillNull.bind(EmptyOptions), &[x, fill])?, + )) } fn is_strict(&self, _options: &Self::Options) -> bool { @@ -477,48 +478,54 @@ mod tests { use std::sync::LazyLock; use rstest::rstest; + use vortex_array::Canonical; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::DecimalArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::ScalarFnArray; + use vortex_array::arrays::StructArray; + use vortex_array::arrays::VarBinViewArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::DecimalDType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::StructFields; + use vortex_array::scalar::Scalar; + use vortex_array::scalar_fn::ScalarFnVTableExt; + use vortex_array::scalar_fn::VecExecutionArgs; + use vortex_array::scalar_fn::builtin::operators::Operator; + use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; use vortex_error::VortexExpect as _; + use vortex_expr::ArrayExprExt; + use vortex_expr::BoundExpression; + use vortex_expr::Expression; + use vortex_expr::col; + use vortex_expr::eq; + use vortex_expr::get_item; + use vortex_expr::gt; + use vortex_expr::is_not_null; + use vortex_expr::is_null; + use vortex_expr::lit; + use vortex_expr::proto::ExprSerializeProtoExt; + use vortex_expr::root; + use vortex_expr::test_harness; use vortex_session::VortexSession; use super::*; - use crate::Canonical; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::BoolArray; - use crate::arrays::DecimalArray; - use crate::arrays::PrimitiveArray; - use crate::arrays::ScalarFnArray; - use crate::arrays::StructArray; - use crate::arrays::VarBinViewArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::DecimalDType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::StructFields; - use crate::expr::Expression; use crate::expr::case_when; use crate::expr::case_when_no_else; - use crate::expr::col; - use crate::expr::eq; - use crate::expr::get_item; - use crate::expr::gt; - use crate::expr::is_not_null; - use crate::expr::is_null; - use crate::expr::lit; use crate::expr::nested_case_when; - use crate::expr::proto::ExprSerializeProtoExt; - use crate::expr::root; - use crate::expr::test_harness; - use crate::scalar::Scalar; - use crate::scalar_fn::ScalarFnVTableExt; - use crate::scalar_fn::VecExecutionArgs; - use crate::scalar_fn::fns::operators::Operator; - use crate::validity::Validity; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + crate::initialize(&session); + session + }); #[test] fn compact_and_run_merges_agree_for_nested_and_nullable_values() -> VortexResult<()> { diff --git a/vortex-array/src/scalar_fn/fns/dynamic.rs b/vortex-functions/src/dynamic.rs similarity index 89% rename from vortex-array/src/scalar_fn/fns/dynamic.rs rename to vortex-functions/src/dynamic.rs index 7ce5a17dd3b..de023a33c67 100644 --- a/vortex-array/src/scalar_fn/fns/dynamic.rs +++ b/vortex-functions/src/dynamic.rs @@ -9,34 +9,33 @@ use std::hash::Hasher; use std::sync::Arc; use parking_lot::Mutex; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::dtype::DType; +use vortex_array::scalar::Scalar; +use vortex_array::scalar::ScalarValue; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::VecExecutionArgs; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::Operator; +use vortex_array::scalar_fn::display::ExprDisplay; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_expr::BoundExpression; +use vortex_expr::traversal::NodeExt; +use vortex_expr::traversal::NodeVisitor; +use vortex_expr::traversal::TraversalOrder; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::ConstantArray; -use crate::dtype::DType; -use crate::expr::BoundExpression; -use crate::expr::display::ExprDisplay; -use crate::expr::traversal::NodeExt; -use crate::expr::traversal::NodeVisitor; -use crate::expr::traversal::TraversalOrder; -use crate::scalar::Scalar; -use crate::scalar::ScalarValue; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::VecExecutionArgs; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::operators::CompareOperator; -use crate::scalar_fn::fns::operators::Operator; - /// A dynamic comparison expression can be used to capture a comparison to a value that can change /// during the execution of a query, such as when a compute engine pushes down an ORDER BY + LIMIT /// operation and is able to progressively tighten the bounds of the filter. @@ -302,22 +301,29 @@ mod tests { use std::sync::atomic::AtomicI32; use std::sync::atomic::Ordering; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::root; use vortex_session::VortexSession; use super::*; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::BoolArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; use crate::expr::dynamic; - use crate::expr::root; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); #[test] fn is_not_strict() { diff --git a/vortex-functions/src/expr.rs b/vortex-functions/src/expr.rs new file mode 100644 index 00000000000..780ca417837 --- /dev/null +++ b/vortex-functions/src/expr.rs @@ -0,0 +1,764 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Expression constructors for this crate's scalar functions. +//! +//! These complement the constructors in `vortex_expr`. + +use std::sync::Arc; + +use vortex_array::aggregate_fn::NumericalAggregateOpts; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::scalar::Scalar; +use vortex_array::scalar::ScalarValue; +use vortex_array::scalar_fn::EmptyOptions; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; +use vortex_error::vortex_panic; +use vortex_expr::BoundExpression; +use vortex_expr::Expression; +use vortex_expr::ScalarFnExprExt; +use vortex_expr::lit; + +use crate::between::Between; +use crate::between::BetweenOptions; +use crate::between::StrictComparison; +use crate::byte_length::ByteLength; +use crate::case_when::CaseWhen; +use crate::case_when::CaseWhenOptions; +use crate::dynamic::DynamicComparison; +use crate::dynamic::DynamicComparisonExpr; +use crate::dynamic::Rhs; +use crate::ext_storage::ExtStorage; +use crate::is_nan::IsNan; +use crate::like::Like; +use crate::like::LikeOptions; +use crate::list_contains::ListContains; +use crate::list_contains::ListContainsOptions; +use crate::list_length::ListLength; +use crate::list_sum::ListSum; +use crate::merge::DuplicateHandling; +use crate::merge::Merge; +use crate::replace_time_zone::ReplaceTimeZone; +use crate::replace_time_zone::ReplaceTimeZoneOptions; +use crate::variant_get::VariantGet; +use crate::variant_get::VariantGetOptions; +use crate::variant_get::VariantPath; +use crate::zip::Zip; + +/// Creates a CASE WHEN expression with one WHEN/THEN pair and an ELSE value. +pub fn case_when( + condition: Expression, + then_value: Expression, + else_value: Expression, +) -> Expression { + let options = CaseWhenOptions { + num_when_then_pairs: 1, + has_else: true, + }; + CaseWhen.new_expr(options, [condition, then_value, else_value]) +} + +/// Creates a bound CASE WHEN expression with one WHEN/THEN pair and an ELSE value. +pub fn bound_case_when( + condition: BoundExpression, + then_value: BoundExpression, + else_value: BoundExpression, +) -> BoundExpression { + let options = CaseWhenOptions { + num_when_then_pairs: 1, + has_else: true, + }; + CaseWhen + .try_new_bound_expr(options, [condition, then_value, else_value]) + .vortex_expect("case expressions must have boolean conditions and matching branch dtypes") +} + +/// Creates a CASE WHEN expression with one WHEN/THEN pair and no ELSE value. +pub fn case_when_no_else(condition: Expression, then_value: Expression) -> Expression { + let options = CaseWhenOptions { + num_when_then_pairs: 1, + has_else: false, + }; + CaseWhen.new_expr(options, [condition, then_value]) +} + +/// Creates a bound CASE WHEN expression with one WHEN/THEN pair and no ELSE value. +pub fn bound_case_when_no_else( + condition: BoundExpression, + then_value: BoundExpression, +) -> BoundExpression { + let options = CaseWhenOptions { + num_when_then_pairs: 1, + has_else: false, + }; + CaseWhen + .try_new_bound_expr(options, [condition, then_value]) + .vortex_expect("case expressions must have boolean conditions") +} + +/// Creates an n-ary CASE WHEN expression from WHEN/THEN pairs and an optional ELSE value. +pub fn nested_case_when( + when_then_pairs: Vec<(Expression, Expression)>, + else_value: Option, +) -> Expression { + assert!( + !when_then_pairs.is_empty(), + "nested_case_when requires at least one when/then pair" + ); + + let has_else = else_value.is_some(); + let mut children = Vec::with_capacity(when_then_pairs.len() * 2 + usize::from(has_else)); + for (condition, then_value) in &when_then_pairs { + children.push(condition.clone()); + children.push(then_value.clone()); + } + if let Some(else_expr) = else_value { + children.push(else_expr); + } + + let Ok(num_when_then_pairs) = u32::try_from(when_then_pairs.len()) else { + vortex_panic!("nested_case_when has too many when/then pairs"); + }; + let options = CaseWhenOptions { + num_when_then_pairs, + has_else, + }; + CaseWhen.new_expr(options, children) +} + +/// Creates a bound n-ary CASE WHEN expression from WHEN/THEN pairs and an optional ELSE value. +pub fn bound_nested_case_when( + when_then_pairs: Vec<(BoundExpression, BoundExpression)>, + else_value: Option, +) -> BoundExpression { + assert!( + !when_then_pairs.is_empty(), + "nested_case_when requires at least one when/then pair" + ); + + let Ok(num_when_then_pairs) = u32::try_from(when_then_pairs.len()) else { + vortex_panic!("nested_case_when has too many when/then pairs"); + }; + let has_else = else_value.is_some(); + let mut children = Vec::with_capacity(when_then_pairs.len() * 2 + usize::from(has_else)); + for (condition, then_value) in when_then_pairs { + children.push(condition); + children.push(then_value); + } + if let Some(else_expr) = else_value { + children.push(else_expr); + } + + let options = CaseWhenOptions { + num_when_then_pairs, + has_else, + }; + CaseWhen + .try_new_bound_expr(options, children) + .vortex_expect("case expressions must have boolean conditions and matching branch dtypes") +} + +/// Creates an expression that merges struct expressions into a single struct. +/// +/// Combines fields from all input expressions. If field names are duplicated, +/// later expressions win. Fields are not recursively merged. +/// +/// ```rust +/// # use vortex_array::dtype::Nullability; +/// # use vortex_functions::expr::merge; +/// # use vortex_expr::{get_item, root}; +/// let expr = merge([get_item("a", root()), get_item("b", root())]); +/// ``` +pub fn merge(elements: impl IntoIterator>) -> Expression { + use itertools::Itertools as _; + let values = elements.into_iter().map(|value| value.into()).collect_vec(); + Merge.new_expr(DuplicateHandling::default(), values) +} + +/// Creates a bound expression that merges struct expressions. +pub fn bound_merge(elements: impl IntoIterator) -> BoundExpression { + bound_merge_opts(elements, DuplicateHandling::default()) +} + +/// Creates a merge expression with explicit duplicate handling. +pub fn merge_opts( + elements: impl IntoIterator>, + duplicate_handling: DuplicateHandling, +) -> Expression { + use itertools::Itertools as _; + let values = elements.into_iter().map(|value| value.into()).collect_vec(); + Merge.new_expr(duplicate_handling, values) +} + +/// Creates a bound merge expression with explicit duplicate handling. +pub fn bound_merge_opts( + elements: impl IntoIterator, + duplicate_handling: DuplicateHandling, +) -> BoundExpression { + Merge + .try_new_bound_expr(duplicate_handling, elements) + .vortex_expect("merge expressions require non-nullable struct children") +} + +/// Creates an expression that computes the byte length of each element. +/// This is akin to ANSI SQL OCTET_LENGTH(), or DuckDB's strlen(). +/// +/// ```rust +/// # use vortex_functions::expr::byte_length; +/// # use vortex_expr::root; +/// let expr = byte_length(root()); +/// ``` +pub fn byte_length(input: Expression) -> Expression { + ByteLength.new_expr(EmptyOptions, [input]) +} + +/// Creates a bound expression that computes each element's byte length. +pub fn bound_byte_length(input: BoundExpression) -> BoundExpression { + ByteLength + .try_new_bound_expr(EmptyOptions, [input]) + .vortex_expect("byte-length expressions require a variable-length binary child") +} + +/// Creates an expression that extracts the storage values from an extension array. +/// +/// ```rust +/// # use vortex_functions::expr::ext_storage; +/// # use vortex_expr::root; +/// let expr = ext_storage(root()); +/// ``` +pub fn ext_storage(input: Expression) -> Expression { + ExtStorage.new_expr(EmptyOptions, [input]) +} + +/// Creates a bound expression that extracts an extension array's storage values. +pub fn bound_ext_storage(input: BoundExpression) -> BoundExpression { + ExtStorage + .try_new_bound_expr(EmptyOptions, [input]) + .vortex_expect("extension-storage expressions require an extension child") +} + +/// Creates an expression that computes the number of elements in each list +/// for `List` and `FixedSizeList` inputs. This is akin to ANSI SQL `CARDINALITY()`, +/// or DuckDB's `len()`/`array_length()`. +/// +/// ```rust +/// # use vortex_functions::expr::list_length; +/// # use vortex_expr::root; +/// let expr = list_length(root()); +/// ``` +pub fn list_length(input: Expression) -> Expression { + ListLength.new_expr(EmptyOptions, [input]) +} + +/// Creates a bound expression that computes the number of elements in each list. +pub fn bound_list_length(input: BoundExpression) -> BoundExpression { + ListLength + .try_new_bound_expr(EmptyOptions, [input]) + .vortex_expect("list-length expressions require a list child") +} + +/// Creates an expression that sums the elements of each list for `List` and +/// `FixedSizeList` inputs, akin to DuckDB's `list_sum()`. +/// +/// Follows SQL `SUM` semantics per list: null lists, empty lists, and lists whose elements are +/// all null yield null; null elements are skipped; integer and decimal overflow yields a null +/// value. The result dtype follows `sum`'s widening rules and is always nullable. NaN float +/// elements are skipped by default; see [`list_sum_opts`] for the NaN-including variant. +/// +/// ```rust +/// # use vortex_functions::expr::list_sum; +/// # use vortex_expr::root; +/// let expr = list_sum(root()); +/// ``` +pub fn list_sum(input: Expression) -> Expression { + ListSum.new_expr(NumericalAggregateOpts::default(), [input]) +} + +/// Creates a bound expression that sums the elements of each list. +pub fn bound_list_sum(input: BoundExpression) -> BoundExpression { + ListSum + .try_new_bound_expr(NumericalAggregateOpts::default(), [input]) + .vortex_expect("list-sum expressions require a numeric list child") +} + +/// Creates a [`list_sum`] expression with explicit [`NumericalAggregateOpts`], controlling +/// whether NaN float elements are skipped (the default) or poison the list's sum to NaN. +pub fn list_sum_opts(input: Expression, options: NumericalAggregateOpts) -> Expression { + ListSum.new_expr(options, [input]) +} + +/// Creates a bound list-sum expression with explicit aggregate options. +pub fn bound_list_sum_opts( + input: BoundExpression, + options: NumericalAggregateOpts, +) -> BoundExpression { + ListSum + .try_new_bound_expr(options, [input]) + .vortex_expect("list-sum expressions require a numeric list child") +} + +/// Reinterpret timestamp wall times in `options.time_zone`, preserving the time unit. +/// +/// `ambiguous` is a UTF-8 expression containing `raise`, `earliest`, `latest`, or `null`. +/// Removing the timezone preserves wall time in a timezone-naive timestamp. +pub fn replace_time_zone( + input: Expression, + ambiguous: Expression, + options: ReplaceTimeZoneOptions, +) -> Expression { + ReplaceTimeZone.new_expr(options, [input, ambiguous]) +} + +/// Create a bound timezone replacement expression. +pub fn bound_replace_time_zone( + input: BoundExpression, + ambiguous: BoundExpression, + options: ReplaceTimeZoneOptions, +) -> BoundExpression { + ReplaceTimeZone + .try_new_bound_expr(options, [input, ambiguous]) + .vortex_expect("timezone replacement requires a timestamp and a UTF-8 ambiguity policy") +} + +/// Constructors for expressions whose children have already been bound and type-checked. +/// +/// These mirror the constructors in [`crate::expr`] and panic when the supplied children do not +/// form a well-typed expression. +/// Creates a dynamic comparison expression from its complete options. +pub fn dynamic_with_options(options: DynamicComparisonExpr, lhs: Expression) -> Expression { + DynamicComparison.new_expr(options, [lhs]) +} + +/// Creates a bound dynamic comparison expression from its complete options. +pub fn bound_dynamic_with_options( + options: DynamicComparisonExpr, + lhs: BoundExpression, +) -> BoundExpression { + DynamicComparison + .try_new_bound_expr(options, [lhs]) + .vortex_expect("dynamic comparisons require a compatible left-hand dtype") +} + +/// Creates a dynamic comparison expression. +pub fn dynamic( + operator: CompareOperator, + rhs_value: impl Fn() -> Option + Send + Sync + 'static, + rhs_dtype: DType, + default: bool, + lhs: Expression, +) -> Expression { + dynamic_with_options( + DynamicComparisonExpr { + operator, + rhs: Arc::new(Rhs { + value: Arc::new(rhs_value), + dtype: rhs_dtype, + }), + default, + }, + lhs, + ) +} + +/// Creates a bound dynamic comparison expression. +pub fn bound_dynamic( + operator: CompareOperator, + rhs_value: impl Fn() -> Option + Send + Sync + 'static, + rhs_dtype: DType, + default: bool, + lhs: BoundExpression, +) -> BoundExpression { + bound_dynamic_with_options( + DynamicComparisonExpr { + operator, + rhs: Arc::new(Rhs { + value: Arc::new(rhs_value), + dtype: rhs_dtype, + }), + default, + }, + lhs, + ) +} + +// ---- Between ---- + +/// Creates an expression that checks if values are between two bounds. +/// +/// Returns a boolean array indicating which values fall within the specified range. +/// The comparison strictness is controlled by the options parameter. +/// +/// ```rust +/// # use vortex_functions::between::BetweenOptions; +/// # use vortex_functions::between::StrictComparison; +/// # use vortex_expr::{lit, root}; +/// # use vortex_functions::expr::between; +/// let opts = BetweenOptions { +/// lower_strict: StrictComparison::NonStrict, +/// upper_strict: StrictComparison::NonStrict, +/// }; +/// let expr = between(root(), lit(10), lit(20), opts); +/// ``` +pub fn between( + arr: Expression, + lower: Expression, + upper: Expression, + options: BetweenOptions, +) -> Expression { + Between + .try_new_expr(options, [arr, lower, upper]) + .vortex_expect("Failed to create Between expression") +} + +/// Creates a bound expression that checks if values are between two bounds. +pub fn bound_between( + arr: BoundExpression, + lower: BoundExpression, + upper: BoundExpression, + options: BetweenOptions, +) -> BoundExpression { + Between + .try_new_bound_expr(options, [arr, lower, upper]) + .vortex_expect("between expressions require compatible operand dtypes") +} + +// ---- ListContains ---- + +/// Creates an expression that checks whether a list contains a value. +/// +/// A null element never matches anything: a needle that matches no element yields `false`. For +/// SQL's three-valued `IN`, where a null element makes that `null`, use [`in_list`]. +/// +/// ```rust +/// # use vortex_expr::{lit, root}; +/// # use vortex_functions::expr::list_contains; +/// let expr = list_contains(root(), lit(42)); +/// ``` +pub fn list_contains(list: Expression, value: Expression) -> Expression { + list_contains_opts(list, value, ListContainsOptions::default()) +} + +/// Creates an expression that checks whether a list contains a value, with explicit +/// [`ListContainsOptions`]. +pub fn list_contains_opts( + list: Expression, + value: Expression, + options: ListContainsOptions, +) -> Expression { + ListContains.new_expr(options, [list, value]) +} + +/// Creates `needle IN (list)` with SQL null semantics: a null element is an unknown value, so a +/// needle that matches no element yields `null` rather than `false` when the list holds one, and +/// `not(in_list(..))` never admits such a row. +/// +/// ```rust +/// # use vortex_expr::{lit, root}; +/// # use vortex_functions::expr::in_list; +/// let expr = in_list(root(), lit(vec![1, 2, 3])); +/// ``` +pub fn in_list(needle: Expression, list: Expression) -> Expression { + list_contains_opts( + list, + needle, + ListContainsOptions { + sql_null_semantics: true, + }, + ) +} + +/// Creates `needle IN (elements)` with SQL null semantics for a list of constant elements of +/// `element_dtype`, ignoring nullability. An element can also be an untyped null. +/// +/// The nullability of `element_dtype` is not used: the list elements are nullable only when a +/// null element is present. +/// +/// Non-null integers that hold every value between their smallest and largest become a range +/// check, which is cheaper to evaluate and prunes with tighter bounds than a set lookup, as for +/// the keys of a small dimension table that a join pushes down. +/// +/// # Errors +/// +/// Returns an error if an element is not a null and not of `element_dtype`. +/// +/// ```rust +/// # use vortex_array::dtype::{DType, Nullability, PType}; +/// # use vortex_expr::root; +/// # use vortex_functions::expr::in_list_of; +/// let dtype = DType::Primitive(PType::I32, Nullability::NonNullable); +/// let expr = in_list_of(root(), dtype, vec![3i32.into(), 1i32.into(), 2i32.into()])?; +/// assert_eq!(expr.to_string(), "(1i32 <= $ <= 3i32)"); +/// # Ok::<(), vortex_error::VortexError>(()) +/// ``` +pub fn in_list_of( + needle: Expression, + element_dtype: DType, + elements: Vec, +) -> VortexResult { + let element_dtype = if elements.iter().any(Scalar::is_null) { + element_dtype.as_nullable() + } else { + element_dtype.as_nonnullable() + }; + let mut elements = elements + .iter() + .map(|element| { + vortex_ensure!( + element.dtype() == &DType::Null + || element.dtype().eq_ignore_nullability(&element_dtype), + "IN list element {element} is not of type {element_dtype}" + ); + element.cast(&element_dtype) + }) + .collect::>>()?; + + if let Some((low, high)) = contiguous_integer_bounds(&elements) { + let options = BetweenOptions { + lower_strict: StrictComparison::NonStrict, + upper_strict: StrictComparison::NonStrict, + }; + return Ok(between( + needle, + lit(low.clone()), + lit(high.clone()), + options, + )); + } + + // Ordered elements make equal lists from the same set, so that equal filters deduplicate. + if elements + .windows(2) + .all(|pair| pair[0].partial_cmp(&pair[1]).is_some()) + { + elements.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + } + let list = Scalar::list(Arc::new(element_dtype), elements, Nullability::NonNullable); + Ok(in_list(needle, lit(list))) +} + +/// The smallest and largest of non-null integer `scalars` when they hold every integer between +/// those two. +fn contiguous_integer_bounds(scalars: &[Scalar]) -> Option<(&Scalar, &Scalar)> { + let mut values = scalars + .iter() + .map(|scalar| { + let primitive = scalar.as_primitive_opt()?; + if !primitive.ptype().is_int() { + return None; + } + let value = if primitive.ptype().is_unsigned_int() { + primitive.as_opt::()??.into() + } else { + i128::from(primitive.as_opt::()??) + }; + Some((value, scalar)) + }) + .collect::>>()?; + values.sort_unstable_by_key(|(value, _)| *value); + values.dedup_by_key(|(value, _)| *value); + let (low, low_scalar) = values.first()?; + let (high, high_scalar) = values.last()?; + (high - low + 1 == i128::try_from(values.len()).ok()?).then_some((*low_scalar, *high_scalar)) +} + +/// Creates a bound expression that checks whether a list contains a value. +pub fn bound_list_contains(list: BoundExpression, value: BoundExpression) -> BoundExpression { + ListContains + .try_new_bound_expr(ListContainsOptions::default(), [list, value]) + .vortex_expect("list-contains expressions require a list child") +} + +/// Creates a bound `needle IN (list)` with SQL null semantics. See [`in_list`]. +pub fn bound_in_list(needle: BoundExpression, list: BoundExpression) -> BoundExpression { + ListContains + .try_new_bound_expr( + ListContainsOptions { + sql_null_semantics: true, + }, + [list, needle], + ) + .vortex_expect("list-contains expressions require a list child") +} + +// ---- VariantGet ---- + +/// Creates an expression that extracts a path from a Variant expression. +/// +/// Missing paths, traversal mismatches, and failed casts return null. When `dtype` is `None`, +/// results are nullable Variant values; otherwise results are nullable values of `dtype`. +pub fn variant_get( + child: Expression, + path: impl Into, + dtype: Option, +) -> Expression { + VariantGet.new_expr(VariantGetOptions::new(path.into(), dtype), vec![child]) +} + +/// Creates a bound expression that extracts a path from a Variant expression. +pub fn bound_variant_get( + child: BoundExpression, + path: impl Into, + dtype: Option, +) -> BoundExpression { + VariantGet + .try_new_bound_expr(VariantGetOptions::new(path.into(), dtype), [child]) + .vortex_expect("variant-get expressions require a Variant child") +} + +// ---- Like ---- + +/// Creates a SQL LIKE expression. +pub fn like(child: Expression, pattern: Expression) -> Expression { + Like.new_expr( + LikeOptions { + negated: false, + case_insensitive: false, + }, + [child, pattern], + ) +} + +/// Creates a bound SQL LIKE expression. +pub fn bound_like(child: BoundExpression, pattern: BoundExpression) -> BoundExpression { + bound_like_with_options(child, pattern, false, false) +} + +/// Creates a case-insensitive SQL ILIKE expression. +pub fn ilike(child: Expression, pattern: Expression) -> Expression { + Like.new_expr( + LikeOptions { + negated: false, + case_insensitive: true, + }, + [child, pattern], + ) +} + +/// Creates a bound case-insensitive SQL ILIKE expression. +pub fn bound_ilike(child: BoundExpression, pattern: BoundExpression) -> BoundExpression { + bound_like_with_options(child, pattern, false, true) +} + +/// Creates a negated SQL NOT LIKE expression. +pub fn not_like(child: Expression, pattern: Expression) -> Expression { + Like.new_expr( + LikeOptions { + negated: true, + case_insensitive: false, + }, + [child, pattern], + ) +} + +/// Creates a bound negated SQL NOT LIKE expression. +pub fn bound_not_like(child: BoundExpression, pattern: BoundExpression) -> BoundExpression { + bound_like_with_options(child, pattern, true, false) +} + +/// Creates a negated case-insensitive SQL NOT ILIKE expression. +pub fn not_ilike(child: Expression, pattern: Expression) -> Expression { + Like.new_expr( + LikeOptions { + negated: true, + case_insensitive: true, + }, + [child, pattern], + ) +} + +/// Creates a bound negated case-insensitive SQL NOT ILIKE expression. +pub fn bound_not_ilike(child: BoundExpression, pattern: BoundExpression) -> BoundExpression { + bound_like_with_options(child, pattern, true, true) +} + +fn bound_like_with_options( + child: BoundExpression, + pattern: BoundExpression, + negated: bool, + case_insensitive: bool, +) -> BoundExpression { + Like.try_new_bound_expr( + LikeOptions { + negated, + case_insensitive, + }, + [child, pattern], + ) + .vortex_expect("like expressions require UTF-8 or binary operands") +} + +// ---- IsNan ---- + +/// Creates an expression that checks for NaN values. +/// +/// The expression is strict: null inputs produce null outputs, so the output nullability +/// follows the input. Only primitive float inputs are supported. +/// +/// ```rust +/// # use vortex_expr::root; +/// # use vortex_functions::expr::is_nan; +/// let expr = is_nan(root()); +/// ``` +pub fn is_nan(child: Expression) -> Expression { + IsNan.new_expr(EmptyOptions, vec![child]) +} + +/// Creates a bound expression that checks for NaN values. +pub fn bound_is_nan(child: BoundExpression) -> BoundExpression { + IsNan + .try_new_bound_expr(EmptyOptions, [child]) + .vortex_expect("is-nan expressions are always well-typed") +} + +// ---- Zip ---- + +/// Creates a zip expression that conditionally selects between two arrays. +/// +/// ```rust +/// # use vortex_expr::{lit, root}; +/// # use vortex_functions::expr::zip_expr; +/// let expr = zip_expr(lit(true), root(), lit(0i32)); +/// ``` +pub fn zip_expr(mask: Expression, if_true: Expression, if_false: Expression) -> Expression { + Zip.new_expr(EmptyOptions, [if_true, if_false, mask]) +} + +/// Creates a bound zip expression that conditionally selects between two arrays. +pub fn bound_zip_expr( + mask: BoundExpression, + if_true: BoundExpression, + if_false: BoundExpression, +) -> BoundExpression { + Zip.try_new_bound_expr(EmptyOptions, [if_true, if_false, mask]) + .vortex_expect("zip expressions require a boolean mask and compatible value dtypes") +} + +pub mod bound { + pub use super::bound_between as between; + pub use super::bound_byte_length as byte_length; + pub use super::bound_case_when as case_when; + pub use super::bound_case_when_no_else as case_when_no_else; + pub use super::bound_dynamic as dynamic; + pub use super::bound_dynamic_with_options as dynamic_with_options; + pub use super::bound_ext_storage as ext_storage; + pub use super::bound_ilike as ilike; + pub use super::bound_in_list as in_list; + pub use super::bound_is_nan as is_nan; + pub use super::bound_like as like; + pub use super::bound_list_contains as list_contains; + pub use super::bound_list_length as list_length; + pub use super::bound_list_sum as list_sum; + pub use super::bound_list_sum_opts as list_sum_opts; + pub use super::bound_merge as merge; + pub use super::bound_merge_opts as merge_opts; + pub use super::bound_nested_case_when as nested_case_when; + pub use super::bound_not_ilike as not_ilike; + pub use super::bound_not_like as not_like; + pub use super::bound_replace_time_zone as replace_time_zone; + pub use super::bound_variant_get as variant_get; + pub use super::bound_zip_expr as zip_expr; +} diff --git a/vortex-array/src/scalar_fn/fns/ext_storage.rs b/vortex-functions/src/ext_storage.rs similarity index 78% rename from vortex-array/src/scalar_fn/fns/ext_storage.rs rename to vortex-functions/src/ext_storage.rs index 1e708cf2414..75bfd5d8c9a 100644 --- a/vortex-array/src/scalar_fn/fns/ext_storage.rs +++ b/vortex-functions/src/ext_storage.rs @@ -1,25 +1,24 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::dtype::DType; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::EmptyOptions; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::ConstantArray; -use crate::arrays::ExtensionArray; -use crate::arrays::extension::ExtensionArrayExt; -use crate::dtype::DType; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::EmptyOptions; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; - /// Extract the storage values from an extension array. #[derive(Clone)] pub struct ExtStorage; @@ -97,27 +96,34 @@ impl ScalarFnVTable for ExtStorage { mod tests { use std::sync::LazyLock; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::ExtensionArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::extension::ExtDTypeRef; + use vortex_array::extension::datetime::TimeUnit; + use vortex_array::extension::datetime::Timestamp; + use vortex_array::scalar::Scalar; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::root; use vortex_session::VortexSession; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::ConstantArray; - use crate::arrays::ExtensionArray; - use crate::arrays::PrimitiveArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::extension::ExtDTypeRef; use crate::expr::ext_storage; - use crate::expr::root; - use crate::extension::datetime::TimeUnit; - use crate::extension::datetime::Timestamp; - use crate::scalar::Scalar; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); fn ext_dtype(nullability: Nullability) -> ExtDTypeRef { Timestamp::new(TimeUnit::Nanoseconds, nullability).erased() diff --git a/vortex-array/src/scalar_fn/fns/is_nan.rs b/vortex-functions/src/is_nan.rs old mode 100755 new mode 100644 similarity index 88% rename from vortex-array/src/scalar_fn/fns/is_nan.rs rename to vortex-functions/src/is_nan.rs index c8059a4e6ab..4f3464a167c --- a/vortex-array/src/scalar_fn/fns/is_nan.rs +++ b/vortex-functions/src/is_nan.rs @@ -1,17 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::dtype::DType; +use vortex_array::match_each_float_ptype; +use vortex_array::scalar_fn::EmptyOptions; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::unstable::row::RowFn; +use vortex_array::scalar_fn::unstable::row::RowVisitor; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_session::registry::CachedId; -use crate::dtype::DType; -use crate::match_each_float_ptype; -use crate::scalar_fn::EmptyOptions; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::unstable::row::RowFn; -use crate::scalar_fn::unstable::row::RowVisitor; - /// Expression that checks for NaN values. /// /// The function is strict: null inputs produce null outputs. Only primitive float inputs @@ -50,23 +49,28 @@ impl RowFn for IsNan { mod tests { use std::sync::LazyLock; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::scalar::Scalar; + use vortex_array::validity::Validity; use vortex_buffer::buffer; + use vortex_expr::ArrayExprExt; + use vortex_expr::get_item; + use vortex_expr::lit; + use vortex_expr::root; use vortex_session::VortexSession; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::PrimitiveArray; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::expr::get_item; use crate::expr::is_nan; - use crate::expr::lit; - use crate::expr::root; - use crate::scalar::Scalar; - use crate::validity::Validity; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + crate::initialize(&session); + session + }); #[test] fn dtype() { diff --git a/vortex-functions/src/lib.rs b/vortex-functions/src/lib.rs new file mode 100644 index 00000000000..8f3cbd95a0b --- /dev/null +++ b/vortex-functions/src/lib.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Scalar functions built on the vortex-array scalar function framework that the core array +//! crate does not itself depend on. +//! +//! The scalar functions that canonical arrays, validity, the optimizer and expression analysis +//! rely on (cast, fill_null, mask, binary operators, get_item, pack, select, literal, ...) live in +//! `vortex_array::scalar_fn::builtin`. This crate holds struct merging ([`merge`]), conditionals +//! ([`case_when`], [`zip`]), list reductions ([`list_length`], [`list_sum`]), [`byte_length`], +//! [`ext_storage`] and [`replace_time_zone`], string and range predicates ([`like`], +//! [`between`], [`list_contains`], [`is_nan`]) and the scan-time [`dynamic`] comparison against +//! a value that changes during a scan, along with their expression constructors in [`expr`] and +//! their statistics-based [`pruning`] rewrites. +//! +//! Call [`initialize`] to register them with a session. + +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::builtin::get_item::GetItem; +use vortex_array::scalar_fn::builtin::literal::Literal; +use vortex_array::scalar_fn::session::ScalarFnSessionExt; +use vortex_session::VortexSession; + +pub mod between; +pub mod builtins; +pub mod byte_length; +pub mod case_when; +pub mod dynamic; +pub mod expr; +pub mod ext_storage; +pub mod is_nan; +pub mod like; +pub mod list_contains; +pub mod list_length; +pub mod list_sum; +pub mod merge; +pub mod pruning; +pub mod replace_time_zone; +pub mod variant_get; +pub mod zip; + +#[cfg(test)] +mod tests; + +/// Register this crate's scalar functions with the given session. +pub fn initialize(session: &VortexSession) { + between::initialize(session); + like::initialize(session); + list_contains::initialize(session); + variant_get::initialize(session); + zip::initialize(session); + pruning::initialize(session); + let scalar_fns = session.scalar_fns(); + scalar_fns.register(byte_length::ByteLength); + scalar_fns.register(case_when::CaseWhen); + scalar_fns.register(ext_storage::ExtStorage); + scalar_fns.register(is_nan::IsNan); + scalar_fns.register(list_length::ListLength); + scalar_fns.register(list_sum::ListSum); + scalar_fns.register(merge::Merge); + scalar_fns.register(replace_time_zone::ReplaceTimeZone); +} + +/// A scalar function has a negative cost if applying it to an array and +/// canonicalizing is cheaper than canonicalizing an array and applying it. +/// +/// Example of negative cost expressions are byte_length(), ext_storage(), and get_item() since +/// they don't depend on input size. +/// +/// Example of non-negative cost expression is like() as it's linear over +/// individual input. +pub fn is_negative_cost(id: ScalarFnId) -> bool { + id == ScalarFnVTable::id(&byte_length::ByteLength) + || id == ScalarFnVTable::id(&ext_storage::ExtStorage) + || id == ScalarFnVTable::id(&GetItem) + || id == ScalarFnVTable::id(&Literal) +} diff --git a/vortex-array/src/arrays/dict/compute/like.rs b/vortex-functions/src/like/dict.rs similarity index 69% rename from vortex-array/src/arrays/dict/compute/like.rs rename to vortex-functions/src/like/dict.rs index ba4bf2f9f8c..50e5c565076 100644 --- a/vortex-array/src/arrays/dict/compute/like.rs +++ b/vortex-functions/src/like/dict.rs @@ -1,20 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::IntoArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::DictArray; +use vortex_array::arrays::dict::DictArrayExt; +use vortex_array::arrays::dict::DictArraySlotsExt; +use vortex_array::optimizer::ArrayOptimizer; use vortex_error::VortexResult; use super::Dict; -use super::DictArray; -use crate::ArrayRef; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::ConstantArray; -use crate::arrays::dict::DictArrayExt; -use crate::arrays::dict::DictArraySlotsExt; -use crate::optimizer::ArrayOptimizer; -use crate::scalar_fn::fns::like::Like; -use crate::scalar_fn::fns::like::LikeOptions; -use crate::scalar_fn::fns::like::LikeReduce; +use crate::like::Like; +use crate::like::LikeOptions; +use crate::like::LikeReduce; impl LikeReduce for Dict { fn like( @@ -53,22 +53,28 @@ impl LikeReduce for Dict { mod tests { use std::sync::LazyLock; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::DictArray; + use vortex_array::arrays::VarBinArray; + use vortex_array::assert_arrays_eq; + use vortex_array::optimizer::ArrayOptimizer; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::BoolArray; - use crate::arrays::DictArray; - use crate::arrays::VarBinArray; - use crate::arrays::dict::compute::like::ConstantArray; - use crate::assert_arrays_eq; - use crate::optimizer::ArrayOptimizer; - use crate::scalar_fn::fns::like::Like; - use crate::scalar_fn::fns::like::LikeOptions; + use crate::like::Like; + use crate::like::LikeOptions; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); #[test] fn like_reduce_dict() -> VortexResult<()> { diff --git a/vortex-array/src/scalar_fn/fns/like/kernel.rs b/vortex-functions/src/like/kernel.rs similarity index 86% rename from vortex-array/src/scalar_fn/fns/like/kernel.rs rename to vortex-functions/src/like/kernel.rs index b3b683212ff..d586590da0f 100644 --- a/vortex-array/src/scalar_fn/fns/like/kernel.rs +++ b/vortex-functions/src/like/kernel.rs @@ -1,21 +1,21 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::VTable; +use vortex_array::arrays::ScalarFn; +use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::kernel::ExecuteParentKernel; +use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::array::ArrayView; -use crate::array::VTable; -use crate::arrays::ScalarFn; -use crate::arrays::scalar_fn::ExactScalarFn; -use crate::arrays::scalar_fn::ScalarFnArrayExt; -use crate::arrays::scalar_fn::ScalarFnArrayView; -use crate::kernel::ExecuteParentKernel; -use crate::optimizer::rules::ArrayParentReduceRule; -use crate::scalar_fn::fns::like::Like as LikeExpr; -use crate::scalar_fn::fns::like::LikeOptions; +use crate::like::Like as LikeExpr; +use crate::like::LikeOptions; /// Like pattern matching on an array without reading buffers. /// diff --git a/vortex-array/src/scalar_fn/fns/like/mod.rs b/vortex-functions/src/like/mod.rs similarity index 90% rename from vortex-array/src/scalar_fn/fns/like/mod.rs rename to vortex-functions/src/like/mod.rs index da0d094c201..fb8b06320a4 100644 --- a/vortex-array/src/scalar_fn/fns/like/mod.rs +++ b/vortex-functions/src/like/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod dict; mod kernel; mod pattern; @@ -10,6 +11,32 @@ use std::fmt::Formatter; pub use kernel::*; use pattern::LikePattern; use prost::Message; +use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Dict; +use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::varbinview::BinaryView; +use vortex_array::arrays::varbinview::ResolvedViews; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::optimizer::kernels::ArrayKernelsExt; +use vortex_array::optimizer::kernels::apply_reduce_parent_rule; +use vortex_array::proto::expr as pb; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::display::ExprDisplay; +use vortex_array::scalar_fn::session::ScalarFnSessionExt; use vortex_buffer::BitBuffer; use vortex_buffer::BitBufferMut; use vortex_buffer::BufferAllocatorRef; @@ -19,27 +46,21 @@ use vortex_error::vortex_err; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::Canonical; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::BoolArray; -use crate::arrays::ConstantArray; -use crate::arrays::ScalarFnArray; -use crate::arrays::VarBinViewArray; -use crate::arrays::varbinview::BinaryView; -use crate::arrays::varbinview::ResolvedViews; -use crate::dtype::DType; -use crate::dtype::Nullability; -use crate::expr::display::ExprDisplay; -use crate::proto::expr as pb; -use crate::scalar::Scalar; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::ScalarFnVTableExt; +/// Registers [`Like`] and its rule for dictionary-encoded arrays. +pub(crate) fn initialize(session: &VortexSession) { + session.scalar_fns().register(Like); + session + .kernels() + .register_reduce_parent(Like.id(), Dict.id(), &[dict_like]); +} + +fn dict_like( + child: &ArrayRef, + parent: &ArrayRef, + child_idx: usize, +) -> VortexResult> { + apply_reduce_parent_rule(&LikeReduceAdaptor(Dict), child, parent, child_idx) +} /// Options for SQL LIKE function #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -423,37 +444,43 @@ mod tests { use std::sync::LazyLock; use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::VarBinArray; + use vortex_array::arrays::VarBinViewArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::scalar::Scalar; + use vortex_expr::get_item; + use vortex_expr::lit; + use vortex_expr::root; use vortex_session::VortexSession; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::BoolArray; - use crate::arrays::ConstantArray; - use crate::arrays::VarBinArray; - use crate::arrays::VarBinViewArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::expr::get_item; use crate::expr::like; - use crate::expr::lit; use crate::expr::not_ilike; - use crate::expr::root; - use crate::scalar::Scalar; - use crate::scalar_fn::fns::like::Like; - use crate::scalar_fn::fns::like::LikeOptions; - use crate::scalar_fn::fns::like::is_ascii; - use crate::scalar_fn::fns::like::suffix_bytes_unchecked; + use crate::like::Like; + use crate::like::LikeOptions; + use crate::like::is_ascii; + use crate::like::suffix_bytes_unchecked; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + session + }); const LONG: &str = "a value far too long to live inside its own view"; fn run_like( - array: crate::ArrayRef, - pattern: crate::ArrayRef, + array: vortex_array::ArrayRef, + pattern: vortex_array::ArrayRef, options: LikeOptions, - ) -> crate::ArrayRef { + ) -> vortex_array::ArrayRef { Like::try_new(array, pattern, options).unwrap().into_array() } diff --git a/vortex-array/src/scalar_fn/fns/like/pattern.rs b/vortex-functions/src/like/pattern.rs similarity index 100% rename from vortex-array/src/scalar_fn/fns/like/pattern.rs rename to vortex-functions/src/like/pattern.rs diff --git a/vortex-array/src/arrays/chunked/compute/list_contains.rs b/vortex-functions/src/list_contains/chunked.rs similarity index 74% rename from vortex-array/src/arrays/chunked/compute/list_contains.rs rename to vortex-functions/src/list_contains/chunked.rs index 2c59d5c0dae..0c6d75585c3 100644 --- a/vortex-array/src/arrays/chunked/compute/list_contains.rs +++ b/vortex-functions/src/list_contains/chunked.rs @@ -1,20 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::IntoArray; +use vortex_array::arrays::Chunked; +use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::chunked::ChunkedArrayExt; +use vortex_array::dtype::DType; +use vortex_array::optimizer::ArrayOptimizer; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::Chunked; -use crate::arrays::ChunkedArray; -use crate::arrays::chunked::ChunkedArrayExt; -use crate::dtype::DType; -use crate::optimizer::ArrayOptimizer; -use crate::scalar_fn::fns::list_contains::ListContains; -use crate::scalar_fn::fns::list_contains::ListContainsElementReduce; -use crate::scalar_fn::fns::list_contains::ListContainsOptions; -use crate::scalar_fn::fns::list_contains::PreparedSet; +use crate::list_contains::ListContains; +use crate::list_contains::ListContainsElementReduce; +use crate::list_contains::ListContainsOptions; +use crate::list_contains::PreparedSet; /// Pushes `list_contains` of a prepared set into each chunk of the needles. /// diff --git a/vortex-array/src/array/haystack/mod.rs b/vortex-functions/src/list_contains/haystack.rs similarity index 94% rename from vortex-array/src/array/haystack/mod.rs rename to vortex-functions/src/list_contains/haystack.rs index f1e04f9c686..89dbca6bbfe 100644 --- a/vortex-array/src/array/haystack/mod.rs +++ b/vortex-functions/src/list_contains/haystack.rs @@ -7,6 +7,26 @@ use std::hash::BuildHasher; use num_traits::ToPrimitive; use num_traits::WrappingSub; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::RecursiveCanonical; +use vortex_array::arrays::DecimalArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::decimal::converted_buffer; +use vortex_array::arrays::primitive::PrimitiveArrayExt; +use vortex_array::arrays::varbinview::BinaryView; +use vortex_array::dtype::DType; +use vortex_array::dtype::DecimalType; +use vortex_array::dtype::NativeDecimalType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::PType; +use vortex_array::dtype::i256; +use vortex_array::match_each_decimal_value_type; +use vortex_array::match_each_integer_ptype; +use vortex_array::scalar_fn::builtin::binary::build_row_comparator; +use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::BitBufferMut; use vortex_buffer::Buffer; @@ -20,27 +40,6 @@ use vortex_utils::aliases::hash_map::HashTable; use vortex_utils::aliases::hash_map::HashTableEntry; use vortex_utils::aliases::hash_map::RandomState; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::RecursiveCanonical; -use crate::arrays::DecimalArray; -use crate::arrays::PrimitiveArray; -use crate::arrays::VarBinViewArray; -use crate::arrays::decimal::converted_buffer; -use crate::arrays::primitive::PrimitiveArrayExt; -use crate::arrays::varbinview::BinaryView; -use crate::dtype::DType; -use crate::dtype::DecimalType; -use crate::dtype::NativeDecimalType; -use crate::dtype::NativePType; -use crate::dtype::PType; -use crate::dtype::i256; -use crate::match_each_decimal_value_type; -use crate::match_each_integer_ptype; -use crate::scalar_fn::fns::binary::build_row_comparator; -use crate::validity::Validity; - /// A set whose span of values needs at most this many bits per element is looked up through a /// bitmap over the span, bounding the bitmap to a few words per element. const BITMAP_BITS_PER_ELEMENT: u128 = 64; @@ -49,7 +48,7 @@ const BITMAP_MIN_BITS: u128 = 1 << 12; /// Looks up needles among the elements of an array, for many batches of needles. /// -/// The haystack counterpart of [`RepeatedArrayProbe`](crate::RepeatedArrayProbe): where a probe +/// The haystack counterpart of [`RepeatedArrayProbe`](vortex_array::RepeatedArrayProbe): where a probe /// reads the rows of an array by index, a haystack answers whether each row of a batch of /// needles is an element of the array. A haystack pays off only over repeated lookups, so it /// has no one-off form. It is built eagerly. A holder that must build it lazily, or share it diff --git a/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs b/vortex-functions/src/list_contains/kernel.rs similarity index 78% rename from vortex-array/src/scalar_fn/fns/list_contains/kernel.rs rename to vortex-functions/src/list_contains/kernel.rs index 28830b71dd0..8eaf59c604e 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs +++ b/vortex-functions/src/list_contains/kernel.rs @@ -1,21 +1,21 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::VTable; +use vortex_array::arrays::ScalarFn; +use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::kernel::ExecuteParentKernel; +use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::array::ArrayView; -use crate::array::VTable; -use crate::arrays::ScalarFn; -use crate::arrays::scalar_fn::ExactScalarFn; -use crate::arrays::scalar_fn::ScalarFnArrayExt; -use crate::arrays::scalar_fn::ScalarFnArrayView; -use crate::kernel::ExecuteParentKernel; -use crate::optimizer::rules::ArrayParentReduceRule; -use crate::scalar_fn::fns::list_contains::ListContains as ListContainsExpr; -use crate::scalar_fn::fns::list_contains::ListContainsOptions; +use crate::list_contains::ListContains as ListContainsExpr; +use crate::list_contains::ListContainsOptions; /// Check list-contains without reading buffers (metadata-only). /// @@ -43,11 +43,11 @@ pub trait ListContainsElementReduce: VTable { /// [`PreparedSetData::contains`] or [`PreparedSetData::contains_scalar`], or make its result from /// matches it found another way with [`PreparedSetData::result_from_bits`]. /// -/// [`PreparedSet`]: crate::scalar_fn::fns::list_contains::PreparedSet -/// [`PreparedSet::set_of`]: crate::scalar_fn::fns::list_contains::PreparedSet::set_of -/// [`PreparedSetData::contains`]: crate::scalar_fn::fns::list_contains::PreparedSetData::contains -/// [`PreparedSetData::contains_scalar`]: crate::scalar_fn::fns::list_contains::PreparedSetData::contains_scalar -/// [`PreparedSetData::result_from_bits`]: crate::scalar_fn::fns::list_contains::PreparedSetData::result_from_bits +/// [`PreparedSet`]: crate::list_contains::PreparedSet +/// [`PreparedSet::set_of`]: crate::list_contains::PreparedSet::set_of +/// [`PreparedSetData::contains`]: crate::list_contains::PreparedSetData::contains +/// [`PreparedSetData::contains_scalar`]: crate::list_contains::PreparedSetData::contains_scalar +/// [`PreparedSetData::result_from_bits`]: crate::list_contains::PreparedSetData::result_from_bits pub trait ListContainsElementKernel: VTable { fn list_contains( list: &ArrayRef, diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-functions/src/list_contains/mod.rs similarity index 94% rename from vortex-array/src/scalar_fn/fns/list_contains/mod.rs rename to vortex-functions/src/list_contains/mod.rs index 5d92baac6f6..2c3a6d7e585 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-functions/src/list_contains/mod.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod chunked; +mod haystack; mod kernel; mod prepared; @@ -16,6 +18,44 @@ use num_traits::Zero; pub use prepared::PreparedSet; pub use prepared::PreparedSetData; use prost::Message; +use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; +use vortex_array::Columnar; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::Chunked; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::ListViewArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::arrays::listview::ListViewArraySlotsExt; +use vortex_array::arrays::primitive::PrimitiveArrayExt; +use vortex_array::dtype::DType; +use vortex_array::dtype::IntegerPType; +use vortex_array::dtype::Nullability; +use vortex_array::match_each_integer_ptype; +use vortex_array::match_each_unsigned_integer_ptype; +use vortex_array::optimizer::kernels::ArrayKernelsExt; +use vortex_array::optimizer::kernels::apply_reduce_parent_rule; +use vortex_array::proto::expr as pb; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ReduceNode; +use vortex_array::scalar_fn::ReduceNodeValidity; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::literal::Literal; +use vortex_array::scalar_fn::builtin::operators::Operator; +use vortex_array::scalar_fn::is_not_null_node; +use vortex_array::scalar_fn::session::ScalarFnSessionExt; +use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; @@ -26,39 +66,31 @@ use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::Columnar; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::BoolArray; -use crate::arrays::Constant; -use crate::arrays::ConstantArray; -use crate::arrays::ListViewArray; -use crate::arrays::PrimitiveArray; -use crate::arrays::ScalarFnArray; -use crate::arrays::bool::BoolArrayExt; -use crate::arrays::listview::ListViewArraySlotsExt; -use crate::arrays::primitive::PrimitiveArrayExt; -use crate::dtype::DType; -use crate::dtype::IntegerPType; -use crate::dtype::Nullability; -use crate::match_each_integer_ptype; -use crate::match_each_unsigned_integer_ptype; -use crate::proto::expr as pb; -use crate::scalar::Scalar; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ReduceNode; -use crate::scalar_fn::ReduceNodeValidity; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::binary::Binary; -use crate::scalar_fn::fns::literal::Literal; -use crate::scalar_fn::fns::operators::Operator; -use crate::scalar_fn::is_not_null_node; -use crate::validity::Validity; +/// Registers [`ListContains`], [`PreparedSet`] and the chunked-element rule for +/// [`ListContains`]. +pub(crate) fn initialize(session: &VortexSession) { + let scalar_fns = session.scalar_fns(); + scalar_fns.register(ListContains); + scalar_fns.register(PreparedSet); + session.kernels().register_reduce_parent( + ListContains.id(), + Chunked.id(), + &[chunked_list_contains], + ); +} + +fn chunked_list_contains( + child: &ArrayRef, + parent: &ArrayRef, + child_idx: usize, +) -> VortexResult> { + apply_reduce_parent_rule( + &ListContainsElementReduceAdaptor(Chunked), + child, + parent, + child_idx, + ) +} #[derive(Clone)] pub struct ListContains; @@ -818,53 +850,60 @@ mod tests { use itertools::Itertools; use num_traits::PrimInt; use rstest::rstest; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::ChunkedArray; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::DictArray; + use vortex_array::arrays::ListArray; + use vortex_array::arrays::ListViewArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::VarBinArray; + use vortex_array::arrays::VarBinViewArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::NativePType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::PType::I32; + use vortex_array::dtype::StructFields; + use vortex_array::optimizer::ArrayOptimizer; + use vortex_array::scalar::PValue; + use vortex_array::scalar::Scalar; + use vortex_array::scalar_fn::builtin::literal::Literal; + use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::Buffer; use vortex_buffer::buffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::Expression; + use vortex_expr::bound; + use vortex_expr::col; + use vortex_expr::get_item; + use vortex_expr::lit; + use vortex_expr::root; use vortex_session::VortexSession; use super::process_matches; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::BoolArray; - use crate::arrays::ChunkedArray; - use crate::arrays::ConstantArray; - use crate::arrays::DictArray; - use crate::arrays::ListArray; - use crate::arrays::ListViewArray; - use crate::arrays::PrimitiveArray; - use crate::arrays::VarBinArray; - use crate::arrays::VarBinViewArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::NativePType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::PType::I32; - use crate::dtype::StructFields; - use crate::expr::Expression; - use crate::expr::bound; - use crate::expr::col; - use crate::expr::get_item; use crate::expr::in_list; use crate::expr::list_contains; use crate::expr::list_contains_opts; - use crate::expr::lit; - use crate::expr::root; - use crate::optimizer::ArrayOptimizer; - use crate::scalar::PValue; - use crate::scalar::Scalar; - use crate::scalar_fn::fns::list_contains::ListContains; - use crate::scalar_fn::fns::list_contains::ListContainsOptions; - use crate::scalar_fn::fns::list_contains::PreparedSet; - use crate::scalar_fn::fns::list_contains::PreparedSetData; - use crate::scalar_fn::fns::literal::Literal; - use crate::validity::Validity; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + use crate::list_contains::ListContains; + use crate::list_contains::ListContainsOptions; + use crate::list_contains::PreparedSet; + use crate::list_contains::PreparedSetData; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); fn test_array() -> ArrayRef { let mut ctx = SESSION.create_execution_ctx(); @@ -1541,7 +1580,7 @@ mod tests { [None, Some(true), None, Some(true)], )?; assert_result( - needles.apply(&crate::expr::not(in_list(root(), lit(set)))), + needles.apply(&vortex_expr::not(in_list(root(), lit(set)))), [None, Some(false), None, Some(false)], ) } @@ -1972,7 +2011,7 @@ mod tests { #[test] fn options_round_trip_through_serde() -> VortexResult<()> { - use crate::scalar_fn::ScalarFnVTable; + use vortex_array::scalar_fn::ScalarFnVTable; for options in [ListContainsOptions::default(), SQL] { let bytes = ListContains .serialize(&options)? diff --git a/vortex-array/src/scalar_fn/fns/list_contains/prepared/data.rs b/vortex-functions/src/list_contains/prepared/data.rs similarity index 93% rename from vortex-array/src/scalar_fn/fns/list_contains/prepared/data.rs rename to vortex-functions/src/list_contains/prepared/data.rs index aaaf047c1d9..2c9913d83fb 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/prepared/data.rs +++ b/vortex-functions/src/list_contains/prepared/data.rs @@ -9,6 +9,19 @@ use std::hash::Hasher; use std::sync::Arc; use std::sync::OnceLock; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::ListViewArray; +use vortex_array::arrays::listview::ListViewArraySlotsExt; +use vortex_array::dtype::DType; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::ScalarFnRef; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::builtin::literal::Literal; +use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_error::SharedVortexResult; use vortex_error::VortexResult; @@ -16,21 +29,8 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; -use crate::ArrayHaystack; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::BoolArray; -use crate::arrays::ConstantArray; -use crate::arrays::ListViewArray; -use crate::arrays::listview::ListViewArraySlotsExt; -use crate::dtype::DType; -use crate::scalar::Scalar; -use crate::scalar_fn::ScalarFnRef; -use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::list_contains::ListContainsOptions; -use crate::scalar_fn::fns::literal::Literal; -use crate::validity::Validity; +use crate::list_contains::ListContainsOptions; +use crate::list_contains::haystack::ArrayHaystack; /// The options of a [`PreparedSet`]: the list that every row holds, and its set. /// @@ -38,7 +38,7 @@ use crate::validity::Validity; /// slice or filter of the array it gives, looks up needles in the one set. The haystack needs /// an [`ExecutionCtx`] to materialize the elements, so it is built on the first lookup. /// -/// [`PreparedSet`]: crate::scalar_fn::fns::list_contains::PreparedSet +/// [`PreparedSet`]: crate::list_contains::PreparedSet #[derive(Clone)] pub struct PreparedSetData(Arc); diff --git a/vortex-array/src/scalar_fn/fns/list_contains/prepared/function.rs b/vortex-functions/src/list_contains/prepared/function.rs similarity index 80% rename from vortex-array/src/scalar_fn/fns/list_contains/prepared/function.rs rename to vortex-functions/src/list_contains/prepared/function.rs index eb18ed35372..de1c316d2c9 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/prepared/function.rs +++ b/vortex-functions/src/list_contains/prepared/function.rs @@ -4,30 +4,30 @@ use std::fmt::Formatter; use prost::Message; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::ScalarFn; +use vortex_array::arrays::ScalarFnArray; +use vortex_array::dtype::DType; +use vortex_array::proto::scalar as pb; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ReduceNode; +use vortex_array::scalar_fn::ReduceNodeValidity; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::display::ExprDisplay; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_session::VortexSession; use vortex_session::registry::CachedId; use super::PreparedSetData; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::ConstantArray; -use crate::arrays::ScalarFn; -use crate::arrays::ScalarFnArray; -use crate::dtype::DType; -use crate::expr::display::ExprDisplay; -use crate::proto::scalar as pb; -use crate::scalar::Scalar; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ReduceNode; -use crate::scalar_fn::ReduceNodeValidity; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::ScalarFnVTableExt; /// A constant, non-null list whose elements are prepared as a set for membership lookups. /// @@ -45,9 +45,9 @@ use crate::scalar_fn::ScalarFnVTableExt; /// /// It serializes as the list, and deserializes into a set prepared again. /// -/// [`ListContains`]: crate::scalar_fn::fns::list_contains::ListContains -/// [`ListContainsElementKernel`]: crate::scalar_fn::fns::list_contains::ListContainsElementKernel -/// [`ListContainsElementReduce`]: crate::scalar_fn::fns::list_contains::ListContainsElementReduce +/// [`ListContains`]: crate::list_contains::ListContains +/// [`ListContainsElementKernel`]: crate::list_contains::ListContainsElementKernel +/// [`ListContainsElementReduce`]: crate::list_contains::ListContainsElementReduce #[derive(Clone, Debug)] pub struct PreparedSet; @@ -57,8 +57,8 @@ impl PreparedSet { /// [`ListContains`] prepares a constant list itself. A kernel crate can use this to test its /// [`ListContainsElementKernel`] against a prepared set. /// - /// [`ListContains`]: crate::scalar_fn::fns::list_contains::ListContains - /// [`ListContainsElementKernel`]: crate::scalar_fn::fns::list_contains::ListContainsElementKernel + /// [`ListContains`]: crate::list_contains::ListContains + /// [`ListContainsElementKernel`]: crate::list_contains::ListContainsElementKernel /// /// # Errors /// diff --git a/vortex-array/src/scalar_fn/fns/list_contains/prepared/mod.rs b/vortex-functions/src/list_contains/prepared/mod.rs similarity index 74% rename from vortex-array/src/scalar_fn/fns/list_contains/prepared/mod.rs rename to vortex-functions/src/list_contains/prepared/mod.rs index b077ab2e4db..5d02735f49d 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/prepared/mod.rs +++ b/vortex-functions/src/list_contains/prepared/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors //! Membership in a constant set. [`PreparedSet`] holds a constant list together with the -//! [`ArrayHaystack`](crate::ArrayHaystack) of its elements, so that the kernels of a needle +//! [`ArrayHaystack`](crate::list_contains::haystack::ArrayHaystack) of its elements, so that the kernels of a needle //! encoding can look up their own values. mod data; diff --git a/vortex-array/src/scalar_fn/fns/list_contains/prepared/tests.rs b/vortex-functions/src/list_contains/prepared/tests.rs similarity index 83% rename from vortex-array/src/scalar_fn/fns/list_contains/prepared/tests.rs rename to vortex-functions/src/list_contains/prepared/tests.rs index fe3c6b46da6..8b7f0daf55b 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/prepared/tests.rs +++ b/vortex-functions/src/list_contains/prepared/tests.rs @@ -4,52 +4,62 @@ use std::sync::LazyLock; use rstest::rstest; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::Bool; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::Chunked; +use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::DecimalArray; +use vortex_array::arrays::FilterArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::ListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::ScalarFn; +use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::StructArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::chunked::ChunkedArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::assert_arrays_eq; +use vortex_array::builders::builder_with_capacity_in; +use vortex_array::dtype::DType; +use vortex_array::dtype::DecimalDType; +use vortex_array::dtype::DecimalType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::dtype::i256; +use vortex_array::match_each_decimal_value_type; +use vortex_array::optimizer::ArrayOptimizer; +use vortex_array::scalar::DecimalValue; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::TypedScalarFnInstance; +use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::buffer; use vortex_error::VortexResult; +use vortex_expr::ArrayExprExt; +use vortex_expr::lit; +use vortex_expr::root; use vortex_mask::Mask; use vortex_session::VortexSession; use super::PreparedSet; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::VortexSessionExecute; -use crate::arrays::Bool; -use crate::arrays::BoolArray; -use crate::arrays::Chunked; -use crate::arrays::ChunkedArray; -use crate::arrays::ConstantArray; -use crate::arrays::DecimalArray; -use crate::arrays::FixedSizeListArray; -use crate::arrays::ListArray; -use crate::arrays::PrimitiveArray; -use crate::arrays::ScalarFn; -use crate::arrays::StructArray; -use crate::arrays::VarBinViewArray; -use crate::arrays::chunked::ChunkedArrayExt; -use crate::arrays::scalar_fn::ScalarFnArrayExt; -use crate::assert_arrays_eq; -use crate::builders::builder_with_capacity_in; -use crate::dtype::DType; -use crate::dtype::DecimalDType; -use crate::dtype::DecimalType; -use crate::dtype::Nullability; -use crate::dtype::PType; -use crate::dtype::i256; use crate::expr::list_contains; -use crate::expr::lit; -use crate::expr::root; -use crate::match_each_decimal_value_type; -use crate::optimizer::ArrayOptimizer; -use crate::scalar::DecimalValue; -use crate::scalar::Scalar; -use crate::scalar_fn::fns::list_contains::ListContains; -use crate::scalar_fn::fns::list_contains::ListContainsOptions; -use crate::scalar_fn::fns::list_contains::PreparedSetData; -use crate::validity::Validity; - -static SESSION: LazyLock = LazyLock::new(VortexSession::empty); +use crate::list_contains::ListContains; +use crate::list_contains::ListContainsOptions; +use crate::list_contains::PreparedSetData; + +static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session +}); /// Prepares the non-null list scalar `list`. fn prepare(list: &Scalar) -> VortexResult { @@ -535,7 +545,7 @@ fn test_constant_list_over_chunked_needles( // The list is prepared before the push-down, so every chunk gets a slice of the one set. let array = ListContains::try_new_opts(list, needles, options)? .into_array() - .optimize()?; + .optimize_ctx(&SESSION)?; assert_chunks_share_one_set(&array); assert_arrays_eq!(array, BoolArray::from_iter(expected), &mut ctx); @@ -550,7 +560,7 @@ fn test_optimized_expression_over_chunked_needles() -> VortexResult<()> { .bind(needles.dtype())? .optimize()?; - let array = needles.apply_bound(&expr)?; + let array = needles.apply_bound(&expr)?.optimize_ctx(&SESSION)?; assert_chunks_share_one_set(&array); assert_arrays_eq!( @@ -560,3 +570,56 @@ fn test_optimized_expression_over_chunked_needles() -> VortexResult<()> { ); Ok(()) } + +/// A prepared set has no children, so its `ScalarFnArray` takes an explicit length. +#[test] +fn test_scalar_fn_without_children_requires_explicit_len() -> VortexResult<()> { + let list = Scalar::list( + DType::Primitive(PType::I32, Nullability::NonNullable), + vec![Scalar::from(1i32)], + Nullability::NonNullable, + ); + let scalar_fn = + TypedScalarFnInstance::new(PreparedSet, PreparedSetData::try_new(list.clone())?).erased(); + + let Err(err) = ScalarFnArray::try_new(scalar_fn.clone(), vec![]) else { + panic!("ScalarFnArray::try_new should reject zero children"); + }; + assert!( + err.to_string() + .contains("ScalarFnArray length cannot be inferred without children") + ); + + let scalar_fn_array = ScalarFnArray::try_new_with_len(scalar_fn, vec![], 3)?; + assert_eq!(scalar_fn_array.len(), 3); + assert_eq!(scalar_fn_array.child_count(), 0); + assert_arrays_eq!( + scalar_fn_array, + ConstantArray::new(list, 3), + &mut SESSION.create_execution_ctx() + ); + Ok(()) +} + +/// Filtering a prepared set, which has no children, keeps a prepared set of the filtered length. +#[test] +fn test_filter_pushdown_without_children() -> VortexResult<()> { + let list = Scalar::list( + DType::Primitive(PType::I32, Nullability::NonNullable), + vec![Scalar::from(7i32)], + Nullability::NonNullable, + ); + let array = PreparedSet::try_new_array(list.clone(), 3)?; + let result = FilterArray::try_new(array, Mask::from_iter([true, false, true]))? + .into_array() + .optimize()?; + + assert_eq!(result.as_::().nchildren(), 0); + assert_eq!(result.len(), 2); + assert_arrays_eq!( + result, + ConstantArray::new(list, 2), + &mut SESSION.create_execution_ctx() + ); + Ok(()) +} diff --git a/vortex-array/src/scalar_fn/fns/list_length.rs b/vortex-functions/src/list_length.rs similarity index 86% rename from vortex-array/src/scalar_fn/fns/list_length.rs rename to vortex-functions/src/list_length.rs index f88e2baf80d..38ce31a3aec 100644 --- a/vortex-array/src/scalar_fn/fns/list_length.rs +++ b/vortex-functions/src/list_length.rs @@ -2,38 +2,37 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use num_traits::AsPrimitive; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::FixedSizeList; +use vortex_array::arrays::List; +use vortex_array::arrays::ListView; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::list::ListArrayExt; +use vortex_array::arrays::list::ListArraySlotsExt; +use vortex_array::arrays::listview::ListViewArrayExt; +use vortex_array::arrays::listview::ListViewArraySlotsExt; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::matcher::Matcher; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::EmptyOptions; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::ConstantArray; -use crate::arrays::FixedSizeList; -use crate::arrays::List; -use crate::arrays::ListView; -use crate::arrays::fixed_size_list::FixedSizeListArrayExt; -use crate::arrays::list::ListArrayExt; -use crate::arrays::list::ListArraySlotsExt; -use crate::arrays::listview::ListViewArrayExt; -use crate::arrays::listview::ListViewArraySlotsExt; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::Nullability; -use crate::dtype::PType; -use crate::matcher::Matcher; -use crate::scalar::Scalar; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::EmptyOptions; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::operators::Operator; - /// Number of elements in each list of a `List` or `FixedSizeList` typed array. /// /// This is computed purely from the list's offsets (`ListArray`), sizes (`ListViewArray`), or @@ -187,30 +186,37 @@ mod tests { use std::sync::LazyLock; use rstest::rstest; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::FixedSizeListArray; + use vortex_array::arrays::ListArray; + use vortex_array::arrays::ListViewArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::scalar::Scalar; + use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::cast; + use vortex_expr::root; use vortex_session::VortexSession; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::BoolArray; - use crate::arrays::ConstantArray; - use crate::arrays::FixedSizeListArray; - use crate::arrays::ListArray; - use crate::arrays::ListViewArray; - use crate::arrays::PrimitiveArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::expr::cast; use crate::expr::list_length; - use crate::expr::root; - use crate::scalar::Scalar; - use crate::validity::Validity; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); fn create_list_elements() -> ArrayRef { PrimitiveArray::from_option_iter::([ diff --git a/vortex-array/src/scalar_fn/fns/list_sum.rs b/vortex-functions/src/list_sum.rs similarity index 90% rename from vortex-array/src/scalar_fn/fns/list_sum.rs rename to vortex-functions/src/list_sum.rs index 0e83a22ed3b..46160563406 100644 --- a/vortex-array/src/scalar_fn/fns/list_sum.rs +++ b/vortex-functions/src/list_sum.rs @@ -1,30 +1,29 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::Columnar; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::aggregate_fn::AggregateFnVTable; +use vortex_array::aggregate_fn::DynGroupedAccumulator; +use vortex_array::aggregate_fn::GroupedAccumulator; +use vortex_array::aggregate_fn::NumericalAggregateOpts; +use vortex_array::aggregate_fn::fns::sum_v2::SumV2; +use vortex_array::arrays::ConstantArray; +use vortex_array::dtype::DType; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::Canonical; -use crate::Columnar; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::aggregate_fn::AggregateFnVTable; -use crate::aggregate_fn::DynGroupedAccumulator; -use crate::aggregate_fn::GroupedAccumulator; -use crate::aggregate_fn::NumericalAggregateOpts; -use crate::aggregate_fn::fns::sum_v2::SumV2; -use crate::arrays::ConstantArray; -use crate::dtype::DType; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; - /// Sum of the elements in each list of a `List` or `FixedSizeList` typed array. /// /// Follows SQL `SUM` semantics per list, matching DuckDB's `list_sum`: null lists, empty @@ -142,36 +141,43 @@ mod tests { use prost::Message; use rstest::rstest; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::aggregate_fn::NumericalAggregateOpts; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::FixedSizeListArray; + use vortex_array::arrays::ListArray; + use vortex_array::arrays::ListViewArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::proto::expr as pb; + use vortex_array::scalar::Scalar; + use vortex_array::scalar_fn::ScalarFnVTable; + use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::Expression; + use vortex_expr::proto::ExprSerializeProtoExt; + use vortex_expr::root; use vortex_session::VortexSession; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::aggregate_fn::NumericalAggregateOpts; - use crate::arrays::BoolArray; - use crate::arrays::ConstantArray; - use crate::arrays::FixedSizeListArray; - use crate::arrays::ListArray; - use crate::arrays::ListViewArray; - use crate::arrays::PrimitiveArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::expr::Expression; use crate::expr::list_sum; use crate::expr::list_sum_opts; - use crate::expr::proto::ExprSerializeProtoExt; - use crate::expr::root; - use crate::proto::expr as pb; - use crate::scalar::Scalar; - use crate::scalar_fn::ScalarFnVTable; - use crate::scalar_fn::fns::list_sum::ListSum; - use crate::validity::Validity; + use crate::list_sum::ListSum; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + session + }); fn create_list_elements() -> ArrayRef { PrimitiveArray::from_option_iter::([ @@ -564,7 +570,9 @@ mod tests { let proto = expr.serialize_proto()?; let buf = proto.encode_to_vec(); let decoded = pb::Expr::decode(buf.as_slice())?; - let deser = Expression::from_proto(&decoded, &SESSION)?; + let session = VortexSession::empty(); + crate::initialize(&session); + let deser = Expression::from_proto(&decoded, &session)?; assert_eq!(expr, deser); } Ok(()) diff --git a/vortex-array/src/scalar_fn/fns/merge.rs b/vortex-functions/src/merge.rs similarity index 80% rename from vortex-array/src/scalar_fn/fns/merge.rs rename to vortex-functions/src/merge.rs index 4913b120000..0a55256217c 100644 --- a/vortex-array/src/scalar_fn/fns/merge.rs +++ b/vortex-functions/src/merge.rs @@ -7,6 +7,26 @@ use std::hash::Hash; use std::sync::Arc; use itertools::Itertools as _; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray as _; +use vortex_array::arrays::StructArray; +use vortex_array::arrays::struct_::StructArrayExt; +use vortex_array::dtype::DType; +use vortex_array::dtype::FieldNames; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::StructFields; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ReduceNode; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::builtin::get_item::GetItem; +use vortex_array::scalar_fn::builtin::pack::Pack; +use vortex_array::scalar_fn::builtin::pack::PackOptions; +use vortex_array::validity::Validity; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -14,27 +34,6 @@ use vortex_session::VortexSession; use vortex_session::registry::CachedId; use vortex_utils::aliases::hash_set::HashSet; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray as _; -use crate::arrays::StructArray; -use crate::arrays::struct_::StructArrayExt; -use crate::dtype::DType; -use crate::dtype::FieldNames; -use crate::dtype::Nullability; -use crate::dtype::StructFields; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ReduceNode; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::get_item::GetItem; -use crate::scalar_fn::fns::pack::Pack; -use crate::scalar_fn::fns::pack::PackOptions; -use crate::validity::Validity; - /// Merge zero or more expressions that ALL return structs. /// /// If any field names are duplicated, the field from later expressions wins. @@ -256,33 +255,48 @@ impl Display for DuplicateHandling { mod tests { use std::sync::LazyLock; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::struct_::StructArrayExt; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::FieldNames; + use vortex_array::dtype::Nullability::NonNullable; + use vortex_array::dtype::PType::I32; + use vortex_array::dtype::PType::I64; + use vortex_array::dtype::PType::U32; + use vortex_array::dtype::PType::U64; + use vortex_array::dtype::StructFields; + use vortex_array::scalar_fn::builtin::pack::Pack; use vortex_buffer::buffer; + use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; + use vortex_expr::ArrayExprExt; + use vortex_expr::BoundExpression; + use vortex_expr::Expression; + use vortex_expr::analysis::make_bound_free_field_annotator; + use vortex_expr::col; + use vortex_expr::get_item; + use vortex_expr::pack; + use vortex_expr::root; + use vortex_expr::transform::partition_bound; use vortex_session::VortexSession; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::PrimitiveArray; - use crate::arrays::struct_::StructArrayExt; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability::NonNullable; - use crate::dtype::PType::I32; - use crate::dtype::PType::I64; - use crate::dtype::PType::U32; - use crate::dtype::PType::U64; - use crate::expr::Expression; - use crate::expr::get_item; use crate::expr::merge; use crate::expr::merge_opts; - use crate::expr::root; - use crate::scalar_fn::fns::merge::DuplicateHandling; - use crate::scalar_fn::fns::merge::StructArray; - use crate::scalar_fn::fns::pack::Pack; + use crate::merge::DuplicateHandling; + use crate::merge::StructArray; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + session + }); fn field_array(array: &ArrayRef, field_path: &[&str]) -> VortexResult { let mut ctx = SESSION.create_execution_ctx(); @@ -444,7 +458,7 @@ mod tests { Default::default(), vec![], 3, - crate::validity::Validity::NonNullable, + vortex_array::validity::Validity::NonNullable, ) .unwrap(); assert_arrays_eq!(actual_array, expected, &mut SESSION.create_execution_ctx()); @@ -591,4 +605,65 @@ mod tests { &DType::struct_([("a", I32), ("b", U32), ("c", U64)], NonNullable) ); } + + fn partition_root_dtype(names: &FieldNames, partitions: &[BoundExpression]) -> DType { + DType::Struct( + StructFields::new( + names.clone(), + partitions + .iter() + .map(|partition| partition.dtype().clone()) + .collect(), + ), + NonNullable, + ) + } + + #[test] + fn partitions_merge_by_field() -> VortexResult<()> { + let dtype = DType::Struct( + StructFields::from_iter([ + ( + "a", + DType::Struct( + StructFields::from_iter([("x", DType::from(I32)), ("y", DType::from(I32))]), + NonNullable, + ), + ), + ("b", DType::from(I32)), + ("c", DType::from(I32)), + ]), + NonNullable, + ); + let fields = dtype.as_struct_fields_opt().vortex_expect("struct"); + let expr = merge([col("a"), pack([("b", col("b"))], NonNullable)]); + + let partitioned = + partition_bound(expr.bind(&dtype)?, make_bound_free_field_annotator(fields))?; + let expected = merge([get_item("a_0", col("a")), get_item("b_0", col("b"))]); + let root_dtype = + partition_root_dtype(&partitioned.partition_names, &partitioned.partitions); + assert_eq!( + partitioned.root, + expected.bind(&root_dtype)?, + "{} {}", + partitioned.root, + expected + ); + + assert_eq!(partitioned.partitions.len(), 2); + + let part_a = partitioned + .find_partition(&"a".into()) + .vortex_expect("partition a"); + let expected_a = pack([("a_0", col("a"))], NonNullable); + assert_eq!(part_a, &expected_a.bind(&dtype)?, "{part_a} {expected_a}"); + + let part_b = partitioned + .find_partition(&"b".into()) + .vortex_expect("partition b"); + let expected_b = pack([("b_0", pack([("b", col("b"))], NonNullable))], NonNullable); + assert_eq!(part_b, &expected_b.bind(&dtype)?, "{part_b} {expected_b}"); + Ok(()) + } } diff --git a/vortex-expr/src/stats/rewrite/like_variant.rs b/vortex-functions/src/pruning/like_variant.rs similarity index 100% rename from vortex-expr/src/stats/rewrite/like_variant.rs rename to vortex-functions/src/pruning/like_variant.rs diff --git a/vortex-functions/src/pruning/mod.rs b/vortex-functions/src/pruning/mod.rs new file mode 100644 index 00000000000..ed113f54a9d --- /dev/null +++ b/vortex-functions/src/pruning/mod.rs @@ -0,0 +1,711 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Statistics-based pruning rewrites for this crate's scalar functions. + +use std::cmp::Ordering; + +use vortex_array::expr::stats::Stat; +use vortex_array::scalar::Scalar; +use vortex_array::scalar::StringLike; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::builtin::literal::Literal; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; +use vortex_error::VortexResult; +use vortex_expr::BoundExpression; +use vortex_expr::bound::and; +use vortex_expr::bound::and_collect; +use vortex_expr::bound::binary; +use vortex_expr::bound::eq; +use vortex_expr::bound::gt; +use vortex_expr::bound::gt_eq; +use vortex_expr::bound::lit; +use vortex_expr::bound::lt; +use vortex_expr::bound::or; +use vortex_expr::stats::rewrite::StatsRewriteRule; +use vortex_expr::stats::rewrite::falsify; +use vortex_expr::stats::rewrite::helpers::*; +use vortex_expr::stats::session::StatsSessionExt; +use vortex_session::VortexSession; + +use self::like_variant::LikeVariant; +use crate::between::Between; +use crate::dynamic::DynamicComparison; +use crate::expr::bound::dynamic_with_options; +use crate::is_nan::IsNan; +use crate::like::Like; +use crate::list_contains::ListContains; + +mod like_variant; + +/// Register the stats rewrite rules for this crate's scalar functions. +pub fn initialize(session: &VortexSession) { + let stats = session.stats(); + stats.register_rewrite(BetweenStatsRewrite); + stats.register_rewrite(IsNanNaNCountStatsRewrite); + stats.register_rewrite(IsNanAllNonNanStatsRewrite); + stats.register_rewrite(IsNanAllNanStatsRewrite); + stats.register_rewrite(LikeStatsRewrite); + stats.register_rewrite(ListContainsNanCountStatsRewrite); + stats.register_rewrite(ListContainsAllNonNanStatsRewrite); + stats.register_rewrite(DynamicComparisonNanCountStatsRewrite); + stats.register_rewrite(DynamicComparisonAllNonNanStatsRewrite); +} + +#[derive(Debug)] +struct BetweenStatsRewrite; + +impl StatsRewriteRule for BetweenStatsRewrite { + fn scalar_fn_id(&self) -> ScalarFnId { + Between.id() + } + + fn falsify( + &self, + expr: &BoundExpression, + session: &VortexSession, + ) -> VortexResult> { + let options = expr.as_::(); + let arr = expr.child(0).clone(); + let lower = expr.child(1).clone(); + let upper = expr.child(2).clone(); + + let lhs = binary(options.lower_strict.to_operator(), lower, arr.clone()); + let rhs = binary(options.upper_strict.to_operator(), arr, upper); + falsify(&and(lhs, rhs), session) + } +} + +/// Rewrites `is_nan` using the `NaNCount` stat: the predicate is falsified when a zone +/// contains no NaN values, and satisfied when every row in the zone is NaN. +#[derive(Debug)] +struct IsNanNaNCountStatsRewrite; + +impl StatsRewriteRule for IsNanNaNCountStatsRewrite { + fn scalar_fn_id(&self) -> ScalarFnId { + IsNan.id() + } + + fn falsify( + &self, + expr: &BoundExpression, + _session: &VortexSession, + ) -> VortexResult> { + Ok(nan_count(expr.child(0)).map(|nan_count| eq(nan_count, lit(0u64)))) + } + + fn satisfy( + &self, + expr: &BoundExpression, + _session: &VortexSession, + ) -> VortexResult> { + Ok(nan_count(expr.child(0)).map(|nan_count| eq(nan_count, row_count()))) + } +} + +/// Falsifies `is_nan` when the `AllNonNan` pruning stat proves that no value in the zone +/// is NaN. +#[derive(Debug)] +struct IsNanAllNonNanStatsRewrite; + +impl StatsRewriteRule for IsNanAllNonNanStatsRewrite { + fn scalar_fn_id(&self) -> ScalarFnId { + IsNan.id() + } + + fn falsify( + &self, + expr: &BoundExpression, + _session: &VortexSession, + ) -> VortexResult> { + Ok(Some(all_non_nan(expr.child(0)))) + } +} + +/// Satisfies `is_nan` when the `AllNan` pruning stat proves that every value in the zone +/// is NaN. +#[derive(Debug)] +struct IsNanAllNanStatsRewrite; + +impl StatsRewriteRule for IsNanAllNanStatsRewrite { + fn scalar_fn_id(&self) -> ScalarFnId { + IsNan.id() + } + + fn satisfy( + &self, + expr: &BoundExpression, + _session: &VortexSession, + ) -> VortexResult> { + Ok(Some(all_nan(expr.child(0)))) + } +} + +#[derive(Debug)] +struct LikeStatsRewrite; + +impl StatsRewriteRule for LikeStatsRewrite { + fn scalar_fn_id(&self) -> ScalarFnId { + Like.id() + } + + fn falsify( + &self, + expr: &BoundExpression, + _session: &VortexSession, + ) -> VortexResult> { + let like_options = expr.as_::(); + if like_options.negated || like_options.case_insensitive { + return Ok(None); + } + + let Some(pattern) = expr.child(1).as_opt::() else { + return Ok(None); + }; + let Some(pattern) = pattern.as_utf8().value() else { + return Ok(None); + }; + + let source = expr.child(0); + Ok(match LikeVariant::from_str(pattern) { + Some(LikeVariant::Exact(text)) => { + min(source) + .zip(max(source)) + .map(|(source_min, source_max)| { + or( + gt(source_min, lit(text.as_ref())), + lt(source_max, lit(text.as_ref())), + ) + }) + } + Some(LikeVariant::Prefix(prefix)) => { + let Some(successor) = prefix.to_string().increment().ok() else { + return Ok(None); + }; + min(source) + .zip(max(source)) + .map(|(source_min, source_max)| { + or( + gt_eq(source_min, lit(successor)), + lt(source_max, lit(prefix.as_ref())), + ) + }) + } + None => None, + }) + } +} + +#[derive(Debug)] +struct ListContainsNanCountStatsRewrite; + +impl StatsRewriteRule for ListContainsNanCountStatsRewrite { + fn scalar_fn_id(&self) -> ScalarFnId { + ListContains.id() + } + + fn falsify( + &self, + expr: &BoundExpression, + _session: &VortexSession, + ) -> VortexResult> { + list_contains_falsify::(expr) + } +} + +#[derive(Debug)] +struct ListContainsAllNonNanStatsRewrite; + +impl StatsRewriteRule for ListContainsAllNonNanStatsRewrite { + fn scalar_fn_id(&self) -> ScalarFnId { + ListContains.id() + } + + fn falsify( + &self, + expr: &BoundExpression, + _session: &VortexSession, + ) -> VortexResult> { + list_contains_falsify::(expr) + } +} + +/// Lists with more elements falsify `needle IN list` with the bounds of the list rather than +/// with a test per element, which keeps the pruning expression, and its evaluation per zone, +/// small for the large lists of dynamic join filters. +const MAX_LIST_ELEMENT_FALSIFICATIONS: usize = 16; + +/// The smallest and largest non-null elements, or `None` when there are none or two elements +/// are unordered. +fn non_null_bounds(elements: &[Scalar]) -> Option<(&Scalar, &Scalar)> { + let mut non_null = elements.iter().filter(|element| !element.is_null()); + let first = non_null.next()?; + non_null.try_fold((first, first), |(low, high), element| { + let low = match element.partial_cmp(low)? { + Ordering::Less => element, + Ordering::Equal | Ordering::Greater => low, + }; + let high = match element.partial_cmp(high)? { + Ordering::Greater => element, + Ordering::Equal | Ordering::Less => high, + }; + Some((low, high)) + }) +} + +fn list_contains_falsify( + expr: &BoundExpression, +) -> VortexResult> { + let list = expr.child(0); + let needle = expr.child(1); + + let Some(list_scalar) = literal_stat(list, Stat::Min) else { + return Ok(None); + }; + let elements = list_scalar + .as_opt::() + .and_then(|literal| literal.as_list_opt()) + .and_then(|list| list.elements()); + let Some(elements) = elements else { + return Ok(None); + }; + if elements.is_empty() { + return Ok(P::EMIT_UNGUARDED_REWRITES.then(|| lit(true))); + } + + let Some(value_max) = max(needle) else { + return Ok(None); + }; + let Some(value_min) = min(needle) else { + return Ok(None); + }; + + let value_predicate = if elements.len() > MAX_LIST_ELEMENT_FALSIFICATIONS + && let Some((low, high)) = non_null_bounds(&elements) + { + Some(or( + lt(value_max, lit(low.clone())), + gt(value_min, lit(high.clone())), + )) + } else { + and_collect(elements.iter().map(|value| { + or( + lt(value_max.clone(), lit(value.clone())), + gt(value_min.clone(), lit(value.clone())), + ) + })) + }; + value_predicate + .map(|value_predicate| with_non_nan_guards::

([needle], value_predicate)) + .transpose() + .map(Option::flatten) +} + +#[derive(Debug)] +struct DynamicComparisonNanCountStatsRewrite; + +impl StatsRewriteRule for DynamicComparisonNanCountStatsRewrite { + fn scalar_fn_id(&self) -> ScalarFnId { + DynamicComparison.id() + } + + fn falsify( + &self, + expr: &BoundExpression, + _session: &VortexSession, + ) -> VortexResult> { + dynamic_comparison_falsify::(expr) + } +} + +#[derive(Debug)] +struct DynamicComparisonAllNonNanStatsRewrite; + +impl StatsRewriteRule for DynamicComparisonAllNonNanStatsRewrite { + fn scalar_fn_id(&self) -> ScalarFnId { + DynamicComparison.id() + } + + fn falsify( + &self, + expr: &BoundExpression, + _session: &VortexSession, + ) -> VortexResult> { + dynamic_comparison_falsify::(expr) + } +} + +fn dynamic_comparison_falsify( + expr: &BoundExpression, +) -> VortexResult> { + let dynamic = expr.as_::(); + let lhs = expr.child(0); + + let Some((operator, lhs_stat)) = (match dynamic.operator() { + CompareOperator::Eq | CompareOperator::NotEq => None, + CompareOperator::Gt => max(lhs).map(|lhs_stat| (CompareOperator::Lte, lhs_stat)), + CompareOperator::Gte => max(lhs).map(|lhs_stat| (CompareOperator::Lt, lhs_stat)), + CompareOperator::Lt => min(lhs).map(|lhs_stat| (CompareOperator::Gte, lhs_stat)), + CompareOperator::Lte => min(lhs).map(|lhs_stat| (CompareOperator::Gt, lhs_stat)), + }) else { + return Ok(None); + }; + + let value_predicate = dynamic_with_options( + dynamic.with_operator(operator, !dynamic.default()), + lhs_stat, + ); + with_non_nan_guards::

([lhs], value_predicate) +} + +#[cfg(test)] +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::LazyLock; + + use vortex_array::aggregate_fn::AggregateFnRef; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::StructFields; + use vortex_array::expr::stats::Stat; + use vortex_array::scalar::Scalar; + use vortex_array::scalar_fn::EmptyOptions; + use vortex_array::scalar_fn::builtin::operators::CompareOperator; + use vortex_error::VortexResult; + use vortex_expr::BoundExpression; + use vortex_expr::Expression; + use vortex_expr::ScalarFnExprExt; + use vortex_expr::and; + use vortex_expr::col; + use vortex_expr::eq; + use vortex_expr::get_item; + use vortex_expr::gt; + use vortex_expr::gt_eq; + use vortex_expr::lambda::row_count::RowCount; + use vortex_expr::lit; + use vortex_expr::lt; + use vortex_expr::or; + use vortex_expr::stats::expr::StatFn; + use vortex_expr::stats::expr::StatOptions; + use vortex_expr::stats::rewrite::BoundExpressionPruneExt; + use vortex_session::VortexSession; + + use crate::between::BetweenOptions; + use crate::between::StrictComparison; + use crate::dynamic::DynamicComparison; + use crate::expr::between; + use crate::expr::dynamic; + use crate::expr::is_nan; + use crate::expr::like; + use crate::expr::list_contains; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + crate::initialize(&session); + session + }); + + fn stat(expr: Expression, stat: Stat) -> Expression { + let aggregate_fn = stat.aggregate_fn().expect("stat should have aggregate fn"); + stat_fn(expr, aggregate_fn) + } + + fn stat_fn(expr: Expression, aggregate_fn: AggregateFnRef) -> Expression { + StatFn.new_expr(StatOptions::new(aggregate_fn), [expr]) + } + + fn test_scope() -> DType { + DType::Struct( + StructFields::from_iter([ + ("a", DType::Primitive(PType::I32, Nullability::NonNullable)), + ("b", DType::Primitive(PType::I32, Nullability::NonNullable)), + ("f", DType::Primitive(PType::F32, Nullability::NonNullable)), + ("s", DType::Utf8(Nullability::NonNullable)), + ("t", DType::Utf8(Nullability::NonNullable)), + ("n", nested_struct_dtype()), + ]), + Nullability::NonNullable, + ) + } + + fn nested_struct_dtype() -> DType { + DType::Struct( + StructFields::from_iter([("x", DType::Primitive(PType::F32, Nullability::Nullable))]), + Nullability::NonNullable, + ) + } + + fn falsify(expr: &Expression) -> VortexResult> { + expr.bind(&test_scope())?.falsify(&SESSION) + } + + fn satisfy(expr: &Expression) -> VortexResult> { + expr.bind(&test_scope())?.satisfy(&SESSION) + } + + fn bind_expected(expr: Option) -> VortexResult> { + expr.map(|expr| expr.bind(&test_scope())).transpose() + } + + fn all_nan(expr: &Expression) -> Expression { + vortex_expr::stats::all_nan(expr.clone()) + } + + fn all_non_nan(expr: &Expression) -> Expression { + vortex_expr::stats::all_non_nan(expr.clone()) + } + + macro_rules! assert_rewrite_eq { + ($actual:expr, $expected:expr) => { + assert_eq!($actual, bind_expected($expected)?) + }; + } + + #[test] + fn rewrites_between_falsifier() -> VortexResult<()> { + let expr = between( + col("a"), + lit(10), + lit(50), + BetweenOptions { + lower_strict: StrictComparison::NonStrict, + upper_strict: StrictComparison::NonStrict, + }, + ); + + assert_rewrite_eq!( + falsify(&expr)?, + Some(or( + gt(lit(10), stat(col("a"), Stat::Max)), + gt(stat(col("a"), Stat::Min), lit(50)), + )) + ); + Ok(()) + } + + #[test] + fn rewrites_nan_falsifiers() -> VortexResult<()> { + assert_rewrite_eq!( + falsify(&is_nan(col("f")))?, + Some(or( + eq(stat(col("f"), Stat::NaNCount), lit(0u64)), + all_non_nan(&col("f")), + )) + ); + + // Nullable floats prune on nan_count just the same. + assert_rewrite_eq!( + falsify(&is_nan(get_item("x", col("n"))))?, + Some(or( + eq(stat(get_item("x", col("n")), Stat::NaNCount), lit(0u64)), + all_non_nan(&get_item("x", col("n"))), + )) + ); + + // NaN literals fold through the nan_count stat. + assert_rewrite_eq!( + falsify(&is_nan(lit(f32::NAN)))?, + Some(or(eq(lit(1u64), lit(0u64)), all_non_nan(&lit(f32::NAN)))) + ); + Ok(()) + } + + #[test] + fn rewrites_nan_satisfiers() -> VortexResult<()> { + assert_rewrite_eq!( + satisfy(&is_nan(col("f")))?, + Some(or( + eq( + stat(col("f"), Stat::NaNCount), + RowCount.new_expr(EmptyOptions, []) + ), + all_nan(&col("f")), + )) + ); + + assert_rewrite_eq!( + satisfy(&is_nan(lit(f32::NAN)))?, + Some(or( + eq(lit(1u64), RowCount.new_expr(EmptyOptions, [])), + all_nan(&lit(f32::NAN)), + )) + ); + Ok(()) + } + + #[test] + fn rewrites_list_contains_falsifier() -> VortexResult<()> { + let list = Scalar::list( + Arc::new(DType::Primitive(PType::I32, Nullability::NonNullable)), + vec![1.into(), 2.into(), 3.into()], + Nullability::NonNullable, + ); + let expr = list_contains(lit(list), col("a")); + + assert_rewrite_eq!( + falsify(&expr)?, + Some(and( + and( + or( + lt(stat(col("a"), Stat::Max), lit(1i32)), + gt(stat(col("a"), Stat::Min), lit(1i32)), + ), + or( + lt(stat(col("a"), Stat::Max), lit(2i32)), + gt(stat(col("a"), Stat::Min), lit(2i32)), + ), + ), + or( + lt(stat(col("a"), Stat::Max), lit(3i32)), + gt(stat(col("a"), Stat::Min), lit(3i32)), + ), + )) + ); + Ok(()) + } + + #[test] + fn rewrites_large_list_contains_falsifier_to_bounds() -> VortexResult<()> { + let list = Scalar::list( + Arc::new(DType::Primitive(PType::I32, Nullability::Nullable)), + (0..20) + .rev() + .map(|value| Scalar::from(Some(value))) + .chain([Scalar::null(DType::Primitive( + PType::I32, + Nullability::Nullable, + ))]) + .collect(), + Nullability::NonNullable, + ); + let expr = list_contains(lit(list), col("a")); + + assert_rewrite_eq!( + falsify(&expr)?, + Some(or( + lt(stat(col("a"), Stat::Max), lit(Scalar::from(Some(0i32)))), + gt(stat(col("a"), Stat::Min), lit(Scalar::from(Some(19i32)))), + )) + ); + Ok(()) + } + + #[test] + fn rewrites_like_falsifier() -> VortexResult<()> { + let expr = like(col("s"), lit("prefix%")); + assert_rewrite_eq!( + falsify(&expr)?, + Some(or( + gt_eq(stat(col("s"), Stat::Min), lit("prefiy")), + lt(stat(col("s"), Stat::Max), lit("prefix")), + )) + ); + + let expr = like(col("s"), lit(r"\%%")); + assert_rewrite_eq!( + falsify(&expr)?, + Some(or( + gt_eq(stat(col("s"), Stat::Min), lit("&")), + lt(stat(col("s"), Stat::Max), lit("%")), + )) + ); + + let expr = like(col("s"), lit("pref%ix%")); + assert_rewrite_eq!( + falsify(&expr)?, + Some(or( + gt_eq(stat(col("s"), Stat::Min), lit("preg")), + lt(stat(col("s"), Stat::Max), lit("pref")), + )) + ); + + let expr = like(col("s"), lit("pref_ix_")); + assert_rewrite_eq!( + falsify(&expr)?, + Some(or( + gt_eq(stat(col("s"), Stat::Min), lit("preg")), + lt(stat(col("s"), Stat::Max), lit("pref")), + )) + ); + + let expr = like(col("s"), lit("exact")); + assert_rewrite_eq!( + falsify(&expr)?, + Some(or( + gt(stat(col("s"), Stat::Min), lit("exact")), + lt(stat(col("s"), Stat::Max), lit("exact")), + )) + ); + + let expr = like(col("s"), lit("%suffix")); + assert_rewrite_eq!(falsify(&expr)?, None); + Ok(()) + } + + #[test] + fn rewrites_dynamic_comparison_falsifier() -> VortexResult<()> { + let expr = dynamic( + CompareOperator::Gt, + || Some(10i32.into()), + DType::Primitive(PType::I32, Nullability::NonNullable), + true, + col("a"), + ); + let dynamic = expr.as_::(); + + assert_rewrite_eq!( + falsify(&expr)?, + Some(DynamicComparison.new_expr( + dynamic.with_operator(CompareOperator::Lte, false), + [stat(col("a"), Stat::Max)], + )) + ); + Ok(()) + } + + #[test] + fn list_falsification() -> VortexResult<()> { + let expr = list_contains( + lit(Scalar::list( + Arc::new(DType::Primitive(PType::I32, Nullability::NonNullable)), + vec![1.into(), 2.into(), 3.into()], + Nullability::NonNullable, + )), + col("a"), + ); + let scope = DType::Struct( + StructFields::new( + ["a"].into(), + vec![DType::Primitive(PType::I32, Nullability::NonNullable)], + ), + Nullability::NonNullable, + ); + + assert_eq!( + expr.bind(&scope)?.falsify(&SESSION)?, + Some( + and( + and( + or( + lt(stat(col("a"), Stat::Max), lit(1i32)), + gt(stat(col("a"), Stat::Min), lit(1i32)), + ), + or( + lt(stat(col("a"), Stat::Max), lit(2i32)), + gt(stat(col("a"), Stat::Min), lit(2i32)), + ) + ), + or( + lt(stat(col("a"), Stat::Max), lit(3i32)), + gt(stat(col("a"), Stat::Min), lit(3i32)), + ) + ) + .bind(&scope)? + ) + ); + Ok(()) + } +} diff --git a/vortex-array/src/scalar_fn/fns/replace_time_zone.rs b/vortex-functions/src/replace_time_zone.rs similarity index 87% rename from vortex-array/src/scalar_fn/fns/replace_time_zone.rs rename to vortex-functions/src/replace_time_zone.rs index a07029cfe70..393b8524794 100644 --- a/vortex-array/src/scalar_fn/fns/replace_time_zone.rs +++ b/vortex-functions/src/replace_time_zone.rs @@ -10,6 +10,30 @@ use jiff::Timestamp as JiffTimestamp; use jiff::tz::AmbiguousOffset; use jiff::tz::TimeZone; use prost::Message; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::extension::datetime::TimeUnit; +use vortex_array::extension::datetime::Timestamp; +use vortex_array::extension::datetime::TimestampOptions; +use vortex_array::extension::datetime::resolve_time_zone as resolve_zone; +use vortex_array::proto::expr as pb; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ReduceNode; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::validity::Validity; use vortex_buffer::BitBufferMut; use vortex_buffer::BufferMut; use vortex_error::VortexResult; @@ -18,33 +42,6 @@ use vortex_error::vortex_err; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::ConstantArray; -use crate::arrays::ExtensionArray; -use crate::arrays::PrimitiveArray; -use crate::arrays::VarBinViewArray; -use crate::arrays::extension::ExtensionArrayExt; -use crate::dtype::DType; -use crate::dtype::Nullability; -use crate::dtype::PType; -use crate::expr::BoundExpression; -use crate::expr::bound; -use crate::extension::datetime::TimeUnit; -use crate::extension::datetime::Timestamp; -use crate::extension::datetime::TimestampOptions; -use crate::extension::datetime::resolve_time_zone as resolve_zone; -use crate::proto::expr as pb; -use crate::scalar::Scalar; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::literal::Literal; -use crate::validity::Validity; - /// Timezone replacement options. The ambiguity policy is a UTF-8 expression child. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct ReplaceTimeZoneOptions { @@ -112,20 +109,21 @@ impl ScalarFnVTable for ReplaceTimeZone { } } - fn simplify( + fn simplify( &self, options: &Self::Options, - expr: &BoundExpression, - ) -> VortexResult> { - let (Some(value), Some(policy)) = ( - expr.child(0).as_opt::(), - expr.child(1).as_opt::(), - ) else { + node: &T, + ) -> VortexResult> { + let (Some(value), Some(policy)) = + (node.child(0).as_constant(), node.child(1).as_constant()) + else { return Ok(None); }; - // Fold to a literal so stats pruning sees a bare literal operand. A failing replacement + // Fold to a constant so stats pruning sees a bare literal operand. A failing replacement // (e.g. a raised ambiguity) is left in place so the error surfaces at execution time. - Ok(replace_scalar(options, value, policy).ok().map(bound::lit)) + Ok(replace_scalar(options, &value, &policy) + .ok() + .map(|scalar| node.new_constant(scalar))) } fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult { @@ -316,35 +314,42 @@ mod tests { use std::sync::LazyLock; use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::ExtensionArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::StructArray; + use vortex_array::arrays::VarBinViewArray; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::StructFields; + use vortex_array::extension::datetime::TimeUnit; + use vortex_array::extension::datetime::Timestamp; + use vortex_array::scalar::Scalar; + use vortex_array::scalar_fn::ScalarFnVTable; + use vortex_array::scalar_fn::builtin::literal::Literal; use vortex_error::VortexResult; use vortex_error::vortex_err; + use vortex_expr::ArrayExprExt; + use vortex_expr::get_item; + use vortex_expr::lit; + use vortex_expr::root; use vortex_session::VortexSession; use super::ReplaceTimeZone; use super::ReplaceTimeZoneOptions; use super::replace; use super::resolve_zone; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::ExtensionArray; - use crate::arrays::PrimitiveArray; - use crate::arrays::StructArray; - use crate::arrays::VarBinViewArray; - use crate::assert_arrays_eq; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::StructFields; - use crate::expr::get_item; - use crate::expr::lit; use crate::expr::replace_time_zone; - use crate::expr::root; - use crate::extension::datetime::TimeUnit; - use crate::extension::datetime::Timestamp; - use crate::scalar::Scalar; - use crate::scalar_fn::ScalarFnVTable; - use crate::scalar_fn::fns::literal::Literal; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); #[rstest] #[case(TimeUnit::Seconds, 1)] diff --git a/vortex-functions/src/tests.rs b/vortex-functions/src/tests.rs new file mode 100644 index 00000000000..c6fdffce363 --- /dev/null +++ b/vortex-functions/src/tests.rs @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Expression display, binding and optimizer tests that involve this crate's functions. + +use vortex_array::dtype::DType; +use vortex_array::dtype::FieldName; +use vortex_array::dtype::FieldNames; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::dtype::StructFields; +use vortex_array::scalar::Scalar; +use vortex_error::VortexResult; +use vortex_expr::Expression; +use vortex_expr::and; +use vortex_expr::bound; +use vortex_expr::cast; +use vortex_expr::col; +use vortex_expr::eq; +use vortex_expr::get_item; +use vortex_expr::gt; +use vortex_expr::gt_eq; +use vortex_expr::lit; +use vortex_expr::lt; +use vortex_expr::lt_eq; +use vortex_expr::not; +use vortex_expr::not_eq; +use vortex_expr::or; +use vortex_expr::pack; +use vortex_expr::root; +use vortex_expr::select; +use vortex_expr::select_exclude; + +use crate::between::BetweenOptions; +use crate::between::StrictComparison; +use crate::expr::between; +use crate::expr::bound::case_when as bound_case_when; +use crate::expr::case_when; +use crate::expr::merge; + +#[test] +fn test_display_tree_select() { + use insta::assert_snapshot; + let select_expr = select(["name", "age"], root()); + assert_snapshot!(select_expr.display_tree().to_string(), @r" + vortex.select({name, age}) + └── child: vortex.root() + "); +} + +#[test] +fn test_display_tree_select_exclude() { + use insta::assert_snapshot; + let select_exclude_expr = select_exclude(["internal_id", "metadata"], root()); + assert_snapshot!(select_exclude_expr.display_tree().to_string(), @r" + vortex.select(~{internal_id, metadata}) + └── child: vortex.root() + "); +} + +#[test] +fn test_display_tree_nested() { + use insta::assert_snapshot; + let nested_expr = select( + ["result"], + cast( + between( + get_item("score", root()), + lit(50), + lit(100), + BetweenOptions { + lower_strict: StrictComparison::Strict, + upper_strict: StrictComparison::NonStrict, + }, + ), + DType::Bool(Nullability::NonNullable), + ), + ); + assert_snapshot!(nested_expr.display_tree().to_string(), @r" + vortex.select({result}) + └── child: vortex.cast(bool) + └── input: vortex.between(lower_strict: <, upper_strict: <=) + ├── array: vortex.get_item(score) + │ └── input: vortex.root() + ├── lower: vortex.literal(50i32) + └── upper: vortex.literal(100i32) + "); +} + +#[test] +fn test_display_tree_pack() { + use insta::assert_snapshot; + let select_from_pack_expr = select( + ["fizz", "buzz"], + pack( + [ + ("fizz", root()), + ("bar", lit(5)), + ("buzz", eq(lit(42), get_item("answer", root()))), + ], + Nullability::Nullable, + ), + ); + assert_snapshot!(select_from_pack_expr.display_tree().to_string(), @r" + vortex.select({fizz, buzz}) + └── child: vortex.pack(names: [fizz, bar, buzz], nullability: Nullable) + ├── fizz: vortex.root() + ├── bar: vortex.literal(5i32) + └── buzz: vortex.binary(=) + ├── lhs: vortex.literal(42i32) + └── rhs: vortex.get_item(answer) + └── input: vortex.root() + "); +} + +/// `Select` over a `pack` rewrites to a `pack` of the selected child fields. It must read +/// those fields out of the child `pack` itself, because the optimizer visits children before +/// their parent and so never revisits the children a rule introduces. +#[test] +fn optimize_select_of_pack_reads_pack_fields() -> VortexResult<()> { + let field_names = (0..4) + .map(|idx| FieldName::from(format!("field_{idx}"))) + .collect::>(); + let fields = StructFields::new( + field_names.clone().into(), + vec![DType::Primitive(PType::U64, Nullability::NonNullable); field_names.len()], + ); + let scope = DType::Struct(fields, Nullability::NonNullable); + + let root_pack = pack( + field_names + .iter() + .map(|name| (name.clone(), get_item(name.clone(), root()))) + .collect::>(), + Nullability::NonNullable, + ); + let expr = select( + field_names + .iter() + .cloned() + .chain(["input_row_idx".into(), "input_file_idx".into()]) + .collect::>(), + merge([ + root_pack, + pack( + [ + ("input_row_idx", lit(0_u64)), + ("input_file_idx", lit(1_u64)), + ], + Nullability::NonNullable, + ), + ]), + ); + + let expected = pack( + field_names + .into_iter() + .map(|name| (name.clone(), get_item(name, root()))) + .chain([ + ("input_row_idx".into(), lit(0_u64)), + ("input_file_idx".into(), lit(1_u64)), + ]) + .collect::>(), + Nullability::NonNullable, + ); + + let expected = expected.bind(&scope)?; + + let optimized = expr.bind(&scope)?.optimize_recursive()?; + assert_eq!(optimized, expected, "got {optimized}"); + assert_eq!(optimized.optimize_recursive()?, expected); + + Ok(()) +} + +/// `Merge` reduces to a `pack` that reads each field out of its children, so it must also +/// read straight out of a `pack` child. +#[test] +fn optimize_merge_of_packs_reads_pack_fields() -> VortexResult<()> { + let scope = DType::Struct(StructFields::empty(), Nullability::NonNullable); + let expr = merge([ + pack([("a", lit(0_u64))], Nullability::NonNullable), + pack([("b", lit(1_u64))], Nullability::NonNullable), + ]); + + let expected = pack( + [("a", lit(0_u64)), ("b", lit(1_u64))], + Nullability::NonNullable, + ); + + let expected = expected.bind(&scope)?; + + let optimized = expr.bind(&scope)?.optimize_recursive()?; + assert_eq!(optimized, expected, "got {optimized}"); + assert_eq!(optimized.optimize_recursive()?, expected); + + Ok(()) +} + +#[test] +fn bound_constructors_preserve_order_and_types() -> VortexResult<()> { + let value_dtype = DType::Primitive(PType::I32, Nullability::NonNullable); + let scope = DType::Struct( + StructFields::from_iter([("value", value_dtype.clone())]), + Nullability::NonNullable, + ); + + let root = bound::root(scope.clone()); + let value = bound::get_item("value", root); + let literal = bound::lit(5i32); + let condition = bound::gt(value.clone(), literal.clone()); + assert_eq!(condition.dtype(), &DType::Bool(Nullability::NonNullable)); + assert_eq!(condition.children(), &[value.clone(), literal.clone()]); + + let case = bound_case_when(condition.clone(), value.clone(), literal.clone()); + assert_eq!(case.dtype(), &value_dtype); + assert_eq!(case.children(), &[condition.clone(), value, literal]); + + let packed = bound::pack( + [("condition", condition.clone()), ("value", case.clone())], + Nullability::NonNullable, + ); + assert_eq!(packed.children(), &[condition, case.clone()]); + assert_eq!( + packed.dtype(), + &DType::Struct( + StructFields::from_iter([ + ("condition", DType::Bool(Nullability::NonNullable)), + ("value", value_dtype), + ]), + Nullability::NonNullable, + ) + ); + + let unbound = case_when(gt(col("value"), lit(5i32)), col("value"), lit(5i32)); + assert_eq!(unbound.bind(&scope)?, case); + Ok(()) +} + +#[test] +fn expr_display() { + assert_eq!(col("a").to_string(), "$.a"); + assert_eq!(root().to_string(), "$"); + + let col1: Expression = col("col1"); + let col2: Expression = col("col2"); + assert_eq!( + and(col1.clone(), col2.clone()).to_string(), + "($.col1 and $.col2)" + ); + assert_eq!( + or(col1.clone(), col2.clone()).to_string(), + "($.col1 or $.col2)" + ); + assert_eq!( + eq(col1.clone(), col2.clone()).to_string(), + "($.col1 = $.col2)" + ); + assert_eq!( + not_eq(col1.clone(), col2.clone()).to_string(), + "($.col1 != $.col2)" + ); + assert_eq!( + gt(col1.clone(), col2.clone()).to_string(), + "($.col1 > $.col2)" + ); + assert_eq!( + gt_eq(col1.clone(), col2.clone()).to_string(), + "($.col1 >= $.col2)" + ); + assert_eq!( + lt(col1.clone(), col2.clone()).to_string(), + "($.col1 < $.col2)" + ); + assert_eq!( + lt_eq(col1.clone(), col2.clone()).to_string(), + "($.col1 <= $.col2)" + ); + + assert_eq!( + or(lt(col1.clone(), col2.clone()), not_eq(col1.clone(), col2),).to_string(), + "(($.col1 < $.col2) or ($.col1 != $.col2))" + ); + + assert_eq!(not(col1).to_string(), "vortex.not($.col1)"); + + assert_eq!( + select(vec![FieldName::from("col1")], root()).to_string(), + "${col1}" + ); + assert_eq!( + select( + vec![FieldName::from("col1"), FieldName::from("col2")], + root() + ) + .to_string(), + "${col1, col2}" + ); + assert_eq!( + select_exclude( + vec![FieldName::from("col1"), FieldName::from("col2")], + root() + ) + .to_string(), + "${~ col1, col2}" + ); + + assert_eq!(lit(Scalar::from(0u8)).to_string(), "0u8"); + assert_eq!(lit(Scalar::from(0.0f32)).to_string(), "0f32"); + assert_eq!( + lit(Scalar::from(i64::MAX)).to_string(), + "9223372036854775807i64" + ); + assert_eq!(lit(Scalar::from(true)).to_string(), "true"); + assert_eq!( + lit(Scalar::null(DType::Bool(Nullability::Nullable))).to_string(), + "null" + ); + + assert_eq!( + lit(Scalar::struct_( + DType::Struct( + StructFields::new( + FieldNames::from(["dog", "cat"]), + vec![ + DType::Primitive(PType::U32, Nullability::NonNullable), + DType::Utf8(Nullability::NonNullable) + ], + ), + Nullability::NonNullable + ), + vec![Scalar::from(32_u32), Scalar::from("rufus".to_string())] + )) + .to_string(), + "{dog: 32u32, cat: \"rufus\"}" + ); +} diff --git a/vortex-array/src/scalar_fn/fns/variant_get/mod.rs b/vortex-functions/src/variant_get/mod.rs similarity index 82% rename from vortex-array/src/scalar_fn/fns/variant_get/mod.rs rename to vortex-functions/src/variant_get/mod.rs index ef493d0146e..8f5cc0a53f1 100644 --- a/vortex-array/src/scalar_fn/fns/variant_get/mod.rs +++ b/vortex-functions/src/variant_get/mod.rs @@ -1,11 +1,34 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod variant; use std::fmt; use std::fmt::Display; use std::fmt::Formatter; use prost::Message; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::VariantArray; +use vortex_array::builders::builder_with_capacity_in; +use vortex_array::dtype::DType; +use vortex_array::dtype::FieldName; +use vortex_array::dtype::Nullability; +use vortex_array::proto::expr as pb; +use vortex_array::proto::expr::variant_path_element; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::display::ExprDisplay; +use vortex_array::scalar_fn::session::ScalarFnSessionExt; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -13,27 +36,11 @@ use vortex_session::VortexSession; use vortex_session::registry::CachedId; use vortex_utils::aliases::StringEscape; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::ChunkedArray; -use crate::arrays::ConstantArray; -use crate::arrays::ScalarFnArray; -use crate::arrays::VariantArray; -use crate::builders::builder_with_capacity_in; -use crate::dtype::DType; -use crate::dtype::FieldName; -use crate::dtype::Nullability; -use crate::expr::display::ExprDisplay; -use crate::proto::expr as pb; -use crate::proto::expr::variant_path_element; -use crate::scalar::Scalar; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::ScalarFnVTableExt; +/// Registers [`VariantGet`] and its kernel for canonical variant arrays. +pub(crate) fn initialize(session: &VortexSession) { + session.scalar_fns().register(VariantGet); + variant::initialize(session); +} /// Extracts a field/index path from Variant values. /// @@ -411,43 +418,51 @@ impl From for VariantPathElement { mod tests { use std::sync::LazyLock; + use vortex_array::ArrayRef; + use vortex_array::Canonical; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Chunked; + use vortex_array::arrays::ChunkedArray; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::StructArray; + use vortex_array::arrays::VariantArray; + use vortex_array::arrays::variant::VariantArraySlotsExt; + use vortex_array::assert_arrays_eq; + use vortex_array::assert_nth_scalar_is_null; + use vortex_array::dtype::DType; + use vortex_array::dtype::FieldName; + use vortex_array::dtype::FieldNames; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::StructFields; + use vortex_array::scalar::Scalar; + use vortex_array::scalar::ScalarValue; + use vortex_array::scalar_fn::ScalarFnVTable; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; + use vortex_expr::ArrayExprExt; + use vortex_expr::Expression; + use vortex_expr::proto::ExprSerializeProtoExt; + use vortex_expr::root; use vortex_session::VortexSession; - use crate::ArrayRef; - use crate::Canonical; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::Chunked; - use crate::arrays::ChunkedArray; - use crate::arrays::ConstantArray; - use crate::arrays::PrimitiveArray; - use crate::arrays::VariantArray; - use crate::arrays::variant::VariantArraySlotsExt; - use crate::assert_arrays_eq; - use crate::assert_nth_scalar_is_null; - use crate::dtype::DType; - use crate::dtype::FieldName; - use crate::dtype::FieldNames; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::StructFields; - use crate::expr::Expression; - use crate::expr::proto::ExprSerializeProtoExt; - use crate::expr::root; use crate::expr::variant_get; - use crate::scalar::Scalar; - use crate::scalar::ScalarValue; - use crate::scalar_fn::ScalarFnVTable; - use crate::scalar_fn::fns::variant_get::VariantGet; - use crate::scalar_fn::fns::variant_get::VariantGetOptions; - use crate::scalar_fn::fns::variant_get::VariantPath; - use crate::scalar_fn::fns::variant_get::VariantPathElement; + use crate::variant_get::VariantGet; + use crate::variant_get::VariantGetOptions; + use crate::variant_get::VariantPath; + use crate::variant_get::VariantPathElement; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + session + }); fn variant_object(fields: impl IntoIterator) -> Scalar { let fields = fields.into_iter().collect::>(); @@ -848,4 +863,85 @@ mod tests { } Ok(()) } + + fn row_storage(values: impl IntoIterator) -> VortexResult { + Ok(ChunkedArray::try_new( + values.into_iter().map(|value| { + ConstantArray::new( + Scalar::variant(Scalar::primitive(value, Nullability::NonNullable)), + 1, + ) + .into_array() + }), + DType::Variant(Nullability::NonNullable), + )? + .into_array()) + } + + #[test] + fn variant_get_keeps_valid_shredded_rows_for_matching_dtype() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let core_storage = row_storage([1, 2, 3])?; + let shredded = StructArray::try_from_iter([( + "a", + PrimitiveArray::from_iter([10i32, 20, 30]).into_array(), + )])?; + let variant = VariantArray::try_new(core_storage, Some(shredded.into_array()))?; + let expr = variant_get( + root(), + VariantPath::field("a"), + Some(DType::Primitive(PType::I32, Nullability::NonNullable)), + ); + + let result = variant.into_array().apply(&expr)?; + + assert_arrays_eq!( + result, + PrimitiveArray::from_option_iter([Some(10i32), Some(20), Some(30)]), + &mut ctx + ); + Ok(()) + } + + #[test] + fn variant_get_treats_value_and_typed_value_as_logical_field_names() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let core_storage = row_storage([1, 2, 3])?; + let shredded = StructArray::try_from_iter([ + ( + "value", + PrimitiveArray::from_iter([10i32, 20, 30]).into_array(), + ), + ( + "typed_value", + PrimitiveArray::from_iter([40i32, 50, 60]).into_array(), + ), + ])?; + let variant = VariantArray::try_new(core_storage, Some(shredded.into_array()))?; + + let value_expr = variant_get( + root(), + VariantPath::field("value"), + Some(DType::Primitive(PType::I32, Nullability::NonNullable)), + ); + let value_result = variant.clone().into_array().apply(&value_expr)?; + assert_arrays_eq!( + value_result, + PrimitiveArray::from_option_iter([Some(10i32), Some(20), Some(30)]), + &mut ctx + ); + + let typed_value_expr = variant_get( + root(), + VariantPath::field("typed_value"), + Some(DType::Primitive(PType::I32, Nullability::NonNullable)), + ); + let typed_value_result = variant.into_array().apply(&typed_value_expr)?; + assert_arrays_eq!( + typed_value_result, + PrimitiveArray::from_option_iter([Some(40i32), Some(50), Some(60)]), + &mut ctx + ); + Ok(()) + } } diff --git a/vortex-array/src/arrays/variant/vtable/kernel.rs b/vortex-functions/src/variant_get/variant.rs similarity index 86% rename from vortex-array/src/arrays/variant/vtable/kernel.rs rename to vortex-functions/src/variant_get/variant.rs index 22d52ecd6a7..8815d57339b 100644 --- a/vortex-array/src/arrays/variant/vtable/kernel.rs +++ b/vortex-functions/src/variant_get/variant.rs @@ -1,35 +1,36 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::Array; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Struct; +use vortex_array::arrays::Variant; +use vortex_array::arrays::VariantArray; +use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::arrays::struct_::StructArrayExt; +use vortex_array::arrays::variant::VariantArraySlotsExt; +use vortex_array::arrays::variant::merge_typed_scalar_as_variant; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::kernel::ExecuteParentKernel; +use vortex_array::optimizer::kernels::ArrayKernelsExt; +use vortex_array::scalar_fn::ScalarFnVTable; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_session::VortexSession; -use super::merge_typed_scalar_as_variant; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::Array; -use crate::array::ArrayView; -use crate::arrays::ChunkedArray; -use crate::arrays::ConstantArray; -use crate::arrays::Struct; -use crate::arrays::Variant; -use crate::arrays::VariantArray; -use crate::arrays::scalar_fn::ExactScalarFn; -use crate::arrays::scalar_fn::ScalarFnArrayView; -use crate::arrays::struct_::StructArrayExt; -use crate::arrays::variant::VariantArraySlotsExt; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::Nullability; -use crate::kernel::ExecuteParentKernel; -use crate::optimizer::kernels::ArrayKernelsExt; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::fns::variant_get::VariantGet; -use crate::scalar_fn::fns::variant_get::VariantGetOptions; -use crate::scalar_fn::fns::variant_get::VariantPath; -use crate::scalar_fn::fns::variant_get::VariantPathElement; +use crate::builtins::ArrayFunctions; +use crate::variant_get::VariantGet; +use crate::variant_get::VariantGetOptions; +use crate::variant_get::VariantPath; +use crate::variant_get::VariantPathElement; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); diff --git a/vortex-array/src/arrays/bool/compute/zip.rs b/vortex-functions/src/zip/bool.rs similarity index 88% rename from vortex-array/src/arrays/bool/compute/zip.rs rename to vortex-functions/src/zip/bool.rs index 70e22a27752..70144c2c243 100644 --- a/vortex-array/src/arrays/bool/compute/zip.rs +++ b/vortex-functions/src/zip/bool.rs @@ -1,21 +1,21 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Bool; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::builtins::ArrayBuiltins; use vortex_buffer::BitBuffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; use vortex_mask::Mask; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::Bool; -use crate::arrays::BoolArray; -use crate::arrays::bool::BoolArrayExt; -use crate::builtins::ArrayBuiltins; -use crate::scalar_fn::fns::zip::ZipKernel; -use crate::scalar_fn::fns::zip::zip_validity; +use crate::zip::ZipKernel; +use crate::zip::zip_validity; /// A branchless boolean zip kernel that blends the two value bitmaps with the mask in one pass. /// @@ -86,20 +86,26 @@ fn zip_value_bits(if_true: &BitBuffer, if_false: &BitBuffer, mask: &BitBuffer) - mod tests { use std::sync::LazyLock; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Bool; + use vortex_array::arrays::BoolArray; + use vortex_array::assert_arrays_eq; use vortex_error::VortexResult; use vortex_mask::Mask; use vortex_session::VortexSession; use super::zip_value_bits; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::Bool; - use crate::arrays::BoolArray; - use crate::assert_arrays_eq; - use crate::builtins::ArrayBuiltins; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + use crate::builtins::ArrayFunctions; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); #[test] fn blend_value_bits_boundaries() { diff --git a/vortex-array/src/arrays/chunked/compute/zip.rs b/vortex-functions/src/zip/chunked.rs similarity index 75% rename from vortex-array/src/arrays/chunked/compute/zip.rs rename to vortex-functions/src/zip/chunked.rs index dc43043b6c7..44b0ce36359 100644 --- a/vortex-array/src/arrays/chunked/compute/zip.rs +++ b/vortex-functions/src/zip/chunked.rs @@ -1,18 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Chunked; +use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::chunked::ChunkedArrayExt; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::Chunked; -use crate::arrays::ChunkedArray; -use crate::arrays::chunked::ChunkedArrayExt; -use crate::arrays::chunked::paired_chunks::PairedChunksExt; -use crate::builtins::ArrayBuiltins; -use crate::scalar_fn::fns::zip::ZipKernel; +use crate::builtins::ArrayFunctions; +use crate::zip::ZipKernel; +use crate::zip::paired_chunks::PairedChunksExt; // Push down the zip call to the chunks. Without this rule // the default implementation canonicalises the chunked array @@ -48,23 +48,29 @@ impl ZipKernel for Chunked { mod tests { use std::sync::LazyLock; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Chunked; + use vortex_array::arrays::ChunkedArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::chunked::ChunkedArrayExt; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; use vortex_buffer::buffer; use vortex_mask::Mask; use vortex_session::VortexSession; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::Chunked; - use crate::arrays::ChunkedArray; - use crate::arrays::PrimitiveArray; - use crate::arrays::chunked::ChunkedArrayExt; - use crate::builtins::ArrayBuiltins; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + use crate::builtins::ArrayFunctions; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); #[test] fn test_chunked_zip_aligns_across_boundaries() { diff --git a/vortex-array/src/scalar_fn/fns/zip/kernel.rs b/vortex-functions/src/zip/kernel.rs similarity index 87% rename from vortex-array/src/scalar_fn/fns/zip/kernel.rs rename to vortex-functions/src/zip/kernel.rs index 575c2f4c55a..d0db3cc07aa 100644 --- a/vortex-array/src/scalar_fn/fns/zip/kernel.rs +++ b/vortex-functions/src/zip/kernel.rs @@ -1,20 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::VTable; +use vortex_array::arrays::ScalarFn; +use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::kernel::ExecuteParentKernel; +use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::array::ArrayView; -use crate::array::VTable; -use crate::arrays::ScalarFn; -use crate::arrays::scalar_fn::ExactScalarFn; -use crate::arrays::scalar_fn::ScalarFnArrayExt; -use crate::arrays::scalar_fn::ScalarFnArrayView; -use crate::kernel::ExecuteParentKernel; -use crate::optimizer::rules::ArrayParentReduceRule; -use crate::scalar_fn::fns::zip::Zip as ZipExpr; +use crate::zip::Zip as ZipExpr; /// Zip two arrays using a mask without reading buffers. /// diff --git a/vortex-array/src/arrays/listview/compute/zip.rs b/vortex-functions/src/zip/listview.rs similarity index 81% rename from vortex-array/src/arrays/listview/compute/zip.rs rename to vortex-functions/src/zip/listview.rs index a50f1909395..2b34455d25f 100644 --- a/vortex-array/src/arrays/listview/compute/zip.rs +++ b/vortex-functions/src/zip/listview.rs @@ -6,33 +6,33 @@ use std::ops::BitAnd; use std::ops::BitOr; use std::ops::Not; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Chunked; +use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::ListView; +use vortex_array::arrays::ListViewArray; +use vortex_array::arrays::chunked::ChunkedArrayExt; +use vortex_array::arrays::listview::ListViewArraySlotsExt; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::Chunked; -use crate::arrays::ChunkedArray; -use crate::arrays::ListView; -use crate::arrays::ListViewArray; -use crate::arrays::chunked::ChunkedArrayExt; -use crate::arrays::listview::ListViewArraySlotsExt; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::Nullability; -use crate::dtype::PType; -use crate::scalar_fn::fns::zip::ZipKernel; -use crate::validity::Validity; +use crate::zip::ZipKernel; /// Zip two [`ListViewArray`]s by selecting whole list views per row. /// /// A [`ListViewArray`] addresses each list by an `(offset, size)` pair into a shared `elements` -/// array, and unlike [`ListArray`](crate::arrays::ListArray) it does not require lists to be stored +/// array, and unlike [`ListArray`](vortex_array::arrays::ListArray) it does not require lists to be stored /// contiguously or in order. Zipping two list views is therefore a metadata-only operation over the /// `offsets`, `sizes` and `validity` child arrays: we concatenate the two `elements` arrays /// (without rewriting them) and, for each row, select the `(offset, size)` pair from `if_true` or @@ -236,33 +236,40 @@ mod tests { clippy::cast_possible_truncation, reason = "test fixtures use small indices that fit the target widths" )] - + use std::sync::Arc; use std::sync::LazyLock; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::Chunked; + use vortex_array::arrays::ChunkedArray; + use vortex_array::arrays::ListView; + use vortex_array::arrays::ListViewArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::chunked::ChunkedArrayExt; + use vortex_array::arrays::listview::ListViewArraySlotsExt; + use vortex_array::assert_arrays_eq; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_mask::Mask; use vortex_session::VortexSession; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::BoolArray; - use crate::arrays::Chunked; - use crate::arrays::ChunkedArray; - use crate::arrays::ListView; - use crate::arrays::ListViewArray; - use crate::arrays::chunked::ChunkedArrayExt; - use crate::arrays::listview::ListViewArraySlotsExt; - use crate::assert_arrays_eq; - use crate::builtins::ArrayBuiltins; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::validity::Validity; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + use crate::builtins::ArrayFunctions; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); fn list_view( elements: ArrayRef, @@ -583,4 +590,102 @@ mod tests { assert_arrays_eq!(result, expected, &mut ctx); Ok(()) } + + #[test] + fn test_zip_widens_false_element_nullability() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + // [[1, 2], [3], [4]] + let if_true = ListViewArray::new( + buffer![1i32, 2, 3, 4].into_array(), + buffer![0u32, 2, 3].into_array(), + buffer![2u32, 1, 1].into_array(), + Validity::NonNullable, + &mut ctx, + ) + .into_array(); + // [[10, null], [30], [40]] + let if_false = ListViewArray::new( + PrimitiveArray::from_option_iter([Some(10i32), None, Some(30), Some(40)]).into_array(), + buffer![0u32, 2, 3].into_array(), + buffer![2u32, 1, 1].into_array(), + Validity::NonNullable, + &mut ctx, + ) + .into_array(); + let mask = Mask::from_iter([false, true, false]); + + let result = mask + .into_array() + .zip(if_true, if_false)? + .execute::(&mut SESSION.create_execution_ctx())?; + assert!(result.is::()); + assert_eq!( + result.dtype(), + &DType::List( + Arc::new(DType::Primitive(PType::I32, Nullability::Nullable)), + Nullability::NonNullable, + ) + ); + + // [[10, null], [3], [40]] + let expected = ListViewArray::new( + PrimitiveArray::from_option_iter([Some(10i32), None, Some(3), Some(40)]).into_array(), + buffer![0u32, 2, 3].into_array(), + buffer![2u32, 1, 1].into_array(), + Validity::NonNullable, + &mut ctx, + ) + .into_array(); + assert_arrays_eq!(result, expected, &mut ctx); + Ok(()) + } + + #[test] + fn test_zip_widens_true_element_nullability() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + // [[1, null], [3], [4]] + let if_true = ListViewArray::new( + PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), Some(4)]).into_array(), + buffer![0u32, 2, 3].into_array(), + buffer![2u32, 1, 1].into_array(), + Validity::NonNullable, + &mut ctx, + ) + .into_array(); + // [[10], [20], [30]] + let if_false = ListViewArray::new( + buffer![10i32, 20, 30].into_array(), + buffer![0u32, 1, 2].into_array(), + buffer![1u32, 1, 1].into_array(), + Validity::NonNullable, + &mut ctx, + ) + .into_array(); + let mask = Mask::from_iter([true, false, true]); + + let result = mask + .into_array() + .zip(if_true, if_false)? + .execute::(&mut SESSION.create_execution_ctx())?; + assert!(result.is::()); + assert_eq!( + result.dtype(), + &DType::List( + Arc::new(DType::Primitive(PType::I32, Nullability::Nullable)), + Nullability::NonNullable, + ) + ); + + // [[1, null], [20], [4]] + let expected = ListViewArray::new( + PrimitiveArray::from_option_iter([Some(1i32), None, Some(20), Some(4)]).into_array(), + buffer![0u32, 2, 3].into_array(), + buffer![2u32, 1, 1].into_array(), + Validity::NonNullable, + &mut ctx, + ) + .into_array(); + assert_arrays_eq!(result, expected, &mut ctx); + Ok(()) + } } diff --git a/vortex-array/src/scalar_fn/fns/zip/mod.rs b/vortex-functions/src/zip/mod.rs similarity index 79% rename from vortex-array/src/scalar_fn/fns/zip/mod.rs rename to vortex-functions/src/zip/mod.rs index fa46512a4f0..da719d85244 100644 --- a/vortex-array/src/scalar_fn/fns/zip/mod.rs +++ b/vortex-functions/src/zip/mod.rs @@ -1,13 +1,50 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod bool; +mod chunked; mod kernel; +mod listview; +mod paired_chunks; +mod primitive; +mod struct_; +mod varbinview; use std::fmt::Display; use std::fmt::Formatter; use std::sync::Arc; pub use kernel::*; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Bool; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::Chunked; +use vortex_array::arrays::ListView; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::Struct; +use vortex_array::arrays::VarBinView; +use vortex_array::arrays::bool::BoolArrayExt; +use vortex_array::builders::ArrayBuilder; +use vortex_array::builders::builder_with_capacity_in; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::StructFields; +use vortex_array::optimizer::kernels::ArrayKernelsExt; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::EmptyOptions; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ReduceNode; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::builtin::cast::Cast; +use vortex_array::scalar_fn::display::ExprDisplay; +use vortex_array::scalar_fn::session::ScalarFnSessionExt; +use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -16,29 +53,19 @@ use vortex_mask::MaskValues; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::BoolArray; -use crate::arrays::ScalarFnArray; -use crate::arrays::bool::BoolArrayExt; -use crate::builders::ArrayBuilder; -use crate::builders::builder_with_capacity_in; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::StructFields; -use crate::expr::BoundExpression; -use crate::expr::bound; -use crate::expr::display::ExprDisplay; -use crate::scalar_fn::Arity; -use crate::scalar_fn::ChildName; -use crate::scalar_fn::EmptyOptions; -use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnVTable; -use crate::scalar_fn::ScalarFnVTableExt; -use crate::scalar_fn::fns::literal::Literal; -use crate::validity::Validity; +use crate::builtins::ArrayFunctions; + +/// Registers [`Zip`] and its kernels for vortex-array's canonical encodings. +pub(crate) fn initialize(session: &VortexSession) { + session.scalar_fns().register(Zip); + let kernels = session.kernels(); + kernels.register_execute_parent_kernel(Zip.id(), Bool, ZipExecuteAdaptor(Bool)); + kernels.register_execute_parent_kernel(Zip.id(), Chunked, ZipExecuteAdaptor(Chunked)); + kernels.register_execute_parent_kernel(Zip.id(), ListView, ZipExecuteAdaptor(ListView)); + kernels.register_execute_parent_kernel(Zip.id(), Primitive, ZipExecuteAdaptor(Primitive)); + kernels.register_execute_parent_kernel(Zip.id(), Struct, ZipExecuteAdaptor(Struct)); + kernels.register_execute_parent_kernel(Zip.id(), VarBinView, ZipExecuteAdaptor(VarBinView)); +} /// An expression that conditionally selects between two arrays based on a boolean mask. /// @@ -159,27 +186,28 @@ impl ScalarFnVTable for Zip { zip_impl(&if_true, &if_false, &mask, ctx) } - fn simplify( + fn simplify( &self, _options: &Self::Options, - expr: &BoundExpression, - ) -> VortexResult> { - let Some(mask_lit) = expr.child(2).as_opt::() else { + node: &T, + ) -> VortexResult> { + let Some(mask_constant) = node.child(2).as_constant() else { return Ok(None); }; - if let Some(mask_val) = mask_lit.as_bool().value() { - let selected = if mask_val { - expr.child(0).clone() + if let Some(mask_value) = mask_constant.as_bool().value() { + let selected = if mask_value { + node.child(0) } else { - expr.child(1).clone() + node.child(1) }; // The selected branch can be less nullable than the zip result, which unions // both branches' nullability, so keep the dtype that binding inferred. - return Ok(Some(if selected.dtype() == expr.dtype() { + let dtype = node.node_dtype()?; + return Ok(Some(if selected.node_dtype()? == dtype { selected } else { - bound::cast(selected, expr.dtype().clone()) + selected.new_node(Cast.bind(dtype), std::slice::from_ref(&selected))? })); } @@ -237,7 +265,7 @@ fn zip_return_dtype(if_true: &DType, if_false: &DType) -> VortexResult { /// Unions the nullability of two dtypes at every nesting level. /// /// Returns `None` when the dtypes differ in more than nullability. -pub(crate) fn zip_nullability_union(lhs: &DType, rhs: &DType) -> Option { +pub fn zip_nullability_union(lhs: &DType, rhs: &DType) -> Option { let nullability = lhs.nullability() | rhs.nullability(); match (lhs, rhs) { @@ -344,35 +372,42 @@ pub(crate) fn zip_validity( mod tests { use std::sync::LazyLock; + use vortex_array::ArrayRef; + use vortex_array::Columnar; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::Struct; + use vortex_array::arrays::StructArray; + use vortex_array::arrays::VarBinView; + use vortex_array::arrays::VarBinViewArray; + use vortex_array::assert_arrays_eq; + use vortex_array::builders::ArrayBuilder; + use vortex_array::builders::BufferGrowthStrategy; + use vortex_array::builders::VarBinViewBuilder; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::scalar::Scalar; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::Expression; + use vortex_expr::lit; + use vortex_expr::root; use vortex_mask::Mask; use vortex_session::VortexSession; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::ConstantArray; - use crate::arrays::PrimitiveArray; - use crate::arrays::Struct; - use crate::arrays::StructArray; - use crate::arrays::VarBinView; - use crate::arrays::VarBinViewArray; - use crate::assert_arrays_eq; - use crate::builders::ArrayBuilder; - use crate::builders::BufferGrowthStrategy; - use crate::builders::VarBinViewBuilder; - use crate::builtins::ArrayBuiltins; - use crate::columnar::Columnar; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::expr::lit; - use crate::expr::root; + use crate::builtins::ArrayFunctions; use crate::expr::zip_expr; - use crate::scalar::Scalar; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); #[test] fn dtype() { @@ -578,4 +613,23 @@ mod tests { })); assert_arrays_eq!(zipped.array().clone(), expected, &mut ctx); } + + #[rstest::rstest] + #[case::zip_true( + zip_expr(lit(true), lit(1i32), root()), + DType::Primitive(PType::I32, Nullability::Nullable) + )] + #[case::zip_false( + zip_expr(lit(false), root(), lit(1i32)), + DType::Primitive(PType::I32, Nullability::Nullable) + )] + fn literal_simplification_preserves_result_dtype( + #[case] expr: Expression, + #[case] scope: DType, + ) -> VortexResult<()> { + let original = expr.bind(&scope)?; + let optimized = expr.bind(&scope)?.optimize_recursive()?; + assert_eq!(optimized.dtype(), original.dtype()); + Ok(()) + } } diff --git a/vortex-array/src/arrays/chunked/paired_chunks.rs b/vortex-functions/src/zip/paired_chunks.rs similarity index 92% rename from vortex-array/src/arrays/chunked/paired_chunks.rs rename to vortex-functions/src/zip/paired_chunks.rs index 796daee1b37..12f23b806d8 100644 --- a/vortex-array/src/arrays/chunked/paired_chunks.rs +++ b/vortex-functions/src/zip/paired_chunks.rs @@ -3,11 +3,10 @@ use std::ops::Range; +use vortex_array::ArrayRef; +use vortex_array::arrays::chunked::ChunkedArrayExt; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::arrays::chunked::ChunkedArrayExt; - pub(crate) struct AlignedPair { pub left: ArrayRef, pub right: ArrayRef, @@ -119,20 +118,26 @@ impl Iterator for PairedChunks { mod tests { use std::sync::LazyLock; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::ChunkedArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::ChunkedArray; - use crate::arrays::PrimitiveArray; - use crate::arrays::chunked::paired_chunks::PairedChunksExt; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; + use crate::zip::paired_chunks::PairedChunksExt; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + session + }); fn i32_dtype() -> DType { DType::Primitive(PType::I32, Nullability::NonNullable) diff --git a/vortex-array/src/arrays/primitive/compute/zip.rs b/vortex-functions/src/zip/primitive.rs similarity index 89% rename from vortex-array/src/arrays/primitive/compute/zip.rs rename to vortex-functions/src/zip/primitive.rs index a653f01099e..293af7fe814 100644 --- a/vortex-array/src/arrays/primitive/compute/zip.rs +++ b/vortex-functions/src/zip/primitive.rs @@ -3,23 +3,23 @@ use std::mem::MaybeUninit; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Primitive; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::NativePType; +use vortex_array::match_each_native_ptype; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_mask::Mask; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::Primitive; -use crate::arrays::PrimitiveArray; -use crate::builtins::ArrayBuiltins; -use crate::dtype::NativePType; -use crate::match_each_native_ptype; -use crate::scalar_fn::fns::zip::ZipKernel; -use crate::scalar_fn::fns::zip::zip_validity; +use crate::zip::ZipKernel; +use crate::zip::zip_validity; /// A dedicated primitive zip kernel that selects values branchlessly per row. /// @@ -150,19 +150,25 @@ mod tests { use std::sync::LazyLock; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Primitive; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; use vortex_error::VortexResult; use vortex_mask::Mask; use vortex_session::VortexSession; - use crate::ArrayRef; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::Primitive; - use crate::arrays::PrimitiveArray; - use crate::assert_arrays_eq; - use crate::builtins::ArrayBuiltins; + use crate::builtins::ArrayFunctions; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); /// The branchless kernel must agree with the scalar reference across the chunk boundary (index /// 63/64) and the trailing remainder, for non-nullable inputs. diff --git a/vortex-array/src/arrays/struct_/compute/zip.rs b/vortex-functions/src/zip/struct_.rs similarity index 84% rename from vortex-array/src/arrays/struct_/compute/zip.rs rename to vortex-functions/src/zip/struct_.rs index a6dc944d019..fde3ba4f380 100644 --- a/vortex-array/src/arrays/struct_/compute/zip.rs +++ b/vortex-functions/src/zip/struct_.rs @@ -5,19 +5,20 @@ use std::ops::BitAnd; use std::ops::BitOr; use std::ops::Not; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Struct; +use vortex_array::arrays::StructArray; +use vortex_array::arrays::struct_::StructArrayExt; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_mask::Mask; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::Struct; -use crate::arrays::StructArray; -use crate::arrays::struct_::StructArrayExt; -use crate::builtins::ArrayBuiltins; -use crate::scalar_fn::fns::zip::ZipKernel; -use crate::validity::Validity; +use crate::builtins::ArrayFunctions; +use crate::zip::ZipKernel; impl ZipKernel for Struct { fn zip( @@ -72,18 +73,24 @@ impl ZipKernel for Struct { mod tests { use std::sync::LazyLock; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::StructArray; + use vortex_array::dtype::FieldNames; + use vortex_array::validity::Validity; use vortex_mask::Mask; use vortex_session::VortexSession; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::PrimitiveArray; - use crate::arrays::StructArray; - use crate::builtins::ArrayBuiltins; - use crate::dtype::FieldNames; - use crate::validity::Validity; + use crate::builtins::ArrayFunctions; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); #[test] fn test_validity_zip_both_validity_array() { diff --git a/vortex-array/src/arrays/varbinview/compute/zip.rs b/vortex-functions/src/zip/varbinview.rs similarity index 90% rename from vortex-array/src/arrays/varbinview/compute/zip.rs rename to vortex-functions/src/zip/varbinview.rs index fdcfccbd0d5..6a42731a9bf 100644 --- a/vortex-array/src/arrays/varbinview/compute/zip.rs +++ b/vortex-functions/src/zip/varbinview.rs @@ -3,24 +3,24 @@ use std::ops::Range; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::VarBinView; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::varbinview::BinaryView; +use vortex_array::arrays::varbinview::VarBinViewArrayExt; +use vortex_array::builders::DeduplicatedBuffers; +use vortex_array::builders::LazyBitBufferBuilder; +use vortex_array::builtins::ArrayBuiltins; use vortex_buffer::BufferMut; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_mask::AllOr; use vortex_mask::Mask; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::VarBinView; -use crate::arrays::VarBinViewArray; -use crate::arrays::varbinview::BinaryView; -use crate::arrays::varbinview::VarBinViewArrayExt; -use crate::builders::DeduplicatedBuffers; -use crate::builders::LazyBitBufferBuilder; -use crate::builtins::ArrayBuiltins; -use crate::scalar_fn::fns::zip::ZipKernel; +use crate::zip::ZipKernel; // A dedicated VarBinView zip kernel that builds the result directly by adjusting views and validity, // instead of routing through the generic builder (which would redo buffer lookups per mask slice). @@ -212,18 +212,24 @@ fn push_view( mod tests { use std::sync::LazyLock; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::VarBinViewArray; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; use vortex_error::VortexResult; use vortex_mask::Mask; use vortex_session::VortexSession; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::VarBinViewArray; - use crate::builtins::ArrayBuiltins; - use crate::dtype::DType; - use crate::dtype::Nullability; + use crate::builtins::ArrayFunctions; - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty(); + + crate::initialize(&session); + + session + }); #[test] fn zip_varbinview_kernel_zips() -> VortexResult<()> { diff --git a/vortex-jni/src/expression.rs b/vortex-jni/src/expression.rs index afcefe990c4..7a7ea8f135b 100644 --- a/vortex-jni/src/expression.rs +++ b/vortex-jni/src/expression.rs @@ -40,6 +40,7 @@ use vortex::encodings::uuid::Uuid; use vortex::encodings::uuid::UuidMetadata; use vortex::error::vortex_err; use vortex::expr::Expression; +use vortex::expr::ScalarFnExprExt; use vortex::expr::and_collect; use vortex::expr::between; use vortex::expr::get_item; @@ -57,21 +58,20 @@ use vortex::extension::datetime::Date; use vortex::extension::datetime::Time; use vortex::extension::datetime::TimeUnit; use vortex::extension::datetime::Timestamp; +use vortex::functions::between::BetweenOptions; +use vortex::functions::between::StrictComparison; +use vortex::functions::like::Like; +use vortex::functions::like::LikeOptions; +use vortex::functions::list_contains::ListContainsOptions; +use vortex::functions::merge::DuplicateHandling; use vortex::layout::layouts::row_idx::row_idx; use vortex::scalar::DecimalValue; use vortex::scalar::Scalar; use vortex::scalar::ScalarValue; use vortex::scalar_fn::EmptyOptions; -use vortex::scalar_fn::ScalarFnVTableExt; -use vortex::scalar_fn::fns::between::BetweenOptions; -use vortex::scalar_fn::fns::between::StrictComparison; -use vortex::scalar_fn::fns::binary::Binary; -use vortex::scalar_fn::fns::like::Like; -use vortex::scalar_fn::fns::like::LikeOptions; -use vortex::scalar_fn::fns::list_contains::ListContainsOptions; -use vortex::scalar_fn::fns::literal::Literal; -use vortex::scalar_fn::fns::merge::DuplicateHandling; -use vortex::scalar_fn::fns::operators::Operator; +use vortex::scalar_fn::builtin::binary::Binary; +use vortex::scalar_fn::builtin::literal::Literal; +use vortex::scalar_fn::builtin::operators::Operator; use vortex::session::VortexSession; use vortex_spatial::extension::native_geometry_scalar_from_wkb; use vortex_spatial::scalar_fn::area::SpatialArea; diff --git a/vortex-json/Cargo.toml b/vortex-json/Cargo.toml index 27c8c86fea2..21c3f7c02cb 100644 --- a/vortex-json/Cargo.toml +++ b/vortex-json/Cargo.toml @@ -23,6 +23,8 @@ vortex-array = { workspace = true, default-features = false } vortex-arrow = { workspace = true } vortex-edition = { workspace = true } vortex-error = { workspace = true, default-features = false } +vortex-expr = { workspace = true } +vortex-functions = { workspace = true } vortex-session = { workspace = true } [dev-dependencies] diff --git a/vortex-json/src/json_to_variant.rs b/vortex-json/src/json_to_variant.rs index 32538b7ad3d..911af87c2d1 100644 --- a/vortex-json/src/json_to_variant.rs +++ b/vortex-json/src/json_to_variant.rs @@ -15,7 +15,6 @@ use vortex_array::arrays::Extension; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::ScalarFnArray; use vortex_array::dtype::DType; -use vortex_array::expr::Expression; use vortex_array::proto::expr as pb; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; @@ -23,12 +22,14 @@ use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; use vortex_array::scalar_fn::ScalarFnVTableExt; -use vortex_array::scalar_fn::fns::variant_get::VariantPath; -use vortex_array::scalar_fn::fns::variant_get::VariantPathElement; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_expr::Expression; +use vortex_expr::ScalarFnExprExt; +use vortex_functions::variant_get::VariantPath; +use vortex_functions::variant_get::VariantPathElement; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -314,11 +315,11 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::session::DTypeSession; - use vortex_array::expr::proto::ExprSerializeProtoExt; - use vortex_array::expr::root; use vortex_array::scalar_fn::session::ScalarFnSession; use vortex_array::scalar_fn::session::ScalarFnSessionExt; use vortex_array::session::ArraySession; + use vortex_expr::proto::ExprSerializeProtoExt; + use vortex_expr::root; use super::*; diff --git a/vortex-layout/Cargo.toml b/vortex-layout/Cargo.toml index 81cd54e9a38..a5dc9506093 100644 --- a/vortex-layout/Cargo.toml +++ b/vortex-layout/Cargo.toml @@ -48,6 +48,7 @@ vortex-btrblocks = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-expr = { workspace = true } +vortex-functions = { workspace = true } vortex-io = { workspace = true } vortex-mask = { workspace = true } vortex-metrics = { workspace = true } diff --git a/vortex-layout/benches/zone_map_prune.rs b/vortex-layout/benches/zone_map_prune.rs index ebd4ece8218..d3fb468fe3c 100644 --- a/vortex-layout/benches/zone_map_prune.rs +++ b/vortex-layout/benches/zone_map_prune.rs @@ -29,18 +29,18 @@ use vortex_array::arrays::StructArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::Expression; -use vortex_array::expr::eq; -use vortex_array::expr::gt; -use vortex_array::expr::is_not_null; -use vortex_array::expr::lit; -use vortex_array::expr::or; -use vortex_array::expr::root; use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; +use vortex_expr::BoundExpression; +use vortex_expr::Expression; +use vortex_expr::eq; +use vortex_expr::gt; +use vortex_expr::is_not_null; +use vortex_expr::lit; +use vortex_expr::or; +use vortex_expr::root; use vortex_expr::stats::rewrite::BoundExpressionPruneExt; use vortex_expr::stats::session::StatsSession; use vortex_layout::layouts::zoned::zone_map::ZoneMap; diff --git a/vortex-layout/examples/reverse_index/main.rs b/vortex-layout/examples/reverse_index/main.rs index a03a16d2d20..26b2e57295a 100644 --- a/vortex-layout/examples/reverse_index/main.rs +++ b/vortex-layout/examples/reverse_index/main.rs @@ -38,16 +38,12 @@ use vortex_array::dtype::FieldNames; use vortex_array::dtype::FieldPath; use vortex_array::dtype::Nullability::NonNullable; use vortex_array::dtype::StructFields; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::col; -use vortex_array::expr::eq; -use vortex_array::expr::lit; use vortex_array::memory::BufferAllocatorRef; use vortex_array::memory::MemorySessionExt; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::literal::Literal; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::literal::Literal; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::search_sorted::SearchSorted; use vortex_array::search_sorted::SearchSortedArray; use vortex_array::search_sorted::SearchSortedSide; @@ -63,6 +59,10 @@ use vortex_edition::EditionSessionExt; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; +use vortex_expr::BoundExpression; +use vortex_expr::col; +use vortex_expr::eq; +use vortex_expr::lit; use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder; diff --git a/vortex-layout/examples/reverse_index/pruning_tests.rs b/vortex-layout/examples/reverse_index/pruning_tests.rs index cf09ed781aa..07e1f840ef0 100644 --- a/vortex-layout/examples/reverse_index/pruning_tests.rs +++ b/vortex-layout/examples/reverse_index/pruning_tests.rs @@ -23,11 +23,6 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::FieldPath; -use vortex_array::expr::col; -use vortex_array::expr::eq; -use vortex_array::expr::lit; -use vortex_array::expr::root; -use vortex_array::expr::select; use vortex_array::stream::ArrayStreamExt; use vortex_arrow::ArrowSessionExt; use vortex_edition::Edition; @@ -36,6 +31,11 @@ use vortex_edition::EditionId; use vortex_edition::EditionMember; use vortex_edition::EditionSession; use vortex_edition::EditionSessionExt; +use vortex_expr::col; +use vortex_expr::eq; +use vortex_expr::lit; +use vortex_expr::root; +use vortex_expr::select; use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder; diff --git a/vortex-layout/examples/reverse_index/tests.rs b/vortex-layout/examples/reverse_index/tests.rs index 041973b782d..0560697f9eb 100644 --- a/vortex-layout/examples/reverse_index/tests.rs +++ b/vortex-layout/examples/reverse_index/tests.rs @@ -22,10 +22,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::expr::col; -use vortex_array::expr::eq; -use vortex_array::expr::lit; -use vortex_array::expr::root; use vortex_array::stream::ArrayStreamExt; use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; @@ -37,6 +33,10 @@ use vortex_edition::EditionSession; use vortex_edition::EditionSessionExt; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::col; +use vortex_expr::eq; +use vortex_expr::lit; +use vortex_expr::root; use vortex_file::OpenOptionsSessionExt; use vortex_file::VortexFile; use vortex_file::WriteOptionsSessionExt; diff --git a/vortex-layout/src/layouts/chunked/reader.rs b/vortex-layout/src/layouts/chunked/reader.rs index c658a83bb46..96a639886d6 100644 --- a/vortex-layout/src/layouts/chunked/reader.rs +++ b/vortex-layout/src/layouts/chunked/reader.rs @@ -20,11 +20,11 @@ use vortex_array::MaskFuture; use vortex_array::arrays::ChunkedArray; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; -use vortex_array::expr::BoundExpression; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; +use vortex_expr::BoundExpression; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -453,8 +453,8 @@ mod test { use vortex_array::dtype::FieldMask; use vortex_array::dtype::Nullability::NonNullable; use vortex_array::dtype::PType; - use vortex_array::expr::root; use vortex_buffer::buffer; + use vortex_expr::root; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; use vortex_mask::Mask; diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 0beab32f938..748b24f90b2 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -19,18 +19,19 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::dtype::Nullability; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::ExactBoundExpr; -use vortex_array::expr::bound::pack as bound_pack; -use vortex_array::expr::root; use vortex_array::optimizer::ArrayOptimizer; -use vortex_array::scalar_fn::is_negative_cost; use vortex_error::VortexError; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::ArrayExprExt; +use vortex_expr::BoundExpression; +use vortex_expr::ExactBoundExpr; +use vortex_expr::bound::pack as bound_pack; use vortex_expr::direct_bound_annotations; use vortex_expr::label_bound_tree; +use vortex_expr::root; use vortex_expr::transform::partition_bound_annotations; +use vortex_functions::is_negative_cost; use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_utils::aliases::dash_map::DashMap; @@ -374,21 +375,22 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; - use vortex_array::expr::BoundExpression; - use vortex_array::expr::Expression; - use vortex_array::expr::bound::pack as bound_pack; - use vortex_array::expr::byte_length; - use vortex_array::expr::cast; - use vortex_array::expr::eq; - use vortex_array::expr::get_item; - use vortex_array::expr::is_not_null; - use vortex_array::expr::like; - use vortex_array::expr::lit; - use vortex_array::expr::pack; - use vortex_array::expr::root; use vortex_array::validity::Validity; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_error::VortexResult; + use vortex_expr::ArrayExprExt; + use vortex_expr::BoundExpression; + use vortex_expr::Expression; + use vortex_expr::bound::pack as bound_pack; + use vortex_expr::cast; + use vortex_expr::eq; + use vortex_expr::get_item; + use vortex_expr::is_not_null; + use vortex_expr::lit; + use vortex_expr::pack; + use vortex_expr::root; + use vortex_functions::expr::byte_length; + use vortex_functions::expr::like; use vortex_io::runtime::Handle; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSession; diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index 38a9bf0218d..9d0b03f1e38 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -16,10 +16,11 @@ use vortex_array::VortexSessionExecute; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; -use vortex_array::expr::BoundExpression; use vortex_array::serde::SerializedArray; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::ArrayExprExt; +use vortex_expr::BoundExpression; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -245,12 +246,12 @@ mod test { use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; - use vortex_array::expr::gt; - use vortex_array::expr::lit; - use vortex_array::expr::root; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::gt; + use vortex_expr::lit; + use vortex_expr::root; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index ac8bd44045a..188ba8989e9 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -189,7 +189,6 @@ mod tests { use vortex_array::dtype::FieldName; use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability; - use vortex_array::expr::root; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; use vortex_array::expr::stats::StatsProviderExt; @@ -197,6 +196,7 @@ mod tests { use vortex_buffer::BitBufferMut; use vortex_buffer::buffer; use vortex_error::VortexExpect; + use vortex_expr::root; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; use vortex_mask::AllOr; diff --git a/vortex-layout/src/layouts/indexed/index.rs b/vortex-layout/src/layouts/indexed/index.rs index ac1981b84f2..1a24a414e93 100644 --- a/vortex-layout/src/layouts/indexed/index.rs +++ b/vortex-layout/src/layouts/indexed/index.rs @@ -13,9 +13,9 @@ use roaring::RoaringBitmap; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::dtype::DType; -use vortex_array::expr::BoundExpression; use vortex_buffer::BitBufferMut; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_session::registry::Id; diff --git a/vortex-layout/src/layouts/indexed/reader.rs b/vortex-layout/src/layouts/indexed/reader.rs index f2a28b80e77..b131b214936 100644 --- a/vortex-layout/src/layouts/indexed/reader.rs +++ b/vortex-layout/src/layouts/indexed/reader.rs @@ -21,11 +21,11 @@ use vortex_array::MaskFuture; use vortex_array::VortexSessionExecute; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; -use vortex_array::expr::BoundExpression; use vortex_buffer::BitBufferMut; use vortex_error::SharedVortexResult; use vortex_error::VortexError; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_utils::aliases::dash_map::DashMap; diff --git a/vortex-layout/src/layouts/indexed/tests.rs b/vortex-layout/src/layouts/indexed/tests.rs index f2f28b54509..9597f054d16 100644 --- a/vortex-layout/src/layouts/indexed/tests.rs +++ b/vortex-layout/src/layouts/indexed/tests.rs @@ -26,16 +26,16 @@ use vortex_array::arrays::VarBinViewArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::eq; -use vortex_array::expr::like; -use vortex_array::expr::lit; -use vortex_array::expr::root; use vortex_array::stream::ArrayStreamExt; use vortex_array::test_harness::check_metadata; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_expr::BoundExpression; +use vortex_expr::eq; +use vortex_expr::lit; +use vortex_expr::root; +use vortex_functions::expr::like; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -1088,17 +1088,17 @@ mod exact_value { use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability::NonNullable; use vortex_array::dtype::StructFields; - use vortex_array::expr::BoundExpression; - use vortex_array::expr::col; - use vortex_array::expr::eq; - use vortex_array::expr::lit; - use vortex_array::scalar_fn::fns::binary::Binary; - use vortex_array::scalar_fn::fns::literal::Literal; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::binary::Binary; + use vortex_array::scalar_fn::builtin::literal::Literal; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; + use vortex_expr::BoundExpression; + use vortex_expr::col; + use vortex_expr::eq; + use vortex_expr::lit; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -1485,11 +1485,11 @@ mod fixed_superset { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::BoundExpression; - use vortex_array::expr::eq; - use vortex_array::expr::lit; - use vortex_array::expr::root; use vortex_error::VortexResult; + use vortex_expr::BoundExpression; + use vortex_expr::eq; + use vortex_expr::lit; + use vortex_expr::root; use vortex_session::VortexSession; use crate::layouts::indexed::IndexBuilder; diff --git a/vortex-layout/src/layouts/list/expr.rs b/vortex-layout/src/layouts/list/expr.rs index 8c10e400ad8..e42f562b3d7 100644 --- a/vortex-layout/src/layouts/list/expr.rs +++ b/vortex-layout/src/layouts/list/expr.rs @@ -3,12 +3,12 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::bound::not; -use vortex_array::scalar_fn::fns::is_not_null::IsNotNull; -use vortex_array::scalar_fn::fns::is_null::IsNull; -use vortex_array::scalar_fn::fns::list_length::ListLength; +use vortex_array::scalar_fn::builtin::is_not_null::IsNotNull; +use vortex_array::scalar_fn::builtin::is_null::IsNull; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; +use vortex_expr::bound::not; +use vortex_functions::list_length::ListLength; /// The minimal set of list children an expression needs for evaluation. /// diff --git a/vortex-layout/src/layouts/list/reader.rs b/vortex-layout/src/layouts/list/reader.rs index fafa48346bd..8ba377cc9f9 100644 --- a/vortex-layout/src/layouts/list/reader.rs +++ b/vortex-layout/src/layouts/list/reader.rs @@ -19,12 +19,13 @@ use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::root; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_array::validity::Validity; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::ArrayExprExt; +use vortex_expr::BoundExpression; +use vortex_expr::root; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -626,17 +627,16 @@ mod tests { use vortex_array::arrays::ListArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; - use vortex_array::expr::Expression; - use vortex_array::expr::cast; - use vortex_array::expr::gt; - use vortex_array::expr::is_not_null; - use vortex_array::expr::is_null; - use vortex_array::expr::list_length; - use vortex_array::expr::lit; use vortex_buffer::buffer; + use vortex_expr::Expression; + use vortex_expr::cast; + use vortex_expr::gt; + use vortex_expr::is_not_null; + use vortex_expr::is_null; + use vortex_expr::lit; + use vortex_functions::expr::list_length; use vortex_io::session::RuntimeSession; use vortex_io::session::RuntimeSessionExt; - use vortex_session::VortexSession; use super::*; use crate::LayoutRef; diff --git a/vortex-layout/src/layouts/list/writer.rs b/vortex-layout/src/layouts/list/writer.rs index 8e3a25cf407..d599b2e6bf3 100644 --- a/vortex-layout/src/layouts/list/writer.rs +++ b/vortex-layout/src/layouts/list/writer.rs @@ -22,7 +22,7 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::matcher::Matcher; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; diff --git a/vortex-layout/src/layouts/partitioned.rs b/vortex-layout/src/layouts/partitioned.rs index ee1e675b149..9f439b0e9ac 100644 --- a/vortex-layout/src/layouts/partitioned.rs +++ b/vortex-layout/src/layouts/partitioned.rs @@ -14,10 +14,11 @@ use vortex_array::arrays::StructArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::expr::BoundExpression; use vortex_array::validity::Validity; use vortex_error::VortexError; use vortex_error::VortexResult; +use vortex_expr::ArrayExprExt; +use vortex_expr::BoundExpression; use vortex_expr::transform::BoundPartitionedExpr; use vortex_mask::Mask; use vortex_session::VortexSession; diff --git a/vortex-layout/src/layouts/row_idx/expr.rs b/vortex-layout/src/layouts/row_idx/expr.rs index 659eb1f69d7..f81b7533dec 100644 --- a/vortex-layout/src/layouts/row_idx/expr.rs +++ b/vortex-layout/src/layouts/row_idx/expr.rs @@ -7,17 +7,17 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::expr::Expression; -use vortex_array::expr::display::ExprDisplay; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::ScalarFnVTableExt; +use vortex_array::scalar_fn::display::ExprDisplay; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_expr::Expression; +use vortex_expr::ScalarFnExprExt; use vortex_session::registry::CachedId; #[derive(Clone)] @@ -74,9 +74,9 @@ pub fn row_idx() -> Expression { #[cfg(test)] mod tests { - use vortex_array::expr::Lambda; - use vortex_array::expr::checked_add; - use vortex_array::expr::var; + use vortex_expr::Lambda; + use vortex_expr::checked_add; + use vortex_expr::var; use super::row_idx; diff --git a/vortex-layout/src/layouts/row_idx/mod.rs b/vortex-layout/src/layouts/row_idx/mod.rs index f9641cbd6cb..7e90edb2ee4 100644 --- a/vortex-layout/src/layouts/row_idx/mod.rs +++ b/vortex-layout/src/layouts/row_idx/mod.rs @@ -24,16 +24,17 @@ use vortex_array::dtype::FieldMask; use vortex_array::dtype::FieldName; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::ExactBoundExpr; -use vortex_array::expr::traversal::NodeExt; -use vortex_array::expr::traversal::Transformed; -use vortex_array::expr::traversal::TraversalOrder; use vortex_array::scalar::PValue; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::ArrayExprExt; +use vortex_expr::BoundExpression; +use vortex_expr::ExactBoundExpr; use vortex_expr::transform::BoundPartitionedExpr; use vortex_expr::transform::partition_bound; +use vortex_expr::traversal::NodeExt; +use vortex_expr::traversal::Transformed; +use vortex_expr::traversal::TraversalOrder; use vortex_mask::Mask; use vortex_sequence::Sequence; use vortex_sequence::SequenceArray; @@ -358,12 +359,12 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::eq; - use vortex_array::expr::gt; - use vortex_array::expr::lit; - use vortex_array::expr::or; - use vortex_array::expr::root; use vortex_buffer::buffer; + use vortex_expr::eq; + use vortex_expr::gt; + use vortex_expr::lit; + use vortex_expr::or; + use vortex_expr::root; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; use vortex_mask::Mask; diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index 9d9630473c8..5b5dfd2805d 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -19,32 +19,32 @@ use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability; use vortex_array::dtype::RESERVED_VALIDITY_FIELD_NAME; use vortex_array::dtype::StructFields; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::ExactBoundExpr; -use vortex_array::expr::bound::get_item; -use vortex_array::expr::bound::is_not_null; -use vortex_array::expr::bound::mask; -use vortex_array::expr::bound::not; -use vortex_array::expr::bound::pack; -use vortex_array::expr::traversal::NodeExt; -use vortex_array::expr::traversal::Transformed; -use vortex_array::expr::traversal::TraversalOrder; -use vortex_array::scalar_fn::fns::get_item::GetItem; -use vortex_array::scalar_fn::fns::is_not_null::IsNotNull; -use vortex_array::scalar_fn::fns::is_null::IsNull; -use vortex_array::scalar_fn::fns::mask::Mask as MaskFn; -use vortex_array::scalar_fn::fns::pack::Pack; -use vortex_array::scalar_fn::fns::select::FieldSelection; -use vortex_array::scalar_fn::fns::select::Select; +use vortex_array::scalar_fn::builtin::get_item::GetItem; +use vortex_array::scalar_fn::builtin::is_not_null::IsNotNull; +use vortex_array::scalar_fn::builtin::is_null::IsNull; +use vortex_array::scalar_fn::builtin::mask::Mask as MaskFn; +use vortex_array::scalar_fn::builtin::pack::Pack; +use vortex_array::scalar_fn::builtin::select::FieldSelection; +use vortex_array::scalar_fn::builtin::select::Select; use vortex_array::validity::Validity; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_expr::BoundExpression; +use vortex_expr::ExactBoundExpr; use vortex_expr::analysis::AnnotationFn; +use vortex_expr::bound::get_item; +use vortex_expr::bound::is_not_null; +use vortex_expr::bound::mask; +use vortex_expr::bound::not; +use vortex_expr::bound::pack; use vortex_expr::make_bound_free_field_annotator; use vortex_expr::transform::BoundPartitionedExpr; use vortex_expr::transform::partition_bound; +use vortex_expr::traversal::NodeExt; +use vortex_expr::traversal::Transformed; +use vortex_expr::traversal::TraversalOrder; use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_utils::aliases::dash_map::DashMap; @@ -719,24 +719,24 @@ mod tests { use vortex_array::dtype::PType; use vortex_array::dtype::RESERVED_VALIDITY_FIELD_NAME; use vortex_array::dtype::StructFields; - use vortex_array::expr::Expression; - use vortex_array::expr::and; - use vortex_array::expr::col; - use vortex_array::expr::eq; - use vortex_array::expr::get_item; - use vortex_array::expr::gt; - use vortex_array::expr::is_not_null; - use vortex_array::expr::is_null; - use vortex_array::expr::lit; - use vortex_array::expr::or; - use vortex_array::expr::pack; - use vortex_array::expr::root; - use vortex_array::expr::select; - use vortex_array::expr::select_exclude; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::Expression; + use vortex_expr::and; + use vortex_expr::col; + use vortex_expr::eq; + use vortex_expr::get_item; + use vortex_expr::gt; + use vortex_expr::is_not_null; + use vortex_expr::is_null; + use vortex_expr::lit; + use vortex_expr::or; + use vortex_expr::pack; + use vortex_expr::root; + use vortex_expr::select; + use vortex_expr::select_exclude; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; use vortex_mask::Mask; diff --git a/vortex-layout/src/layouts/zoned/aggregates/bloom_filter/scalar_fn.rs b/vortex-layout/src/layouts/zoned/aggregates/bloom_filter/scalar_fn.rs index fc8f91627cb..d2e580f6908 100644 --- a/vortex-layout/src/layouts/zoned/aggregates/bloom_filter/scalar_fn.rs +++ b/vortex-layout/src/layouts/zoned/aggregates/bloom_filter/scalar_fn.rs @@ -23,18 +23,15 @@ use vortex_array::arrays::varbinview::BinaryView; use vortex_array::arrays::varbinview::VarBinViewArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::bound::not; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::ScalarFnVTableExt; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::literal::Literal; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::literal::Literal; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_buffer::BitBufferMut; use vortex_buffer::Buffer; use vortex_error::VortexExpect; @@ -42,6 +39,9 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; +use vortex_expr::BoundExpression; +use vortex_expr::ScalarFnExprExt; +use vortex_expr::bound::not; use vortex_expr::stats::expr::bound::stat as bound_stat; use vortex_expr::stats::rewrite::StatsRewriteRule; use vortex_session::VortexSession; @@ -296,9 +296,6 @@ mod tests { use vortex_array::dtype::DecimalDType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::bound::eq; - use vortex_array::expr::bound::lit; - use vortex_array::expr::bound::root; use vortex_array::scalar::DecimalValue; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::ScalarFnVTable; @@ -307,6 +304,9 @@ mod tests { use vortex_array::scalar_fn::session::ScalarFnSessionExt; use vortex_array::validity::Validity; use vortex_error::VortexResult; + use vortex_expr::bound::eq; + use vortex_expr::bound::lit; + use vortex_expr::bound::root; use vortex_expr::stats::StatsSessionExt; use vortex_expr::stats::rewrite::BoundExpressionPruneExt; use vortex_expr::stats::rewrite::StatsRewriteRule; diff --git a/vortex-layout/src/layouts/zoned/pruning.rs b/vortex-layout/src/layouts/zoned/pruning.rs index 6e39823689e..c113b247634 100644 --- a/vortex-layout/src/layouts/zoned/pruning.rs +++ b/vortex-layout/src/layouts/zoned/pruning.rs @@ -18,14 +18,14 @@ use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::AggregateFnRef; use vortex_array::arrays::StructArray; use vortex_array::dtype::DType; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::ExactBoundExpr; -use vortex_array::expr::root; -use vortex_array::scalar_fn::fns::dynamic::DynamicExprUpdates; use vortex_error::SharedVortexResult; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; +use vortex_expr::ExactBoundExpr; +use vortex_expr::root; use vortex_expr::stats::rewrite::BoundExpressionPruneExt; +use vortex_functions::dynamic::DynamicExprUpdates; use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_utils::aliases::dash_map::DashMap; diff --git a/vortex-layout/src/layouts/zoned/reader.rs b/vortex-layout/src/layouts/zoned/reader.rs index 46f1d229c46..176abc44759 100644 --- a/vortex-layout/src/layouts/zoned/reader.rs +++ b/vortex-layout/src/layouts/zoned/reader.rs @@ -12,12 +12,12 @@ use vortex_array::ArrayRef; use vortex_array::MaskFuture; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; -use vortex_array::expr::BoundExpression; use vortex_buffer::BitBufferMut; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; +use vortex_expr::BoundExpression; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -257,14 +257,14 @@ mod test { use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; - use vortex_array::expr::gt; - use vortex_array::expr::is_not_null; - use vortex_array::expr::lit; - use vortex_array::expr::root; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; + use vortex_expr::gt; + use vortex_expr::is_not_null; + use vortex_expr::lit; + use vortex_expr::root; use vortex_io::runtime::Handle; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSession; diff --git a/vortex-layout/src/layouts/zoned/schema.rs b/vortex-layout/src/layouts/zoned/schema.rs index 0e4d9db318a..ddcfb2db808 100644 --- a/vortex-layout/src/layouts/zoned/schema.rs +++ b/vortex-layout/src/layouts/zoned/schema.rs @@ -11,15 +11,15 @@ use vortex_array::aggregate_fn::session::AggregateFnSessionExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::StructFields; -use vortex_array::expr::Expression; -use vortex_array::expr::cast; -use vortex_array::expr::get_item; -use vortex_array::expr::pack; -use vortex_array::expr::root; use vortex_array::expr::stats::Stat; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_expr::Expression; +use vortex_expr::cast; +use vortex_expr::get_item; +use vortex_expr::pack; +use vortex_expr::root; use vortex_session::VortexSession; pub const MAX_IS_TRUNCATED: &str = "max_is_truncated"; diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index 2b9011e330c..ac4b0787853 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -24,23 +24,24 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::Expression; -use vortex_array::expr::eq; -use vortex_array::expr::get_item; -use vortex_array::expr::lit; -use vortex_array::expr::root; use vortex_array::expr::stats::Stat; use vortex_array::scalar_fn::EmptyOptions; -use vortex_array::scalar_fn::ScalarFnVTableExt; -use vortex_array::scalar_fn::internal::row_count::RowCount; -use vortex_array::scalar_fn::internal::row_count::contains_row_count; -use vortex_array::scalar_fn::internal::row_count::substitute_row_count; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure_eq; +use vortex_expr::ArrayExprExt; +use vortex_expr::BoundExpression; +use vortex_expr::Expression; +use vortex_expr::ScalarFnExprExt; +use vortex_expr::eq; +use vortex_expr::get_item; +use vortex_expr::lambda::row_count::RowCount; +use vortex_expr::lambda::row_count::contains_row_count; +use vortex_expr::lambda::row_count::substitute_row_count; +use vortex_expr::lit; +use vortex_expr::root; use vortex_expr::stats::bind::StatBinder; use vortex_expr::stats::bind::bind_stats; use vortex_mask::Mask; @@ -423,30 +424,30 @@ mod tests { use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::BoundExpression; - use vortex_array::expr::Expression; - use vortex_array::expr::cast; - use vortex_array::expr::dynamic; - use vortex_array::expr::gt; - use vortex_array::expr::gt_eq; - use vortex_array::expr::is_not_null; - use vortex_array::expr::is_null; - use vortex_array::expr::lit; - use vortex_array::expr::lt; - use vortex_array::expr::not_eq; - use vortex_array::expr::root; use vortex_array::expr::stats::Stat; use vortex_array::scalar::ScalarValue; - use vortex_array::scalar_fn::fns::operators::CompareOperator; + use vortex_array::scalar_fn::builtin::operators::CompareOperator; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::BoundExpression; + use vortex_expr::Expression; + use vortex_expr::cast; + use vortex_expr::gt; + use vortex_expr::gt_eq; + use vortex_expr::is_not_null; + use vortex_expr::is_null; + use vortex_expr::lit; + use vortex_expr::lt; + use vortex_expr::not_eq; + use vortex_expr::root; use vortex_expr::stats::all_nan; use vortex_expr::stats::all_non_nan; use vortex_expr::stats::all_non_null; use vortex_expr::stats::all_null; use vortex_expr::stats::rewrite::BoundExpressionPruneExt; + use vortex_functions::expr::dynamic; use vortex_mask::Mask; use crate::layouts::zoned::zone_map::ZoneMap; diff --git a/vortex-layout/src/plan/physical/operators/eval.rs b/vortex-layout/src/plan/physical/operators/eval.rs index 9ef61e039a5..4d498940c1f 100644 --- a/vortex-layout/src/plan/physical/operators/eval.rs +++ b/vortex-layout/src/plan/physical/operators/eval.rs @@ -3,8 +3,9 @@ //! Applying an expression to each batch. -use vortex_array::expr::BoundExpression; use vortex_error::VortexResult; +use vortex_expr::ArrayExprExt; +use vortex_expr::BoundExpression; use crate::plan::physical::operator::Cx; use crate::plan::physical::operator::Input; diff --git a/vortex-layout/src/plan/physical/pipeline/tests.rs b/vortex-layout/src/plan/physical/pipeline/tests.rs index da9af0ab96c..b0cdfae9b4b 100644 --- a/vortex-layout/src/plan/physical/pipeline/tests.rs +++ b/vortex-layout/src/plan/physical/pipeline/tests.rs @@ -20,10 +20,6 @@ use vortex_array::arrays::VarBinViewArray; use vortex_array::assert_arrays_eq; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::StructFields; -use vortex_array::expr::get_item; -use vortex_array::expr::gt; -use vortex_array::expr::lit; -use vortex_array::expr::root; use vortex_array::serde::SerializeOptions; use vortex_buffer::Alignment; use vortex_buffer::Buffer; @@ -31,6 +27,10 @@ use vortex_buffer::ByteBufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; +use vortex_expr::get_item; +use vortex_expr::gt; +use vortex_expr::lit; +use vortex_expr::root; use vortex_mask::Mask; use vortex_session::registry::ReadContext; diff --git a/vortex-layout/src/plan/physical/plans/eval.rs b/vortex-layout/src/plan/physical/plans/eval.rs index b6a2f1ff620..1be60d15861 100644 --- a/vortex-layout/src/plan/physical/plans/eval.rs +++ b/vortex-layout/src/plan/physical/plans/eval.rs @@ -8,14 +8,14 @@ use std::ops::Range; use vortex_array::EmptyMetadata; use vortex_array::dtype::Nullability; use vortex_array::dtype::StructFields; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::traversal::NodeExt; -use vortex_array::expr::traversal::Transformed; -use vortex_array::expr::traversal::TraversalOrder; -use vortex_array::scalar_fn::fns::pack::Pack; +use vortex_array::scalar_fn::builtin::pack::Pack; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_expr::BoundExpression; +use vortex_expr::traversal::NodeExt; +use vortex_expr::traversal::Transformed; +use vortex_expr::traversal::TraversalOrder; use vortex_mask::Mask; use vortex_session::registry::CachedId; diff --git a/vortex-layout/src/plan/physical/plans/pack.rs b/vortex-layout/src/plan/physical/plans/pack.rs index b37f50934e8..b01505d6399 100644 --- a/vortex-layout/src/plan/physical/plans/pack.rs +++ b/vortex-layout/src/plan/physical/plans/pack.rs @@ -10,24 +10,24 @@ use vortex_array::dtype::FieldName; use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability; use vortex_array::dtype::StructFields; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::ExactBoundExpr; -use vortex_array::expr::traversal::NodeExt; -use vortex_array::expr::traversal::Transformed; -use vortex_array::expr::traversal::TraversalOrder; use vortex_array::scalar_fn::ScalarFnVTableExt; -use vortex_array::scalar_fn::fns::get_item::GetItem; -use vortex_array::scalar_fn::fns::pack::Pack as PackFn; -use vortex_array::scalar_fn::fns::pack::PackOptions; -use vortex_array::scalar_fn::fns::select::Select; +use vortex_array::scalar_fn::builtin::get_item::GetItem; +use vortex_array::scalar_fn::builtin::pack::Pack as PackFn; +use vortex_array::scalar_fn::builtin::pack::PackOptions; +use vortex_array::scalar_fn::builtin::select::Select; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_expr::BoundExpression; +use vortex_expr::ExactBoundExpr; use vortex_expr::descendent_bound_annotations; use vortex_expr::make_bound_free_field_annotator; use vortex_expr::transform::partition_bound; +use vortex_expr::traversal::NodeExt; +use vortex_expr::traversal::Transformed; +use vortex_expr::traversal::TraversalOrder; use vortex_mask::Mask; use vortex_session::registry::CachedId; diff --git a/vortex-layout/src/plan/physical/plans/row_idx.rs b/vortex-layout/src/plan/physical/plans/row_idx.rs index 5c45d2cae4f..7e81f9a67dd 100644 --- a/vortex-layout/src/plan/physical/plans/row_idx.rs +++ b/vortex-layout/src/plan/physical/plans/row_idx.rs @@ -12,16 +12,16 @@ use vortex_array::dtype::FieldName; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::traversal::NodeExt; -use vortex_array::expr::traversal::Transformed; -use vortex_array::expr::traversal::TraversalOrder; -use vortex_array::scalar_fn::fns::pack::Pack as PackFn; +use vortex_array::scalar_fn::builtin::pack::Pack as PackFn; use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; +use vortex_expr::BoundExpression; use vortex_expr::transform::partition_bound; +use vortex_expr::traversal::NodeExt; +use vortex_expr::traversal::Transformed; +use vortex_expr::traversal::TraversalOrder; use vortex_mask::Mask; use vortex_session::registry::CachedId; diff --git a/vortex-layout/src/plan/physical/plans/take.rs b/vortex-layout/src/plan/physical/plans/take.rs index b2a1703ff8a..2d8fe291bb2 100644 --- a/vortex-layout/src/plan/physical/plans/take.rs +++ b/vortex-layout/src/plan/physical/plans/take.rs @@ -8,13 +8,13 @@ use vortex_array::Canonical; use vortex_array::EmptyMetadata; use vortex_array::IntoArray; use vortex_array::dtype::DType; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::ExactBoundExpr; -use vortex_array::scalar_fn::fns::dynamic::DynamicComparison; -use vortex_array::scalar_fn::is_negative_cost; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; use vortex_expr::BoundLabels; +use vortex_expr::ExactBoundExpr; use vortex_expr::label_bound_tree; +use vortex_functions::dynamic::DynamicComparison; +use vortex_functions::is_negative_cost; use vortex_mask::Mask; use vortex_session::registry::CachedId; diff --git a/vortex-layout/src/plan/physical/plans/zoned.rs b/vortex-layout/src/plan/physical/plans/zoned.rs index d1a17702ebf..e7a300235fc 100644 --- a/vortex-layout/src/plan/physical/plans/zoned.rs +++ b/vortex-layout/src/plan/physical/plans/zoned.rs @@ -16,19 +16,19 @@ use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::StructArray; use vortex_array::dtype::DType; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::traversal::NodeExt; -use vortex_array::expr::traversal::Transformed; -use vortex_array::expr::traversal::TraversalOrder; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::dynamic::DynamicExprUpdates; -use vortex_array::scalar_fn::fns::stat::StatFn; use vortex_array::validity::Validity; use vortex_buffer::BitBufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_expr::BoundExpression; +use vortex_expr::stats::stat_fn::StatFn; +use vortex_expr::traversal::NodeExt; +use vortex_expr::traversal::Transformed; +use vortex_expr::traversal::TraversalOrder; +use vortex_functions::dynamic::DynamicExprUpdates; use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_session::registry::CachedId; diff --git a/vortex-layout/src/plan/physical/tests.rs b/vortex-layout/src/plan/physical/tests.rs index 47bad2dd94f..d4d526aa15a 100644 --- a/vortex-layout/src/plan/physical/tests.rs +++ b/vortex-layout/src/plan/physical/tests.rs @@ -18,26 +18,27 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; -use vortex_array::expr::Expression; -use vortex_array::expr::and; -use vortex_array::expr::bound::and as bound_and; -use vortex_array::expr::byte_length; -use vortex_array::expr::cast; -use vortex_array::expr::checked_add; -use vortex_array::expr::dynamic; -use vortex_array::expr::get_item; -use vortex_array::expr::gt; -use vortex_array::expr::is_null; -use vortex_array::expr::lit; -use vortex_array::expr::pack; -use vortex_array::expr::root; use vortex_array::expr::stats::Stat; use vortex_array::scalar::ScalarValue; -use vortex_array::scalar_fn::fns::operators::CompareOperator; +use vortex_array::scalar_fn::builtin::operators::CompareOperator; use vortex_array::stats::as_stat_bitset_bytes; use vortex_error::VortexResult; use vortex_error::vortex_err; +use vortex_expr::ArrayExprExt; +use vortex_expr::Expression; +use vortex_expr::and; +use vortex_expr::bound::and as bound_and; +use vortex_expr::cast; +use vortex_expr::checked_add; +use vortex_expr::get_item; +use vortex_expr::gt; +use vortex_expr::is_null; +use vortex_expr::lit; +use vortex_expr::pack; +use vortex_expr::root; use vortex_expr::stats::rewrite::BoundExpressionPruneExt; +use vortex_functions::expr::byte_length; +use vortex_functions::expr::dynamic; use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_session::registry::CachedId; diff --git a/vortex-layout/src/reader.rs b/vortex-layout/src/reader.rs index 91dda0e23c5..86ffdc6b3af 100644 --- a/vortex-layout/src/reader.rs +++ b/vortex-layout/src/reader.rs @@ -14,9 +14,9 @@ use vortex_array::MaskFuture; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; -use vortex_array::expr::BoundExpression; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_expr::BoundExpression; use vortex_mask::Mask; use vortex_session::VortexSession; diff --git a/vortex-layout/src/scan/execution/filter.rs b/vortex-layout/src/scan/execution/filter.rs index f89acf597c0..25fe0db00d1 100644 --- a/vortex-layout/src/scan/execution/filter.rs +++ b/vortex-layout/src/scan/execution/filter.rs @@ -6,12 +6,12 @@ use std::sync::Arc; use bit_vec::BitVec; use vortex_array::VortexSessionExecute; use vortex_array::builtins::ArrayBuiltins; -use vortex_array::expr::BoundExpression; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; +use vortex_expr::BoundExpression; use vortex_expr::stats::rewrite::BoundExpressionPruneExt; -use vortex_expr::transform::find_between; +use vortex_functions::between::find_between; use vortex_io::request::IoConsumer; use vortex_io::request::IoIntent; use vortex_io::request::IoRequest; diff --git a/vortex-layout/src/scan/filter.rs b/vortex-layout/src/scan/filter.rs index 2cb36068913..4e6313d194d 100644 --- a/vortex-layout/src/scan/filter.rs +++ b/vortex-layout/src/scan/filter.rs @@ -7,13 +7,13 @@ use bit_vec::BitVec; use itertools::Itertools; use parking_lot::RwLock; use sketches_ddsketch::DDSketch; -use vortex_array::expr::BoundExpression; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::dynamic::DynamicExprUpdates; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexExpect; use vortex_error::vortex_err; use vortex_error::vortex_panic; +use vortex_expr::BoundExpression; +use vortex_functions::dynamic::DynamicExprUpdates; /// The selectivity histogram quantile to use for reordering conjuncts. Where 0 == no rows match. const DEFAULT_SELECTIVITY_QUANTILE: f64 = 0.1; @@ -161,11 +161,11 @@ impl FilterExpr { mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; - use vortex_array::expr::and; - use vortex_array::expr::lit; - use vortex_array::expr::not; - use vortex_array::expr::root; use vortex_error::VortexResult; + use vortex_expr::and; + use vortex_expr::lit; + use vortex_expr::not; + use vortex_expr::root; use super::FilterExpr; diff --git a/vortex-layout/src/scan/layout.rs b/vortex-layout/src/scan/layout.rs index ae670958ff4..095b847d71a 100644 --- a/vortex-layout/src/scan/layout.rs +++ b/vortex-layout/src/scan/layout.rs @@ -18,7 +18,6 @@ use vortex_array::arrays::ConstantArray; use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; use vortex_array::dtype::Nullability; -use vortex_array::expr::BoundExpression; use vortex_array::expr::stats::Precision; use vortex_array::scalar::Scalar; use vortex_array::stats::StatsSet; @@ -27,6 +26,7 @@ use vortex_array::stream::ArrayStreamExt; use vortex_array::stream::SendableArrayStream; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_expr::BoundExpression; use vortex_mask::Mask; use vortex_metrics::MetricsRegistry; use vortex_scan::DataSource; @@ -431,9 +431,9 @@ mod tests { use futures::StreamExt; use futures::TryStreamExt; use rstest::rstest; - use vortex_array::expr::root; use vortex_buffer::Buffer; use vortex_error::VortexResult; + use vortex_expr::root; use vortex_io::runtime::BlockingRuntime; use vortex_io::runtime::single::SingleThreadRuntime; use vortex_scan::DataSource; diff --git a/vortex-layout/src/scan/multi.rs b/vortex-layout/src/scan/multi.rs index c4a0d56fe3b..b01283a6059 100644 --- a/vortex-layout/src/scan/multi.rs +++ b/vortex-layout/src/scan/multi.rs @@ -37,7 +37,6 @@ use itertools::Itertools; use tracing::Instrument; use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; -use vortex_array::expr::BoundExpression; use vortex_array::expr::stats::Precision; use vortex_array::stats::StatsSet; use vortex_array::stream::ArrayStreamAdapter; @@ -47,6 +46,7 @@ use vortex_error::SharedVortexResult; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_expr::BoundExpression; use vortex_io::session::RuntimeSessionExt; use vortex_mask::Mask; use vortex_scan::DataSource; @@ -627,10 +627,10 @@ mod tests { use rstest::rstest; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::eq; - use vortex_array::expr::lit; - use vortex_array::expr::root; use vortex_error::VortexResult; + use vortex_expr::eq; + use vortex_expr::lit; + use vortex_expr::root; use vortex_io::runtime::BlockingRuntime; use vortex_io::runtime::single::SingleThreadRuntime; use vortex_scan::DataSource; diff --git a/vortex-layout/src/scan/repeated_scan.rs b/vortex-layout/src/scan/repeated_scan.rs index 4200d00614a..0f33eff5612 100644 --- a/vortex-layout/src/scan/repeated_scan.rs +++ b/vortex-layout/src/scan/repeated_scan.rs @@ -20,7 +20,6 @@ use itertools::Either; use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::dtype::DType; -use vortex_array::expr::BoundExpression; use vortex_array::iter::ArrayIterator; use vortex_array::iter::ArrayIteratorAdapter; use vortex_array::stream::ArrayStream; @@ -28,6 +27,7 @@ use vortex_array::stream::ArrayStreamAdapter; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_expr::BoundExpression; use vortex_io::runtime::BlockingRuntime; use vortex_io::session::RuntimeSessionExt; use vortex_scan::row_mask::RowMask; diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs index 21dc22c04ca..6f5cb8b7906 100644 --- a/vortex-layout/src/scan/scan_builder.rs +++ b/vortex-layout/src/scan/scan_builder.rs @@ -15,15 +15,15 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; -use vortex_array::expr::BoundExpression; use vortex_array::iter::ArrayIterator; use vortex_array::iter::ArrayIteratorAdapter; use vortex_array::stream::ArrayStream; use vortex_array::stream::ArrayStreamAdapter; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; use vortex_expr::analysis::referenced_field_paths; -use vortex_expr::transform::find_between; +use vortex_functions::between::find_between; use vortex_io::runtime::BlockingRuntime; use vortex_io::runtime::Task; use vortex_io::session::RuntimeSessionExt; @@ -491,23 +491,23 @@ mod test { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; - use vortex_array::expr::BoundExpression; - use vortex_array::expr::ExactBoundExpr; - use vortex_array::expr::and; - use vortex_array::expr::between; - use vortex_array::expr::col; - use vortex_array::expr::eq; - use vortex_array::expr::get_item; - use vortex_array::expr::gt_eq; - use vortex_array::expr::is_not_null; - use vortex_array::expr::lit; - use vortex_array::expr::lt; - use vortex_array::expr::root; - use vortex_array::scalar_fn::fns::between::BetweenOptions; - use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; + use vortex_expr::BoundExpression; + use vortex_expr::ExactBoundExpr; + use vortex_expr::and; + use vortex_expr::col; + use vortex_expr::eq; + use vortex_expr::get_item; + use vortex_expr::gt_eq; + use vortex_expr::is_not_null; + use vortex_expr::lit; + use vortex_expr::lt; + use vortex_expr::root; + use vortex_functions::between::BetweenOptions; + use vortex_functions::between::StrictComparison; + use vortex_functions::expr::between; use vortex_io::runtime::BlockingRuntime; use vortex_io::runtime::single::SingleThreadRuntime; use vortex_io::runtime::tokio::TokioRuntime; diff --git a/vortex-layout/src/scan/split_by.rs b/vortex-layout/src/scan/split_by.rs index b995e46222f..61ee2788559 100644 --- a/vortex-layout/src/scan/split_by.rs +++ b/vortex-layout/src/scan/split_by.rs @@ -164,9 +164,9 @@ mod test { use vortex_array::dtype::FieldPath; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::BoundExpression; use vortex_buffer::buffer; use vortex_error::vortex_bail; + use vortex_expr::BoundExpression; use vortex_io::runtime::single::block_on; use vortex_mask::Mask; diff --git a/vortex-layout/src/scan/tasks.rs b/vortex-layout/src/scan/tasks.rs index d10c1c656ec..b915f007944 100644 --- a/vortex-layout/src/scan/tasks.rs +++ b/vortex-layout/src/scan/tasks.rs @@ -12,8 +12,8 @@ use futures::FutureExt; use futures::future::BoxFuture; use vortex_array::ArrayRef; use vortex_array::MaskFuture; -use vortex_array::expr::BoundExpression; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; use vortex_mask::Mask; use vortex_scan::row_mask::RowMask; diff --git a/vortex-layout/src/scan/test.rs b/vortex-layout/src/scan/test.rs index 7d030f2c987..ea956c5f0fe 100644 --- a/vortex-layout/src/scan/test.rs +++ b/vortex-layout/src/scan/test.rs @@ -16,9 +16,9 @@ use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::expr::BoundExpression; use vortex_error::VortexResult; use vortex_error::vortex_err; +use vortex_expr::BoundExpression; use vortex_io::runtime::BlockingRuntime; use vortex_io::runtime::Handle; use vortex_io::runtime::single::SingleThreadRuntime; diff --git a/vortex-layout/src/scan/v2/cache.rs b/vortex-layout/src/scan/v2/cache.rs index 3daa0975a0e..4b02fd6e6f6 100644 --- a/vortex-layout/src/scan/v2/cache.rs +++ b/vortex-layout/src/scan/v2/cache.rs @@ -7,8 +7,8 @@ use std::sync::Arc; use parking_lot::Mutex; use rustc_hash::FxHashMap; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::ExactBoundExpr; +use vortex_expr::BoundExpression; +use vortex_expr::ExactBoundExpr; use super::io::SegmentRanges; use crate::plan::physical::PlanRef; diff --git a/vortex-layout/src/scan/v2/mod.rs b/vortex-layout/src/scan/v2/mod.rs index c6b359fa328..5ff226c1047 100644 --- a/vortex-layout/src/scan/v2/mod.rs +++ b/vortex-layout/src/scan/v2/mod.rs @@ -40,9 +40,9 @@ use futures::stream::BoxStream; use vortex_array::ArrayRef; use vortex_array::VortexSessionExecute; use vortex_array::dtype::DType; -use vortex_array::expr::BoundExpression; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; use vortex_io::request::IoService; use vortex_io::session::RuntimeSessionExt; use vortex_scan::planning::planner::WorkScope; diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index 29fbcf22d11..37775b4cc25 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -28,22 +28,22 @@ use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; -use vortex_array::expr::Expression; -use vortex_array::expr::and; -use vortex_array::expr::eq; -use vortex_array::expr::get_item; -use vortex_array::expr::gt; -use vortex_array::expr::is_null; -use vortex_array::expr::lit; -use vortex_array::expr::lt; -use vortex_array::expr::pack; -use vortex_array::expr::root; use vortex_array::validity::Validity; use vortex_arrow::ArrowSessionExt; use vortex_buffer::Alignment; use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_err; +use vortex_expr::Expression; +use vortex_expr::and; +use vortex_expr::eq; +use vortex_expr::get_item; +use vortex_expr::gt; +use vortex_expr::is_null; +use vortex_expr::lit; +use vortex_expr::lt; +use vortex_expr::pack; +use vortex_expr::root; use vortex_io::session::RuntimeSessionExt; use vortex_scan::strict_sorted_buffer::StrictSortedBuffer; use vortex_session::VortexSession; diff --git a/vortex-layout/src/test.rs b/vortex-layout/src/test.rs index 29eeb8c3eba..c2151edb5a1 100644 --- a/vortex-layout/src/test.rs +++ b/vortex-layout/src/test.rs @@ -12,9 +12,11 @@ use crate::session::LayoutSession; pub static SESSION: LazyLock = LazyLock::new(|| new_session().with_tokio()); pub fn new_session() -> VortexSession { - VortexSession::empty() + let session = VortexSession::empty() .with::() - .with::() + .with::(); + vortex_functions::initialize(&session); + session } /// A child strategy that fails with [`FailingStrategy::MESSAGE`] without reading its stream, as a diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index 842c6c694ae..86b9a71912f 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -49,10 +49,11 @@ use vortex::buffer::ByteBuffer; use vortex::dtype::DType; use vortex::dtype::Nullability; use vortex::error::VortexResult; +use vortex::expr::ArrayExprExt; use vortex::flatbuffers::WriteFlatBufferExt; use vortex::ipc::messages::EncoderMessage; use vortex::ipc::messages::MessageEncoder; -use vortex::scalar_fn::fns::operators::Operator; +use vortex::scalar_fn::builtin::operators::Operator; use vortex_arrow::ArrowSessionExt; use vortex_python_abi::BUFFER_EXPORT_CAPSULE_NAME; use vortex_python_abi::VORTEX_BUFFER_EXPORT_VERSION; diff --git a/vortex-python/src/expr/mod.rs b/vortex-python/src/expr/mod.rs index b126ba39343..a63a291499a 100644 --- a/vortex-python/src/expr/mod.rs +++ b/vortex-python/src/expr/mod.rs @@ -15,19 +15,19 @@ use vortex::dtype::FieldNames; use vortex::dtype::Nullability; use vortex::expr; use vortex::expr::Expression; +use vortex::expr::ScalarFnExprExt; use vortex::expr::lit; use vortex::expr::proto::ExprSerializeProtoExt; +use vortex::functions::between::BetweenOptions; +use vortex::functions::between::StrictComparison; +use vortex::functions::list_contains::ListContainsOptions; +use vortex::functions::merge::DuplicateHandling; +use vortex::functions::replace_time_zone::ReplaceTimeZoneOptions; +use vortex::functions::variant_get::VariantPath; +use vortex::functions::variant_get::VariantPathElement; use vortex::proto::expr as pb; -use vortex::scalar_fn::ScalarFnVTableExt; -use vortex::scalar_fn::fns::between::BetweenOptions; -use vortex::scalar_fn::fns::between::StrictComparison; -use vortex::scalar_fn::fns::binary::Binary; -use vortex::scalar_fn::fns::list_contains::ListContainsOptions; -use vortex::scalar_fn::fns::merge::DuplicateHandling; -use vortex::scalar_fn::fns::operators::Operator; -use vortex::scalar_fn::fns::replace_time_zone::ReplaceTimeZoneOptions; -use vortex::scalar_fn::fns::variant_get::VariantPath; -use vortex::scalar_fn::fns::variant_get::VariantPathElement; +use vortex::scalar_fn::builtin::binary::Binary; +use vortex::scalar_fn::builtin::operators::Operator; use crate::dtype::PyDType; use crate::error::PyVortexResult; diff --git a/vortex-scan/Cargo.toml b/vortex-scan/Cargo.toml index d12c207b049..2e5e94b472f 100644 --- a/vortex-scan/Cargo.toml +++ b/vortex-scan/Cargo.toml @@ -18,6 +18,7 @@ all-features = true vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } +vortex-expr = { workspace = true } vortex-io = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/vortex-scan/README.md b/vortex-scan/README.md index 025772e2873..5c690af3388 100644 --- a/vortex-scan/README.md +++ b/vortex-scan/README.md @@ -37,7 +37,7 @@ The `vortex-scan` crate provides efficient scanning operations over Vortex array ```rust use vortex_scan::ScanBuilder; -use vortex_array::expr::lit; +use vortex_expr::lit; // Create a scan that reads specific columns with a filter let scan = ScanBuilder::new(layout_reader) @@ -167,7 +167,8 @@ This controls how many splits are processed concurrently. Core dependencies: -- `vortex-array`: Core array types and operations (includes expression evaluation framework) +- `vortex-array`: Core array types and operations +- `vortex-expr`: Expression trees and the analyses and rewrites built on them - `vortex-layout`: Layout reader abstraction - `futures`: Async runtime abstractions - `arrow-array` (optional): Arrow integration diff --git a/vortex-scan/src/lib.rs b/vortex-scan/src/lib.rs index a15b49d4a67..280c14fdf2b 100644 --- a/vortex-scan/src/lib.rs +++ b/vortex-scan/src/lib.rs @@ -36,13 +36,13 @@ use futures::stream::BoxStream; use selection::Selection; use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; -use vortex_array::expr::Expression; -use vortex_array::expr::root; use vortex_array::expr::stats::Precision; use vortex_array::stats::StatsSet; use vortex_array::stream::SendableArrayStream; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_expr::Expression; +use vortex_expr::root; use vortex_session::VortexSession; /// A sendable stream of partitions. diff --git a/vortex-spatial/Cargo.toml b/vortex-spatial/Cargo.toml index b48831cf879..bd604217afd 100644 --- a/vortex-spatial/Cargo.toml +++ b/vortex-spatial/Cargo.toml @@ -26,6 +26,7 @@ vortex-buffer = { workspace = true } vortex-edition = { workspace = true } vortex-error = { workspace = true } vortex-expr = { workspace = true } +vortex-functions = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } wkb = { workspace = true } diff --git a/vortex-spatial/src/prune/distance.rs b/vortex-spatial/src/prune/distance.rs index 7417a23d042..30f89f85eaa 100644 --- a/vortex-spatial/src/prune/distance.rs +++ b/vortex-spatial/src/prune/distance.rs @@ -4,13 +4,13 @@ //! `ST_Distance(geom, const) radius` pruning. use geo::Rect as SpatialRect; -use vortex_array::expr::BoundExpression; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; -use vortex_array::scalar_fn::fns::binary::Binary; -use vortex_array::scalar_fn::fns::literal::Literal; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::binary::Binary; +use vortex_array::scalar_fn::builtin::literal::Literal; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; use vortex_expr::stats::rewrite::StatsRewriteRule; use vortex_session::VortexSession; @@ -130,17 +130,17 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::BoundExpression; - use vortex_array::expr::gt_eq; - use vortex_array::expr::lit; - use vortex_array::expr::lt_eq; - use vortex_array::expr::root; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::EmptyOptions; - use vortex_array::scalar_fn::ScalarFnVTableExt; - use vortex_array::scalar_fn::fns::binary::Binary; - use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::scalar_fn::builtin::binary::Binary; + use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; + use vortex_expr::BoundExpression; + use vortex_expr::ScalarFnExprExt; + use vortex_expr::gt_eq; + use vortex_expr::lit; + use vortex_expr::lt_eq; + use vortex_expr::root; use vortex_expr::stats::rewrite::BoundExpressionPruneExt; use vortex_expr::stats::rewrite::StatsRewriteRule; diff --git a/vortex-spatial/src/prune/intersects.rs b/vortex-spatial/src/prune/intersects.rs index 71c1236255f..11799918da1 100644 --- a/vortex-spatial/src/prune/intersects.rs +++ b/vortex-spatial/src/prune/intersects.rs @@ -3,10 +3,10 @@ //! `ST_Intersects(geom, const)` pruning. -use vortex_array::expr::BoundExpression; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; use vortex_expr::stats::rewrite::StatsRewriteRule; use vortex_session::VortexSession; @@ -56,13 +56,13 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::expr::BoundExpression; - use vortex_array::expr::lit; - use vortex_array::expr::root; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::EmptyOptions; - use vortex_array::scalar_fn::ScalarFnVTableExt; use vortex_error::VortexResult; + use vortex_expr::BoundExpression; + use vortex_expr::ScalarFnExprExt; + use vortex_expr::lit; + use vortex_expr::root; use vortex_expr::stats::rewrite::BoundExpressionPruneExt; use vortex_expr::stats::rewrite::StatsRewriteRule; diff --git a/vortex-spatial/src/prune/mod.rs b/vortex-spatial/src/prune/mod.rs index dd1a6951503..86b5d2127be 100644 --- a/vortex-spatial/src/prune/mod.rs +++ b/vortex-spatial/src/prune/mod.rs @@ -22,22 +22,22 @@ pub use intersects::SpatialIntersectsPrune; use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::AggregateFnVTableExt; use vortex_array::aggregate_fn::EmptyOptions; -use vortex_array::expr::BoundExpression; -use vortex_array::expr::bound::binary; -use vortex_array::expr::bound::case_when; -use vortex_array::expr::bound::checked_add; -use vortex_array::expr::bound::ext_storage; -use vortex_array::expr::bound::get_item; -use vortex_array::expr::bound::gt; -use vortex_array::expr::bound::gt_eq; -use vortex_array::expr::bound::lit; -use vortex_array::expr::bound::lt; -use vortex_array::expr::bound::lt_eq; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::literal::Literal; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::literal::Literal; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; +use vortex_expr::BoundExpression; +use vortex_expr::bound::binary; +use vortex_expr::bound::checked_add; +use vortex_expr::bound::get_item; +use vortex_expr::bound::gt; +use vortex_expr::bound::gt_eq; +use vortex_expr::bound::lit; +use vortex_expr::bound::lt; +use vortex_expr::bound::lt_eq; use vortex_expr::stats::bound::stat; +use vortex_functions::expr::bound::case_when; +use vortex_functions::expr::bound::ext_storage; use vortex_session::VortexSession; use crate::aggregate_fn::GeometryAabb; diff --git a/vortex-tensor/src/vector_search.rs b/vortex-tensor/src/vector_search.rs index 3603691a1c6..f436e764734 100644 --- a/vortex-tensor/src/vector_search.rs +++ b/vortex-tensor/src/vector_search.rs @@ -40,7 +40,7 @@ use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::scalar_fn::builtin::operators::Operator; use vortex_error::VortexResult; use crate::scalar_fns::cosine_similarity::CosineSimilarity; diff --git a/vortex-test/compat-gen/src/adapter.rs b/vortex-test/compat-gen/src/adapter.rs index f1c518a1186..37a16aa42c4 100644 --- a/vortex-test/compat-gen/src/adapter.rs +++ b/vortex-test/compat-gen/src/adapter.rs @@ -14,7 +14,7 @@ use tokio::runtime::Runtime; use vortex::VortexSessionDefault; use vortex::array::ArrayRef; use vortex::array::MaskFuture; -use vortex::array::expr::root; +use vortex::expr::root; use vortex::file::OpenOptionsSessionExt; use vortex::file::WriteOptionsSessionExt; use vortex::io::session::RuntimeSessionExt; diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index cff73ed08ae..5fd973db292 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -34,6 +34,7 @@ vortex-expr = { workspace = true } vortex-fastlanes = { workspace = true } vortex-file = { workspace = true, optional = true } vortex-fsst = { workspace = true } +vortex-functions = { workspace = true } vortex-io = { workspace = true } vortex-ipc = { workspace = true } vortex-layout = { workspace = true } diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 8cc8ee52c1c..75a8fb9fbb9 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -66,7 +66,7 @@ //! use vortex::VortexSessionDefault; //! use vortex::array::{IntoArray, stream::ArrayStreamExt}; //! use vortex::array::arrays::PrimitiveArray; -//! use vortex::array::expr::{gt, lit, root}; +//! use vortex::expr::{gt, lit, root}; //! use vortex::array::validity::Validity; //! use vortex::buffer::{ByteBufferMut, buffer}; //! use vortex::file::{OpenOptionsSessionExt, WriteOptionsSessionExt}; @@ -105,18 +105,22 @@ pub use vortex_array::aggregate_fn; use vortex_array::aggregate_fn::session::AggregateFnSession; pub use vortex_array::compute; use vortex_array::dtype::session::DTypeSession; -// vortex::expr is in the process of having its dependencies inverted, and will eventually be -// pulled back out into a vortex_expr crate. pub mod expr { //! Expression trees, their optimizer, and the analyses and rewrites built on them. - pub use vortex_array::expr::*; pub use vortex_expr::*; + pub use vortex_functions::expr::*; /// Statistics, stat expressions, and statistics-based pruning rewrites. pub mod stats { pub use vortex_array::expr::stats::*; pub use vortex_expr::stats::*; } + + /// Constructors for expressions whose children have already been bound and type-checked. + pub mod bound { + pub use vortex_expr::bound::*; + pub use vortex_functions::expr::bound::*; + } } use vortex_array::memory::MemorySession; use vortex_array::optimizer::kernels::KernelSession; @@ -125,6 +129,7 @@ use vortex_array::scalar_fn::session::ScalarFnSession; use vortex_array::session::ArraySession; use vortex_btrblocks::CompressionSession; use vortex_expr::stats::session::StatsSession; +pub use vortex_functions as functions; use vortex_io::session::RuntimeSession; use vortex_layout::session::LayoutSession; use vortex_session::VortexSession; @@ -346,6 +351,8 @@ impl VortexSessionDefault for VortexSession { .with::() .with::() .with::(); + vortex_expr::initialize(&session); + vortex_functions::initialize(&session); vortex_arrow::initialize(&session); #[cfg(feature = "parquet-variant")] vortex_parquet_variant::initialize(&session); @@ -383,15 +390,15 @@ mod test { use vortex_array::arrays::StructArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::FieldNames; - use vortex_array::expr::gt; - use vortex_array::expr::lit; - use vortex_array::expr::root; - use vortex_array::expr::select; use vortex_array::stream::ArrayStreamExt; use vortex_array::validity::Validity; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_expr::gt; + use vortex_expr::lit; + use vortex_expr::root; + use vortex_expr::select; use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder;